diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index d0eb11083..8996b3cd4 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -42,13 +42,22 @@ jobs: - name: Run flake checks run: | - time nix flake check --fallback --print-build-logs + start="$SECONDS" + nix flake check --fallback --print-build-logs + elapsed="$((SECONDS - start))" + printf 'Nix flake check: %dm %02ds\n' "$((elapsed / 60))" "$((elapsed % 60))" | tee -a "$GITHUB_STEP_SUMMARY" + # On macOS release validation builds the same native CLI dependency + # closure, so run its smoke test there rather than building it twice. - name: Build and smoke-test native CLI + if: matrix.os == 'ubuntu-latest' run: | - time nix build .#sce --fallback --out-link result --print-build-logs + start="$SECONDS" + nix build .#sce --fallback --out-link result --print-build-logs ./result/bin/sce --help ./result/bin/sce version + elapsed="$((SECONDS - start))" + printf 'Native CLI build + smoke: %dm %02ds\n' "$((elapsed / 60))" "$((elapsed % 60))" | tee -a "$GITHUB_STEP_SUMMARY" release-validation: name: Release validation (${{ matrix.os }}) @@ -78,4 +87,19 @@ jobs: - name: Build release package and audit portability run: | - time nix build .#ci-checks --fallback --out-link result-ci-checks --print-build-logs + start="$SECONDS" + nix build .#ci-checks --fallback --out-link result-ci-checks --print-build-logs + elapsed="$((SECONDS - start))" + printf 'Release validation: %dm %02ds\n' "$((elapsed / 60))" "$((elapsed % 60))" | tee -a "$GITHUB_STEP_SUMMARY" + + # On Darwin, .#sce-release is the native CLI with commit metadata. + # Smoke-test that exact binary, which is already built by .#ci-checks. + - name: Smoke-test macOS release CLI + if: matrix.os == 'macos-latest' + run: | + start="$SECONDS" + ./result-ci-checks/sce-release/bin/sce --help + ./result-ci-checks/sce-release/bin/sce version + elapsed="$((SECONDS - start))" + printf 'macOS release CLI smoke: %dm %02ds\n' "$((elapsed / 60))" "$((elapsed % 60))" | tee -a "$GITHUB_STEP_SUMMARY" + diff --git a/.github/workflows/quint.yml b/.github/workflows/quint.yml index 18395f894..7f4916bc1 100644 --- a/.github/workflows/quint.yml +++ b/.github/workflows/quint.yml @@ -55,8 +55,8 @@ jobs: needs: detect if: needs.detect.outputs.quint == 'true' runs-on: ubuntu-latest - # The Quint Connect Nix check compiles the CLI crate, so this needs more - # headroom than the pure-Quint steps alone required. + # The Rust/Quint Connect MBT suite already runs in the Nix CI cli-tests + # check on both Linux and macOS. Keep this workflow spec-only. timeout-minutes: 30 steps: - name: Harden the runner (Audit all outbound calls) @@ -70,12 +70,6 @@ jobs: - name: Install Nix uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 - - name: Enable Magic Nix Cache - uses: DeterminateSystems/magic-nix-cache-action@908b263ff629f4cc17666315b7fd3ec127c6244d # v14 - with: - use-flakehub: false - use-gha-cache: true - - name: Typecheck Quint spec run: nix run .#quint -- typecheck spec/mutation_cursor.qnt @@ -95,13 +89,6 @@ jobs: --max-samples=5000 --max-steps=20 - - name: Quint Connect model-based tests (Nix-pinned Rust + Quint) - # The entire MBT invocation goes through this Nix check rather than - # `cargo test` on the runner's own Cargo, so both the Rust toolchain - # and the Quint binary come from the repository's pinned flake - # inputs, never a second, unpinned Rust installation. - run: nix build .#checks.x86_64-linux.mutation-trace-quint-connect --print-build-logs - gate: name: Quint gate if: always() diff --git a/.gitignore b/.gitignore index e84719570..123b8b2cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .direnv/ node_modules/ -cli/target/ +cli/target*/ cli/package-fallback/ integrations/**/target/ result* diff --git a/.sce/config.json b/.sce/config.json index f9789ab7f..08495f79a 100644 --- a/.sce/config.json +++ b/.sce/config.json @@ -6,8 +6,8 @@ "integrations": { "optional_workflows": [], "target": [ - "claude", "opencode", + "claude", "pi", "codex" ] diff --git a/AGENTS.md b/AGENTS.md index cb700cc01..022929f93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -135,7 +135,7 @@ Run from repo root through Nix (do not call bare `bun` on the host). Working dir ## Testing notes -- Rust tests live inline in source files and in module test files such as `cli/src/services/setup/tests.rs`. +- Rust tests live inline in source files and in module test files such as `cli/src/services/mutation_trace/mbt/tests.rs`. - Repository Cargo builds must use `scripts/run-cli-cargo.sh`; it generates a fresh temporary Pkl payload, passes `SCE_CLI_GENERATED_INPUT_DIR` to Cargo, and cleans up the payload afterward. - Rust/Cargo commands should be executed through `nix develop`, even for one-off builds, tests, fmt, and clippy runs. - Prefer `nix flake check` for routine verification and avoid bare `cargo test` / `cargo check` / `cargo fmt --check` unless the user explicitly asks. @@ -258,7 +258,6 @@ Run from repo root through Nix (do not call bare `bun` on the host). Working dir - `context/overview.md` - `cli/Cargo.toml` - `cli/src/app.rs` -- `cli/src/services/setup/tests.rs` - `config/lib/bash-policy-plugin/package.json` - `config/lib/bash-policy-plugin/bash-policy-runtime.test.ts` - `config/lib/bash-policy-plugin/opencode-bash-policy-plugin.ts` diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 61826d60b..b16e47124 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -43,7 +43,7 @@ reqwest = { version = "0.13", default-features = false, features = ["json", "for serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.11" -tokio = { version = "1", default-features = false, features = ["rt", "io-util", "sync", "time", "macros", "rt-multi-thread"] } +tokio = { version = "1", default-features = false, features = ["rt", "io-util", "sync", "time", "macros", "rt-multi-thread", "process", "fs"] } toml = "0.9" tracing = "0.1" diff --git a/cli/build.rs b/cli/build.rs index afe0da848..1082ed3bc 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -187,7 +187,11 @@ fn stage_generated_input( let output_root = out_dir.join(PKL_OUTPUT_DIR); remove_path_if_exists(&output_root)?; - copy_tree(&generated_input_root.join(PKL_OUTPUT_DIR), &output_root)?; + copy_tree( + &generated_input_root.join(PKL_OUTPUT_DIR), + &output_root, + true, + )?; Ok(()) } @@ -257,7 +261,7 @@ fn stage_packaged_fallback(manifest_dir: &Path, out_dir: &Path) -> io::Result<() let source = fallback_root.join(directory); let destination = out_dir.join(directory); remove_path_if_exists(&destination)?; - copy_tree(&source, &destination)?; + copy_tree(&source, &destination, true)?; } Ok(()) @@ -342,14 +346,17 @@ fn stage_static_inputs( copy_tree( &manifest_dir.join("assets/hooks"), &static_root.join("hooks"), + true, )?; copy_tree( &manifest_dir.join(MIGRATIONS_ROOT), &static_root.join(MIGRATIONS_ROOT), + true, )?; copy_file( &repository_root.join("config/schema/agent-trace.schema.json"), &static_root.join("schema/agent-trace.schema.json"), + true, ) } @@ -366,33 +373,39 @@ fn stage_codex_target(out_dir: &Path) -> io::Result<()> { copy_tree( &pkl_output_root.join(CODEX_AGENTS_SOURCE_DIR), &destination_root.join(".agents"), + false, )?; copy_tree( &pkl_output_root.join(CODEX_HOOKS_SOURCE_DIR), &destination_root.join(".codex"), + false, ) } -fn copy_tree(source_root: &Path, destination_root: &Path) -> io::Result<()> { - println!("cargo:rerun-if-changed={}", source_root.display()); +fn copy_tree(source_root: &Path, destination_root: &Path, track_sources: bool) -> io::Result<()> { + if track_sources { + println!("cargo:rerun-if-changed={}", source_root.display()); + } let mut files = Vec::new(); collect_files(source_root, source_root, &mut files)?; files.sort_unstable_by(|left, right| left.relative_path.cmp(&right.relative_path)); for file in files { - println!("cargo:rerun-if-changed={}", file.absolute_path.display()); copy_file( &file.absolute_path, &destination_root.join(&file.relative_path), + track_sources, )?; } Ok(()) } -fn copy_file(source: &Path, destination: &Path) -> io::Result<()> { - println!("cargo:rerun-if-changed={}", source.display()); +fn copy_file(source: &Path, destination: &Path, track_source: bool) -> io::Result<()> { + if track_source { + println!("cargo:rerun-if-changed={}", source.display()); + } let parent = destination .parent() .ok_or_else(|| invalid_data(&"staged file must have a parent directory"))?; diff --git a/cli/src/app.rs b/cli/src/app.rs index 580be1507..b651cc964 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -57,21 +57,16 @@ pub(crate) trait HasLogger { fn logger(&self) -> &Self::Logger; } -#[allow(dead_code)] pub(crate) trait HasTelemetry { type Telemetry: TelemetryTrait; fn telemetry(&self) -> &Self::Telemetry; } -#[allow(dead_code)] pub(crate) trait HasFs { type Fs: services::capabilities::FsOps; - - fn fs(&self) -> &Self::Fs; } -#[allow(dead_code)] pub(crate) trait HasGit { type Git: services::capabilities::GitOps; @@ -113,23 +108,12 @@ where HasLogger::logger(self) } - #[allow(dead_code)] - pub(crate) fn fs(&self) -> &F { - HasFs::fs(self) - } - - #[allow(dead_code)] - pub(crate) fn git(&self) -> &G { - HasGit::git(self) - } - fn telemetry(&self) -> &T { HasTelemetry::telemetry(self) } /// Returns a context for a command-scoped repository root while preserving /// the runtime logger, telemetry, and capability dependencies. - #[allow(dead_code)] pub(crate) fn with_repo_root(&self, repo_root: impl Into) -> Self { Self { logger: self.logger, @@ -139,15 +123,6 @@ where repo_root: Some(repo_root.into()), } } - - /// Returns the resolved repository root path when available. - /// - /// Lifecycle providers use this during setup to avoid re-resolving - /// the repository root independently. - #[allow(dead_code)] - pub fn repo_root(&self) -> Option<&Path> { - HasRepoRoot::repo_root(self) - } } impl HasLogger for AppContext<'_, L, T, F, G> @@ -186,10 +161,6 @@ where G: services::capabilities::GitOps, { type Fs = F; - - fn fs(&self) -> &Self::Fs { - self.fs - } } impl HasGit for AppContext<'_, L, T, F, G> @@ -415,6 +386,9 @@ where Ok(command) } +#[cfg(test)] +mod doctor_reconciliation_cli_tests; + #[cfg(test)] mod tests { use std::future::{poll_fn, Future}; @@ -590,12 +564,12 @@ mod tests { let fs = services::capabilities::StdFsOps; let git = services::capabilities::ProcessGitOps; let context = AppContext::new(&logger, &telemetry, &fs, &git, None); - let result = run_command_lifecycle_with_context( + let result = Box::pin(run_command_lifecycle_with_context( args.into_iter().map(String::from), &services::command_registry::CommandRegistry::default(), &context, &mut Vec::new(), - ) + )) .await; if let Some(class) = expected_error { let error = result.unwrap_err(); @@ -695,7 +669,7 @@ mod tests { stdout.clear(); stderr.clear(); let missing_config = std::env::current_dir().unwrap().join("missing-config.json"); - let code = run_with_dependency_check_and_streams( + let code = Box::pin(run_with_dependency_check_and_streams( [ "sce", "config", @@ -707,7 +681,7 @@ mod tests { || Ok(()), &mut stdout, &mut stderr, - ) + )) .await; assert_eq!(code, ExitCode::from(4)); assert_eq!(stdout, b""); diff --git a/cli/src/app/doctor_reconciliation_cli_tests.rs b/cli/src/app/doctor_reconciliation_cli_tests.rs new file mode 100644 index 000000000..ed1dfff00 --- /dev/null +++ b/cli/src/app/doctor_reconciliation_cli_tests.rs @@ -0,0 +1,596 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitCode}; + +use serde_json::Value; + +use super::run_with_dependency_check_and_streams; +use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::default_paths::agent_trace_db_path_for_repository_at; +use crate::services::repository_identity::resolve::resolve_repository_identity; + +const ARGS_ENV: &str = "SCE_DOCTOR_RECONCILIATION_CLI_ARGS"; +const OUT_ENV: &str = "SCE_DOCTOR_RECONCILIATION_CLI_OUT"; +const CHILD_TEST: &str = "app::doctor_reconciliation_cli_tests::doctor_reconciliation_cli_child"; +const STATE_FILE: &str = "ref-maintenance.json"; +const LOCK_FILE: &str = "mutation-cursor.lock"; +const RECONCILIATION_DETAIL_PREFIX: &str = "Snapshot ref reconciliation"; + +const EXIT_SUCCESS: &str = "success"; +const EXIT_FAILURE: &str = "failure"; + +struct CliRun { + stdout: String, + stderr: String, +} + +impl CliRun { + fn json(&self) -> Value { + serde_json::from_str(&self.stdout) + .unwrap_or_else(|error| panic!("doctor stdout is not JSON ({error}):\n{}", self.stdout)) + } +} + +struct Sandbox { + dir: tempfile::TempDir, + repo: PathBuf, + repository_id: String, +} + +fn git(cwd: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git command"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +impl Sandbox { + fn new(remote: &str) -> Self { + let dir = tempfile::tempdir().expect("sandbox"); + let repo = dir.path().join("repo"); + std::fs::create_dir_all(&repo).expect("repo dir"); + git(&repo, &["init", "-q"]); + git(&repo, &["config", "user.email", "t@example.invalid"]); + git(&repo, &["config", "user.name", "T"]); + git(&repo, &["config", "commit.gpgsign", "false"]); + git(&repo, &["remote", "add", "origin", remote]); + std::fs::write(repo.join("tracked.txt"), "tracked\n").expect("tracked file"); + git(&repo, &["add", "tracked.txt"]); + git(&repo, &["commit", "-q", "-m", "initial"]); + let repository_id = resolve_repository_identity(&repo, None, "origin") + .expect("identity") + .identity + .repository_id; + Self { + dir, + repo, + repository_id, + } + } + + fn state_root(&self) -> PathBuf { + if cfg!(target_os = "macos") { + self.dir.path().join("Library").join("Application Support") + } else { + self.dir.path().join("state") + } + } + + fn db_path(&self) -> PathBuf { + agent_trace_db_path_for_repository_at(&self.state_root(), &self.repository_id) + .expect("db path") + } + + fn sce_dir(&self) -> PathBuf { + self.repo.join(".git").join("sce") + } + + fn state_file(&self) -> PathBuf { + self.sce_dir().join(STATE_FILE) + } + + async fn create_db(&self, repository_id: &str) { + let db = RepositoryAgentTraceDb::new_at(self.db_path()) + .await + .expect("create db"); + db.verify_or_initialize_repository_metadata(repository_id) + .await + .expect("initialize metadata"); + } + + fn orphan_pin(&self) -> String { + let tree = git(&self.repo, &["rev-parse", "HEAD^{tree}"]); + git( + &self.repo, + &[ + "update-ref", + &format!("refs/sce/mutation-cursor/main/{tree}"), + &tree, + ], + ); + tree + } + + fn refs(&self) -> String { + git( + &self.repo, + &[ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/sce/", + ], + ) + } + + fn state_bytes(&self) -> Option> { + std::fs::read(self.state_file()).ok() + } + + fn state_json(&self) -> Value { + serde_json::from_slice(&self.state_bytes().expect("maintenance state was written")) + .expect("state JSON") + } + + fn run(&self, args: &[&str]) -> CliRun { + let out = self.dir.path().join(format!("out-{}", args.join("-"))); + std::fs::create_dir_all(&out).expect("out dir"); + let output = Command::new(std::env::current_exe().expect("current exe")) + .args(["--exact", CHILD_TEST, "--nocapture"]) + .current_dir(&self.repo) + .env(ARGS_ENV, args.join(" ")) + .env(OUT_ENV, &out) + .env("HOME", self.dir.path()) + .env("XDG_CONFIG_HOME", self.dir.path().join("config")) + .env("XDG_STATE_HOME", self.state_root()) + .env("XDG_CACHE_HOME", self.dir.path().join("cache")) + .env("NO_COLOR", "1") + .env_remove("SCE_CONFIG_FILE") + .output() + .expect("child runs"); + assert!( + output.status.success(), + "child harness failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let run = CliRun { + stdout: std::fs::read_to_string(out.join("stdout")).expect("stdout"), + stderr: std::fs::read_to_string(out.join("stderr")).expect("stderr"), + }; + let exit = std::fs::read_to_string(out.join("exit")).expect("exit"); + assert_eq!( + exit, + EXIT_SUCCESS, + "sce {} exit status:\n{}\n{}", + args.join(" "), + run.stdout, + run.stderr + ); + run + } +} + +fn non_reconciliation_fix_results(report: &Value, sandbox_root: &Path) -> Vec { + let root = sandbox_root.to_string_lossy().into_owned(); + report["fix_results"] + .as_array() + .expect("fix_results array") + .iter() + .filter(|entry| { + !entry["detail"] + .as_str() + .expect("detail") + .starts_with(RECONCILIATION_DETAIL_PREFIX) + }) + .map(|entry| { + let mut entry = entry.clone(); + let detail = entry["detail"] + .as_str() + .expect("detail") + .replace(&root, ""); + entry["detail"] = Value::String(detail); + entry + }) + .collect() +} + +fn reconciliation_fix_result(report: &Value) -> Value { + let matches: Vec = report["fix_results"] + .as_array() + .expect("fix_results array") + .iter() + .filter(|entry| { + entry["detail"] + .as_str() + .expect("detail") + .starts_with(RECONCILIATION_DETAIL_PREFIX) + }) + .cloned() + .collect(); + assert_eq!(matches.len(), 1, "reconciliation reports exactly once"); + matches.into_iter().next().expect("one entry") +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_reconciliation_cli_child() { + let (Some(args), Some(out)) = ( + std::env::var_os(ARGS_ENV), + std::env::var_os(OUT_ENV).map(PathBuf::from), + ) else { + return; + }; + let mut command_line = vec!["sce".to_string()]; + command_line.extend( + args.to_string_lossy() + .split(' ') + .filter(|part| !part.is_empty()) + .map(str::to_string), + ); + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let exit_code = Box::pin(run_with_dependency_check_and_streams( + command_line, + || Ok(()), + &mut stdout, + &mut stderr, + )) + .await; + let exit = if exit_code == ExitCode::SUCCESS { + EXIT_SUCCESS + } else { + EXIT_FAILURE + }; + std::fs::write(out.join("exit"), exit).expect("exit"); + std::fs::write(out.join("stdout"), stdout).expect("stdout"); + std::fs::write(out.join("stderr"), stderr).expect("stderr"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_inspection_is_read_only_and_fix_reconciles_the_orphan_pin_once() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + let tree = sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + assert!(refs_before.contains(&tree)); + + let inspection = sandbox.run(&["doctor", "--format", "json"]); + let report = inspection.json(); + assert!( + report["ref_reconciliation_fix"].is_null(), + "plain doctor never reconciles: {}", + inspection.stderr + ); + assert_eq!(sandbox.refs(), refs_before); + assert_eq!(sandbox.state_bytes(), None); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "completed", "{}", fix.stdout); + assert_eq!(reconciliation["counts"]["deleted"], 1); + assert_eq!(reconciliation["counts"]["retained"], 0); + assert_eq!(reconciliation["counts"]["local_required"], 0); + assert!(reconciliation["failure"].is_null()); + assert!(reconciliation["state_warning"].is_null()); + assert_eq!(sandbox.refs(), ""); + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "fixed"); + assert_eq!(entry["category"], "mutation_scope_health"); + assert_eq!( + entry["detail"], + "Snapshot ref reconciliation completed: deleted 1, retained 0, locally required 0." + ); + + let state = sandbox.state_json(); + assert!(state["last_success"].is_number()); + assert_eq!(state["last_report"]["deleted"], 1); + assert!(state.get("consecutive_failures").is_none_or(|v| v == 0)); + + let again = sandbox.run(&["doctor", "--fix", "--format", "json"]).json(); + assert_eq!(again["ref_reconciliation_fix"]["counts"]["deleted"], 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_missing_database_without_creating_it_or_deleting_refs() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + let db_path = sandbox.db_path(); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "failed", "{}", fix.stdout); + assert_eq!(reconciliation["failure"]["kind"], "agent_trace_db_missing"); + assert!(reconciliation["counts"].is_null()); + assert_eq!(sandbox.refs(), refs_before); + assert!(!db_path.exists()); + let bootstrap: Vec = non_reconciliation_fix_results(&report, sandbox.dir.path()) + .iter() + .map(|entry| entry["detail"].as_str().expect("detail").to_string()) + .filter(|detail| detail.starts_with("Agent trace DB parent directory bootstrapped")) + .collect(); + assert_eq!( + bootstrap.len(), + 1, + "the parent directory comes from the separate global-state repair, not from reconciliation" + ); + for suffix in ["-wal", "-shm", "-tshm"] { + let mut sidecar = db_path.clone().into_os_string(); + sidecar.push(suffix); + assert!(!PathBuf::from(sidecar).exists()); + } + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "failed"); + assert!(entry["detail"] + .as_str() + .expect("detail") + .starts_with("Snapshot ref reconciliation failed (agent_trace_db_missing):")); + + let state = sandbox.state_json(); + assert_eq!(state["consecutive_failures"], 1); + assert!(state.get("last_success").is_none()); + assert!(report["fix_results"].is_array()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_repository_mismatch_without_touching_the_database_or_refs() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + let foreign = "some-other-repository"; + sandbox.create_db(foreign).await; + sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "failed", "{}", fix.stdout); + assert_eq!( + reconciliation["failure"]["kind"], + "agent_trace_db_repository_mismatch" + ); + assert!(reconciliation["failure"]["message"] + .as_str() + .expect("message") + .contains(foreign)); + assert_eq!(sandbox.refs(), refs_before); + assert_eq!(sandbox.state_json()["consecutive_failures"], 1); + + let (_, metadata) = + RepositoryAgentTraceDb::open_verified_existing_at(sandbox.db_path(), foreign) + .await + .expect("the foreign repository identity is untouched"); + assert_eq!(metadata.repository_id, foreign); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_busy_when_the_worktree_lock_is_held_and_keeps_other_repairs() { + let reference = Sandbox::new("https://example.invalid/org/repo-a.git"); + reference.create_db(&reference.repository_id).await; + reference.orphan_pin(); + let reference_report = reference + .run(&["doctor", "--fix", "--format", "json"]) + .json(); + + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + sandbox.orphan_pin(); + let refs_before = sandbox.refs(); + std::fs::create_dir_all(sandbox.sce_dir()).expect("sce dir"); + let held = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(sandbox.sce_dir().join(LOCK_FILE)) + .expect("lock file"); + held.try_lock().expect("hold the worktree lock externally"); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + let report = fix.json(); + drop(held); + + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!(reconciliation["outcome"], "busy", "{}", fix.stdout); + assert!(reconciliation["counts"].is_null()); + assert!(reconciliation["failure"].is_null()); + assert_eq!(sandbox.refs(), refs_before); + assert_eq!(sandbox.state_bytes(), None); + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "skipped"); + assert!(entry["detail"] + .as_str() + .expect("detail") + .starts_with("Snapshot ref reconciliation skipped:")); + assert_eq!( + non_reconciliation_fix_results(&report, sandbox.dir.path()), + non_reconciliation_fix_results(&reference_report, reference.dir.path()), + "independent doctor repairs are preserved" + ); +} + +const FUTURE_SKEW_TOLERANCE_MS: i64 = 5 * 60 * 1000; +const STATE_TIMESTAMP_FIELDS: [&str; 4] = + ["anchor", "last_success", "last_attempt", "last_advised"]; + +fn now_ms() -> i64 { + i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock") + .as_millis(), + ) + .expect("unix ms") +} + +impl Sandbox { + fn seed_state(&self, bytes: &[u8]) { + std::fs::create_dir_all(self.sce_dir()).expect("sce dir"); + std::fs::write(self.state_file(), bytes).expect("seed state"); + } + + fn make_state_unreadable(&self) { + std::fs::create_dir_all(self.state_file()).expect("directory in place of the state file"); + } +} + +fn assert_unreadable_state_outcome(fix: &CliRun, sandbox: &Sandbox, label: &str) { + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!( + reconciliation["outcome"], "completed_state_persist_failed", + "{label}: {}", + fix.stdout + ); + assert_eq!(reconciliation["counts"]["deleted"], 1, "{label}"); + assert_eq!(reconciliation["counts"]["retained"], 0, "{label}"); + assert_eq!(reconciliation["counts"]["local_required"], 0, "{label}"); + assert!(reconciliation["failure"].is_null(), "{label}"); + let warning = &reconciliation["state_warning"]; + assert_eq!(warning["kind"], "previous_state_unreadable", "{label}"); + assert!(warning["phase"].is_null(), "{label}"); + assert_eq!( + warning["message"], "Maintenance state could not be read; the update was not attempted.", + "{label}" + ); + assert!( + !warning["cause"].as_str().expect("cause").is_empty(), + "{label}" + ); + + let entry = reconciliation_fix_result(&report); + assert_eq!(entry["outcome"], "fixed", "{label}"); + let detail = entry["detail"].as_str().expect("detail"); + assert!( + detail.starts_with( + "Snapshot ref reconciliation completed: deleted 1, retained 0, locally required 0. Warning: Maintenance state could not be read; the update was not attempted. Cause: " + ), + "{label}: {detail}" + ); + assert!(!detail.contains("failed"), "{label}: {detail}"); + + assert_eq!(sandbox.refs(), "", "{label}: orphan pin must be deleted"); + assert!( + sandbox.state_file().is_dir(), + "{label}: the state path must stay a directory" + ); + assert_eq!( + std::fs::read_dir(sandbox.state_file()) + .expect("state directory") + .count(), + 0, + "{label}: the state directory must stay untouched" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable() { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + sandbox.orphan_pin(); + sandbox.make_state_unreadable(); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + + assert_unreadable_state_outcome(&fix, &sandbox, "unreadable"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn doctor_fix_recovers_from_every_initial_maintenance_state_condition() { + let now = now_ms(); + let day_ms = 24 * 60 * 60 * 1000; + let year_ms = 365 * day_ms; + let failed_streak = serde_json::json!({ + "version": 1, + "anchor": now - 10 * day_ms, + "last_attempt": now - 60_000, + "last_attempt_outcome": "failed", + "consecutive_failures": 3, + "last_failure": {"kind": "agent_trace_db_missing", "message": "seeded failure"}, + }); + let expired_advisory = serde_json::json!({ + "version": 1, + "anchor": now - 90 * day_ms, + "last_success": now - 60 * day_ms, + "last_advised": now - 30 * day_ms, + }); + let invalid_timestamps = serde_json::json!({ + "version": 1, + "anchor": now + 10 * year_ms, + "last_success": now + 10 * year_ms, + "last_attempt": now + 10 * year_ms, + "last_advised": now + 10 * year_ms, + "last_attempt_outcome": "failed", + "consecutive_failures": 2, + }); + let cases: Vec<(&str, Vec)> = vec![ + ("failed_streak", failed_streak.to_string().into_bytes()), + ( + "expired_advisory", + expired_advisory.to_string().into_bytes(), + ), + ( + "invalid_timestamps", + invalid_timestamps.to_string().into_bytes(), + ), + ("corrupt", b"not json".to_vec()), + ("oversized", vec![b' '; 5000]), + ]; + + for (label, bytes) in cases { + let sandbox = Sandbox::new("https://example.invalid/org/repo-a.git"); + sandbox.create_db(&sandbox.repository_id).await; + sandbox.orphan_pin(); + sandbox.seed_state(&bytes); + + let fix = sandbox.run(&["doctor", "--fix", "--format", "json"]); + + let report = fix.json(); + let reconciliation = &report["ref_reconciliation_fix"]; + assert_eq!( + reconciliation["outcome"], "completed", + "{label}: {}", + fix.stdout + ); + assert_eq!(reconciliation["counts"]["deleted"], 1, "{label}"); + assert!(reconciliation["failure"].is_null(), "{label}"); + assert!(reconciliation["state_warning"].is_null(), "{label}"); + assert_eq!(sandbox.refs(), "", "{label}: orphan pin must be reclaimed"); + + let after = now_ms(); + let state = sandbox.state_json(); + assert_eq!(state["version"], 1, "{label}"); + assert!(state["last_success"].is_number(), "{label}"); + assert_eq!(state["last_attempt_outcome"], "completed", "{label}"); + assert!( + state.get("consecutive_failures").is_none_or(|v| v == 0), + "{label}: failure streak must be cleared" + ); + assert!(state.get("last_failure").is_none(), "{label}"); + assert_eq!(state["last_report"]["deleted"], 1, "{label}"); + for field in STATE_TIMESTAMP_FIELDS { + if let Some(value) = state.get(field) { + let timestamp = value.as_i64().expect("timestamp"); + assert!( + timestamp <= after + FUTURE_SKEW_TOLERANCE_MS, + "{label}: {field} must be normalized, got {timestamp}" + ); + } + } + + let again = sandbox.run(&["doctor", "--fix", "--format", "json"]); + assert_eq!( + again.json()["ref_reconciliation_fix"]["outcome"], + "completed", + "{label}: the rewritten state is readable" + ); + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 8ad8eac70..313c38b66 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,7 +1,7 @@ +#![recursion_limit = "256"] mod app; mod cli_schema; mod command_surface; -#[allow(dead_code)] mod generated_migrations { include!(concat!(env!("OUT_DIR"), "/generated_migrations.rs")); } diff --git a/cli/src/services/agent_trace.rs b/cli/src/services/agent_trace.rs index 15c2bebf3..0120affdf 100644 --- a/cli/src/services/agent_trace.rs +++ b/cli/src/services/agent_trace.rs @@ -188,34 +188,22 @@ fn parse_commit_timestamp(commit_timestamp: &str) -> Result = OnceLock::new(); #[derive(Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) enum AgentTraceValidationError { - FileRead { path: String, message: String }, - InvalidJson { message: String }, SchemaValidation { errors: Vec }, } impl fmt::Display for AgentTraceValidationError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::FileRead { path, message } => { - write!(f, "Agent Trace JSON file could not be read at '{path}': {message}") - } - Self::InvalidJson { message } => { - write!(f, "Agent Trace JSON must be valid JSON: {message}") - } Self::SchemaValidation { errors } => write!( f, "Agent Trace JSON failed schema validation against embedded schema '{AGENT_TRACE_SCHEMA_PATH}': {}", @@ -227,7 +215,6 @@ impl fmt::Display for AgentTraceValidationError { impl Error for AgentTraceValidationError {} -#[allow(dead_code)] fn agent_trace_schema_validator() -> &'static Validator { AGENT_TRACE_SCHEMA_VALIDATOR.get_or_init(|| { let schema: Value = serde_json::from_str(AGENT_TRACE_SCHEMA_JSON) @@ -236,7 +223,6 @@ fn agent_trace_schema_validator() -> &'static Validator { }) } -#[allow(dead_code)] pub(crate) fn validate_agent_trace_value(value: &Value) -> Result<(), AgentTraceValidationError> { let mut errors = agent_trace_schema_validator() .iter_errors(value) @@ -252,26 +238,6 @@ pub(crate) fn validate_agent_trace_value(value: &Value) -> Result<(), AgentTrace Err(AgentTraceValidationError::SchemaValidation { errors }) } -#[allow(dead_code)] -pub(crate) fn validate_agent_trace_json(raw: &str) -> Result<(), AgentTraceValidationError> { - let value: Value = - serde_json::from_str(raw).map_err(|error| AgentTraceValidationError::InvalidJson { - message: error.to_string(), - })?; - - validate_agent_trace_value(&value) -} - -#[allow(dead_code)] -pub(crate) fn validate_agent_trace_file(path: &Path) -> Result<(), AgentTraceValidationError> { - let raw = - std::fs::read_to_string(path).map_err(|error| AgentTraceValidationError::FileRead { - path: path.display().to_string(), - message: error.to_string(), - })?; - - validate_agent_trace_json(&raw) -} /// Classification of a single hunk's origin relative to the AI candidate patch. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] @@ -305,7 +271,6 @@ pub struct Conversation { /// A related resource for a conversation entry. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] -#[allow(dead_code)] pub struct ConversationRelated { /// Free-form related resource type. #[serde(rename = "type")] @@ -371,40 +336,6 @@ pub struct AgentTrace { pub files: Vec, } -/// Classify a single `post_commit_patch` hunk against the corresponding -/// `intersection_patch` hunk (if any). -/// -/// Two hunks correspond when they share the same `old_start` value within the -/// same file. This is the slot-matching rule that aligns `intersection_patch` -/// hunks to `post_commit_patch` hunks for comparison. -/// -/// Returns: -/// - `HunkContributor::Ai` when the `intersection_patch` hunk exists and its -/// touched lines match the `post_commit_patch` hunk's touched lines exactly -/// (same count, same kind, same `line_number`, same content, in the same order). -/// - `HunkContributor::Mixed` when the `intersection_patch` hunk exists but its -/// touched lines differ from the `post_commit_patch` hunk's touched lines. -/// - `HunkContributor::Unknown` when no `intersection_patch` hunk with the same -/// `old_start` exists for this `post_commit_patch` hunk. -#[allow(dead_code)] -pub fn classify_hunk( - post_commit_hunk: &PatchHunk, - intersection_hunks: &[PatchHunk], -) -> HunkContributor { - let Some(intersection_hunk) = intersection_hunks - .iter() - .find(|h| h.old_start == post_commit_hunk.old_start) - else { - return HunkContributor::Unknown; - }; - - if hunks_match_exactly(post_commit_hunk, intersection_hunk) { - HunkContributor::Ai - } else { - HunkContributor::Mixed - } -} - type CoverageLineKey<'a> = (TouchedLineKind, u64, &'a str); fn coverage_line_key(line: &TouchedLine) -> CoverageLineKey<'_> { @@ -473,7 +404,6 @@ fn combined_model_id( } } -#[allow(dead_code)] pub(crate) fn patches_have_overlap( candidate_patch: &ParsedPatch, target_patch: &ParsedPatch, @@ -490,17 +420,6 @@ pub(crate) fn patch_has_touched_lines(patch: &ParsedPatch) -> bool { .any(|file| file.hunks.iter().any(|hunk| !hunk.lines.is_empty())) } -/// Check whether two hunks have identical touched lines in the same order. -fn hunks_match_exactly(left: &PatchHunk, right: &PatchHunk) -> bool { - if left.lines.len() != right.lines.len() { - return false; - } - left.lines.iter().zip(right.lines.iter()).all(|(ll, rl)| { - ll.kind == rl.kind && ll.line_number == rl.line_number && ll.content == rl.content - }) -} - -#[allow(dead_code)] pub(crate) fn range_content_hash(hunk: &PatchHunk) -> String { let mut input = Vec::new(); input.extend_from_slice(RANGE_CONTENT_HASH_INPUT_VERSION); @@ -650,25 +569,6 @@ fn build_trace_file( }) } -#[allow(dead_code)] -pub fn build_agent_trace( - constructed_patch: &ParsedPatch, - post_commit_patch: &ParsedPatch, - metadata: AgentTraceMetadataInput<'_>, -) -> Result { - let empty_mutation_ai_patch = ParsedPatch { files: Vec::new() }; - - build_agent_trace_from_evidence( - AgentTraceEvidence { - direct_patch: constructed_patch, - mutation_ai_patch: &empty_mutation_ai_patch, - }, - post_commit_patch, - metadata, - ) -} - -#[allow(dead_code)] pub fn build_agent_trace_from_evidence( evidence: AgentTraceEvidence<'_>, post_commit_patch: &ParsedPatch, diff --git a/cli/src/services/agent_trace/tests.rs b/cli/src/services/agent_trace/tests.rs index bf1f3b1fb..22ae65678 100644 --- a/cli/src/services/agent_trace/tests.rs +++ b/cli/src/services/agent_trace/tests.rs @@ -1,14 +1,15 @@ use super::{ - build_agent_trace, build_agent_trace_from_evidence, patches_have_overlap, - validate_agent_trace_value, AgentTraceEvidence, AgentTraceMetadataInput, AgentTraceVcsType, - LineRange, AGENT_TRACE_VERSION, + build_agent_trace_from_evidence, validate_agent_trace_value, AgentTraceEvidence, + AgentTraceMetadataInput, AgentTraceVcsType, AGENT_TRACE_VERSION, }; use crate::services::{ agent_trace::agent_trace_conversation_url, patch::{combine_patches, parse_patch, ParsedPatch}, - structured_patch::{derive_claude_structured_patch, ClaudeStructuredPatchDerivationResult}, }; -use serde_json::{json, Value}; +use serde_json::Value; + +const TEST_COMMIT_TIMESTAMP: &str = "2026-04-23T10:20:30Z"; +const TEST_COMMIT_REVISION: &str = "a0b1c2d3e4f5a6b7c8d9e0f11223344556677889"; #[derive(Clone, Copy)] struct AgentTraceScenario { @@ -17,9 +18,6 @@ struct AgentTraceScenario { golden: &'static str, } -const TEST_COMMIT_TIMESTAMP: &str = "2026-04-23T10:20:30Z"; -const TEST_COMMIT_REVISION: &str = "a0b1c2d3e4f5a6b7c8d9e0f11223344556677889"; - fn parse_fixtures(fixtures: &[&str]) -> Vec { fixtures .iter() @@ -27,53 +25,6 @@ fn parse_fixtures(fixtures: &[&str]) -> Vec { .collect() } -fn parse_fixture(fixture: &str) -> ParsedPatch { - parse_patch(fixture, None).expect("fixture patch should parse") -} - -fn build_evidence_trace_with_mutation_provenance( - direct_fixture: &str, - mutation_fixture: &str, - post_commit_fixture: &str, - mutation_model_id: Option<&str>, - mutation_session_ids: &[&str], -) -> super::AgentTrace { - let mut direct_patch = parse_patch(direct_fixture, Some(EVIDENCE_DIRECT_SESSION_ID)) - .expect("direct fixture patch should parse"); - for file in &mut direct_patch.files { - for hunk in &mut file.hunks { - hunk.model_id = Some(String::from(EVIDENCE_DIRECT_MODEL_ID)); - } - } - - let mut mutation_ai_patch = parse_fixture(mutation_fixture); - for (line, session_id) in mutation_ai_patch.files[0].hunks[0] - .lines - .iter_mut() - .zip(mutation_session_ids.iter().copied()) - { - line.session_id = Some(String::from(session_id)); - } - mutation_ai_patch.files[0].hunks[0].model_id = mutation_model_id.map(str::to_owned); - - let post_commit_patch = parse_fixture(post_commit_fixture); - build_agent_trace_from_evidence( - AgentTraceEvidence { - direct_patch: &direct_patch, - mutation_ai_patch: &mutation_ai_patch, - }, - &post_commit_patch, - AgentTraceMetadataInput { - commit_timestamp: TEST_COMMIT_TIMESTAMP, - commit_revision: TEST_COMMIT_REVISION, - vcs_type: Some(AgentTraceVcsType::Git), - tool_name: Some(EVIDENCE_TOOL_NAME), - tool_version: Some(EVIDENCE_TOOL_VERSION), - }, - ) - .expect("agent trace should build") -} - const TEXT_FILE_LIFECYCLE_RECONSTRUCTION_INCREMENTALS: &[&str] = &[ include_str!("fixtures/text_file_lifecycle_reconstruction/incremental_01.patch"), include_str!("fixtures/text_file_lifecycle_reconstruction/incremental_02.patch"), @@ -109,8 +60,12 @@ fn assert_builds_expected_agent_trace(scenario: AgentTraceScenario) { parse_patch(scenario.post_commit, None).expect("fixture patch should parse"); let golden: Value = serde_json::from_str(scenario.golden).expect("golden json should load"); validate_agent_trace_value(&golden).expect("golden json should validate against schema"); - let actual = build_agent_trace( - &constructed_patch, + let empty_mutation_ai_patch = ParsedPatch { files: Vec::new() }; + let actual = build_agent_trace_from_evidence( + AgentTraceEvidence { + direct_patch: &constructed_patch, + mutation_ai_patch: &empty_mutation_ai_patch, + }, &post_commit_patch, AgentTraceMetadataInput { commit_timestamp: TEST_COMMIT_TIMESTAMP, @@ -162,66 +117,6 @@ fn assert_builds_expected_agent_trace(scenario: AgentTraceScenario) { assert_eq!(actual_json["files"], expected_files); } -#[test] -fn patch_overlap_predicate_detects_matching_touched_lines() { - let candidate_patch = parse_fixture(include_str!( - "fixtures/hello_world_reconstruction/incremental_01.patch" - )); - let target_patch = parse_fixture(include_str!( - "fixtures/hello_world_reconstruction/post_commit.patch" - )); - - assert!(patches_have_overlap(&candidate_patch, &target_patch)); -} - -#[test] -fn patch_overlap_predicate_rejects_unrelated_touched_lines() { - let candidate_patch = parse_fixture(include_str!( - "fixtures/hello_world_reconstruction/incremental_01.patch" - )); - let target_patch = parse_fixture(include_str!( - "fixtures/poem_write_reconstruction/post_commit.patch" - )); - - assert!(!patches_have_overlap(&candidate_patch, &target_patch)); -} - -#[test] -fn patch_overlap_predicate_rejects_empty_or_untouched_patches() { - let candidate_patch = parse_fixture(include_str!( - "fixtures/hello_world_reconstruction/incremental_01.patch" - )); - let untouched_patch = parse_fixture(include_str!( - "../structured_patch/fixtures/write_create_empty/expected.patch" - )); - let empty_patch = parse_fixture(""); - - assert!(!patches_have_overlap(&candidate_patch, &untouched_patch)); - assert!(!patches_have_overlap(&untouched_patch, &candidate_patch)); - assert!(!patches_have_overlap(&empty_patch, &candidate_patch)); - assert!(!patches_have_overlap(&candidate_patch, &empty_patch)); -} - -#[test] -fn patch_overlap_predicate_accepts_claude_structured_patch_derivation() { - let payload: Value = serde_json::from_str(include_str!( - "../structured_patch/fixtures/edit_single_hunk/claude-post-tool-use.json" - )) - .expect("Claude structured fixture should parse"); - let expected_patch = parse_fixture(include_str!( - "../structured_patch/fixtures/edit_single_hunk/expected.patch" - )); - let derived_patch = match derive_claude_structured_patch("PostToolUse", &payload, 1, None) { - ClaudeStructuredPatchDerivationResult::Derived(derived) => derived.patch, - ClaudeStructuredPatchDerivationResult::Skipped(reason) => { - panic!("Claude structured fixture should derive a patch, got {reason}") - } - }; - - assert_eq!(derived_patch, expected_patch); - assert!(patches_have_overlap(&derived_patch, &expected_patch)); -} - #[test] fn average_age_reconstruction_matches_golden_agent_trace() { assert_builds_expected_agent_trace(AgentTraceScenario { @@ -276,103 +171,6 @@ fn poem_edit_reconstruction_matches_golden_agent_trace() { }); } -#[test] -fn poem_edit_reconstruction_maps_each_hunk_to_one_range() { - let mut constructed_patch = combine_patches(&parse_fixtures(&[ - include_str!("fixtures/poem_edit_reconstruction/incremental_01.patch"), - include_str!("fixtures/poem_edit_reconstruction/incremental_02.patch"), - ])); - let post_commit_patch = parse_patch( - include_str!("fixtures/poem_edit_reconstruction/post_commit.patch"), - None, - ) - .expect("fixture patch should parse"); - - let first_hunk_lines = &mut constructed_patch.files[0].hunks[0].lines; - first_hunk_lines[0].session_id = Some(String::from("session-z")); - first_hunk_lines[1].session_id = Some(String::from("session-a")); - - let agent_trace = build_agent_trace( - &constructed_patch, - &post_commit_patch, - AgentTraceMetadataInput { - commit_timestamp: TEST_COMMIT_TIMESTAMP, - commit_revision: TEST_COMMIT_REVISION, - vcs_type: Some(AgentTraceVcsType::Git), - tool_name: None, - tool_version: None, - }, - ) - .expect("agent trace should build"); - - let actual_json = serde_json::to_value(&agent_trace).expect("agent trace should serialize"); - validate_agent_trace_value(&actual_json).expect("actual json should validate against schema"); - - assert_eq!(agent_trace.files.len(), 1); - assert_eq!(agent_trace.files[0].path, "poem.md"); - assert_eq!(agent_trace.files[0].conversations.len(), 3); - assert_eq!( - agent_trace.files[0].conversations[0].related, - Some(vec![ - super::ConversationRelated { - kind: String::from("session"), - url: String::from("https://sce.crocoder.dev/sessions/session-a"), - }, - super::ConversationRelated { - kind: String::from("session"), - url: String::from("https://sce.crocoder.dev/sessions/session-z"), - }, - ]) - ); - assert_eq!(agent_trace.files[0].conversations[1].related, None); - assert_eq!(agent_trace.files[0].conversations[2].related, None); - assert_eq!( - actual_json["files"][0]["conversations"][0]["related"], - json!([ - { - "type": "session", - "url": "https://sce.crocoder.dev/sessions/session-a" - }, - { - "type": "session", - "url": "https://sce.crocoder.dev/sessions/session-z" - } - ]) - ); - assert!( - actual_json["files"][0]["conversations"][1]["related"].is_null(), - "conversations without session-backed lines should omit related" - ); - assert!( - actual_json["files"][0]["conversations"][2]["related"].is_null(), - "conversations without session-backed lines should omit related" - ); - assert_eq!( - agent_trace.files[0] - .conversations - .iter() - .map(|conversation| conversation.ranges.as_slice()) - .collect::>(), - vec![ - &[LineRange { - start_line: 1, - end_line: 8, - content_hash: "murmur3:25e05a40".to_string(), - }][..], - &[LineRange { - start_line: 10, - end_line: 16, - content_hash: "murmur3:bc5d346b".to_string(), - }][..], - &[LineRange { - start_line: 21, - end_line: 24, - content_hash: "murmur3:c8621bcb".to_string(), - }][..], - ] - ); -} - #[test] fn poem_write_reconstruction_matches_golden_agent_trace() { assert_builds_expected_agent_trace(AgentTraceScenario { @@ -404,40 +202,6 @@ fn file_rename_reconstruction_matches_golden_agent_trace() { }); } -#[test] -fn schema_validation_allows_agent_trace_without_vcs() { - let value = json!({ - "version": AGENT_TRACE_VERSION, - "id": "0196f25d-cf7f-7ca8-a652-8562c8a9f1d5", - "timestamp": TEST_COMMIT_TIMESTAMP, - "files": [] - }); - - validate_agent_trace_value(&value) - .expect("agent trace without vcs should validate against schema"); -} - -#[test] -fn schema_validation_rejects_vcs_missing_revision() { - let value = json!({ - "version": AGENT_TRACE_VERSION, - "id": "0196f25d-cf7f-7ca8-a652-8562c8a9f1d5", - "timestamp": TEST_COMMIT_TIMESTAMP, - "vcs": { - "type": "git" - }, - "files": [] - }); - - let error = validate_agent_trace_value(&value) - .expect_err("agent trace with vcs missing revision should fail validation"); - let rendered = error.to_string(); - assert!( - rendered.contains("\"revision\" is a required property"), - "expected vcs/revision validation failure, got: {rendered}" - ); -} - #[derive(Clone, Copy)] struct EvidenceScenario { direct: &'static str, @@ -573,164 +337,3 @@ fn mutation_only_no_provenance_evidence_matches_golden_agent_trace() { golden: include_str!("fixtures/mutation_only_no_provenance/golden.json"), }); } - -#[test] -fn mutation_only_evidence_emits_mutation_model_and_session() { - let trace = build_evidence_trace_with_mutation_provenance( - include_str!("fixtures/exclusive_without_direct/direct.patch"), - include_str!("fixtures/exclusive_without_direct/mutation_ai.patch"), - include_str!("fixtures/exclusive_without_direct/post_commit.patch"), - Some("gpt-5.6-sol"), - &[ - "cx-session-1", - "cx-session-1", - "cx-session-1", - "cx-session-1", - ], - ); - - let conversation = &trace.files[0].conversations[0]; - assert_eq!(conversation.contributor.kind, super::HunkContributor::Ai); - assert_eq!( - conversation.contributor.model_id.as_deref(), - Some("gpt-5.6-sol") - ); - assert_eq!( - conversation.related, - Some(vec![super::ConversationRelated { - kind: String::from("session"), - url: String::from("https://sce.crocoder.dev/sessions/cx-session-1"), - }]) - ); - validate_agent_trace_value( - &serde_json::to_value(&trace).expect("agent trace should serialize"), - ) - .expect("mutation-only agent trace should validate against schema"); -} - -#[test] -fn combined_evidence_unions_sessions_and_requires_model_agreement() { - let matching_trace = build_evidence_trace_with_mutation_provenance( - include_str!("fixtures/direct_plus_mutation/direct.patch"), - include_str!("fixtures/direct_plus_mutation/mutation_ai.patch"), - include_str!("fixtures/direct_plus_mutation/post_commit.patch"), - Some(EVIDENCE_DIRECT_MODEL_ID), - &["sess-a", "sess-z"], - ); - let matching_conversation = &matching_trace.files[0].conversations[0]; - assert_eq!( - matching_conversation.contributor.model_id.as_deref(), - Some(EVIDENCE_DIRECT_MODEL_ID) - ); - assert_eq!( - matching_conversation.related, - Some(vec![ - super::ConversationRelated { - kind: String::from("session"), - url: String::from("https://sce.crocoder.dev/sessions/sess-a"), - }, - super::ConversationRelated { - kind: String::from("session"), - url: String::from("https://sce.crocoder.dev/sessions/sess-direct"), - }, - super::ConversationRelated { - kind: String::from("session"), - url: String::from("https://sce.crocoder.dev/sessions/sess-z"), - }, - ]) - ); - - let conflicting_trace = build_evidence_trace_with_mutation_provenance( - include_str!("fixtures/direct_plus_mutation/direct.patch"), - include_str!("fixtures/direct_plus_mutation/mutation_ai.patch"), - include_str!("fixtures/direct_plus_mutation/post_commit.patch"), - Some("claude-opus-5"), - &["sess-a", "sess-direct"], - ); - assert_eq!( - conflicting_trace.files[0].conversations[0] - .contributor - .model_id, - None - ); - assert_eq!( - conflicting_trace.files[0].conversations[0] - .related - .as_ref() - .expect("conflicting evidence should retain related sessions") - .len(), - 2 - ); - - let unknown_trace = build_evidence_trace_with_mutation_provenance( - include_str!("fixtures/direct_plus_mutation/direct.patch"), - include_str!("fixtures/direct_plus_mutation/mutation_ai.patch"), - include_str!("fixtures/direct_plus_mutation/post_commit.patch"), - None, - &["sess-a", "sess-z"], - ); - assert_eq!( - unknown_trace.files[0].conversations[0].contributor.model_id, - None - ); - - for trace in [&matching_trace, &conflicting_trace, &unknown_trace] { - validate_agent_trace_value( - &serde_json::to_value(trace).expect("agent trace should serialize"), - ) - .expect("combined agent trace should validate against schema"); - } -} - -#[test] -fn direct_only_evidence_equals_direct_only_build_agent_trace() { - let direct = include_str!("fixtures/direct_only/direct.patch"); - let post_commit = include_str!("fixtures/direct_only/post_commit.patch"); - let empty_mutation_ai = include_str!("fixtures/direct_only/mutation_ai.patch"); - - let constructed_patch = parse_fixture(direct); - let post_commit_patch = parse_fixture(post_commit); - let mutation_ai_patch = parse_fixture(empty_mutation_ai); - - let metadata = AgentTraceMetadataInput { - commit_timestamp: TEST_COMMIT_TIMESTAMP, - commit_revision: TEST_COMMIT_REVISION, - vcs_type: Some(AgentTraceVcsType::Git), - tool_name: Some(EVIDENCE_TOOL_NAME), - tool_version: Some(EVIDENCE_TOOL_VERSION), - }; - - let compat = build_agent_trace(&constructed_patch, &post_commit_patch, metadata) - .expect("compat agent trace should build"); - let evidence = build_agent_trace_from_evidence( - AgentTraceEvidence { - direct_patch: &constructed_patch, - mutation_ai_patch: &mutation_ai_patch, - }, - &post_commit_patch, - metadata, - ) - .expect("evidence agent trace should build"); - - assert_eq!( - without_generated_identifiers(serde_json::to_value(&compat).expect("compat serializes")), - without_generated_identifiers( - serde_json::to_value(&evidence).expect("evidence serializes") - ) - ); -} - -fn without_generated_identifiers(mut trace: Value) -> Value { - trace["id"] = Value::Null; - if let Some(files) = trace["files"].as_array_mut() { - for conversation in files.iter_mut().flat_map(|file| { - file["conversations"] - .as_array_mut() - .expect("conversations should be an array") - .iter_mut() - }) { - conversation["url"] = Value::Null; - } - } - trace -} diff --git a/cli/src/services/agent_trace_db/lifecycle.rs b/cli/src/services/agent_trace_db/lifecycle.rs index e257f9d23..b00acce28 100644 --- a/cli/src/services/agent_trace_db/lifecycle.rs +++ b/cli/src/services/agent_trace_db/lifecycle.rs @@ -8,27 +8,29 @@ use crate::services::db::{bootstrap_db_parent, collect_db_path_health, DbSpec}; use crate::services::default_paths::agent_trace_db_path_for_repository; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; use crate::services::repository_identity::resolve::{ resolve_repository_identity, RepositoryIdentitySource, }; -use super::repository::{RepositoryAgentTraceDb, RepositoryAgentTraceDbSpec}; +use super::repository::{ + ExistingRepositoryDbError, RepositoryAgentTraceDb, RepositoryAgentTraceDbSpec, +}; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct AgentTraceDbLifecycle; impl ServiceLifecycle for AgentTraceDbLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::AgentTraceDb - } - - fn diagnose(&self, ctx: &C) -> Vec { - diagnose_agent_trace_db_health(ctx.repo_root()) + async fn diagnose(&self, ctx: &C) -> Vec { + diagnose_agent_trace_db_health(ctx.repo_root()).await } - fn fix(&self, ctx: &C, problems: &[HealthProblem]) -> Vec { + async fn fix( + &self, + ctx: &C, + problems: &[HealthProblem], + ) -> Vec { let should_bootstrap_parent = problems.iter().any(|problem| { problem.category == HealthCategory::GlobalState && problem.fixability == HealthFixability::AutoFixable @@ -56,11 +58,15 @@ impl ServiceLifecycle for AgentTraceDbLifecycle { } } - fn setup(&self, ctx: &C) -> Result { + async fn setup(&self, ctx: &C) -> Result { let repository_setup = match ctx.repo_root() { - Some(repo_root) => Some(initialize_repository_agent_trace_db(repo_root).context( - "Agent trace DB lifecycle setup failed while initializing repository database", - )?), + Some(repo_root) => Some( + initialize_repository_agent_trace_db(repo_root) + .await + .context( + "Agent trace DB lifecycle setup failed while initializing repository database", + )?, + ), None => None, }; @@ -84,7 +90,7 @@ struct RepositoryDatabaseSetup { database_path: PathBuf, } -fn initialize_repository_agent_trace_db(repo_root: &Path) -> Result { +async fn initialize_repository_agent_trace_db(repo_root: &Path) -> Result { let storage_config = config::resolve_agent_trace_storage_runtime_config(repo_root) .context("failed to resolve Agent Trace repository storage config")?; let storage_context = AgentTraceStorageContext { @@ -92,7 +98,7 @@ fn initialize_repository_agent_trace_db(repo_root: &Path) -> Result (String::from("explicit_config"), None), @@ -128,7 +134,7 @@ fn format_repository_storage_setup_message(setup: &RepositoryDatabaseSetup) -> S ) } -pub fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec { +pub async fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec { let mut problems = Vec::new(); let db_path = match resolve_lifecycle_agent_trace_db_path(repo_root) { @@ -153,11 +159,24 @@ pub fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec { - if let Err(error) = db.ensure_schema_ready_for_hooks() { - problems.push(HealthProblem { + problems.extend(inspect_existing_db_schema(&db_path, || {}).await); + problems +} + +async fn inspect_existing_db_schema( + db_path: &Path, + before_open: impl FnOnce(), +) -> Vec { + let mut problems = Vec::new(); + if db_path.exists() && !db_path.is_file() { + return problems; + } + + match RepositoryAgentTraceDb::open_existing_schema_ready_at(db_path, before_open).await { + Ok(_db) => {} + Err(ExistingRepositoryDbError::Missing { .. }) => {} + Err(ExistingRepositoryDbError::IncompatibleSchema(error)) => { + problems.push(HealthProblem { kind: HealthProblemKind::AgentTraceDbSchemaNotReady, category: HealthCategory::GlobalState, severity: HealthSeverity::Error, @@ -171,10 +190,13 @@ pub fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec { - problems.push(HealthProblem { + } + Err(error) => { + let error = match error { + ExistingRepositoryDbError::Unreadable(source) => source.to_string(), + other => other.to_string(), + }; + problems.push(HealthProblem { kind: HealthProblemKind::AgentTraceDbConnectionFailed, category: HealthCategory::GlobalState, severity: HealthSeverity::Error, @@ -188,7 +210,6 @@ pub fn diagnose_agent_trace_db_health(repo_root: Option<&Path>) -> Vec) -> Result) -> RepositoryDatabaseSetup { - RepositoryDatabaseSetup { - repository_id: String::from("repo-123"), - canonical_identity: String::from("github.com/example/repo"), - identity_source: String::from("remote_url"), - configured_remote: configured_remote.map(String::from), - source_instance_id: String::from("11111111-2222-3333-4444-555555555555"), - database_path: PathBuf::from("/tmp/agent-trace.db"), - } + fn dir_entries(dir: &Path) -> Vec { + let mut names = std::fs::read_dir(dir) + .expect("read dir") + .map(|entry| { + entry + .expect("entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect::>(); + names.sort(); + names + } + + async fn schema_objects(path: &Path) -> Vec { + let db = RepositoryAgentTraceDb::open_existing_without_migrations_at(path) + .await + .expect("read-only open"); + let mut names = db + .query_map("SELECT name FROM sqlite_master ORDER BY name", (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("schema query"); + names.sort(); + names } - #[test] - fn formatted_message_includes_source_instance_id_line() { - let message = format_repository_storage_setup_message(&setup(Some("origin"))); + async fn count_rows(path: &Path, table: &str) -> i64 { + let db = RepositoryAgentTraceDb::open_existing_without_migrations_at(path) + .await + .expect("read-only open"); + db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .expect("count query")[0] + } - assert!( - message - .contains("Agent Trace source-instance ID: 11111111-2222-3333-4444-555555555555"), - "expected message to report the source-instance ID, got: {message}" + #[tokio::test(flavor = "multi_thread")] + async fn missing_database_is_not_created_and_leaves_no_parent_directory() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir + .path() + .join("state") + .join("repos") + .join("id") + .join("agent-trace.db"); + + let problems = inspect_existing_db_schema(&path, || {}).await; + let again = inspect_existing_db_schema(&path, || {}).await; + + assert!(problems.is_empty()); + assert!(again.is_empty()); + assert!(!path.exists()); + assert!(dir_entries(dir.path()).is_empty()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn database_removed_immediately_before_open_is_not_recreated() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + RepositoryAgentTraceDb::new_at(&path).await.expect("create"); + assert!(path.is_file()); + + let problems = inspect_existing_db_schema(&path, || { + for entry in std::fs::read_dir(dir.path()).expect("read dir") { + std::fs::remove_file(entry.expect("entry").path()).expect("remove"); + } + }) + .await; + + assert!(problems.is_empty()); + assert!(dir_entries(dir.path()).is_empty()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn valid_database_is_inspected_without_problems_or_logical_change() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + let db = RepositoryAgentTraceDb::new_at(&path).await.expect("create"); + db.verify_or_initialize_repository_metadata("repo") + .await + .expect("metadata"); + drop(db); + let objects_before = schema_objects(&path).await; + let migrations_before = count_rows(&path, "__sce_migrations").await; + let metadata_before = count_rows(&path, "repository_metadata").await; + + let problems = inspect_existing_db_schema(&path, || {}).await; + let first_files = dir_entries(dir.path()); + let repeat = inspect_existing_db_schema(&path, || {}).await; + + assert!(problems.is_empty()); + assert!(repeat.is_empty()); + assert_eq!(dir_entries(dir.path()), first_files); + assert_eq!(schema_objects(&path).await, objects_before); + assert_eq!( + count_rows(&path, "__sce_migrations").await, + migrations_before ); - assert!( - message.contains("Agent Trace repository ID: repo-123"), - "expected message to still report the repository ID, got: {message}" + assert_eq!( + count_rows(&path, "repository_metadata").await, + metadata_before ); - assert!( - message.contains("Agent Trace configured remote: origin"), - "expected message to still report the configured remote, got: {message}" + } + + #[tokio::test(flavor = "multi_thread")] + async fn incompatible_schema_is_diagnosed_without_initializing_or_migrating() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + let raw = RepositoryAgentTraceDb::open_without_migrations_at(&path) + .await + .expect("create empty"); + raw.execute("CREATE TABLE unrelated (id INTEGER)", ()) + .await + .expect("unrelated table"); + drop(raw); + let objects_before = schema_objects(&path).await; + + let problems = inspect_existing_db_schema(&path, || {}).await; + + assert_eq!(problems.len(), 1); + assert_eq!( + problems[0].kind, + HealthProblemKind::AgentTraceDbSchemaNotReady ); + assert!(problems[0].summary.contains("is not ready")); + assert_eq!(schema_objects(&path).await, objects_before); + assert!(!objects_before.iter().any(|name| name == "__sce_migrations")); } - #[test] - fn formatted_message_omits_remote_line_but_keeps_source_instance_id() { - let message = format_repository_storage_setup_message(&setup(None)); + #[tokio::test(flavor = "multi_thread")] + async fn non_database_file_is_unreadable_and_left_byte_identical() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + std::fs::write( + &path, + b"this is not a sqlite database at all, just some text bytes....", + ) + .expect("write"); + let before = std::fs::read(&path).expect("read"); + + let problems = inspect_existing_db_schema(&path, || {}).await; - assert!(!message.contains("Agent Trace configured remote:")); - assert!(message - .contains("Agent Trace source-instance ID: 11111111-2222-3333-4444-555555555555")); + assert_eq!(problems.len(), 1); + assert_eq!( + problems[0].kind, + HealthProblemKind::AgentTraceDbConnectionFailed + ); + assert_eq!(std::fs::read(&path).expect("read"), before); + assert_eq!(dir_entries(dir.path()), vec!["agent-trace.db".to_string()]); } } diff --git a/cli/src/services/agent_trace_db/lock_contention_tests.rs b/cli/src/services/agent_trace_db/lock_contention_tests.rs deleted file mode 100644 index 6e3e225b5..000000000 --- a/cli/src/services/agent_trace_db/lock_contention_tests.rs +++ /dev/null @@ -1,1355 +0,0 @@ -use std::{ - fs, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Barrier, - }, - thread, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, -}; - -use anyhow::Result; - -use crate::services::db::{ - count_write_contention, count_write_statements, record_write_contention_timeline, - WriteContentionCounts, WriteContentionTimelineEvent, -}; - -use super::repository::RepositoryAgentTraceDb; -use super::{InsertMessageInsert, InsertPartInsert, MessageRole, PartType}; - -const DATABASE_LOCKED_ERROR: &str = "database is locked"; -const WRITE_CONTENTION_ERROR: &str = "under write contention"; -const LOCK_HOLD_DURATIONS_MS: &[u64] = &[ - 100, 250, 500, 750, 1_000, 1_500, 1_750, 2_000, 2_250, 2_500, 3_000, -]; -const RELIABLY_WITHIN_CONTENTION_BUDGET_MS: u64 = 1_000; -const RELIABLY_BEYOND_CONTENTION_BUDGET_MS: u64 = 3_000; -const WRITE_CONTENTION_MAX_ATTEMPTS: u32 = 2; -const BUSY_TIMEOUT_PRODUCTION_HOLD_MS: u64 = 100; -const ROUNDS_ENV: &str = "SCE_LOCK_CONTENTION_ROUNDS"; -const WRITERS_ENV: &str = "SCE_LOCK_CONTENTION_WRITERS"; -const STRICT_ENV: &str = "SCE_LOCK_CONTENTION_STRICT"; -const DEFAULT_ROUNDS: usize = 200; -const DEFAULT_DUPLICATE_WRITER_COUNTS: &[usize] = &[2, 3, 4, 8]; -const DEFAULT_DISTINCT_WRITER_COUNTS: &[usize] = &[2, 3, 4, 8]; -const MEASUREMENT_PREFIX: &str = "SCE_MEAS"; -const SLOW_OPERATION_MS: f64 = 500.0; -const STALL_MONITOR_TICK: Duration = Duration::from_millis(5); -const STALL_MONITOR_REPORT_MS: f64 = 50.0; - -fn unix_ms_now() -> f64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_secs_f64() - * 1_000.0 -} - -fn millis(duration: Duration) -> f64 { - duration.as_secs_f64() * 1_000.0 -} - -fn emit_measurement(record: &serde_json::Value) { - eprintln!("{MEASUREMENT_PREFIX} {record}"); -} - -fn timeline_json( - started_at: Instant, - timeline: &[(Instant, WriteContentionTimelineEvent)], -) -> serde_json::Value { - serde_json::Value::Array( - timeline - .iter() - .map(|(at, event)| { - let at_ms = millis(at.saturating_duration_since(started_at)); - match event { - WriteContentionTimelineEvent::AttemptStart => { - serde_json::json!({"event": "attempt_start", "at_ms": at_ms}) - } - WriteContentionTimelineEvent::AttemptEnd => { - serde_json::json!({"event": "attempt_end", "at_ms": at_ms}) - } - WriteContentionTimelineEvent::BackoffRequested(backoff) => serde_json::json!({ - "event": "backoff_requested", - "at_ms": at_ms, - "backoff_ms": millis(*backoff), - }), - WriteContentionTimelineEvent::BackoffSlept => { - serde_json::json!({"event": "backoff_slept", "at_ms": at_ms}) - } - } - }) - .collect(), - ) -} - -struct StallMonitor { - stop: Arc, - handle: thread::JoinHandle>, -} - -impl StallMonitor { - fn start() -> Self { - let stop = Arc::new(AtomicBool::new(false)); - let handle = { - let stop = Arc::clone(&stop); - thread::spawn(move || { - let mut gaps = Vec::new(); - while !stop.load(Ordering::Relaxed) { - let before = Instant::now(); - thread::sleep(STALL_MONITOR_TICK); - let late_ms = millis(before.elapsed()) - millis(STALL_MONITOR_TICK); - if late_ms >= STALL_MONITOR_REPORT_MS { - gaps.push((unix_ms_now(), late_ms)); - } - } - gaps - }) - }; - Self { stop, handle } - } - - fn finish(self) -> Vec<(f64, f64)> { - self.stop.store(true, Ordering::Relaxed); - self.handle.join().expect("stall monitor should not panic") - } -} - -fn stall_gaps_json(gaps: &[(f64, f64)]) -> serde_json::Value { - serde_json::Value::Array( - gaps.iter() - .map(|(ended_unix_ms, late_ms)| { - serde_json::json!({"ended_unix_ms": ended_unix_ms, "late_ms": late_ms}) - }) - .collect(), - ) -} - -#[derive(Clone, Debug, Eq, PartialEq)] -enum WriteOutcome { - Inserted, - AlreadyPresent, - Locked(String), - Other(String), -} - -impl WriteOutcome { - fn from_result(result: Result) -> Self { - match result { - Ok(true) => Self::Inserted, - Ok(false) => Self::AlreadyPresent, - Err(error) => { - let message = format!("{error:#}"); - if message.contains(DATABASE_LOCKED_ERROR) { - Self::Locked(message) - } else { - Self::Other(message) - } - } - } - } - - fn label(&self) -> &'static str { - match self { - Self::Inserted => "Ok(true)", - Self::AlreadyPresent => "Ok(false)", - Self::Locked(_) => "database is locked", - Self::Other(_) => "other error", - } - } -} - -fn unique_test_db_path(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-lock-contention-{label}-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") -} - -fn remove_test_db(db_path: &Path) { - if let Some(parent) = db_path.parent() { - fs::remove_dir_all(parent).expect("test DB directory should be removed"); - } -} - -fn create_repository_db(db_path: &Path) { - RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should be created up front"); -} - -fn open_production_connection(db_path: &Path) -> RepositoryAgentTraceDb { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("production hook connection should open") -} - -fn conversation_text_event( - session_id: &str, - message_id: &str, -) -> (InsertMessageInsert, InsertPartInsert) { - ( - InsertMessageInsert { - session_id: session_id.to_string(), - message_id: message_id.to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_000, - }, - InsertPartInsert { - part_type: PartType::Text, - text: format!("text for {message_id}"), - session_id: session_id.to_string(), - message_id: message_id.to_string(), - generated_at_unix_ms: 1_000, - }, - ) -} - -fn session_row_count(db: &RepositoryAgentTraceDb, table: &str, session_id: &str) -> i64 { - db.query_map( - &format!("SELECT COUNT(*) FROM {table} WHERE session_id = ?1"), - (session_id,), - |row| row.get::(0).map_err(Into::into), - ) - .expect("count query should succeed") - .into_iter() - .next() - .expect("count row should exist") -} - -fn env_usize(name: &str, default: usize) -> usize { - std::env::var(name) - .ok() - .and_then(|value| value.trim().parse().ok()) - .unwrap_or(default) -} - -fn env_writer_counts(default: &[usize]) -> Vec { - std::env::var(WRITERS_ENV) - .ok() - .map(|value| { - value - .split(',') - .filter_map(|count| count.trim().parse().ok()) - .collect::>() - }) - .filter(|counts| !counts.is_empty()) - .unwrap_or_else(|| default.to_vec()) -} - -fn strict_mode() -> bool { - std::env::var(STRICT_ENV).is_ok_and(|value| value == "1") -} - -struct LockBoundarySample { - hold: Duration, - outcome: WriteOutcome, - elapsed: Duration, - holder_released_after: Duration, - messages: i64, - parts: i64, - contention: WriteContentionCounts, - timeline: serde_json::Value, -} - -fn insert_while_write_lock_is_held(hold: Duration) -> LockBoundarySample { - let db_path = unique_test_db_path(&format!("boundary-{}ms", hold.as_millis())); - create_repository_db(&db_path); - let session_id = "cx_lock-boundary"; - - let lock_acquired = Arc::new(Barrier::new(2)); - let holder = { - let db_path = db_path.clone(); - let lock_acquired = Arc::clone(&lock_acquired); - thread::spawn(move || { - let holder = open_production_connection(&db_path); - holder - .execute("BEGIN IMMEDIATE", ()) - .expect("holder should acquire the write lock"); - lock_acquired.wait(); - let held_since = Instant::now(); - thread::sleep(hold); - holder - .execute("COMMIT", ()) - .expect("holder should release the write lock"); - held_since.elapsed() - }) - }; - - let writer = open_production_connection(&db_path); - lock_acquired.wait(); - let (message, part) = conversation_text_event(session_id, "cx:turn-1:user"); - let started_at = Instant::now(); - let ((result, contention), timeline) = record_write_contention_timeline(|| { - count_write_contention(|| writer.insert_conversation_text_event(message, part)) - }); - let elapsed = started_at.elapsed(); - let outcome = WriteOutcome::from_result(result); - let timeline = timeline_json(started_at, &timeline); - - let holder_released_after = holder.join().expect("holder thread should not panic"); - - let verifier = open_production_connection(&db_path); - let messages = session_row_count(&verifier, "messages", session_id); - let parts = session_row_count(&verifier, "parts", session_id); - drop((writer, verifier)); - remove_test_db(&db_path); - - LockBoundarySample { - hold, - outcome, - elapsed, - holder_released_after, - messages, - parts, - contention, - timeline, - } -} - -#[test] -fn lock_budget_boundary_characterizes_single_writer_blocked_by_begin_immediate_holder() { - let samples: Vec = LOCK_HOLD_DURATIONS_MS - .iter() - .map(|hold_ms| insert_while_write_lock_is_held(Duration::from_millis(*hold_ms))) - .collect(); - - eprintln!( - "\nlock-budget boundary (Agent Trace busy_timeout + write-contention retry + contention deadline)" - ); - eprintln!( - "hold_ms | outcome | elapsed_ms | holder_released_ms | attempts | outer_retries | exhaustions | messages | parts" - ); - for sample in &samples { - eprintln!( - "{:>7} | {:<18} | {:>10} | {:>18} | {:>8} | {:>13} | {:>11} | {:>8} | {:>5}", - sample.hold.as_millis(), - sample.outcome.label(), - sample.elapsed.as_millis(), - sample.holder_released_after.as_millis(), - sample.contention.attempts, - sample.contention.outer_retries, - sample.contention.exhaustions, - sample.messages, - sample.parts, - ); - if let WriteOutcome::Locked(message) | WriteOutcome::Other(message) = &sample.outcome { - eprintln!(" error: {message}"); - } - emit_measurement(&serde_json::json!({ - "kind": "boundary_sample", - "hold_ms": sample.hold.as_millis(), - "outcome": sample.outcome.label(), - "error": match &sample.outcome { - WriteOutcome::Locked(message) | WriteOutcome::Other(message) => Some(message), - WriteOutcome::Inserted | WriteOutcome::AlreadyPresent => None, - }, - "elapsed_ms": millis(sample.elapsed), - "holder_released_ms": millis(sample.holder_released_after), - "attempts": sample.contention.attempts, - "outer_retries": sample.contention.outer_retries, - "exhaustions": sample.contention.exhaustions, - "messages": sample.messages, - "parts": sample.parts, - "timeline": sample.timeline, - })); - } - - for sample in &samples { - let hold_ms = u64::try_from(sample.hold.as_millis()).expect("hold should fit in u64"); - assert_eq!( - sample.messages, sample.parts, - "hold {hold_ms}ms left a partial message/part pair" - ); - match &sample.outcome { - WriteOutcome::Inserted => assert_eq!(sample.messages, 1), - WriteOutcome::Locked(_) | WriteOutcome::Other(_) => assert_eq!(sample.messages, 0), - WriteOutcome::AlreadyPresent => panic!("hold {hold_ms}ms reported a phantom replay"), - } - assert!( - (1..=WRITE_CONTENTION_MAX_ATTEMPTS).contains(&sample.contention.attempts), - "hold {hold_ms}ms made {} attempt(s), outside the contention policy", - sample.contention.attempts - ); - assert_eq!( - sample.contention.outer_retries + 1, - sample.contention.attempts, - "hold {hold_ms}ms: every attempt after the first must be an admitted outer retry" - ); - if hold_ms <= RELIABLY_WITHIN_CONTENTION_BUDGET_MS { - assert_eq!( - sample.outcome, - WriteOutcome::Inserted, - "a {hold_ms}ms lock is well inside the contention budget" - ); - assert_eq!( - sample.contention.exhaustions, 0, - "a {hold_ms}ms lock must not exhaust the contention policy" - ); - } - if hold_ms >= RELIABLY_BEYOND_CONTENTION_BUDGET_MS { - match &sample.outcome { - WriteOutcome::Locked(message) => assert!( - message.contains(WRITE_CONTENTION_ERROR), - "a {hold_ms}ms lock should fail with a contention-exhaustion error, got {message}" - ), - outcome => panic!( - "a {hold_ms}ms lock is expected to exhaust the contention policy, got {outcome:?}" - ), - } - assert_eq!( - sample.contention.exhaustions, 1, - "a {hold_ms}ms lock should exhaust the contention policy exactly once" - ); - } - } -} - -#[test] -fn busy_timeout_production_insert_waits_for_begin_immediate_holder() { - let hold = Duration::from_millis(BUSY_TIMEOUT_PRODUCTION_HOLD_MS); - - let sample = insert_while_write_lock_is_held(hold); - - assert_eq!( - sample.outcome, - WriteOutcome::Inserted, - "insert_conversation_text_event should wait out a {}ms holder", - hold.as_millis() - ); - assert_eq!((sample.messages, sample.parts), (1, 1)); - assert!( - sample.elapsed >= hold / 2, - "the insert should have waited for the holder, took {:?}", - sample.elapsed - ); -} - -#[test] -fn agent_trace_db_write_contention_retry_hundred_ms_hold_succeeds_on_the_first_attempt() { - let hold = Duration::from_millis(BUSY_TIMEOUT_PRODUCTION_HOLD_MS); - - let (sample, counts) = count_write_contention(|| insert_while_write_lock_is_held(hold)); - - assert_eq!(sample.outcome, WriteOutcome::Inserted); - assert_eq!((sample.messages, sample.parts), (1, 1)); - assert_eq!( - (counts.attempts, counts.outer_retries, counts.exhaustions), - (1, 0, 0), - "Turso's busy timeout should absorb a {}ms hold without an outer retry", - hold.as_millis() - ); -} - -struct LatencyPercentiles { - p50: Duration, - p95: Duration, - p99: Duration, - max: Duration, -} - -fn nearest_rank(sorted: &[Duration], percentile: usize) -> Duration { - let rank = (percentile * sorted.len()).div_ceil(100).max(1); - sorted[rank - 1] -} - -fn latency_percentiles(latencies: &[Duration]) -> LatencyPercentiles { - if latencies.is_empty() { - return LatencyPercentiles { - p50: Duration::ZERO, - p95: Duration::ZERO, - p99: Duration::ZERO, - max: Duration::ZERO, - }; - } - let mut sorted = latencies.to_vec(); - sorted.sort_unstable(); - LatencyPercentiles { - p50: nearest_rank(&sorted, 50), - p95: nearest_rank(&sorted, 95), - p99: nearest_rank(&sorted, 99), - max: sorted[sorted.len() - 1], - } -} - -#[test] -fn lock_contention_latency_percentiles_use_nearest_rank() { - let latencies: Vec = (1..=100).rev().map(Duration::from_millis).collect(); - let percentiles = latency_percentiles(&latencies); - assert_eq!(percentiles.p50, Duration::from_millis(50)); - assert_eq!(percentiles.p95, Duration::from_millis(95)); - assert_eq!(percentiles.p99, Duration::from_millis(99)); - assert_eq!(percentiles.max, Duration::from_millis(100)); - - let single = latency_percentiles(&[Duration::from_millis(7)]); - assert_eq!(single.p50, Duration::from_millis(7)); - assert_eq!(single.p99, Duration::from_millis(7)); - - let empty = latency_percentiles(&[]); - assert_eq!(empty.max, Duration::ZERO); -} - -struct SlowOperation { - writer_index: usize, - started_unix_ms: f64, - elapsed: Duration, - outcome: WriteOutcome, - contention: WriteContentionCounts, - timeline: serde_json::Value, -} - -struct RoundResult { - slow_operations: Vec, - outcomes: Vec, - latencies: Vec, - contention: Vec, - max_elapsed: Duration, - messages: i64, - parts: i64, -} - -fn run_concurrent_round( - db_path: &Path, - writers: usize, - session_id: &str, - distinct_events: bool, -) -> RoundResult { - let start_together = Arc::new(Barrier::new(writers)); - let handles: Vec<_> = (0..writers) - .map(|writer_index| { - let db_path = db_path.to_path_buf(); - let start_together = Arc::clone(&start_together); - let session_id = session_id.to_string(); - thread::spawn(move || { - let db = open_production_connection(&db_path); - let message_id = if distinct_events { - format!("cx:turn-{writer_index}:user") - } else { - String::from("cx:turn-shared:user") - }; - let (message, part) = conversation_text_event(&session_id, &message_id); - start_together.wait(); - let started_unix_ms = unix_ms_now(); - let started_at = Instant::now(); - let ((result, contention), timeline) = record_write_contention_timeline(|| { - count_write_contention(|| db.insert_conversation_text_event(message, part)) - }); - let elapsed = started_at.elapsed(); - let outcome = WriteOutcome::from_result(result); - let slow = (millis(elapsed) >= SLOW_OPERATION_MS).then(|| SlowOperation { - writer_index, - started_unix_ms, - elapsed, - outcome: outcome.clone(), - contention, - timeline: timeline_json(started_at, &timeline), - }); - (outcome, elapsed, contention, slow) - }) - }) - .collect(); - - let mut slow_operations = Vec::new(); - let mut outcomes = Vec::with_capacity(writers); - let mut latencies = Vec::with_capacity(writers); - let mut contention = Vec::with_capacity(writers); - let mut max_elapsed = Duration::ZERO; - for handle in handles { - let (outcome, elapsed, counts, slow) = - handle.join().expect("writer thread should not panic"); - slow_operations.extend(slow); - outcomes.push(outcome); - latencies.push(elapsed); - contention.push(counts); - max_elapsed = max_elapsed.max(elapsed); - } - - let verifier = open_production_connection(db_path); - RoundResult { - slow_operations, - outcomes, - latencies, - contention, - max_elapsed, - messages: session_row_count(&verifier, "messages", session_id), - parts: session_row_count(&verifier, "parts", session_id), - } -} - -#[derive(Default)] -struct LevelSummary { - writers: usize, - rounds: usize, - clean_rounds: usize, - rounds_with_lock_exhaustion: usize, - inserted: usize, - already_present: usize, - lock_errors: usize, - other_errors: usize, - lost_events: i64, - orphan_rows: i64, - duplicate_rows: i64, - persisted_rows: i64, - attempts: u64, - outer_retries: u64, - exhaustions: u64, - max_elapsed: Duration, - latencies: Vec, - first_lock_error: Option, - first_other_error: Option, -} - -impl LevelSummary { - fn total_errors(&self) -> usize { - self.lock_errors + self.other_errors - } -} - -#[allow(clippy::too_many_lines)] -fn run_contention_level(writers: usize, rounds: usize, distinct_events: bool) -> LevelSummary { - let mode = if distinct_events { - "distinct" - } else { - "duplicate" - }; - let db_path = unique_test_db_path(&format!("{mode}-{writers}w")); - create_repository_db(&db_path); - - let mut summary = LevelSummary { - writers, - rounds, - ..LevelSummary::default() - }; - - let monitor = StallMonitor::start(); - let level_started_unix_ms = unix_ms_now(); - for round in 0..rounds { - let session_id = format!("cx_{mode}-{writers}w-round-{round}"); - let result = run_concurrent_round(&db_path, writers, &session_id, distinct_events); - - for slow in &result.slow_operations { - emit_measurement(&serde_json::json!({ - "kind": "slow_operation", - "mode": mode, - "writers": writers, - "round": round, - "writer_index": slow.writer_index, - "started_unix_ms": slow.started_unix_ms, - "elapsed_ms": millis(slow.elapsed), - "outcome": slow.outcome.label(), - "attempts": slow.contention.attempts, - "outer_retries": slow.contention.outer_retries, - "exhaustions": slow.contention.exhaustions, - "timeline": slow.timeline, - })); - } - summary.orphan_rows += (result.messages - result.parts).abs(); - let expected_max_rows = if distinct_events { writers } else { 1 }; - summary.duplicate_rows += - (result.messages - i64::try_from(expected_max_rows).expect("count fits i64")).max(0); - summary.persisted_rows += result.messages; - - assert_eq!( - result.messages, result.parts, - "{mode} round {round} with {writers} writers left orphaned rows" - ); - - let inserted = result - .outcomes - .iter() - .filter(|outcome| **outcome == WriteOutcome::Inserted) - .count(); - let already_present = result - .outcomes - .iter() - .filter(|outcome| **outcome == WriteOutcome::AlreadyPresent) - .count(); - let mut round_lock_errors = 0; - let mut round_other_errors = 0; - for outcome in &result.outcomes { - match outcome { - WriteOutcome::Locked(message) => { - round_lock_errors += 1; - summary - .first_lock_error - .get_or_insert_with(|| message.clone()); - } - WriteOutcome::Other(message) => { - round_other_errors += 1; - summary - .first_other_error - .get_or_insert_with(|| message.clone()); - } - WriteOutcome::Inserted | WriteOutcome::AlreadyPresent => {} - } - } - - assert_eq!( - i64::try_from(inserted).expect("count fits i64"), - result.messages, - "{mode} round {round}: Ok(true) count must equal persisted message rows" - ); - - if distinct_events { - assert_eq!( - already_present, 0, - "distinct events must never report Ok(false)" - ); - summary.lost_events += - i64::try_from(writers).expect("count fits i64") - result.messages; - } else { - assert!(inserted <= 1, "duplicate delivery inserted more than once"); - if round_lock_errors + round_other_errors < writers { - assert_eq!( - inserted, 1, - "a duplicate round with at least one completed writer must persist the event" - ); - } - } - - let expected_rows = if distinct_events { writers } else { 1 }; - if round_lock_errors + round_other_errors == 0 - && usize::try_from(result.messages).expect("count fits usize") == expected_rows - { - summary.clean_rounds += 1; - } - if round_lock_errors > 0 { - summary.rounds_with_lock_exhaustion += 1; - } - summary.inserted += inserted; - summary.already_present += already_present; - summary.lock_errors += round_lock_errors; - summary.other_errors += round_other_errors; - for counts in &result.contention { - assert!( - counts.attempts <= WRITE_CONTENTION_MAX_ATTEMPTS, - "{mode} round {round}: a writer made {} attempt(s), beyond the contention policy", - counts.attempts - ); - summary.attempts += u64::from(counts.attempts); - summary.outer_retries += u64::from(counts.outer_retries); - summary.exhaustions += u64::from(counts.exhaustions); - } - summary.max_elapsed = summary.max_elapsed.max(result.max_elapsed); - summary.latencies.extend(result.latencies); - } - - let stall_gaps = monitor.finish(); - let latency = latency_percentiles(&summary.latencies); - emit_measurement(&serde_json::json!({ - "kind": "concurrent_level", - "mode": mode, - "writers": writers, - "rounds": rounds, - "started_unix_ms": level_started_unix_ms, - "ended_unix_ms": unix_ms_now(), - "total_writes": writers * rounds, - "expected_rows": if distinct_events { writers * rounds } else { rounds }, - "persisted_rows": summary.persisted_rows, - "lost_events": summary.lost_events, - "lock_errors": summary.lock_errors, - "other_errors": summary.other_errors, - "orphan_rows": summary.orphan_rows, - "duplicate_rows": summary.duplicate_rows, - "inserted": summary.inserted, - "already_present": summary.already_present, - "attempts": summary.attempts, - "outer_retries": summary.outer_retries, - "exhaustions": summary.exhaustions, - "p50_ms": millis(latency.p50), - "p95_ms": millis(latency.p95), - "p99_ms": millis(latency.p99), - "max_ms": millis(latency.max), - "first_lock_error": summary.first_lock_error, - "first_other_error": summary.first_other_error, - "stall_gaps": stall_gaps_json(&stall_gaps), - })); - - remove_test_db(&db_path); - summary -} - -fn report_levels(title: &str, summaries: &[LevelSummary]) { - eprintln!("\n{title}"); - eprintln!( - "writers | rounds | clean rounds | rounds w/ lock exhaustion | lock errors | other errors | Ok(true) | Ok(false) | lost events | attempts | outer retries | exhaustions | max elapsed ms | p50 ms | p95 ms | p99 ms | max ms" - ); - for summary in summaries { - let latency = latency_percentiles(&summary.latencies); - eprintln!( - "{:>7} | {:>6} | {:>12} | {:>25} | {:>11} | {:>12} | {:>8} | {:>9} | {:>11} | {:>8} | {:>13} | {:>11} | {:>14} | {:>6} | {:>6} | {:>6} | {:>6}", - summary.writers, - summary.rounds, - summary.clean_rounds, - summary.rounds_with_lock_exhaustion, - summary.lock_errors, - summary.other_errors, - summary.inserted, - summary.already_present, - summary.lost_events, - summary.attempts, - summary.outer_retries, - summary.exhaustions, - summary.max_elapsed.as_millis(), - latency.p50.as_millis(), - latency.p95.as_millis(), - latency.p99.as_millis(), - latency.max.as_millis(), - ); - } - for summary in summaries { - if let Some(message) = &summary.first_lock_error { - eprintln!("{} writers first lock error: {message}", summary.writers); - } - if let Some(message) = &summary.first_other_error { - eprintln!("{} writers first other error: {message}", summary.writers); - } - } -} - -fn assert_strict_if_requested(summaries: &[LevelSummary]) { - if !strict_mode() { - return; - } - for summary in summaries { - assert_eq!( - summary.total_errors(), - 0, - "{} concurrent writers produced {} lock error(s) and {} other error(s)", - summary.writers, - summary.lock_errors, - summary.other_errors - ); - assert_eq!( - summary.lost_events, 0, - "{} concurrent writers lost {} distinct event(s)", - summary.writers, summary.lost_events - ); - assert_eq!( - summary.exhaustions, 0, - "{} concurrent writers exhausted the contention policy {} time(s)", - summary.writers, summary.exhaustions - ); - } -} - -#[test] -#[ignore = "lock-contention characterization; run with --ignored --nocapture"] -fn concurrent_duplicate_delivery_persists_each_event_once_under_write_contention() { - let rounds = env_usize(ROUNDS_ENV, DEFAULT_ROUNDS); - let summaries: Vec = env_writer_counts(DEFAULT_DUPLICATE_WRITER_COUNTS) - .into_iter() - .map(|writers| run_contention_level(writers, rounds, false)) - .collect(); - - report_levels( - "concurrent duplicate delivery (same logical event, Agent Trace write-contention policy)", - &summaries, - ); - assert_strict_if_requested(&summaries); -} - -#[test] -#[ignore = "lock-contention characterization; run with --ignored --nocapture"] -fn concurrent_distinct_events_persist_every_event_under_write_contention() { - let rounds = env_usize(ROUNDS_ENV, DEFAULT_ROUNDS); - let summaries: Vec = env_writer_counts(DEFAULT_DISTINCT_WRITER_COUNTS) - .into_iter() - .map(|writers| run_contention_level(writers, rounds, true)) - .collect(); - - report_levels( - "concurrent distinct events (one unique event per writer, Agent Trace write-contention policy)", - &summaries, - ); - assert_strict_if_requested(&summaries); -} - -const SCE_BIN_ENV: &str = "SCE_BIN"; - -struct HookLevelSummary { - writers: usize, - expected: i64, - persisted_messages: i64, - persisted_parts: i64, - nonzero_exits: usize, -} - -fn assert_hook_levels(levels: &[HookLevelSummary], strict: bool) { - for level in levels { - assert!( - level.persisted_messages <= level.expected, - "{} hook writers persisted {} messages for {} distinct events", - level.writers, - level.persisted_messages, - level.expected - ); - assert!( - level.persisted_parts <= level.expected, - "{} hook writers persisted {} parts for {} distinct events", - level.writers, - level.persisted_parts, - level.expected - ); - if strict { - assert_eq!( - level.persisted_messages, level.expected, - "{} hook writers lost message rows", - level.writers - ); - assert_eq!( - level.persisted_parts, level.expected, - "{} hook writers lost part rows", - level.writers - ); - assert_eq!( - level.nonzero_exits, 0, - "{} hook writers had non-zero exits", - level.writers - ); - } - } - - if strict { - let total_lost: i64 = levels - .iter() - .map(|level| level.expected - level.persisted_messages) - .sum(); - let total_nonzero_exits: usize = levels.iter().map(|level| level.nonzero_exits).sum(); - assert_eq!(total_lost, 0, "real hook processes lost distinct events"); - assert_eq!( - total_nonzero_exits, 0, - "real hook processes exited with a non-zero status" - ); - } -} - -#[test] -fn lock_contention_hook_level_assertions_reject_over_persistence() { - let over_persisted = [HookLevelSummary { - writers: 2, - expected: 4, - persisted_messages: 5, - persisted_parts: 5, - nonzero_exits: 0, - }]; - let result = std::panic::catch_unwind(|| assert_hook_levels(&over_persisted, false)); - assert!( - result.is_err(), - "over-persistence must fail even outside strict mode" - ); -} - -#[test] -fn lock_contention_hook_level_assertions_enforce_strict_loss_and_exit_status() { - let lost = [HookLevelSummary { - writers: 2, - expected: 4, - persisted_messages: 3, - persisted_parts: 3, - nonzero_exits: 0, - }]; - assert_hook_levels(&lost, false); - assert!(std::panic::catch_unwind(|| assert_hook_levels(&lost, true)).is_err()); - - let nonzero_exit = [HookLevelSummary { - writers: 2, - expected: 4, - persisted_messages: 4, - persisted_parts: 4, - nonzero_exits: 1, - }]; - assert_hook_levels(&nonzero_exit, false); - assert!(std::panic::catch_unwind(|| assert_hook_levels(&nonzero_exit, true)).is_err()); - - let clean = [HookLevelSummary { - writers: 3, - expected: 6, - persisted_messages: 6, - persisted_parts: 6, - nonzero_exits: 0, - }]; - assert_hook_levels(&clean, true); -} -const DEFAULT_PROCESS_ROUNDS: usize = 50; -const DEFAULT_PROCESS_WRITER_COUNTS: &[usize] = &[2, 3, 4]; - -struct HookRound { - nonzero_exits: usize, - stderr_lines: Vec, - latencies: Vec, - slow_processes: Vec<(usize, f64, Duration)>, -} - -struct HookProcessHarness { - sce: PathBuf, - work: PathBuf, - repo: PathBuf, - db_path: PathBuf, -} - -impl HookProcessHarness { - fn command(&self) -> std::process::Command { - let mut command = std::process::Command::new(&self.sce); - command - .current_dir(&self.repo) - .env("XDG_STATE_HOME", self.work.join("state")) - .env("XDG_CONFIG_HOME", self.work.join("config")) - .env("XDG_DATA_HOME", self.work.join("data")); - command - } - - fn create(sce: PathBuf, label: &str) -> Self { - let work = unique_test_db_path(label) - .parent() - .expect("unique path has a parent") - .to_path_buf(); - let repo = work.join("repo"); - fs::create_dir_all(&repo).expect("repo dir should be created"); - for args in [ - vec!["init", "-q"], - vec![ - "remote", - "add", - "origin", - "https://example.invalid/sce/lock-contention.git", - ], - vec![ - "-c", - "user.name=t", - "-c", - "user.email=t@example.invalid", - "commit", - "-q", - "--allow-empty", - "-m", - "init", - ], - ] { - let status = std::process::Command::new("git") - .args(&args) - .current_dir(&repo) - .status() - .expect("git should spawn"); - assert!(status.success(), "git {args:?} should succeed"); - } - - let mut harness = Self { - sce, - work, - repo, - db_path: PathBuf::new(), - }; - - let setup = harness - .command() - .args(["setup", "--codex", "--non-interactive", "--hooks"]) - .output() - .expect("sce setup should spawn"); - assert!( - setup.status.success(), - "sce setup failed: {}", - String::from_utf8_lossy(&setup.stderr) - ); - - let doctor = harness - .command() - .args(["doctor", "--format", "json"]) - .output() - .expect("sce doctor should spawn"); - let report: serde_json::Value = - serde_json::from_slice(&doctor.stdout).expect("doctor should print JSON"); - harness.db_path = PathBuf::from( - report["agent_trace_db"]["path"] - .as_str() - .expect("doctor should report agent_trace_db.path"), - ); - assert!( - harness.db_path.is_file(), - "repository Agent Trace DB should exist" - ); - harness - } - - fn run_round(&self, writers: usize, session_id: &str) -> HookRound { - use std::io::Write; - use std::process::Stdio; - - let children: Vec<_> = (0..writers) - .map(|_| { - self.command() - .args(["hooks", "codex"]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .expect("sce hooks codex should spawn") - }) - .collect(); - - let release = Arc::new(Barrier::new(writers)); - let feeders: Vec<_> = children - .into_iter() - .enumerate() - .map(|(writer_index, mut child)| { - let mut stdin = child.stdin.take().expect("stdin should be piped"); - let release = Arc::clone(&release); - let payload = serde_json::json!({ - "hook_event_name": "UserPromptSubmit", - "session_id": session_id, - "turn_id": format!("w{writer_index}"), - "prompt": format!("prompt {session_id} w{writer_index}"), - }) - .to_string(); - thread::spawn(move || { - release.wait(); - let started_unix_ms = unix_ms_now(); - let started_at = Instant::now(); - stdin - .write_all(payload.as_bytes()) - .expect("payload should be written"); - drop(stdin); - let output = child - .wait_with_output() - .expect("hook process should finish"); - (output, started_unix_ms, started_at.elapsed()) - }) - }) - .collect(); - - let mut nonzero_exits = 0; - let mut stderr_lines = Vec::new(); - let mut latencies = Vec::with_capacity(writers); - let mut slow_processes = Vec::new(); - for (writer_index, feeder) in feeders.into_iter().enumerate() { - let (output, started_unix_ms, elapsed) = - feeder.join().expect("feeder thread should not panic"); - latencies.push(elapsed); - if millis(elapsed) >= SLOW_OPERATION_MS { - slow_processes.push((writer_index, started_unix_ms, elapsed)); - } - if !output.status.success() { - nonzero_exits += 1; - } - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if !stderr.is_empty() { - stderr_lines.push(stderr); - } - } - HookRound { - nonzero_exits, - stderr_lines, - latencies, - slow_processes, - } - } -} - -#[test] -#[ignore = "end-to-end hook-process lock contention; set SCE_BIN and run with --ignored --nocapture"] -#[allow(clippy::too_many_lines)] -fn concurrent_real_codex_hook_processes_persist_every_distinct_event() { - let Some(sce) = std::env::var_os(SCE_BIN_ENV).map(PathBuf::from) else { - eprintln!("{SCE_BIN_ENV} is not set; skipping end-to-end hook-process reproduction"); - return; - }; - let rounds = env_usize(ROUNDS_ENV, DEFAULT_PROCESS_ROUNDS); - - eprintln!("\nreal `sce hooks codex` UserPromptSubmit processes (distinct events)"); - eprintln!( - "writers | rounds | expected | persisted msgs | persisted parts | lost | rounds w/ loss | non-zero exits | stderr lines | p50 ms | p95 ms | p99 ms | max ms" - ); - let mut levels = Vec::new(); - for writers in env_writer_counts(DEFAULT_PROCESS_WRITER_COUNTS) { - let harness = HookProcessHarness::create(sce.clone(), &format!("hooks-{writers}w")); - let mut persisted_messages = 0; - let mut persisted_parts = 0; - let mut rounds_with_loss = 0; - let mut nonzero_exits = 0; - let mut stderr_samples = Vec::new(); - let mut latencies = Vec::with_capacity(writers * rounds); - let mut fail_open_lost = 0; - let monitor = StallMonitor::start(); - let level_started_unix_ms = unix_ms_now(); - - for round in 0..rounds { - let session_id = format!("lock-contention-{writers}w-r{round}"); - let HookRound { - nonzero_exits: round_nonzero, - stderr_lines: round_stderr, - latencies: round_latencies, - slow_processes, - } = harness.run_round(writers, &session_id); - for (writer_index, started_unix_ms, elapsed) in slow_processes { - emit_measurement(&serde_json::json!({ - "kind": "slow_hook_process", - "writers": writers, - "round": round, - "writer_index": writer_index, - "started_unix_ms": started_unix_ms, - "elapsed_ms": millis(elapsed), - })); - } - for stderr in &round_stderr { - emit_measurement(&serde_json::json!({ - "kind": "hook_stderr", - "writers": writers, - "round": round, - "stderr": stderr, - })); - } - nonzero_exits += round_nonzero; - stderr_samples.extend(round_stderr); - latencies.extend(round_latencies); - - let verifier = open_production_connection(&harness.db_path); - let prefixed = format!("cx_{session_id}"); - let messages = session_row_count(&verifier, "messages", &prefixed); - let parts = session_row_count(&verifier, "parts", &prefixed); - assert_eq!(messages, parts, "hook round left orphaned rows"); - if usize::try_from(messages).expect("count fits usize") != writers { - rounds_with_loss += 1; - let round_lost = i64::try_from(writers).expect("count fits i64") - messages; - let round_fail_open = - (round_lost - i64::try_from(round_nonzero).expect("count fits i64")).max(0); - fail_open_lost += round_fail_open; - emit_measurement(&serde_json::json!({ - "kind": "hook_round_loss", - "writers": writers, - "round": round, - "lost": round_lost, - "nonzero_exits": round_nonzero, - "fail_open_lost": round_fail_open, - "ended_unix_ms": unix_ms_now(), - })); - } - persisted_messages += messages; - persisted_parts += parts; - } - - let expected = i64::try_from(writers * rounds).expect("count fits i64"); - let lost = expected - persisted_messages; - let latency = latency_percentiles(&latencies); - eprintln!( - "{:>7} | {:>6} | {:>8} | {:>14} | {:>15} | {:>4} | {:>14} | {:>14} | {:>12} | {:>6} | {:>6} | {:>6} | {:>6}", - writers, - rounds, - expected, - persisted_messages, - persisted_parts, - lost, - rounds_with_loss, - nonzero_exits, - stderr_samples.len(), - latency.p50.as_millis(), - latency.p95.as_millis(), - latency.p99.as_millis(), - latency.max.as_millis(), - ); - if let Some(sample) = stderr_samples.first() { - eprintln!(" first stderr: {sample}"); - } - let stall_gaps = monitor.finish(); - emit_measurement(&serde_json::json!({ - "kind": "hook_level", - "writers": writers, - "rounds": rounds, - "started_unix_ms": level_started_unix_ms, - "ended_unix_ms": unix_ms_now(), - "expected": expected, - "persisted_messages": persisted_messages, - "persisted_parts": persisted_parts, - "lost": lost, - "fail_open_lost": fail_open_lost, - "rounds_with_loss": rounds_with_loss, - "nonzero_exits": nonzero_exits, - "stderr_outputs": stderr_samples.len(), - "stderr_lines": stderr_samples.iter().map(|sample| sample.lines().count()).sum::(), - "p50_ms": millis(latency.p50), - "p95_ms": millis(latency.p95), - "p99_ms": millis(latency.p99), - "max_ms": millis(latency.max), - "stall_gaps": stall_gaps_json(&stall_gaps), - })); - fs::remove_dir_all(&harness.work).expect("hook harness dir should be removed"); - levels.push(HookLevelSummary { - writers, - expected, - persisted_messages, - persisted_parts, - nonzero_exits, - }); - } - - assert_hook_levels(&levels, strict_mode()); -} - -#[test] -fn initialized_hook_open_succeeds_while_write_lock_is_held() { - let db_path = unique_test_db_path("hook-open-metadata"); - create_repository_db(&db_path); - let repository_id = "lock-contention-repository"; - let initialized = open_production_connection(&db_path) - .verify_or_initialize_repository_metadata(repository_id) - .expect("metadata should initialize before contention"); - - let hold = Duration::from_millis(RELIABLY_BEYOND_CONTENTION_BUDGET_MS); - let lock_acquired = Arc::new(Barrier::new(2)); - let holder = { - let db_path = db_path.clone(); - let lock_acquired = Arc::clone(&lock_acquired); - thread::spawn(move || { - let holder = open_production_connection(&db_path); - holder - .execute("BEGIN IMMEDIATE", ()) - .expect("holder should acquire the write lock"); - lock_acquired.wait(); - thread::sleep(hold); - holder - .execute("COMMIT", ()) - .expect("holder should release the write lock"); - }) - }; - - let hook = open_production_connection(&db_path); - lock_acquired.wait(); - let started_at = Instant::now(); - let ((schema_ready, metadata), writes) = count_write_statements(|| { - ( - hook.ensure_schema_ready_for_hooks(), - hook.verify_or_initialize_repository_metadata(repository_id), - ) - }); - let elapsed = started_at.elapsed(); - holder.join().expect("holder thread should not panic"); - - eprintln!( - "\nhook-open under a {}ms write lock: schema_ready={:?} metadata={} writes={writes} after {}ms", - hold.as_millis(), - schema_ready.as_ref().map_err(|error| format!("{error:#}")), - match &metadata { - Ok(_) => String::from("Ok"), - Err(error) => format!("Err({error:#})"), - }, - elapsed.as_millis() - ); - - assert!( - schema_ready.is_ok(), - "the read-only schema check should not need the write lock" - ); - let metadata = metadata.expect("an initialized hook open should not need the write lock"); - assert_eq!(metadata, initialized); - assert_eq!(writes, 0, "an initialized hook open must issue no writes"); - assert!( - elapsed < hold, - "the initialized hook open must not wait for the held write lock" - ); - drop(hook); - remove_test_db(&db_path); -} diff --git a/cli/src/services/agent_trace_db/mod.rs b/cli/src/services/agent_trace_db/mod.rs index c2c293b17..9c69240ea 100644 --- a/cli/src/services/agent_trace_db/mod.rs +++ b/cli/src/services/agent_trace_db/mod.rs @@ -12,10 +12,12 @@ use crate::services::{ use serde_json::Value; pub mod lifecycle; -#[cfg(test)] -mod lock_contention_tests; + pub mod repository; +#[cfg(test)] +pub(crate) mod transaction_tests; + /// Payload type discriminator for diff trace source payloads. /// /// `OpenCode` normalized diff-trace payloads use [`PAYLOAD_TYPE_PATCH`]. @@ -289,7 +291,10 @@ pub struct InsertPartInsert { pub generated_at_unix_ms: i64, } -fn insert_diff_trace_with(db: &TursoDb, input: DiffTraceInsert<'_>) -> Result { +async fn insert_diff_trace_with( + db: &TursoDb, + input: DiffTraceInsert<'_>, +) -> Result { db.execute( INSERT_DIFF_TRACE_SQL, ( @@ -302,9 +307,10 @@ fn insert_diff_trace_with(db: &TursoDb, input: DiffTraceInsert<'_> input.payload_type, ), ) + .await } -fn insert_post_commit_patch_intersection_with( +async fn insert_post_commit_patch_intersection_with( db: &TursoDb, input: PostCommitPatchIntersectionInsert<'_>, ) -> Result { @@ -320,9 +326,13 @@ fn insert_post_commit_patch_intersection_with( input.intersection_patch, ), ) + .await } -fn insert_agent_trace_with(db: &TursoDb, input: AgentTraceInsert<'_>) -> Result { +async fn insert_agent_trace_with( + db: &TursoDb, + input: AgentTraceInsert<'_>, +) -> Result { db.execute( INSERT_AGENT_TRACE_SQL, ( @@ -334,9 +344,10 @@ fn insert_agent_trace_with(db: &TursoDb, input: AgentTraceInsert<' input.remote_url, ), ) + .await } -fn upsert_claude_model_state_with( +async fn upsert_claude_model_state_with( db: &TursoDb, input: ClaudeModelStateObservation, ) -> Result { @@ -351,18 +362,21 @@ fn upsert_claude_model_state_with( input.observed_at_ms, ), ) + .await } -fn claude_model_state_by_session_and_agent_with( +async fn claude_model_state_by_session_and_agent_with( db: &TursoDb, session_id: &str, agent_id: &str, ) -> Result> { - let rows = db.query_map( - SELECT_CLAUDE_MODEL_STATE_SQL, - (session_id, agent_id), - claude_model_state_observation_from_turso, - )?; + let rows = db + .query_map( + SELECT_CLAUDE_MODEL_STATE_SQL, + (session_id, agent_id), + claude_model_state_observation_from_turso, + ) + .await?; Ok(rows.into_iter().next()) } @@ -394,20 +408,7 @@ fn claude_model_state_observation_from_turso( }) } -#[allow(dead_code)] -fn insert_message_with(db: &TursoDb, input: InsertMessageInsert) -> Result { - db.execute( - INSERT_MESSAGE_SQL, - ( - input.session_id, - input.message_id, - input.role.to_string(), - input.generated_at_unix_ms, - ), - ) -} - -fn insert_messages_with( +async fn insert_messages_with( db: &TursoDb, inputs: Vec, ) -> Result { @@ -432,24 +433,13 @@ fn insert_messages_with( rows.join(", ") ); - db.execute(&sql, params) -} - -#[allow(dead_code)] -fn insert_part_with(db: &TursoDb, input: InsertPartInsert) -> Result { - db.execute( - INSERT_PART_SQL, - ( - input.part_type.to_string(), - input.text, - input.message_id, - input.session_id, - input.generated_at_unix_ms, - ), - ) + db.execute(&sql, params).await } -fn insert_parts_with(db: &TursoDb, inputs: Vec) -> Result { +async fn insert_parts_with( + db: &TursoDb, + inputs: Vec, +) -> Result { if inputs.is_empty() { return Ok(0); } @@ -472,7 +462,7 @@ fn insert_parts_with(db: &TursoDb, inputs: Vec) rows.join(", ") ); - db.execute(&sql, params) + db.execute(&sql, params).await } /// Atomically insert one conversation `messages` row and its one `parts` @@ -481,8 +471,8 @@ fn insert_parts_with(db: &TursoDb, inputs: Vec) /// transaction (`Ok(true)`). `fail_before_part_insert` is a test-only hook /// forcing the transaction to fail after the message insert and before the /// part insert, to prove both roll back together. -fn insert_conversation_text_event_with( - db: &TursoDb, +async fn insert_conversation_text_event_with( + db: &mut TursoDb, message: InsertMessageInsert, part: InsertPartInsert, fail_before_part_insert: bool, @@ -513,6 +503,7 @@ fn insert_conversation_text_event_with( part_params, fail_before_part_insert, ) + .await } fn numbered_placeholders(start: usize, count: usize) -> String { @@ -524,16 +515,18 @@ fn numbered_placeholders(start: usize, count: usize) -> String { format!("({placeholders})") } -fn recent_diff_trace_patches_with( +async fn recent_diff_trace_patches_with( db: &TursoDb, cutoff_time_ms: i64, end_time_ms: i64, ) -> Result { - let rows = db.query_map( - SELECT_RECENT_DIFF_TRACE_PATCHES_SQL, - (cutoff_time_ms, end_time_ms), - diff_trace_patch_row_from_turso, - )?; + let rows = db + .query_map( + SELECT_RECENT_DIFF_TRACE_PATCHES_SQL, + (cutoff_time_ms, end_time_ms), + diff_trace_patch_row_from_turso, + ) + .await?; Ok(parse_recent_diff_trace_patch_rows(rows)) } @@ -621,263 +614,3 @@ fn parse_recent_diff_trace_patch_rows(rows: Vec) -> RecentDif fn skipped_diff_trace_patch_reason(error: &ParseError) -> String { error.to_string() } - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::PathBuf, - time::{SystemTime, UNIX_EPOCH}, - }; - - use super::repository::RepositoryAgentTraceDb; - use super::*; - use crate::services::agent_trace::{build_agent_trace, AgentTraceMetadataInput}; - - fn unique_test_db_path() -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-agent-trace-db-test-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") - } - - fn valid_patch(path: &str, content: &str) -> String { - format!( - "Index: {path}\n===================================================================\n--- {path}\n+++ {path}\n@@ -0,0 +1,1 @@\n+{content}\n" - ) - } - - fn insert_test_diff_trace( - db: &RepositoryAgentTraceDb, - time_ms: i64, - session_id: &str, - patch: &str, - ) { - db.insert_diff_trace(DiffTraceInsert { - time_ms, - session_id, - patch, - model_id: Some("test-provider/test-model"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("diff trace insert should succeed"); - } - - #[test] - fn structured_diff_trace_reconstruction_uses_persisted_model_and_session_provenance() { - let db_path = unique_test_db_path(); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - let payload = - include_str!("../structured_patch/fixtures/edit_multi_hunk/claude-post-tool-use.json"); - - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_000, - session_id: "cc_session-123", - patch: payload, - model_id: Some("claude/claude-sonnet-4-5"), - tool_name: "claude", - tool_version: Some("1.0.0"), - payload_type: PAYLOAD_TYPE_STRUCTURED, - }) - .expect("structured diff trace insert should succeed"); - - let result = db - .recent_diff_trace_patches(0, 2_000) - .expect("structured diff trace should load"); - assert_eq!(result.loaded_count(), 1); - assert_eq!(result.skipped_count(), 0); - - let patch = &result.patches[0].patch; - assert!( - patch - .files - .iter() - .flat_map(|file| &file.hunks) - .all(|hunk| hunk.model_id.as_deref() == Some("claude/claude-sonnet-4-5")), - "every reconstructed hunk should use the persisted row model" - ); - assert!( - patch - .files - .iter() - .flat_map(|file| &file.hunks) - .flat_map(|hunk| &hunk.lines) - .all(|line| line.session_id.as_deref() == Some("cc_session-123")), - "every reconstructed touched line should use the persisted canonical row session" - ); - - drop(db); - if let Some(parent) = db_path.parent() { - fs::remove_dir_all(parent).expect("test DB directory should be removed"); - } - } - - #[test] - fn claude_model_attribution_flows_from_persisted_structured_row_to_agent_trace() { - let db_path = unique_test_db_path(); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - let payload = - include_str!("../structured_patch/fixtures/edit_single_hunk/claude-post-tool-use.json"); - let post_commit_patch = parse_patch( - include_str!("../structured_patch/fixtures/edit_single_hunk/expected.patch"), - None, - ) - .expect("post-commit fixture should parse"); - - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_000, - session_id: "cc_session-123", - patch: payload, - model_id: Some("claude/claude-sonnet-4-5"), - tool_name: "claude", - tool_version: Some("1.0.0"), - payload_type: PAYLOAD_TYPE_STRUCTURED, - }) - .expect("structured diff trace insert should succeed"); - - let mut result = db - .recent_diff_trace_patches(0, 2_000) - .expect("structured diff trace should load"); - let constructed_patch = result - .patches - .pop() - .expect("one structured diff trace should load") - .patch; - let agent_trace = build_agent_trace( - &constructed_patch, - &post_commit_patch, - AgentTraceMetadataInput { - commit_timestamp: "2026-04-23T10:20:30Z", - commit_revision: "a0b1c2d3e4f5a6b7c8d9e0f11223344556677889", - vcs_type: None, - tool_name: Some("claude"), - tool_version: Some("1.0.0"), - }, - ) - .expect("Agent Trace should build"); - let agent_trace_json = - serde_json::to_value(agent_trace).expect("Agent Trace should serialize"); - - assert_eq!( - agent_trace_json["files"][0]["conversations"][0]["contributor"]["model_id"], - "claude/claude-sonnet-4-5" - ); - assert_eq!( - agent_trace_json["files"][0]["conversations"][0]["related"], - serde_json::json!([{ - "type": "session", - "url": "https://sce.crocoder.dev/sessions/cc_session-123" - }]) - ); - - drop(db); - if let Some(parent) = db_path.parent() { - fs::remove_dir_all(parent).expect("test DB directory should be removed"); - } - } - - #[test] - fn recent_diff_trace_patches_applies_bounded_window_ordering_and_parse_accounting() { - let db_path = unique_test_db_path(); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - - let before_cutoff_patch = valid_patch("notes/before.md", "before cutoff"); - let cutoff_patch = valid_patch("notes/cutoff.md", "at cutoff"); - let first_same_time_patch = valid_patch("notes/same-a.md", "same time first"); - let second_same_time_patch = valid_patch("notes/same-b.md", "same time second"); - let end_patch = valid_patch("notes/end.md", "at end"); - let after_end_patch = valid_patch("notes/after.md", "after end"); - - insert_test_diff_trace(&db, 999, "oc_before-cutoff", &before_cutoff_patch); - insert_test_diff_trace(&db, 1000, "oc_at-cutoff", &cutoff_patch); - insert_test_diff_trace( - &db, - 1500, - "oc_malformed", - "Index: notes/malformed.md\n===================================================================\n--- notes/malformed.md\n+++ notes/malformed.md\n@@ malformed @@\n+bad\n", - ); - insert_test_diff_trace(&db, 1500, "oc_same-time-a", &first_same_time_patch); - insert_test_diff_trace(&db, 1500, "oc_same-time-b", &second_same_time_patch); - insert_test_diff_trace(&db, 2000, "oc_at-end", &end_patch); - insert_test_diff_trace(&db, 2001, "oc_after-end", &after_end_patch); - - let result = recent_diff_trace_patches_with(&db, 1000, 2000) - .expect("recent diff trace patches should load"); - - assert_eq!(result.loaded_count(), 4); - assert_eq!(result.skipped_count(), 1); - assert_eq!( - result - .patches - .iter() - .map(|patch| (patch.id, patch.time_ms, patch.session_id.as_str())) - .collect::>(), - vec![ - (2, 1000, "oc_at-cutoff"), - (4, 1500, "oc_same-time-a"), - (5, 1500, "oc_same-time-b"), - (6, 2000, "oc_at-end"), - ] - ); - assert_eq!( - result - .patches - .iter() - .map(|patch| { (patch.tool_name.as_deref(), patch.tool_version.as_deref(),) }) - .collect::>(), - vec![ - (Some("opencode"), Some("1.2.3")), - (Some("opencode"), Some("1.2.3")), - (Some("opencode"), Some("1.2.3")), - (Some("opencode"), Some("1.2.3")), - ] - ); - assert_eq!( - result - .patches - .iter() - .map(|patch| patch.payload_type.as_str()) - .collect::>(), - vec![ - PAYLOAD_TYPE_PATCH, - PAYLOAD_TYPE_PATCH, - PAYLOAD_TYPE_PATCH, - PAYLOAD_TYPE_PATCH - ] - ); - assert_eq!( - result - .patches - .iter() - .map(|patch| patch.patch.files[0].new_path.as_str()) - .collect::>(), - vec![ - "notes/cutoff.md", - "notes/same-a.md", - "notes/same-b.md", - "notes/end.md", - ] - ); - assert_eq!(result.skipped[0].id, 3); - assert_eq!(result.skipped[0].time_ms, 1500); - assert_eq!(result.skipped[0].session_id, "oc_malformed"); - assert!( - result.skipped[0].reason.contains("invalid hunk header"), - "unexpected skipped reason: {}", - result.skipped[0].reason - ); - - drop(db); - if let Some(parent) = db_path.parent() { - fs::remove_dir_all(parent).expect("test DB directory should be removed"); - } - } -} diff --git a/cli/src/services/agent_trace_db/repository.rs b/cli/src/services/agent_trace_db/repository.rs index dbc9e176f..1bfc8cb95 100644 --- a/cli/src/services/agent_trace_db/repository.rs +++ b/cli/src/services/agent_trace_db/repository.rs @@ -5,16 +5,15 @@ use uuid::Uuid; use crate::{ generated_migrations, - services::db::{DbSpec, TursoDb}, + services::db::{DatabaseKind, DbSpec, TursoDb}, }; use super::{ insert_agent_trace_with, insert_conversation_text_event_with, insert_diff_trace_with, - insert_message_with, insert_messages_with, insert_part_with, insert_parts_with, - insert_post_commit_patch_intersection_with, recent_diff_trace_patches_with, - upsert_claude_model_state_with, AgentTraceInsert, ClaudeModelStateObservation, DiffTraceInsert, - InsertMessageInsert, InsertPartInsert, PostCommitPatchIntersectionInsert, - RecentDiffTracePatches, + insert_messages_with, insert_parts_with, insert_post_commit_patch_intersection_with, + recent_diff_trace_patches_with, upsert_claude_model_state_with, AgentTraceInsert, + ClaudeModelStateObservation, DiffTraceInsert, InsertMessageInsert, InsertPartInsert, + PostCommitPatchIntersectionInsert, RecentDiffTracePatches, }; const REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE: &str = "Run 'sce setup'."; @@ -86,6 +85,53 @@ fn ensure_repository_id_matches(stored_repository_id: &str, repository_id: &str) Ok(()) } +#[derive(Debug)] +#[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" +)] +pub enum ExistingRepositoryDbError { + Missing { path: PathBuf }, + Unreadable(anyhow::Error), + IncompatibleSchema(anyhow::Error), + MissingMetadata, + RepositoryMismatch { stored: String, resolved: String }, +} + +impl std::fmt::Display for ExistingRepositoryDbError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExistingRepositoryDbError::Missing { path } => write!( + f, + "repository Agent Trace DB does not exist at '{}'. \ + {REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE}", + path.display() + ), + ExistingRepositoryDbError::Unreadable(source) => { + write!(f, "repository Agent Trace DB is unreadable: {source}") + } + ExistingRepositoryDbError::IncompatibleSchema(source) => { + write!( + f, + "repository Agent Trace DB schema is incompatible: {source}" + ) + } + ExistingRepositoryDbError::MissingMetadata => write!( + f, + "repository Agent Trace DB metadata is missing or has no valid source-instance ID. \ + {REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE}" + ), + ExistingRepositoryDbError::RepositoryMismatch { stored, resolved } => write!( + f, + "repository Agent Trace DB metadata mismatch: stored repository ID {stored} \ + does not match resolved repository ID {resolved}" + ), + } + } +} + +impl std::error::Error for ExistingRepositoryDbError {} + /// Repository-scoped Agent Trace database configuration. pub struct RepositoryAgentTraceDbSpec; @@ -105,9 +151,7 @@ impl DbSpec for RepositoryAgentTraceDbSpec { generated_migrations::AGENT_TRACE_REPOSITORY_MIGRATIONS } - fn db_config_key() -> &'static str { - "agent_trace_db" - } + const KIND: DatabaseKind = DatabaseKind::AgentTrace; } /// Repository-scoped Agent Trace Turso database adapter. @@ -117,13 +161,115 @@ impl RepositoryAgentTraceDb { /// Open a repository-scoped Agent Trace database at an explicit path without /// running migrations, for read-only hook/runtime paths that must not /// migrate from a high-frequency caller. - pub fn open_for_hooks_without_migrations_at(path: impl AsRef) -> Result { - TursoDb::::open_without_migrations_at(path) + pub async fn open_for_hooks_without_migrations_at( + path: impl AsRef, + ) -> Result { + TursoDb::::open_without_migrations_at(path).await + } + + #[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" + )] + pub async fn open_verified_existing_at( + path: impl AsRef, + repository_id: &str, + ) -> std::result::Result<(Self, RepositoryMetadata), ExistingRepositoryDbError> { + Self::open_verified_existing_with(path.as_ref(), repository_id, || {}, || {}).await + } + + #[cfg(test)] + pub(crate) async fn open_verified_existing_at_with_hooks( + path: &std::path::Path, + repository_id: &str, + before_open: impl FnOnce(), + after_open: impl FnOnce(), + ) -> std::result::Result<(Self, RepositoryMetadata), ExistingRepositoryDbError> { + Self::open_verified_existing_with(path, repository_id, before_open, after_open).await + } + + pub async fn open_existing_schema_ready_at( + path: &std::path::Path, + before_open: impl FnOnce(), + ) -> std::result::Result { + Self::open_existing_schema_ready_with(path, before_open, || {}).await + } + + async fn open_existing_schema_ready_with( + path: &std::path::Path, + before_open: impl FnOnce(), + after_open: impl FnOnce(), + ) -> std::result::Result { + before_open(); + let opened = + TursoDb::::open_existing_without_migrations_at(path).await; + after_open(); + let db = match opened { + Ok(db) => db, + Err(source) => { + return Err(match std::fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + ExistingRepositoryDbError::Missing { + path: path.to_path_buf(), + } + } + Ok(_) | Err(_) => ExistingRepositoryDbError::Unreadable(source), + }); + } + }; + db.ensure_schema_ready_for_hooks() + .await + .map_err(ExistingRepositoryDbError::IncompatibleSchema)?; + Ok(db) + } + + async fn open_verified_existing_with( + path: &std::path::Path, + repository_id: &str, + before_open: impl FnOnce(), + after_open: impl FnOnce(), + ) -> std::result::Result<(Self, RepositoryMetadata), ExistingRepositoryDbError> { + let db = Self::open_existing_schema_ready_with(path, before_open, after_open).await?; + let metadata = db + .verify_existing_repository_metadata(repository_id) + .await?; + Ok((db, metadata)) + } + + #[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" + )] + pub async fn verify_existing_repository_metadata( + &self, + repository_id: &str, + ) -> std::result::Result { + let Some((stored_repository_id, source_instance_id)) = self + .select_repository_metadata_row() + .await + .map_err(ExistingRepositoryDbError::Unreadable)? + else { + return Err(ExistingRepositoryDbError::MissingMetadata); + }; + if stored_repository_id != repository_id { + return Err(ExistingRepositoryDbError::RepositoryMismatch { + stored: stored_repository_id, + resolved: repository_id.to_string(), + }); + } + if !is_valid_source_instance_id(&source_instance_id) { + return Err(ExistingRepositoryDbError::MissingMetadata); + } + Ok(RepositoryMetadata { + repository_id: stored_repository_id, + source_instance_id, + }) } /// Verify that the repository-scoped schema baseline already exists. - pub fn ensure_schema_ready_for_hooks(&self) -> Result<()> { + pub async fn ensure_schema_ready_for_hooks(&self) -> Result<()> { self.ensure_schema_ready(REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE) + .await } /// Repair the narrow concurrent-initialization case where the one-file @@ -131,9 +277,9 @@ impl RepositoryAgentTraceDb { /// another first opener. This never creates trace tables; it only records /// the baseline migration after all required repository tables already /// exist. - pub fn repair_missing_repository_schema_migration_metadata(&self) -> Result<()> { + pub async fn repair_missing_repository_schema_migration_metadata(&self) -> Result<()> { for table in REQUIRED_REPOSITORY_SCHEMA_TABLES { - if !self.sqlite_object_exists("table", table)? { + if !self.sqlite_object_exists("table", table).await? { anyhow::bail!( "repository Agent Trace DB schema is incomplete; missing table {table}. \ {REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE}" @@ -141,14 +287,17 @@ impl RepositoryAgentTraceDb { } } - self.execute(RECORD_REPOSITORY_SCHEMA_MIGRATION_SQL, ())?; - self.ensure_schema_ready_for_hooks() + self.execute(RECORD_REPOSITORY_SCHEMA_MIGRATION_SQL, ()) + .await?; + self.ensure_schema_ready_for_hooks().await } - fn sqlite_object_exists(&self, object_type: &str, name: &str) -> Result { - let rows = self.query_map(SELECT_SQLITE_OBJECT_SQL, (object_type, name), |row| { - row.get::(0).map_err(Into::into) - })?; + async fn sqlite_object_exists(&self, object_type: &str, name: &str) -> Result { + let rows = self + .query_map(SELECT_SQLITE_OBJECT_SQL, (object_type, name), |row| { + row.get::(0).map_err(Into::into) + }) + .await?; Ok(!rows.is_empty()) } @@ -163,12 +312,12 @@ impl RepositoryAgentTraceDb { /// WHERE source_instance_id = ''`: concurrent first opens generate their /// own candidate, but only one claim can affect the row, so every caller /// re-reads the row afterward and returns whichever value actually won. - pub fn verify_or_initialize_repository_metadata( + pub async fn verify_or_initialize_repository_metadata( &self, repository_id: &str, ) -> Result { if let Some((stored_repository_id, source_instance_id)) = - self.select_repository_metadata_row()? + self.select_repository_metadata_row().await? { ensure_repository_id_matches(&stored_repository_id, repository_id)?; if is_valid_source_instance_id(&source_instance_id) { @@ -179,10 +328,11 @@ impl RepositoryAgentTraceDb { } } - self.execute_idempotent_write(INSERT_REPOSITORY_METADATA_SQL, (repository_id,))?; + self.execute_idempotent_write(INSERT_REPOSITORY_METADATA_SQL, (repository_id,)) + .await?; let Some((stored_repository_id, source_instance_id)) = - self.select_repository_metadata_row()? + self.select_repository_metadata_row().await? else { anyhow::bail!( "repository Agent Trace DB metadata is missing its repository ID row. \ @@ -200,10 +350,13 @@ impl RepositoryAgentTraceDb { } let candidate = generate_source_instance_id(); - self.execute_idempotent_write(CLAIM_SOURCE_INSTANCE_ID_SQL, (candidate.as_str(),))?; + self.execute_idempotent_write(CLAIM_SOURCE_INSTANCE_ID_SQL, (candidate.as_str(),)) + .await?; - let (final_repository_id, final_source_instance_id) = - self.select_repository_metadata_row()?.ok_or_else(|| { + let (final_repository_id, final_source_instance_id) = self + .select_repository_metadata_row() + .await? + .ok_or_else(|| { anyhow::anyhow!( "repository Agent Trace DB metadata row disappeared after \ source-instance-id claim" @@ -220,79 +373,71 @@ impl RepositoryAgentTraceDb { }) } - fn select_repository_metadata_row(&self) -> Result> { - let rows = self.query_map(SELECT_REPOSITORY_METADATA_SQL, (), |row| { - Ok((row.get::(0)?, row.get::(1)?)) - })?; + async fn select_repository_metadata_row(&self) -> Result> { + let rows = self + .query_map(SELECT_REPOSITORY_METADATA_SQL, (), |row| { + Ok((row.get::(0)?, row.get::(1)?)) + }) + .await?; Ok(rows.into_iter().next()) } /// Insert a diff trace payload into the repository-scoped `diff_traces` /// table. Rows remain repository-level; no checkout provenance is stored. - pub fn insert_diff_trace(&self, input: DiffTraceInsert<'_>) -> Result { - insert_diff_trace_with(self, input) + pub async fn insert_diff_trace(&self, input: DiffTraceInsert<'_>) -> Result { + insert_diff_trace_with(self, input).await } /// Insert a post-commit patch intersection into the repository-scoped /// `post_commit_patch_intersections` table. - pub fn insert_post_commit_patch_intersection( + pub async fn insert_post_commit_patch_intersection( &self, input: PostCommitPatchIntersectionInsert<'_>, ) -> Result { - insert_post_commit_patch_intersection_with(self, input) + insert_post_commit_patch_intersection_with(self, input).await } /// Insert a built Agent Trace payload into the repository-scoped /// `agent_traces` table. - pub fn insert_agent_trace(&self, input: AgentTraceInsert<'_>) -> Result { - insert_agent_trace_with(self, input) + pub async fn insert_agent_trace(&self, input: AgentTraceInsert<'_>) -> Result { + insert_agent_trace_with(self, input).await } - pub fn upsert_claude_model_state(&self, input: ClaudeModelStateObservation) -> Result { - upsert_claude_model_state_with(self, input) + pub async fn upsert_claude_model_state( + &self, + input: ClaudeModelStateObservation, + ) -> Result { + upsert_claude_model_state_with(self, input).await } - pub fn claude_model_state_by_session_and_agent( + pub async fn claude_model_state_by_session_and_agent( &self, session_id: &str, agent_id: &str, ) -> Result> { - super::claude_model_state_by_session_and_agent_with(self, session_id, agent_id) + super::claude_model_state_by_session_and_agent_with(self, session_id, agent_id).await } /// Query and parse recent diff trace patches within the inclusive time /// window for this repository-scoped database. Rows remain repository-level; /// no checkout filter or checkout provenance is applied. - pub fn recent_diff_trace_patches( + pub async fn recent_diff_trace_patches( &self, cutoff_time_ms: i64, end_time_ms: i64, ) -> Result { - recent_diff_trace_patches_with(self, cutoff_time_ms, end_time_ms) - } - - /// Insert a message row, ignoring duplicate `(session_id, message_id)` - /// rows. - #[allow(dead_code)] - pub fn insert_message(&self, input: InsertMessageInsert) -> Result { - insert_message_with(self, input) + recent_diff_trace_patches_with(self, cutoff_time_ms, end_time_ms).await } /// Insert message rows with one multi-row statement, ignoring duplicate /// `(session_id, message_id)` rows. - pub fn insert_messages(&self, inputs: Vec) -> Result { - insert_messages_with(self, inputs) - } - - /// Append a part row (no upsert; multiple rows per message allowed). - #[allow(dead_code)] - pub fn insert_part(&self, input: InsertPartInsert) -> Result { - insert_part_with(self, input) + pub async fn insert_messages(&self, inputs: Vec) -> Result { + insert_messages_with(self, inputs).await } /// Append part rows with one multi-row statement. - pub fn insert_parts(&self, inputs: Vec) -> Result { - insert_parts_with(self, inputs) + pub async fn insert_parts(&self, inputs: Vec) -> Result { + insert_parts_with(self, inputs).await } /// Atomically insert one conversation `messages` row and its one @@ -302,1929 +447,11 @@ impl RepositoryAgentTraceDb { /// text-event handlers (e.g. Codex `UserPromptSubmit`/`Stop`) in place /// of separate `insert_messages`/`insert_parts` calls, so a replayed or /// concurrent duplicate delivery never produces an orphaned `parts` row. - pub fn insert_conversation_text_event( - &self, - message: InsertMessageInsert, - part: InsertPartInsert, - ) -> Result { - insert_conversation_text_event_with(self, message, part, false) - } - - /// Test-only counterpart of [`insert_conversation_text_event`] that - /// forces the transaction to fail after the message insert and before - /// the part insert, proving both statements roll back together. - #[cfg(test)] - pub(crate) fn insert_conversation_text_event_with_injected_failure( - &self, + pub async fn insert_conversation_text_event( + &mut self, message: InsertMessageInsert, part: InsertPartInsert, ) -> Result { - insert_conversation_text_event_with(self, message, part, true) - } -} - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::PathBuf, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, - }; - - use super::pre_health_invariant_fixture::seed_pre_health_invariant_fixture; - use super::*; - use crate::services::agent_trace_db::{ - MessageRole, ObservationKind, PartType, PAYLOAD_TYPE_PATCH, - }; - - fn valid_patch(path: &str, content: &str) -> String { - format!( - "Index: {path}\n===================================================================\n--- {path}\n+++ {path}\n@@ -0,0 +1,1 @@\n+{content}\n" - ) - } - - fn unique_test_db_path(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-repo-agent-trace-db-{label}-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") - } - - fn remove_test_db(db_path: &std::path::Path) { - if let Some(parent) = db_path.parent() { - fs::remove_dir_all(parent).expect("test DB directory should be removed"); - } - } - - fn sqlite_object_exists(db: &RepositoryAgentTraceDb, object_type: &str, name: &str) -> bool { - let rows = db - .query_map( - "SELECT name FROM sqlite_master WHERE type = ?1 AND name = ?2", - (object_type, name), - |row| row.get::(0).map_err(Into::into), - ) - .expect("sqlite_master query should succeed"); - !rows.is_empty() - } - - fn row_count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("count row should exist") - } - - fn table_sql(db: &RepositoryAgentTraceDb, name: &str) -> String { - db.query_map( - "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?1", - (name,), - |row| row.get::(0).map_err(Into::into), - ) - .expect("sqlite_master sql query should succeed") - .into_iter() - .next() - .unwrap_or_else(|| panic!("table '{name}' should exist")) - } - - // These tests verify convergence/idempotency under concurrent writers, not the - // global production lock-retry budget. Retry exhausted lock contention at the - // test boundary so scheduler/load variance does not make those semantic tests - // flaky. Production retry behavior is intentionally unchanged and tracked - // separately. - const DATABASE_LOCKED_ERROR: &str = "database is locked"; - const CONCURRENT_WRITE_DEADLINE: Duration = Duration::from_secs(30); - const CONCURRENT_WRITE_RETRY_BACKOFF: Duration = Duration::from_millis(10); - - fn retry_while_database_locked(mut write: impl FnMut() -> Result) -> Result { - let deadline = Instant::now() + CONCURRENT_WRITE_DEADLINE; - loop { - match write() { - Err(error) - if error.to_string().contains(DATABASE_LOCKED_ERROR) - && Instant::now() < deadline => - { - std::thread::sleep(CONCURRENT_WRITE_RETRY_BACKOFF); - } - outcome => return outcome, - } - } - } - - fn claude_observation( - model_id: &str, - observation_kind: ObservationKind, - source: &str, - observed_at_ms: i64, - ) -> ClaudeModelStateObservation { - ClaudeModelStateObservation { - session_id: String::from("cc_session-1"), - agent_id: String::new(), - model_id: String::from(model_id), - observation_kind, - source: String::from(source), - observed_at_ms, - } - } - - #[test] - fn open_at_initializes_the_full_repository_schema() { - let db_path = unique_test_db_path("baseline"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - for table in [ - "repository_metadata", - "diff_traces", - "post_commit_patch_intersections", - "agent_traces", - "messages", - "parts", - "claude_model_state", - "mutation_trace_worktrees", - "mutation_trace_scopes", - "mutation_trace_processed_events", - "mutation_trace_events", - "mutation_trace_event_active_scopes", - "mutation_trace_scope_provenance", - ] { - assert!( - sqlite_object_exists(&db, "table", table), - "table '{table}' should exist" - ); - } - for index in [ - "idx_diff_traces_time_ms_id", - "idx_agent_traces_agent_trace_id", - "idx_agent_traces_remote_url", - "idx_messages_session_message", - "idx_messages_session_order", - "idx_parts_session_message_order", - "idx_mutation_trace_scopes_worktree", - "idx_mutation_trace_scopes_worktree_status", - ] { - assert!( - sqlite_object_exists(&db, "index", index), - "index '{index}' should exist" - ); - } - for trigger in ["trg_messages_updated_at", "trg_parts_updated_at"] { - assert!( - sqlite_object_exists(&db, "trigger", trigger), - "trigger '{trigger}' should exist" - ); - } - - let applied_ids = db - .query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("migration metadata query should succeed"); - assert_eq!( - applied_ids, - vec![ - String::from("001_repository_schema"), - String::from("002_repository_source_instance_id"), - String::from("003_claude_model_state"), - String::from("004_mutation_trace_protocol"), - String::from("005_mutation_scope_provenance"), - String::from("006_mutation_trace_health_invariant"), - ], - "repository DBs should be initialized from the baseline schema plus \ - its additive source-instance-id, Claude model-state, \ - mutation-trace-protocol, mutation-scope-provenance, and \ - mutation-trace-health-invariant migrations" - ); - - db.ensure_schema_ready_for_hooks() - .expect("fresh repository DB schema should be ready"); - - remove_test_db(&db_path); - } - - #[test] - fn pre_claude_model_state_database_is_upgraded_by_the_additive_migration() { - let db_path = unique_test_db_path("claude-model-state-upgrade"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - db.execute("DROP TABLE claude_model_state", ()) - .expect("test should remove the post-003 table"); - db.execute( - "DELETE FROM __sce_migrations WHERE id = '003_claude_model_state'", - (), - ) - .expect("test should remove the post-003 migration record"); - drop(db); - - let upgraded = - RepositoryAgentTraceDb::new_at(&db_path).expect("pre-003 database should upgrade"); - assert!(sqlite_object_exists( - &upgraded, - "table", - "claude_model_state" - )); - let applied_ids = upgraded - .query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("migration metadata query should succeed"); - assert_eq!( - applied_ids, - vec![ - String::from("001_repository_schema"), - String::from("002_repository_source_instance_id"), - String::from("003_claude_model_state"), - String::from("004_mutation_trace_protocol"), - String::from("005_mutation_scope_provenance"), - String::from("006_mutation_trace_health_invariant"), - ] - ); - - remove_test_db(&db_path); - } - - #[test] - fn claude_model_state_has_exact_scope_and_guarded_deterministic_updates() { - let db_path = unique_test_db_path("claude-model-state"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let initial = claude_observation("claude/A", ObservationKind::SessionStart, "startup", 100); - assert_eq!( - db.upsert_claude_model_state(initial.clone()) - .expect("initial write"), - 1 - ); - assert_eq!( - db.claude_model_state_by_session_and_agent("cc_session-1", "") - .expect("state lookup") - .expect("state should exist"), - initial - ); - assert!(db - .claude_model_state_by_session_and_agent("cc_session-1", "subagent") - .expect("subagent lookup") - .is_none()); - assert!(db - .claude_model_state_by_session_and_agent("cc_other", "") - .expect("other session lookup") - .is_none()); - - assert_eq!( - db.upsert_claude_model_state(claude_observation( - "claude/older", - ObservationKind::PostModelSwitch, - "picker", - 99, - )) - .expect("older write should be guarded"), - 0 - ); - assert_eq!( - db.claude_model_state_by_session_and_agent("cc_session-1", "") - .expect("state lookup") - .expect("state should remain") - .model_id, - "claude/A" - ); - - let switched = - claude_observation("claude/B", ObservationKind::PostModelSwitch, "picker", 101); - assert_eq!( - db.upsert_claude_model_state(switched.clone()) - .expect("newer write"), - 1 - ); - assert_eq!( - db.upsert_claude_model_state(switched.clone()) - .expect("identical replay should be harmless"), - 0 - ); - - assert_eq!( - db.upsert_claude_model_state(claude_observation( - "claude/C", - ObservationKind::SessionStart, - "resume", - 101, - )) - .expect("equal-time lower-priority write should be guarded"), - 0 - ); - assert_eq!( - db.claude_model_state_by_session_and_agent("cc_session-1", "") - .expect("state lookup") - .expect("state should remain") - .model_id, - "claude/B" - ); - - remove_test_db(&db_path); - } - - #[test] - fn equal_time_same_kind_observations_use_a_stable_tie_break_and_concurrent_writes_converge() { - let db_path = unique_test_db_path("claude-model-state-concurrent"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - drop(db); - - let db_path = std::sync::Arc::new(db_path); - let handles: Vec<_> = (0..8) - .map(|index| { - let db_path = std::sync::Arc::clone(&db_path); - std::thread::spawn(move || { - let db = RepositoryAgentTraceDb::open_without_migrations_at(&*db_path) - .expect("repository DB should reopen for concurrent state write"); - retry_while_database_locked(|| { - db.upsert_claude_model_state(claude_observation( - &format!("claude/model-{index}"), - ObservationKind::PostModelSwitch, - "picker", - 500, - )) - }) - .expect("concurrent state writer should eventually complete") - }) - }) - .collect(); - for handle in handles { - handle.join().expect("state writer should not panic"); - } - - let db = RepositoryAgentTraceDb::open_without_migrations_at(&*db_path) - .expect("repository DB should reopen for verification"); - let state = db - .claude_model_state_by_session_and_agent("cc_session-1", "") - .expect("state lookup") - .expect("concurrent writes should leave one state row"); - assert_eq!(state.model_id, "claude/model-7"); - assert_eq!(state.observation_kind, ObservationKind::PostModelSwitch); - assert_eq!(state.observed_at_ms, 500); - - remove_test_db(&db_path); - } - - #[test] - fn mutation_trace_worktrees_revision_must_be_a_blob_not_matching_length_text() { - let db_path = unique_test_db_path("mutation-trace-revision-blob"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let text_revision_error = db - .execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES ('wt-1', 'tree-0', '12345678', 0, 'healthy', 0)", - (), - ) - .expect_err( - "an 8-byte TEXT value must still be rejected by the typeof(revision) = 'blob' check", - ); - assert!( - text_revision_error.to_string().contains("CHECK"), - "unexpected error: {text_revision_error}" - ); - - db.execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES ('wt-1', 'tree-0', X'0000000000000000', 0, 'healthy', 0)", - (), - ) - .expect("an 8-byte BLOB revision should be accepted"); - - remove_test_db(&db_path); - } - - #[test] - fn mutation_trace_events_ai_exclusive_attribution_requires_a_scope_id() { - let db_path = unique_test_db_path("mutation-trace-attribution-check"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let missing_scope_error = db - .execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES ('wt-1', X'0000000000000001', 'tree-0', 'tree-1', 0, 'healthy', - 'ai_exclusive', NULL, 'flush', NULL, NULL)", - (), - ) - .expect_err("ai_exclusive attribution with a NULL attribution_scope_id must be rejected"); - assert!( - missing_scope_error.to_string().contains("CHECK"), - "unexpected error: {missing_scope_error}" - ); - - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES ('wt-1', X'0000000000000001', 'tree-0', 'tree-1', 0, 'healthy', - 'ai_exclusive', 'scope-1', 'start', 'scope-1', 'event-1')", - (), - ) - .expect("ai_exclusive attribution with a scope ID should be accepted"); - - remove_test_db(&db_path); - } - - #[test] - fn mutation_trace_processed_events_identity_is_scope_and_event_only() { - let db_path = unique_test_db_path("mutation-trace-processed-events-identity"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let sql = table_sql(&db, "mutation_trace_processed_events"); - assert!( - !sql.contains("worktree_id"), - "mutation_trace_processed_events must not have a worktree_id column: {sql}" - ); - - db.execute( - "INSERT INTO mutation_trace_processed_events (scope_id, event_id) - VALUES ('scope-1', 'event-1')", - (), - ) - .expect("first (scope_id, event_id) insert should succeed"); - - let duplicate_error = db - .execute( - "INSERT INTO mutation_trace_processed_events (scope_id, event_id) - VALUES ('scope-1', 'event-1')", - (), - ) - .expect_err("a duplicate (scope_id, event_id) pair must be rejected"); - assert!( - duplicate_error.to_string().contains("UNIQUE") - || duplicate_error.to_string().contains("PRIMARY KEY"), - "unexpected error: {duplicate_error}" - ); - - db.execute( - "INSERT INTO mutation_trace_processed_events (scope_id, event_id) - VALUES ('scope-2', 'event-1')", - (), - ) - .expect("the same event_id under a different scope_id should be allowed"); - - db.execute( - "INSERT INTO mutation_trace_processed_events (scope_id, event_id) - VALUES ('scope-1', 'event-2')", - (), - ) - .expect("the same scope_id with a different event_id should be allowed"); - - assert_eq!(row_count(&db, "mutation_trace_processed_events"), 3); - - remove_test_db(&db_path); - } - - #[test] - fn trace_tables_have_no_checkout_id_columns() { - let db_path = unique_test_db_path("no-checkout-id"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - for table in [ - "diff_traces", - "post_commit_patch_intersections", - "agent_traces", - "messages", - "parts", - ] { - let sql = table_sql(&db, table); - assert!( - !sql.contains("checkout_id"), - "table '{table}' must not have a checkout_id column: {sql}" - ); - } - - remove_test_db(&db_path); - } - - #[test] - fn repository_metadata_is_seeded_once_and_validated_on_reopen() { - let db_path = unique_test_db_path("metadata"); - let repository_id = "a".repeat(64); - - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let first = db - .verify_or_initialize_repository_metadata(&repository_id) - .expect("first metadata initialization should succeed"); - assert_eq!(first.repository_id, repository_id); - assert!( - is_valid_source_instance_id(&first.source_instance_id), - "freshly initialized metadata should have a non-empty source-instance ID" - ); - - let repeated = db - .verify_or_initialize_repository_metadata(&repository_id) - .expect("repeated validation with the same repository ID should succeed"); - assert_eq!( - repeated.source_instance_id, first.source_instance_id, - "repeated initialization must not regenerate the source-instance ID" - ); - drop(db); - - let reopened = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) - .expect("repository DB should reopen"); - let reopened_metadata = reopened - .verify_or_initialize_repository_metadata(&repository_id) - .expect("reopen validation with the matching repository ID should succeed"); - assert_eq!( - reopened_metadata.source_instance_id, first.source_instance_id, - "source-instance ID must be stable across reopen" - ); - - remove_test_db(&db_path); - } - - #[test] - fn source_instance_id_is_not_derived_from_repository_id_and_diverges_across_independent_dbs() { - let first_db_path = unique_test_db_path("source-instance-first"); - let second_db_path = unique_test_db_path("source-instance-second"); - let repository_id = "a".repeat(64); - - let first_db = RepositoryAgentTraceDb::new_at(&first_db_path) - .expect("first repository DB should open"); - let second_db = RepositoryAgentTraceDb::new_at(&second_db_path) - .expect("second repository DB should open"); - - let first_metadata = first_db - .verify_or_initialize_repository_metadata(&repository_id) - .expect("first DB metadata initialization should succeed"); - let second_metadata = second_db - .verify_or_initialize_repository_metadata(&repository_id) - .expect("second DB metadata initialization should succeed"); - - assert_eq!(first_metadata.repository_id, second_metadata.repository_id); - assert_ne!( - first_metadata.source_instance_id, second_metadata.source_instance_id, - "two independently created databases for the same logical repository \ - must diverge in source-instance ID" - ); - assert_ne!( - first_metadata.source_instance_id, repository_id, - "source-instance ID must not be derived from repository_id" - ); - - remove_test_db(&first_db_path); - remove_test_db(&second_db_path); - } - - #[test] - fn concurrent_initialization_converges_on_one_source_instance_id() { - use std::sync::Arc; - - let db_path = unique_test_db_path("concurrent-source-instance"); - let repository_id = "a".repeat(64); - - // Create the schema up front so both threads race only on the - // metadata claim, not schema creation. - RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let db_path = Arc::new(db_path); - let handles: Vec<_> = (0..4) - .map(|_| { - let db_path = Arc::clone(&db_path); - let repository_id = repository_id.clone(); - std::thread::spawn(move || { - let db = RepositoryAgentTraceDb::open_without_migrations_at(&*db_path) - .expect("repository DB should reopen for concurrent claim"); - retry_while_database_locked(|| { - db.verify_or_initialize_repository_metadata(&repository_id) - }) - .expect("concurrent metadata initialization worker should eventually complete") - }) - }) - .collect(); - - let results: Vec = handles - .into_iter() - .map(|handle| handle.join().expect("worker thread should not panic")) - .collect(); - - let winning_id = results[0].source_instance_id.clone(); - for result in &results { - assert_eq!( - result.source_instance_id, winning_id, - "all concurrent initializations must converge on one persisted \ - source-instance ID" - ); - } - - remove_test_db(&db_path); - } - - #[test] - fn initialized_repository_metadata_hook_runtime_open_issues_no_writes() { - let db_path = unique_test_db_path("metadata-hook-open-no-writes"); - let repository_id = "a".repeat(64); - - let setup = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let (first, first_writes) = crate::services::db::count_write_statements(|| { - setup.verify_or_initialize_repository_metadata(&repository_id) - }); - let first = first.expect("first metadata initialization should succeed"); - assert!( - first_writes > 0, - "first initialization must seed and claim metadata" - ); - drop(setup); - - let (reopened, writes) = crate::services::db::count_write_statements(|| { - let hook = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path)?; - hook.ensure_schema_ready_for_hooks()?; - hook.verify_or_initialize_repository_metadata(&repository_id) - }); - let reopened = reopened.expect("initialized hook-runtime open should succeed"); - assert_eq!( - writes, 0, - "initialized hook-runtime open must issue no writes" - ); - assert_eq!(reopened, first); - - remove_test_db(&db_path); - } - - #[test] - fn mismatched_repository_metadata_errors_without_writes() { - let db_path = unique_test_db_path("metadata-mismatch-no-writes"); - let stored_repository_id = "a".repeat(64); - let other_repository_id = "b".repeat(64); - - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - db.verify_or_initialize_repository_metadata(&stored_repository_id) - .expect("first metadata initialization should succeed"); - - let (result, writes) = crate::services::db::count_write_statements(|| { - db.verify_or_initialize_repository_metadata(&other_repository_id) - }); - let message = result - .expect_err("mismatched repository ID should fail validation") - .to_string(); - assert!( - message.contains("metadata mismatch"), - "unexpected error: {message}" - ); - assert_eq!(writes, 0, "a rejected mismatch must issue no writes"); - - remove_test_db(&db_path); - } - - #[test] - fn repository_metadata_with_empty_source_instance_id_is_claimed_once() { - let db_path = unique_test_db_path("metadata-empty-source-instance"); - let repository_id = "a".repeat(64); - - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - db.verify_or_initialize_repository_metadata(&repository_id) - .expect("first metadata initialization should succeed"); - db.execute( - "UPDATE repository_metadata SET source_instance_id = '' WHERE id = 1", - (), - ) - .expect("source-instance ID should reset to the empty placeholder"); - - let claimed = db - .verify_or_initialize_repository_metadata(&repository_id) - .expect("an empty source-instance ID should be claimed"); - assert_eq!(claimed.repository_id, repository_id); - assert!(is_valid_source_instance_id(&claimed.source_instance_id)); - - let (reopened, writes) = crate::services::db::count_write_statements(|| { - db.verify_or_initialize_repository_metadata(&repository_id) - }); - assert_eq!( - reopened.expect("claimed metadata should validate"), - claimed, - "a claimed source-instance ID must not be overwritten" - ); - assert_eq!(writes, 0); - - remove_test_db(&db_path); - } - - #[test] - fn mismatched_repository_metadata_errors_on_open() { - let db_path = unique_test_db_path("mismatch"); - let stored_repository_id = "a".repeat(64); - let other_repository_id = "b".repeat(64); - - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let stored = db - .verify_or_initialize_repository_metadata(&stored_repository_id) - .expect("first metadata initialization should succeed"); - - let error = db - .verify_or_initialize_repository_metadata(&other_repository_id) - .expect_err("mismatched repository ID should fail validation"); - let message = error.to_string(); - assert!( - message.contains("metadata mismatch"), - "unexpected error: {message}" - ); - assert!(message.contains(&stored_repository_id)); - assert!(message.contains(&other_repository_id)); - - let unchanged = db - .verify_or_initialize_repository_metadata(&stored_repository_id) - .expect("re-validating with the original repository ID should still succeed"); - assert_eq!( - unchanged.source_instance_id, stored.source_instance_id, - "a rejected mismatched claim must not alter the stored source-instance ID" - ); - - remove_test_db(&db_path); - } - - #[test] - fn baseline_only_fixture_migrates_and_gets_a_stable_source_instance_id() { - let db_path = unique_test_db_path("baseline-only-fixture"); - let repository_id = "a".repeat(64); - - // Simulate a database created before migration 002 existed: build the - // pre-002 `repository_metadata` shape (no `source_instance_id` - // column) and record only migration 001 as applied, so opening with - // the current embedded migration set exercises the real 001-applied, - // 002-pending upgrade path. - let baseline_only = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) - .expect("baseline-only repository DB should open"); - baseline_only - .execute( - "CREATE TABLE IF NOT EXISTS __sce_migrations ( - id TEXT PRIMARY KEY, - applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) -)", - (), - ) - .expect("migration metadata table should create"); - baseline_only - .execute( - "CREATE TABLE IF NOT EXISTS repository_metadata ( - id INTEGER PRIMARY KEY CHECK (id = 1), - repository_id TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) -)", - (), - ) - .expect("baseline repository_metadata table should create"); - baseline_only - .execute( - "INSERT INTO __sce_migrations (id) VALUES ('001_repository_schema')", - (), - ) - .expect("baseline migration record should insert"); - baseline_only - .execute( - "INSERT INTO repository_metadata (id, repository_id) VALUES (1, ?1)", - (repository_id.as_str(),), - ) - .expect("baseline metadata row should seed"); - drop(baseline_only); - - let migrated = - RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should migrate to 002"); - let metadata = migrated - .verify_or_initialize_repository_metadata(&repository_id) - .expect("metadata initialization after migrating from baseline should succeed"); - assert_eq!(metadata.repository_id, repository_id); - assert!( - is_valid_source_instance_id(&metadata.source_instance_id), - "migrating from a baseline-only fixture should populate a valid source-instance ID" - ); - drop(migrated); - - let reopened = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) - .expect("migrated repository DB should reopen"); - let reopened_metadata = reopened - .verify_or_initialize_repository_metadata(&repository_id) - .expect("reopen after migration should succeed"); - assert_eq!( - reopened_metadata.source_instance_id, metadata.source_instance_id, - "source-instance ID populated during migration must remain stable across reopen" - ); - - remove_test_db(&db_path); - } - - fn seed_001_and_002_only_fixture(db_path: &std::path::Path, repository_id: &str) { - let fixture = RepositoryAgentTraceDb::open_without_migrations_at(db_path) - .expect("001+002-only fixture DB should open"); - fixture - .execute( - "CREATE TABLE IF NOT EXISTS __sce_migrations ( - id TEXT PRIMARY KEY, - applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) -)", - (), - ) - .expect("migration metadata table should create"); - fixture - .execute( - "CREATE TABLE IF NOT EXISTS repository_metadata ( - id INTEGER PRIMARY KEY CHECK (id = 1), - repository_id TEXT NOT NULL, - source_instance_id TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) -)", - (), - ) - .expect("post-002 repository_metadata table should create"); - for (table, ddl) in [ - ( - "diff_traces", - "CREATE TABLE IF NOT EXISTS diff_traces (id INTEGER PRIMARY KEY)", - ), - ( - "post_commit_patch_intersections", - "CREATE TABLE IF NOT EXISTS post_commit_patch_intersections (id INTEGER PRIMARY KEY)", - ), - ( - "agent_traces", - "CREATE TABLE IF NOT EXISTS agent_traces (id INTEGER PRIMARY KEY)", - ), - ( - "messages", - "CREATE TABLE IF NOT EXISTS messages (id INTEGER PRIMARY KEY)", - ), - ( - "parts", - "CREATE TABLE IF NOT EXISTS parts (id INTEGER PRIMARY KEY)", - ), - ] { - fixture - .execute(ddl, ()) - .unwrap_or_else(|error| panic!("{table} table should create: {error}")); - } - fixture - .execute( - "INSERT INTO __sce_migrations (id) VALUES ('001_repository_schema')", - (), - ) - .expect("001 migration record should insert"); - fixture - .execute( - "INSERT INTO __sce_migrations (id) VALUES ('002_repository_source_instance_id')", - (), - ) - .expect("002 migration record should insert"); - fixture - .execute( - "INSERT INTO repository_metadata (id, repository_id) VALUES (1, ?1)", - (repository_id,), - ) - .expect("repository_metadata row should seed"); - drop(fixture); - } - - #[test] - fn baseline_and_source_instance_fixture_migrates_to_mutation_trace_protocol_through_setup() { - let db_path = unique_test_db_path("baseline-and-source-instance-fixture"); - let repository_id = "c".repeat(64); - - seed_001_and_002_only_fixture(&db_path, &repository_id); - - let migrated = RepositoryAgentTraceDb::new_at(&db_path) - .expect("repository DB should migrate a 001+002-only fixture through 004"); - - let applied_ids = migrated - .query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("migration metadata query should succeed"); - assert_eq!( - applied_ids, - vec![ - String::from("001_repository_schema"), - String::from("002_repository_source_instance_id"), - String::from("003_claude_model_state"), - String::from("004_mutation_trace_protocol"), - String::from("005_mutation_scope_provenance"), - String::from("006_mutation_trace_health_invariant"), - ], - "an existing 001+002 database should get 003, 004, 005, and 006 applied on top through the setup/lifecycle path, without reapplying 001/002" - ); - - for table in [ - "mutation_trace_worktrees", - "mutation_trace_scopes", - "mutation_trace_processed_events", - "mutation_trace_events", - "mutation_trace_event_active_scopes", - "mutation_trace_scope_provenance", - ] { - assert!( - sqlite_object_exists(&migrated, "table", table), - "table '{table}' should exist after migrating a 001+002-only fixture" - ); - } - - migrated - .ensure_schema_ready_for_hooks() - .expect("migrated repository DB schema should be ready for hooks"); - - let metadata = migrated - .verify_or_initialize_repository_metadata(&repository_id) - .expect("metadata initialization on a migrated 001+002-only fixture should succeed"); - assert_eq!(metadata.repository_id, repository_id); - - remove_test_db(&db_path); - } - - #[test] - fn hook_runtime_path_never_applies_mutation_trace_protocol_migration() { - let db_path = unique_test_db_path("hook-runtime-no-migration"); - let repository_id = "d".repeat(64); - - seed_001_and_002_only_fixture(&db_path, &repository_id); - - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("hook-runtime open without migrations should succeed on an existing DB file"); - - let readiness_error = db.ensure_schema_ready_for_hooks().expect_err( - "a 001+002-only DB should not be schema-ready for the mutation-trace store", - ); - assert!( - readiness_error - .to_string() - .contains(REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE), - "unexpected error: {readiness_error}" - ); - - let repair_error = db - .repair_missing_repository_schema_migration_metadata() - .expect_err("the base-table repair path must not silently mark 003 as applied"); - assert!( - repair_error - .to_string() - .contains(REPOSITORY_AGENT_TRACE_SCHEMA_SETUP_GUIDANCE), - "unexpected error: {repair_error}" - ); - - let applied_ids = db - .query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("migration metadata query should succeed"); - assert_eq!( - applied_ids, - vec![ - String::from("001_repository_schema"), - String::from("002_repository_source_instance_id"), - ], - "the no-migration hook-runtime path must never record or apply 003, 004, 005, or 006" - ); - - for table in [ - "mutation_trace_worktrees", - "mutation_trace_scopes", - "mutation_trace_processed_events", - "mutation_trace_events", - "mutation_trace_event_active_scopes", - "mutation_trace_scope_provenance", - ] { - assert!( - !sqlite_object_exists(&db, "table", table), - "table '{table}' should not exist; the hook-runtime path must not create mutation-trace tables" - ); - } - - drop(db); - remove_test_db(&db_path); - } - - const TABLE_REBUILD_PROBE_SQL: &str = "BEGIN IMMEDIATE; -CREATE TABLE probe_health ( - tainted INTEGER NOT NULL CHECK (tainted IN (0, 1)), - failure_kind TEXT NOT NULL, - CHECK (tainted = CASE WHEN failure_kind = 'healthy' THEN 0 ELSE 1 END) -); -INSERT INTO probe_health (tainted, failure_kind) VALUES (0, 'healthy'); -CREATE TABLE probe_health_v006 ( - tainted INTEGER NOT NULL CHECK (tainted IN (0, 1)), - failure_kind TEXT NOT NULL, - CHECK (tainted = CASE WHEN failure_kind = 'healthy' THEN 0 ELSE 1 END) -); -INSERT INTO probe_health_v006 (tainted, failure_kind) -SELECT tainted, failure_kind FROM probe_health; -DROP TABLE probe_health; -ALTER TABLE probe_health_v006 RENAME TO probe_health; -COMMIT;"; - - const TABLE_REBUILD_PROBE_MIGRATIONS: &[(&str, &str)] = - &[("probe_table_rebuild", TABLE_REBUILD_PROBE_SQL)]; - - struct TableRebuildProbeDbSpec; - - impl DbSpec for TableRebuildProbeDbSpec { - fn db_name() -> &'static str { - "table rebuild probe DB" - } - - fn db_path() -> Result { - anyhow::bail!("table rebuild probe DB has no canonical path") - } - - fn migrations() -> &'static [(&'static str, &'static str)] { - TABLE_REBUILD_PROBE_MIGRATIONS - } - - fn db_config_key() -> &'static str { - "agent_trace_db" - } - } - - #[test] - fn turso_supports_the_transactional_table_rebuild_used_by_migration_006() { - let db_path = unique_test_db_path("table-rebuild-probe"); - let probe = TursoDb::::new_at(&db_path).expect( - "turso should run BEGIN IMMEDIATE, DROP TABLE, ALTER TABLE RENAME and COMMIT in one batch", - ); - - let tables = probe - .query_map( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'probe_health%'", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("sqlite_master query should succeed"); - assert_eq!(tables, vec![String::from("probe_health")]); - - let rows = probe - .query_map( - "SELECT tainted, failure_kind FROM probe_health", - (), - |row| Ok((row.get::(0)?, row.get::(1)?)), - ) - .expect("renamed table should be readable"); - assert_eq!(rows, vec![(0, String::from("healthy"))]); - - probe - .execute( - "INSERT INTO probe_health (tainted, failure_kind) VALUES (1, 'snapshot_failure')", - (), - ) - .expect("a consistent pair should satisfy the CASE check"); - for (tainted, failure_kind) in [(0, "snapshot_failure"), (1, "healthy")] { - let error = probe - .execute( - "INSERT INTO probe_health (tainted, failure_kind) VALUES (?1, ?2)", - (tainted, failure_kind), - ) - .expect_err("an inconsistent pair should violate the CASE check"); - assert!( - error.to_string().contains("CHECK"), - "unexpected error: {error}" - ); - } - - drop(probe); - remove_test_db(&db_path); - } - - const HEALTH_INVARIANT_MIGRATION_ID: &str = "006_mutation_trace_health_invariant"; - const HEALTH_INVARIANT_TABLES: [&str; 2] = - ["mutation_trace_worktrees", "mutation_trace_events"]; - - const WORKTREE_ROWS_SNAPSHOT_SQL: &str = "SELECT worktree_id, quote(cursor_tree) || '|' || quote(revision) || '|' || \ - quote(failure_kind) || '|' || quote(needs_rebaseline) || '|' || quote(created_at) || '|' || \ - quote(updated_at) FROM mutation_trace_worktrees ORDER BY worktree_id"; - const EVENT_ROWS_SNAPSHOT_SQL: &str = "SELECT worktree_id || ':' || hex(revision), quote(before_tree) || '|' || \ - quote(after_tree) || '|' || quote(failure_kind) || '|' || quote(attribution_kind) || '|' || \ - quote(attribution_scope_id) || '|' || quote(boundary_kind) || '|' || quote(boundary_scope_id) || '|' || \ - quote(boundary_event_id) || '|' || quote(created_at) FROM mutation_trace_events \ - ORDER BY worktree_id, revision"; - const WORKTREE_TAINT_SQL: &str = - "SELECT worktree_id, tainted FROM mutation_trace_worktrees ORDER BY worktree_id"; - const EVENT_TAINT_SQL: &str = - "SELECT worktree_id || ':' || hex(revision), tainted FROM mutation_trace_events \ - ORDER BY worktree_id, revision"; - const UNTOUCHED_TABLE_SNAPSHOT_SQL: [&str; 4] = [ - "SELECT scope_id, quote(worktree_id) || '|' || quote(actor_kind) || '|' || quote(status) || '|' || \ - quote(created_at) || '|' || quote(updated_at) FROM mutation_trace_scopes ORDER BY scope_id", - "SELECT scope_id || ':' || event_id, quote(created_at) FROM mutation_trace_processed_events \ - ORDER BY scope_id, event_id", - "SELECT worktree_id || ':' || hex(revision) || ':' || scope_id, '' \ - FROM mutation_trace_event_active_scopes ORDER BY worktree_id, revision, scope_id", - "SELECT scope_id, quote(session_id) || '|' || quote(model_id) || '|' || quote(created_at) \ - FROM mutation_trace_scope_provenance ORDER BY scope_id", - ]; - - const LEGACY_HEALTH_ROWS_SQL: [&str; 12] = [ - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline, created_at, updated_at) - VALUES ('wt-healthy-tainted', 'tree-a', X'0000000000000001', 1, 'healthy', 1, - '2026-01-01T00:00:00.000Z', '2026-01-02T00:00:00.000Z')", - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline, created_at, updated_at) - VALUES ('wt-healthy-untainted', 'tree-b', X'0000000000000002', 0, 'healthy', 0, - '2026-01-03T00:00:00.000Z', '2026-01-04T00:00:00.000Z')", - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline, created_at, updated_at) - VALUES ('wt-snapshot-failure-tainted', 'tree-c', X'0000000000000003', 1, 'snapshot_failure', 0, - '2026-01-05T00:00:00.000Z', '2026-01-06T00:00:00.000Z')", - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline, created_at, updated_at) - VALUES ('wt-snapshot-failure-untainted', 'tree-d', X'00000000000000FF', 0, 'snapshot_failure', 1, - '2026-01-07T00:00:00.000Z', '2026-01-08T00:00:00.000Z')", - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id, created_at) - VALUES ('wt-events', X'0000000000000001', 'tree-0', 'tree-1', 1, 'healthy', - 'ai_exclusive', 'scope-1', 'start', 'scope-1', 'event-1', '2026-02-01T00:00:00.000Z')", - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id, created_at) - VALUES ('wt-events', X'0000000000000002', 'tree-1', 'tree-2', 0, 'healthy', - 'ai_contended', NULL, 'advance', 'scope-1', 'event-2', '2026-02-02T00:00:00.000Z')", - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id, created_at) - VALUES ('wt-events', X'0000000000000003', 'tree-2', 'tree-3', 1, 'snapshot_failure', - 'ineligible_unscoped', NULL, 'flush', NULL, NULL, '2026-02-03T00:00:00.000Z')", - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id, created_at) - VALUES ('wt-events', X'0000000000000004', 'tree-3', 'tree-4', 0, 'snapshot_failure', - 'ineligible_unscoped', NULL, 'close', 'scope-1', 'event-3', '2026-02-04T00:00:00.000Z')", - "INSERT INTO mutation_trace_scopes (scope_id, worktree_id, actor_kind, status, created_at, updated_at) - VALUES ('scope-1', 'wt-events', 'codex', 'closed', - '2026-03-01T00:00:00.000Z', '2026-03-02T00:00:00.000Z')", - "INSERT INTO mutation_trace_processed_events (scope_id, event_id, created_at) - VALUES ('scope-1', 'event-1', '2026-03-03T00:00:00.000Z')", - "INSERT INTO mutation_trace_event_active_scopes (worktree_id, revision, scope_id) - VALUES ('wt-events', X'0000000000000002', 'scope-1')", - "INSERT INTO mutation_trace_scope_provenance (scope_id, session_id, model_id, created_at) - VALUES ('scope-1', 'session-1', 'model-1', '2026-03-04T00:00:00.000Z')", - ]; - - fn keyed_text_rows(db: &RepositoryAgentTraceDb, sql: &str) -> Vec<(String, String)> { - db.query_map(sql, (), |row| { - Ok((row.get::(0)?, row.get::(1)?)) - }) - .expect("snapshot query should succeed") - } - - fn keyed_flag_rows(db: &RepositoryAgentTraceDb, sql: &str) -> Vec<(String, i64)> { - db.query_map(sql, (), |row| { - Ok((row.get::(0)?, row.get::(1)?)) - }) - .expect("flag query should succeed") - } - - fn applied_migration_ids(db: &RepositoryAgentTraceDb) -> Vec { - db.query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("migration metadata query should succeed") - } - - fn health_table_rows(db: &RepositoryAgentTraceDb) -> Vec> { - let mut snapshots = vec![ - keyed_text_rows(db, WORKTREE_ROWS_SNAPSHOT_SQL), - keyed_text_rows(db, EVENT_ROWS_SNAPSHOT_SQL), - ]; - snapshots.extend( - UNTOUCHED_TABLE_SNAPSHOT_SQL - .iter() - .map(|sql| keyed_text_rows(db, sql)), - ); - snapshots - } - - fn insert_health_pair( - db: &RepositoryAgentTraceDb, - table: &str, - tainted: i64, - failure_kind: &str, - ) -> Result { - let sql = match table { - "mutation_trace_worktrees" => { - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES ('wt-1', 'tree-0', X'0000000000000000', ?1, ?2, 0)" - } - "mutation_trace_events" => { - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES ('wt-1', X'0000000000000001', 'tree-0', 'tree-1', ?1, ?2, - 'ineligible_unscoped', NULL, 'flush', NULL, NULL)" - } - other => panic!("unexpected health-invariant table '{other}'"), - }; - db.execute(sql, (tainted, failure_kind)) - } - - fn assert_health_pair_accepted(table: &str, tainted: i64, failure_kind: &str) { - let db_path = unique_test_db_path("mutation-trace-health-accepted"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - insert_health_pair(&db, table, tainted, failure_kind).unwrap_or_else(|error| { - panic!("{table} should accept (tainted={tainted}, '{failure_kind}'): {error}") - }); - assert_eq!(row_count(&db, table), 1); - - remove_test_db(&db_path); - } - - fn assert_health_pair_rejected(table: &str, tainted: i64, failure_kind: &str) { - let db_path = unique_test_db_path("mutation-trace-health-rejected"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let error = insert_health_pair(&db, table, tainted, failure_kind).expect_err(&format!( - "{table} must reject (tainted={tainted}, '{failure_kind}')" - )); - assert!( - error.to_string().contains("CHECK"), - "unexpected error: {error}" - ); - assert_eq!(row_count(&db, table), 0); - - remove_test_db(&db_path); - } - - #[test] - fn mutation_trace_worktrees_accepts_untainted_healthy_pair() { - assert_health_pair_accepted("mutation_trace_worktrees", 0, "healthy"); - } - - #[test] - fn mutation_trace_worktrees_accepts_tainted_snapshot_failure_pair() { - assert_health_pair_accepted("mutation_trace_worktrees", 1, "snapshot_failure"); - } - - #[test] - fn mutation_trace_worktrees_rejects_untainted_snapshot_failure_pair() { - assert_health_pair_rejected("mutation_trace_worktrees", 0, "snapshot_failure"); - } - - #[test] - fn mutation_trace_worktrees_rejects_tainted_healthy_pair() { - assert_health_pair_rejected("mutation_trace_worktrees", 1, "healthy"); - } - - #[test] - fn mutation_trace_events_accepts_untainted_healthy_pair() { - assert_health_pair_accepted("mutation_trace_events", 0, "healthy"); - } - - #[test] - fn mutation_trace_events_accepts_tainted_snapshot_failure_pair() { - assert_health_pair_accepted("mutation_trace_events", 1, "snapshot_failure"); - } - - #[test] - fn mutation_trace_events_rejects_untainted_snapshot_failure_pair() { - assert_health_pair_rejected("mutation_trace_events", 0, "snapshot_failure"); - } - - #[test] - fn mutation_trace_events_rejects_tainted_healthy_pair() { - assert_health_pair_rejected("mutation_trace_events", 1, "healthy"); - } - - #[test] - fn migration_006_keeps_the_existing_mutation_trace_column_checks_and_primary_keys() { - let db_path = unique_test_db_path("mutation-trace-health-existing-checks"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - for (statement, expected) in [ - ( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES ('wt-1', 'tree-0', X'0000000000000000', 2, 'snapshot_failure', 0)", - "CHECK", - ), - ( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES ('wt-1', 'tree-0', X'0000000000000000', 1, 'unknown', 0)", - "CHECK", - ), - ( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES ('wt-1', 'tree-0', X'0000000000000000', 0, 'healthy', 2)", - "CHECK", - ), - ( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES ('wt-1', '12345678', 'tree-0', 'tree-1', 0, 'healthy', - 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", - "CHECK", - ), - ( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES ('wt-1', X'0000000000000001', 'tree-0', 'tree-1', 0, 'healthy', - 'ineligible_unscoped', 'scope-1', 'flush', NULL, NULL)", - "CHECK", - ), - ( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES ('wt-1', X'0000000000000001', 'tree-0', 'tree-1', 0, 'healthy', - 'ineligible_unscoped', NULL, 'flush', 'scope-1', 'event-1')", - "CHECK", - ), - ] { - let error = db - .execute(statement, ()) - .expect_err("a row violating a 004 column check must be rejected after 006"); - assert!( - error.to_string().contains(expected), - "unexpected error: {error}" - ); - } - - insert_health_pair(&db, "mutation_trace_worktrees", 0, "healthy") - .expect("first worktree row should insert"); - insert_health_pair(&db, "mutation_trace_events", 0, "healthy") - .expect("first event row should insert"); - for table in HEALTH_INVARIANT_TABLES { - let error = insert_health_pair(&db, table, 0, "healthy") - .expect_err("a duplicate primary key must be rejected after 006"); - assert!( - error.to_string().contains("UNIQUE") || error.to_string().contains("PRIMARY KEY"), - "unexpected error: {error}" - ); - } - - remove_test_db(&db_path); - } - - #[test] - fn migration_006_normalizes_legacy_health_pairs_from_failure_kind_and_preserves_other_columns() - { - let db_path = unique_test_db_path("mutation-trace-health-legacy"); - seed_pre_health_invariant_fixture(&db_path, &LEGACY_HEALTH_ROWS_SQL); - - let legacy = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) - .expect("pre-006 fixture should reopen without migrations"); - let legacy_rows = health_table_rows(&legacy); - assert_eq!( - keyed_flag_rows(&legacy, WORKTREE_TAINT_SQL), - vec![ - (String::from("wt-healthy-tainted"), 1), - (String::from("wt-healthy-untainted"), 0), - (String::from("wt-snapshot-failure-tainted"), 1), - (String::from("wt-snapshot-failure-untainted"), 0), - ] - ); - drop(legacy); - - let migrated = - RepositoryAgentTraceDb::new_at(&db_path).expect("pre-006 database should upgrade"); - - assert_eq!( - applied_migration_ids(&migrated).last().map(String::as_str), - Some(HEALTH_INVARIANT_MIGRATION_ID) - ); - assert_eq!( - keyed_flag_rows(&migrated, WORKTREE_TAINT_SQL), - vec![ - (String::from("wt-healthy-tainted"), 0), - (String::from("wt-healthy-untainted"), 0), - (String::from("wt-snapshot-failure-tainted"), 1), - (String::from("wt-snapshot-failure-untainted"), 1), - ] - ); - assert_eq!( - keyed_flag_rows(&migrated, EVENT_TAINT_SQL), - vec![ - (String::from("wt-events:0000000000000001"), 0), - (String::from("wt-events:0000000000000002"), 0), - (String::from("wt-events:0000000000000003"), 1), - (String::from("wt-events:0000000000000004"), 1), - ] - ); - assert_eq!( - health_table_rows(&migrated), - legacy_rows, - "006 must preserve every non-tainted column and leave the other mutation-trace tables untouched" - ); - migrated - .ensure_schema_ready_for_hooks() - .expect("upgraded repository DB schema should be ready for hooks"); - - remove_test_db(&db_path); - } - - #[test] - fn fresh_and_upgraded_databases_share_the_migration_006_table_sql() { - let fresh_path = unique_test_db_path("mutation-trace-health-fresh-schema"); - let fresh = RepositoryAgentTraceDb::new_at(&fresh_path).expect("fresh DB should open"); - - let upgraded_path = unique_test_db_path("mutation-trace-health-upgraded-schema"); - seed_pre_health_invariant_fixture(&upgraded_path, &LEGACY_HEALTH_ROWS_SQL); - let upgraded = RepositoryAgentTraceDb::new_at(&upgraded_path) - .expect("pre-006 database should upgrade"); - - for table in HEALTH_INVARIANT_TABLES { - let fresh_sql = table_sql(&fresh, table); - assert!( - fresh_sql.contains("CASE WHEN failure_kind = 'healthy' THEN 0 ELSE 1 END"), - "{table} should carry the cross-column health check: {fresh_sql}" - ); - assert_eq!(fresh_sql, table_sql(&upgraded, table)); - assert!(!sqlite_object_exists( - &fresh, - "table", - &format!("{table}_v006") - )); - assert!(!sqlite_object_exists( - &upgraded, - "table", - &format!("{table}_v006") - )); - } - - remove_test_db(&fresh_path); - remove_test_db(&upgraded_path); - } - - #[test] - fn migration_006_sql_body_reruns_without_changing_schema_or_rows() { - let db_path = unique_test_db_path("mutation-trace-health-rerun"); - seed_pre_health_invariant_fixture(&db_path, &LEGACY_HEALTH_ROWS_SQL); - - let migrated = - RepositoryAgentTraceDb::new_at(&db_path).expect("pre-006 database should upgrade"); - let schema_after_first_run = - HEALTH_INVARIANT_TABLES.map(|table| table_sql(&migrated, table)); - let rows_after_first_run = health_table_rows(&migrated); - let worktree_taint_after_first_run = keyed_flag_rows(&migrated, WORKTREE_TAINT_SQL); - let event_taint_after_first_run = keyed_flag_rows(&migrated, EVENT_TAINT_SQL); - let migrations_after_first_run = applied_migration_ids(&migrated); - migrated - .execute( - "DELETE FROM __sce_migrations WHERE id = ?1", - (HEALTH_INVARIANT_MIGRATION_ID,), - ) - .expect("test should drop the 006 metadata row to force a re-run"); - drop(migrated); - - let rerun = RepositoryAgentTraceDb::new_at(&db_path) - .expect("the 006 SQL body should re-run against an already-rebuilt schema"); - - assert_eq!( - HEALTH_INVARIANT_TABLES.map(|table| table_sql(&rerun, table)), - schema_after_first_run - ); - assert_eq!(health_table_rows(&rerun), rows_after_first_run); - assert_eq!( - keyed_flag_rows(&rerun, WORKTREE_TAINT_SQL), - worktree_taint_after_first_run - ); - assert_eq!( - keyed_flag_rows(&rerun, EVENT_TAINT_SQL), - event_taint_after_first_run - ); - assert_eq!(applied_migration_ids(&rerun), migrations_after_first_run); - - remove_test_db(&db_path); - } - - #[test] - fn repository_scoped_write_methods_insert_all_agent_trace_rows() { - let db_path = unique_test_db_path("writes"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_000, - session_id: "oc_session-1", - patch: "Index: notes.md\n===================================================================\n--- notes.md\n+++ notes.md\n@@ -0,0 +1,1 @@\n+hello\n", - model_id: Some("provider/model"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("diff trace insert should succeed"); - - db.insert_post_commit_patch_intersection(PostCommitPatchIntersectionInsert { - commit_id: "abc123", - post_commit_time_ms: 2_000, - recent_window_cutoff_ms: 1_000, - recent_window_end_ms: 2_000, - loaded_diff_trace_count: 1, - skipped_diff_trace_count: 0, - intersection_patch: "Index: notes.md\n===================================================================\n--- notes.md\n+++ notes.md\n@@ -0,0 +1,1 @@\n+hello\n", - }) - .expect("post-commit intersection insert should succeed"); - - db.insert_agent_trace(AgentTraceInsert { - commit_id: "abc123", - commit_time_ms: 2_000, - trace_json: r#"{"id":"trace-1"}"#, - agent_trace_id: "trace-1", - url: "https://sce.crocoder.dev/agent-trace/trace-1", - remote_url: "https://github.com/acme/widgets", - }) - .expect("agent trace insert should succeed"); - - db.insert_message(InsertMessageInsert { - session_id: "oc_session-1".to_string(), - message_id: "message-1".to_string(), - role: MessageRole::Assistant, - generated_at_unix_ms: 1_000, - }) - .expect("message insert should succeed"); - db.insert_messages(vec![InsertMessageInsert { - session_id: "oc_session-1".to_string(), - message_id: "message-2".to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_001, - }]) - .expect("batch message insert should succeed"); - - db.insert_part(InsertPartInsert { - part_type: PartType::Text, - text: "hello".to_string(), - session_id: "oc_session-1".to_string(), - message_id: "message-1".to_string(), - generated_at_unix_ms: 1_000, - }) - .expect("part insert should succeed"); - db.insert_parts(vec![InsertPartInsert { - part_type: PartType::Patch, - text: "patch text".to_string(), - session_id: "oc_session-1".to_string(), - message_id: "message-2".to_string(), - generated_at_unix_ms: 1_001, - }]) - .expect("batch part insert should succeed"); - - for (table, expected_count) in [ - ("diff_traces", 1_i64), - ("post_commit_patch_intersections", 1), - ("agent_traces", 1), - ("messages", 2), - ("parts", 2), - ] { - let count = db - .query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("count row should exist"); - assert_eq!(count, expected_count, "unexpected row count for {table}"); - } - - remove_test_db(&db_path); - } - - fn conversation_text_event_fixture() -> (InsertMessageInsert, InsertPartInsert) { - ( - InsertMessageInsert { - session_id: "cx_session-1".to_string(), - message_id: "cx:turn-1:user".to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_000, - }, - InsertPartInsert { - part_type: PartType::Text, - text: "hello world".to_string(), - session_id: "cx_session-1".to_string(), - message_id: "cx:turn-1:user".to_string(), - generated_at_unix_ms: 1_000, - }, - ) - } - - #[test] - fn insert_conversation_text_event_inserts_message_and_part_together() { - let db_path = unique_test_db_path("conversation-event-insert"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let (message, part) = conversation_text_event_fixture(); - - let inserted = db - .insert_conversation_text_event(message, part) - .expect("conversation text event insert should succeed"); - - assert!(inserted, "first delivery should insert both rows"); - assert_eq!(row_count(&db, "messages"), 1); - assert_eq!(row_count(&db, "parts"), 1); - - remove_test_db(&db_path); - } - - #[test] - fn insert_conversation_text_event_is_a_no_op_on_sequential_replay() { - let db_path = unique_test_db_path("conversation-event-replay"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let (message, part) = conversation_text_event_fixture(); - let first = db - .insert_conversation_text_event(message, part) - .expect("first delivery should succeed"); - - let (message, part) = conversation_text_event_fixture(); - let second = db - .insert_conversation_text_event(message, part) - .expect("replayed delivery should succeed"); - - assert!(first); - assert!(!second, "a replayed delivery must be a no-op"); - assert_eq!(row_count(&db, "messages"), 1); - assert_eq!(row_count(&db, "parts"), 1); - - remove_test_db(&db_path); - } - - #[test] - fn insert_conversation_text_event_ten_sequential_replays_still_leave_one_row_pair() { - let db_path = unique_test_db_path("conversation-event-replay-ten"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - for _ in 0..10 { - let (message, part) = conversation_text_event_fixture(); - db.insert_conversation_text_event(message, part) - .expect("every replayed delivery should succeed"); - } - - assert_eq!(row_count(&db, "messages"), 1); - assert_eq!(row_count(&db, "parts"), 1); - - remove_test_db(&db_path); - } - - #[test] - fn insert_conversation_text_event_injected_failure_rolls_back_both_rows() { - let db_path = unique_test_db_path("conversation-event-rollback"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let (message, part) = conversation_text_event_fixture(); - - let error = db - .insert_conversation_text_event_with_injected_failure(message, part) - .expect_err("an injected failure before the part insert should propagate as an error"); - assert!(error.to_string().contains("injected failure")); - - assert_eq!( - row_count(&db, "messages"), - 0, - "the message row must roll back along with the failed part insert" - ); - assert_eq!(row_count(&db, "parts"), 0); - - remove_test_db(&db_path); - } - - #[test] - fn insert_conversation_text_event_concurrent_duplicate_delivery_leaves_one_row_pair() { - use std::sync::Arc; - - let db_path = unique_test_db_path("conversation-event-concurrent"); - - // Create the schema up front so every thread races only on the - // conversation text event insert, not schema creation, mirroring - // `concurrent_initialization_converges_on_one_source_instance_id`. - RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let db_path = Arc::new(db_path); - let handles: Vec<_> = (0..4) - .map(|_| { - let db_path = Arc::clone(&db_path); - std::thread::spawn(move || { - let db = RepositoryAgentTraceDb::open_without_migrations_at(&*db_path) - .expect("repository DB should reopen for concurrent delivery"); - retry_while_database_locked(|| { - let (message, part) = conversation_text_event_fixture(); - db.insert_conversation_text_event(message, part) - }) - }) - }) - .collect(); - - let results: Vec = handles - .into_iter() - .map(|handle| { - handle - .join() - .expect("worker thread should not panic") - .expect("every concurrent delivery worker should eventually complete") - }) - .collect(); - - assert_eq!( - results.iter().filter(|inserted| **inserted).count(), - 1, - "exactly one concurrent delivery should have performed the insert" - ); - - let db = RepositoryAgentTraceDb::open_without_migrations_at(&*db_path) - .expect("repository DB should reopen for verification"); - assert_eq!(row_count(&db, "messages"), 1); - assert_eq!(row_count(&db, "parts"), 1); - - remove_test_db(&db_path); - } - - #[test] - fn recent_diff_trace_reads_all_repository_rows_without_checkout_filter() { - let db_path = unique_test_db_path("recent-repository-level"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - db.insert_diff_trace(DiffTraceInsert { - time_ms: 999, - session_id: "oc_before-cutoff", - patch: &valid_patch("notes/before.md", "before"), - model_id: Some("provider/model"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("before-cutoff diff trace insert should succeed"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_000, - session_id: "oc_checkout-a-session", - patch: &valid_patch("notes/a.md", "same repository checkout a"), - model_id: Some("provider/model-a"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("checkout-a diff trace insert should succeed"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_500, - session_id: "pi_checkout-b-session", - patch: &valid_patch("notes/b.md", "same repository checkout b"), - model_id: Some("provider/model-b"), - tool_name: "pi", - tool_version: None, - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("checkout-b diff trace insert should succeed"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: 2_001, - session_id: "oc_after-end", - patch: &valid_patch("notes/after.md", "after"), - model_id: Some("provider/model"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("after-end diff trace insert should succeed"); - - let recent = db - .recent_diff_trace_patches(1_000, 2_000) - .expect("recent repository diff traces should load"); - - assert_eq!(recent.loaded_count(), 2); - assert_eq!(recent.skipped_count(), 0); - assert_eq!( - recent - .patches - .iter() - .map(|patch| ( - patch.time_ms, - patch.session_id.as_str(), - patch.tool_name.as_deref() - )) - .collect::>(), - vec![ - (1_000, "oc_checkout-a-session", Some("opencode")), - (1_500, "pi_checkout-b-session", Some("pi")), - ] - ); - - remove_test_db(&db_path); - } - - #[test] - fn recent_diff_trace_reads_are_isolated_by_repository_db_path() { - let first_db_path = unique_test_db_path("recent-repo-one"); - let second_db_path = unique_test_db_path("recent-repo-two"); - let first_db = RepositoryAgentTraceDb::new_at(&first_db_path) - .expect("first repository DB should open"); - let second_db = RepositoryAgentTraceDb::new_at(&second_db_path) - .expect("second repository DB should open"); - - first_db - .insert_diff_trace(DiffTraceInsert { - time_ms: 1_000, - session_id: "oc_first-repo", - patch: &valid_patch("notes/first.md", "first repository"), - model_id: Some("provider/first"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("first repository diff trace insert should succeed"); - second_db - .insert_diff_trace(DiffTraceInsert { - time_ms: 1_000, - session_id: "oc_second-repo", - patch: &valid_patch("notes/second.md", "second repository"), - model_id: Some("provider/second"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("second repository diff trace insert should succeed"); - - let first_recent = first_db - .recent_diff_trace_patches(0, 2_000) - .expect("first repository recent traces should load"); - let second_recent = second_db - .recent_diff_trace_patches(0, 2_000) - .expect("second repository recent traces should load"); - - assert_eq!(first_recent.loaded_count(), 1); - assert_eq!(second_recent.loaded_count(), 1); - assert_eq!(first_recent.patches[0].session_id, "oc_first-repo"); - assert_eq!(second_recent.patches[0].session_id, "oc_second-repo"); - assert_ne!( - first_recent.patches[0].session_id, - second_recent.patches[0].session_id - ); - - remove_test_db(&first_db_path); - remove_test_db(&second_db_path); - } - - #[test] - fn spec_path_constructor_is_rejected() { - let error = RepositoryAgentTraceDbSpec::db_path() - .expect_err("repository DBs must not have a canonical spec path"); - assert!(error.to_string().contains("explicit-path")); - } -} - -#[cfg(test)] -pub(crate) mod pre_health_invariant_fixture { - use std::path::{Path, PathBuf}; - - use anyhow::Result; - - use crate::{ - generated_migrations, - services::db::{DbSpec, TursoDb}, - }; - - const PRE_HEALTH_INVARIANT_MIGRATION_COUNT: usize = 5; - - struct PreHealthInvariantDbSpec; - - impl DbSpec for PreHealthInvariantDbSpec { - fn db_name() -> &'static str { - "pre-006 repository Agent Trace DB" - } - - fn db_path() -> Result { - anyhow::bail!("pre-006 repository Agent Trace DB has no canonical path") - } - - fn migrations() -> &'static [(&'static str, &'static str)] { - &generated_migrations::AGENT_TRACE_REPOSITORY_MIGRATIONS - [..PRE_HEALTH_INVARIANT_MIGRATION_COUNT] - } - - fn db_config_key() -> &'static str { - "agent_trace_db" - } - } - - pub(crate) fn seed_pre_health_invariant_fixture(db_path: &Path, statements: &[&str]) { - let fixture = TursoDb::::new_at(db_path) - .expect("pre-006 fixture DB should migrate through 005"); - let applied_ids = fixture - .query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("migration metadata query should succeed"); - assert_eq!( - applied_ids.last().map(String::as_str), - Some("005_mutation_scope_provenance"), - "the pre-006 fixture must stop at 005" - ); - for statement in statements { - fixture - .execute(statement, ()) - .unwrap_or_else(|error| panic!("fixture row should insert: {error}")); - } - drop(fixture); + insert_conversation_text_event_with(self, message, part, false).await } } diff --git a/cli/src/services/agent_trace_db/transaction_tests.rs b/cli/src/services/agent_trace_db/transaction_tests.rs new file mode 100644 index 000000000..61e3833b5 --- /dev/null +++ b/cli/src/services/agent_trace_db/transaction_tests.rs @@ -0,0 +1,91 @@ +use std::path::Path; +use std::time::Duration; + +use super::repository::RepositoryAgentTraceDb; +use super::{InsertMessageInsert, InsertPartInsert, MessageRole, PartType}; + +pub(crate) fn message(id: &str) -> InsertMessageInsert { + InsertMessageInsert { + session_id: "session".to_string(), + message_id: id.to_string(), + role: MessageRole::User, + generated_at_unix_ms: 1, + } +} + +pub(crate) fn part(id: &str) -> InsertPartInsert { + InsertPartInsert { + part_type: PartType::Text, + text: "text".to_string(), + session_id: "session".to_string(), + message_id: id.to_string(), + generated_at_unix_ms: 1, + } +} + +async fn count(db: &RepositoryAgentTraceDb, table: &str, id: &str) -> i64 { + let sql = format!("SELECT COUNT(*) FROM {table} WHERE message_id = ?1"); + db.query_map(&sql, (id.to_string(),), |row| { + row.get::(0).map_err(Into::into) + }) + .await + .unwrap()[0] +} + +pub(crate) async fn pair_counts(db: &RepositoryAgentTraceDb, id: &str) -> (i64, i64) { + ( + count(db, "messages", id).await, + count(db, "parts", id).await, + ) +} + +pub(crate) async fn open(path: &Path) -> RepositoryAgentTraceDb { + RepositoryAgentTraceDb::new_at(path).await.unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn write_contention_retries_whole_transaction_once_and_persists_one_pair() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("agent-trace.db"); + let mut writer = open(&path).await; + let holder = open(&path).await; + + holder.execute("BEGIN IMMEDIATE", ()).await.unwrap(); + let release = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1300)).await; + holder.execute("COMMIT", ()).await.unwrap(); + }); + + let inserted = writer + .insert_conversation_text_event(message("contended"), part("contended")) + .await + .unwrap(); + release.await.unwrap(); + + assert!(inserted); + assert_eq!(pair_counts(&writer, "contended").await, (1, 1)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn write_contention_exhausts_bounded_attempts_without_persisting_rows() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("agent-trace.db"); + let mut writer = open(&path).await; + let holder = open(&path).await; + + holder.execute("BEGIN IMMEDIATE", ()).await.unwrap(); + let error = writer + .insert_conversation_text_event(message("exhausted"), part("exhausted")) + .await + .unwrap_err() + .to_string(); + holder.execute("COMMIT", ()).await.unwrap(); + + assert!(error.contains("attempts=2"), "{error}"); + assert!(error.contains("under write contention"), "{error}"); + assert_eq!(pair_counts(&writer, "exhausted").await, (0, 0)); + assert!(writer + .insert_conversation_text_event(message("after"), part("after")) + .await + .unwrap()); +} diff --git a/cli/src/services/agent_trace_export/mod.rs b/cli/src/services/agent_trace_export/mod.rs index 626b3c8b7..64b1e77d1 100644 --- a/cli/src/services/agent_trace_export/mod.rs +++ b/cli/src/services/agent_trace_export/mod.rs @@ -76,6 +76,10 @@ pub struct AgentTracePartExportRow { } /// Owned, wire-compatible export row for the `diff_traces` capture stream. +#[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" +)] #[derive(Clone, Debug, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AgentTraceDiffTraceExportRow { @@ -116,6 +120,10 @@ WHERE id > ?1 ORDER BY id ASC LIMIT ?2"; +#[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" +)] const SELECT_DIFF_TRACES_AFTER_SQL: &str = "SELECT id, session_id, time_ms, patch, model_id, tool_name, tool_version, payload_type FROM diff_traces @@ -145,7 +153,7 @@ impl<'a> AgentTraceExportReader<'a> { /// Read `messages` rows with `id > cursor`, ordered by `id ASC`, capped /// at `limit`. - pub fn read_messages_after( + pub async fn read_messages_after( &self, cursor: i64, limit: usize, @@ -153,11 +161,14 @@ impl<'a> AgentTraceExportReader<'a> { validate_cursor(cursor)?; validate_limit(limit)?; - let rows = self.db.query_map( - SELECT_MESSAGES_AFTER_SQL, - (cursor, limit_as_i64(limit)), - message_export_row_from_turso, - )?; + let rows = self + .db + .query_map( + SELECT_MESSAGES_AFTER_SQL, + (cursor, limit_as_i64(limit)), + message_export_row_from_turso, + ) + .await?; for row in &rows { validate_js_safe_integer(row.source_row_id)?; @@ -169,7 +180,7 @@ impl<'a> AgentTraceExportReader<'a> { /// Read `parts` rows with `id > cursor`, ordered by `id ASC`, capped at /// `limit`. - pub fn read_parts_after( + pub async fn read_parts_after( &self, cursor: i64, limit: usize, @@ -177,11 +188,14 @@ impl<'a> AgentTraceExportReader<'a> { validate_cursor(cursor)?; validate_limit(limit)?; - let rows = self.db.query_map( - SELECT_PARTS_AFTER_SQL, - (cursor, limit_as_i64(limit)), - part_export_row_from_turso, - )?; + let rows = self + .db + .query_map( + SELECT_PARTS_AFTER_SQL, + (cursor, limit_as_i64(limit)), + part_export_row_from_turso, + ) + .await?; for row in &rows { validate_js_safe_integer(row.source_row_id)?; @@ -194,7 +208,11 @@ impl<'a> AgentTraceExportReader<'a> { /// Read `diff_traces` rows with `id > cursor`, ordered by `id ASC`, /// capped at `limit`. `patch` and `payload_type` are returned raw and /// unmodified: no patch parsing or normalization is performed. - pub fn read_diff_traces_after( + #[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" + )] + pub async fn read_diff_traces_after( &self, cursor: i64, limit: usize, @@ -202,11 +220,14 @@ impl<'a> AgentTraceExportReader<'a> { validate_cursor(cursor)?; validate_limit(limit)?; - let rows = self.db.query_map( - SELECT_DIFF_TRACES_AFTER_SQL, - (cursor, limit_as_i64(limit)), - diff_trace_export_row_from_turso, - )?; + let rows = self + .db + .query_map( + SELECT_DIFF_TRACES_AFTER_SQL, + (cursor, limit_as_i64(limit)), + diff_trace_export_row_from_turso, + ) + .await?; for row in &rows { validate_js_safe_integer(row.source_row_id)?; @@ -219,7 +240,7 @@ impl<'a> AgentTraceExportReader<'a> { /// Read `agent_traces` rows with `id > cursor`, ordered by `id ASC`, /// capped at `limit`. `trace_json` is returned as the exact raw string /// from `SQLite`: no parse/reserialize is performed. - pub fn read_agent_traces_after( + pub async fn read_agent_traces_after( &self, cursor: i64, limit: usize, @@ -227,11 +248,14 @@ impl<'a> AgentTraceExportReader<'a> { validate_cursor(cursor)?; validate_limit(limit)?; - let rows = self.db.query_map( - SELECT_AGENT_TRACES_AFTER_SQL, - (cursor, limit_as_i64(limit)), - agent_trace_export_row_from_turso, - )?; + let rows = self + .db + .query_map( + SELECT_AGENT_TRACES_AFTER_SQL, + (cursor, limit_as_i64(limit)), + agent_trace_export_row_from_turso, + ) + .await?; for row in &rows { validate_js_safe_integer(row.source_row_id)?; @@ -271,6 +295,10 @@ fn part_export_row_from_turso(row: &turso::Row) -> Result Result { Ok(AgentTraceDiffTraceExportRow { source_row_id: row.get(0).context("failed to read diff_traces.id")?, @@ -326,1216 +354,3 @@ fn message_role_from_column(value: &str) -> Result { fn limit_as_i64(limit: usize) -> i64 { i64::try_from(limit).expect("validated limit should fit in i64") } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn message_export_row_serializes_to_camel_case_contract() { - let row = AgentTraceMessageExportRow { - source_row_id: 1, - session_id: "sess-1".to_string(), - message_id: "msg-1".to_string(), - role: MessageRole::Assistant, - generated_at_unix_ms: 1_700_000_000_000, - }; - - assert_eq!( - serde_json::to_value(&row).expect("row should serialize"), - serde_json::json!({ - "sourceRowId": 1, - "sessionId": "sess-1", - "messageId": "msg-1", - "role": "assistant", - "generatedAtUnixMs": 1_700_000_000_000_i64, - }) - ); - } - - #[test] - fn message_export_row_serializes_user_role_lowercase() { - let row = AgentTraceMessageExportRow { - source_row_id: 2, - session_id: "sess-2".to_string(), - message_id: "msg-2".to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_700_000_000_001, - }; - - let value = serde_json::to_value(&row).expect("row should serialize"); - assert_eq!(value["role"], serde_json::json!("user")); - } - - #[test] - fn part_export_row_serializes_to_camel_case_contract() { - let row = AgentTracePartExportRow { - source_row_id: 5, - session_id: "sess-1".to_string(), - message_id: "msg-1".to_string(), - part_type: "text".to_string(), - text: "hello".to_string(), - generated_at_unix_ms: 1_700_000_000_002, - }; - - assert_eq!( - serde_json::to_value(&row).expect("row should serialize"), - serde_json::json!({ - "sourceRowId": 5, - "sessionId": "sess-1", - "messageId": "msg-1", - "type": "text", - "text": "hello", - "generatedAtUnixMs": 1_700_000_000_002_i64, - }) - ); - } - - #[test] - fn diff_trace_export_row_serializes_with_all_fields_populated() { - let row = AgentTraceDiffTraceExportRow { - source_row_id: 7, - session_id: "sess-3".to_string(), - time_ms: 1_700_000_000_003, - patch: "Index: a\n".to_string(), - model_id: Some("test-provider/test-model".to_string()), - tool_name: Some("opencode".to_string()), - tool_version: Some("1.2.3".to_string()), - payload_type: "patch".to_string(), - }; - - assert_eq!( - serde_json::to_value(&row).expect("row should serialize"), - serde_json::json!({ - "sourceRowId": 7, - "sessionId": "sess-3", - "timeMs": 1_700_000_000_003_i64, - "patch": "Index: a\n", - "modelId": "test-provider/test-model", - "toolName": "opencode", - "toolVersion": "1.2.3", - "payloadType": "patch", - }) - ); - } - - #[test] - fn diff_trace_export_row_serializes_nullable_fields_as_null() { - let row = AgentTraceDiffTraceExportRow { - source_row_id: 8, - session_id: "sess-4".to_string(), - time_ms: 1_700_000_000_004, - patch: "Index: b\n".to_string(), - model_id: None, - tool_name: None, - tool_version: None, - payload_type: "structured".to_string(), - }; - - assert_eq!( - serde_json::to_value(&row).expect("row should serialize"), - serde_json::json!({ - "sourceRowId": 8, - "sessionId": "sess-4", - "timeMs": 1_700_000_000_004_i64, - "patch": "Index: b\n", - "modelId": null, - "toolName": null, - "toolVersion": null, - "payloadType": "structured", - }) - ); - } - - #[test] - fn agent_trace_export_row_serializes_with_remote_url_null() { - let row = AgentTraceAgentTraceExportRow { - source_row_id: 9, - agent_trace_id: "trace-1".to_string(), - commit_id: "abc123".to_string(), - commit_time_ms: 1_700_000_000_005, - trace_json: "{\"steps\":[]}".to_string(), - url: "https://example.com/trace/1".to_string(), - remote_url: None, - }; - - assert_eq!( - serde_json::to_value(&row).expect("row should serialize"), - serde_json::json!({ - "sourceRowId": 9, - "agentTraceId": "trace-1", - "commitId": "abc123", - "commitTimeMs": 1_700_000_000_005_i64, - "traceJson": "{\"steps\":[]}", - "url": "https://example.com/trace/1", - "remoteUrl": null, - }) - ); - } - - #[test] - fn agent_trace_export_row_serializes_with_remote_url_populated() { - let row = AgentTraceAgentTraceExportRow { - source_row_id: 10, - agent_trace_id: "trace-2".to_string(), - commit_id: "def456".to_string(), - commit_time_ms: 1_700_000_000_006, - trace_json: "{\"steps\":[1]}".to_string(), - url: "https://example.com/trace/2".to_string(), - remote_url: Some("https://github.com/org/repo/commit/def456".to_string()), - }; - - let value = serde_json::to_value(&row).expect("row should serialize"); - assert_eq!( - value["remoteUrl"], - serde_json::json!("https://github.com/org/repo/commit/def456") - ); - } - - #[test] - fn validate_cursor_rejects_negative() { - let error = validate_cursor(-1).expect_err("negative cursor should error"); - assert!(error.to_string().contains("cursor")); - } - - #[test] - fn validate_cursor_accepts_zero_and_positive() { - assert!(validate_cursor(0).is_ok()); - assert!(validate_cursor(42).is_ok()); - } - - #[test] - fn validate_limit_rejects_zero() { - let error = validate_limit(0).expect_err("zero limit should error"); - assert!(error.to_string().contains("limit")); - } - - #[test] - fn validate_limit_rejects_above_batch_size() { - let error = validate_limit(AGENT_TRACE_EXPORT_BATCH_SIZE + 1) - .expect_err("limit above batch size should error"); - assert!(error.to_string().contains("limit")); - } - - #[test] - fn validate_limit_accepts_batch_size() { - assert!(validate_limit(AGENT_TRACE_EXPORT_BATCH_SIZE).is_ok()); - } - - #[test] - fn validate_limit_accepts_one() { - assert!(validate_limit(1).is_ok()); - } - - #[test] - fn validate_js_safe_integer_accepts_zero() { - assert!(validate_js_safe_integer(0).is_ok()); - } - - #[test] - fn validate_js_safe_integer_accepts_max_safe_integer() { - assert!(validate_js_safe_integer(JS_MAX_SAFE_INTEGER).is_ok()); - } - - #[test] - fn validate_js_safe_integer_rejects_above_max_safe_integer() { - let error = validate_js_safe_integer(JS_MAX_SAFE_INTEGER + 1) - .expect_err("value above max safe integer should error"); - assert!(error.to_string().contains("JS-safe-integer")); - } - - #[test] - fn validate_js_safe_integer_rejects_negative() { - let error = validate_js_safe_integer(-1).expect_err("negative value should error"); - assert!(error.to_string().contains("JS-safe-integer")); - } - - use std::{ - fs, - path::PathBuf, - time::{SystemTime, UNIX_EPOCH}, - }; - - use crate::services::agent_trace_db::{ - AgentTraceInsert, DiffTraceInsert, InsertMessageInsert, InsertPartInsert, PartType, - PAYLOAD_TYPE_PATCH, PAYLOAD_TYPE_STRUCTURED, - }; - - fn unique_test_db_path(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-agent-trace-export-{label}-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") - } - - fn remove_test_db(db_path: &std::path::Path) { - if let Some(parent) = db_path.parent() { - fs::remove_dir_all(parent).expect("test DB directory should be removed"); - } - } - - fn row_count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("count row should exist") - } - - fn insert_message_row_with_id(db: &RepositoryAgentTraceDb, id: i64, message_id: &str) { - db.execute( - "INSERT INTO messages (id, session_id, message_id, role, generated_at_unix_ms) VALUES (?1, ?2, ?3, ?4, ?5)", - (id, "sess-1", message_id, "assistant", 1_000 + id), - ) - .expect("direct message insert should succeed"); - } - - fn insert_part_row_with_id(db: &RepositoryAgentTraceDb, id: i64, message_id: &str) { - db.execute( - "INSERT INTO parts (id, type, text, message_id, session_id, generated_at_unix_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - (id, "text", "hello", message_id, "sess-1", 1_000 + id), - ) - .expect("direct part insert should succeed"); - } - - fn insert_diff_trace_row_with_id(db: &RepositoryAgentTraceDb, id: i64, session_id: &str) { - db.execute( - "INSERT INTO diff_traces (id, time_ms, session_id, patch, model_id, tool_name, tool_version, payload_type) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - ( - id, - 1_000 + id, - session_id, - "Index: a\n", - Option::<&str>::None, - Option::<&str>::None, - Option::<&str>::None, - PAYLOAD_TYPE_PATCH, - ), - ) - .expect("direct diff_trace insert should succeed"); - } - - fn insert_agent_trace_row_with_id(db: &RepositoryAgentTraceDb, id: i64, agent_trace_id: &str) { - db.execute( - "INSERT INTO agent_traces (id, commit_id, commit_time_ms, trace_json, agent_trace_id, url, remote_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - ( - id, - "abc123", - 1_000 + id, - "{\"steps\":[]}", - agent_trace_id, - "https://example.com/trace", - Option::<&str>::None, - ), - ) - .expect("direct agent_trace insert should succeed"); - } - - #[test] - fn read_messages_after_returns_rows_after_cursor_in_order() { - let db_path = unique_test_db_path("messages-basic"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.insert_messages(vec![ - InsertMessageInsert { - session_id: "sess-1".to_string(), - message_id: "msg-1".to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_001, - }, - InsertMessageInsert { - session_id: "sess-1".to_string(), - message_id: "msg-2".to_string(), - role: MessageRole::Assistant, - generated_at_unix_ms: 1_002, - }, - InsertMessageInsert { - session_id: "sess-1".to_string(), - message_id: "msg-3".to_string(), - role: MessageRole::Assistant, - generated_at_unix_ms: 1_003, - }, - ]) - .expect("seed messages should insert"); - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_messages_after(1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![2, 3] - ); - assert_eq!(rows[0].message_id, "msg-2"); - assert_eq!(rows[0].role, MessageRole::Assistant); - assert_eq!(rows[0].generated_at_unix_ms, 1_002); - - remove_test_db(&db_path); - } - - #[test] - fn read_messages_after_returns_non_contiguous_ids() { - let db_path = unique_test_db_path("messages-gap"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for (id, message_id) in [ - (11, "msg-11"), - (15, "msg-15"), - (19, "msg-19"), - (30, "msg-30"), - ] { - insert_message_row_with_id(&db, id, message_id); - } - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_messages_after(10, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![11, 15, 19, 30] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_messages_after_limit_truncates_and_follow_up_continues() { - let db_path = unique_test_db_path("messages-limit"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for id in 1..=10 { - db.insert_message(InsertMessageInsert { - session_id: "sess-1".to_string(), - message_id: format!("msg-{id}"), - role: MessageRole::Assistant, - generated_at_unix_ms: 1_000 + id, - }) - .expect("seed message should insert"); - } - - let reader = AgentTraceExportReader::new(&db); - let first_batch = reader - .read_messages_after(0, 3) - .expect("first limited read should succeed"); - assert_eq!( - first_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![1, 2, 3] - ); - - let next_cursor = first_batch - .last() - .expect("first batch non-empty") - .source_row_id; - let second_batch = reader - .read_messages_after(next_cursor, 3) - .expect("follow-up read should succeed"); - assert_eq!( - second_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![4, 5, 6] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_messages_after_returns_empty_at_or_beyond_max_id() { - let db_path = unique_test_db_path("messages-empty-tail"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_message_row_with_id(&db, 1, "msg-1"); - - let reader = AgentTraceExportReader::new(&db); - assert!(reader - .read_messages_after(1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read at max id should succeed") - .is_empty()); - assert!(reader - .read_messages_after(100, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read beyond max id should succeed") - .is_empty()); - - remove_test_db(&db_path); - } - - #[test] - fn read_messages_after_rejects_invalid_cursor_and_limit() { - let db_path = unique_test_db_path("messages-invalid"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - let reader = AgentTraceExportReader::new(&db); - - let cursor_error = reader - .read_messages_after(-1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("negative cursor should error"); - assert!(cursor_error.to_string().contains("cursor")); - - let zero_limit_error = reader - .read_messages_after(0, 0) - .expect_err("zero limit should error"); - assert!(zero_limit_error.to_string().contains("limit")); - - let excess_limit_error = reader - .read_messages_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE + 1) - .expect_err("excess limit should error"); - assert!(excess_limit_error.to_string().contains("limit")); - - remove_test_db(&db_path); - } - - #[test] - fn read_messages_after_rejects_row_above_safe_integer_bound() { - let db_path = unique_test_db_path("messages-unsafe-integer"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.execute( - "INSERT INTO messages (id, session_id, message_id, role, generated_at_unix_ms) VALUES (?1, ?2, ?3, ?4, ?5)", - (1_i64, "sess-1", "msg-1", "assistant", JS_MAX_SAFE_INTEGER + 1), - ) - .expect("direct message insert should succeed"); - - let reader = AgentTraceExportReader::new(&db); - let error = reader - .read_messages_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("row above safe-integer bound should error"); - assert!(error.to_string().contains("JS-safe-integer")); - - remove_test_db(&db_path); - } - - #[test] - fn read_messages_after_performs_no_mutation() { - let db_path = unique_test_db_path("messages-no-mutation"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_message_row_with_id(&db, 1, "msg-1"); - insert_message_row_with_id(&db, 2, "msg-2"); - - let messages_before = row_count(&db, "messages"); - let metadata_before = row_count(&db, "repository_metadata"); - - let reader = AgentTraceExportReader::new(&db); - reader - .read_messages_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read should succeed"); - - assert_eq!(row_count(&db, "messages"), messages_before); - assert_eq!(row_count(&db, "repository_metadata"), metadata_before); - - remove_test_db(&db_path); - } - - #[test] - fn read_parts_after_returns_rows_after_cursor_in_order() { - let db_path = unique_test_db_path("parts-basic"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.insert_parts(vec![ - InsertPartInsert { - part_type: PartType::Text, - text: "first".to_string(), - session_id: "sess-1".to_string(), - message_id: "msg-1".to_string(), - generated_at_unix_ms: 1_001, - }, - InsertPartInsert { - part_type: PartType::Patch, - text: "second".to_string(), - session_id: "sess-1".to_string(), - message_id: "msg-2".to_string(), - generated_at_unix_ms: 1_002, - }, - ]) - .expect("seed parts should insert"); - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_parts_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![1, 2] - ); - assert_eq!(rows[1].part_type, "patch"); - assert_eq!(rows[1].text, "second"); - assert_eq!(rows[1].message_id, "msg-2"); - assert_eq!(rows[1].generated_at_unix_ms, 1_002); - - remove_test_db(&db_path); - } - - #[test] - fn read_parts_after_returns_non_contiguous_ids() { - let db_path = unique_test_db_path("parts-gap"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for (id, message_id) in [ - (11, "msg-11"), - (15, "msg-15"), - (19, "msg-19"), - (30, "msg-30"), - ] { - insert_part_row_with_id(&db, id, message_id); - } - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_parts_after(10, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![11, 15, 19, 30] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_parts_after_limit_truncates_and_follow_up_continues() { - let db_path = unique_test_db_path("parts-limit"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for id in 1..=10 { - insert_part_row_with_id(&db, id, &format!("msg-{id}")); - } - - let reader = AgentTraceExportReader::new(&db); - let first_batch = reader - .read_parts_after(0, 3) - .expect("first limited read should succeed"); - assert_eq!( - first_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![1, 2, 3] - ); - - let next_cursor = first_batch - .last() - .expect("first batch non-empty") - .source_row_id; - let second_batch = reader - .read_parts_after(next_cursor, 3) - .expect("follow-up read should succeed"); - assert_eq!( - second_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![4, 5, 6] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_parts_after_returns_empty_at_or_beyond_max_id() { - let db_path = unique_test_db_path("parts-empty-tail"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_part_row_with_id(&db, 1, "msg-1"); - - let reader = AgentTraceExportReader::new(&db); - assert!(reader - .read_parts_after(1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read at max id should succeed") - .is_empty()); - assert!(reader - .read_parts_after(100, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read beyond max id should succeed") - .is_empty()); - - remove_test_db(&db_path); - } - - #[test] - fn read_parts_after_rejects_invalid_cursor_and_limit() { - let db_path = unique_test_db_path("parts-invalid"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - let reader = AgentTraceExportReader::new(&db); - - let cursor_error = reader - .read_parts_after(-1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("negative cursor should error"); - assert!(cursor_error.to_string().contains("cursor")); - - let zero_limit_error = reader - .read_parts_after(0, 0) - .expect_err("zero limit should error"); - assert!(zero_limit_error.to_string().contains("limit")); - - let excess_limit_error = reader - .read_parts_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE + 1) - .expect_err("excess limit should error"); - assert!(excess_limit_error.to_string().contains("limit")); - - remove_test_db(&db_path); - } - - #[test] - fn read_parts_after_rejects_row_above_safe_integer_bound() { - let db_path = unique_test_db_path("parts-unsafe-integer"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.execute( - "INSERT INTO parts (id, type, text, message_id, session_id, generated_at_unix_ms) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - (1_i64, "text", "hello", "msg-1", "sess-1", JS_MAX_SAFE_INTEGER + 1), - ) - .expect("direct part insert should succeed"); - - let reader = AgentTraceExportReader::new(&db); - let error = reader - .read_parts_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("row above safe-integer bound should error"); - assert!(error.to_string().contains("JS-safe-integer")); - - remove_test_db(&db_path); - } - - #[test] - fn read_parts_after_performs_no_mutation() { - let db_path = unique_test_db_path("parts-no-mutation"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_part_row_with_id(&db, 1, "msg-1"); - insert_part_row_with_id(&db, 2, "msg-2"); - - let parts_before = row_count(&db, "parts"); - let metadata_before = row_count(&db, "repository_metadata"); - - let reader = AgentTraceExportReader::new(&db); - reader - .read_parts_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read should succeed"); - - assert_eq!(row_count(&db, "parts"), parts_before); - assert_eq!(row_count(&db, "repository_metadata"), metadata_before); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_returns_rows_after_cursor_in_order() { - let db_path = unique_test_db_path("diff-traces-basic"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_001, - session_id: "sess-1", - patch: "Index: a\n", - model_id: Some("test-provider/test-model"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("seed diff_trace should insert"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_002, - session_id: "sess-1", - patch: "{\"tool\":\"Edit\"}", - model_id: None, - tool_name: "claude", - tool_version: None, - payload_type: PAYLOAD_TYPE_STRUCTURED, - }) - .expect("seed diff_trace should insert"); - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_diff_traces_after(1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![2] - ); - assert_eq!(rows[0].patch, "{\"tool\":\"Edit\"}"); - assert_eq!(rows[0].payload_type, "structured"); - assert_eq!(rows[0].model_id, None); - assert_eq!(rows[0].tool_name, Some("claude".to_string())); - assert_eq!(rows[0].tool_version, None); - assert_eq!(rows[0].time_ms, 1_002); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_preserves_populated_nullable_fields() { - let db_path = unique_test_db_path("diff-traces-nullable-populated"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_001, - session_id: "sess-1", - patch: "Index: a\n", - model_id: Some("test-provider/test-model"), - tool_name: "opencode", - tool_version: Some("1.2.3"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("seed diff_trace should insert"); - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_diff_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!(rows.len(), 1); - assert_eq!( - rows[0].model_id, - Some("test-provider/test-model".to_string()) - ); - assert_eq!(rows[0].tool_name, Some("opencode".to_string())); - assert_eq!(rows[0].tool_version, Some("1.2.3".to_string())); - assert_eq!(rows[0].patch, "Index: a\n"); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_returns_non_contiguous_ids() { - let db_path = unique_test_db_path("diff-traces-gap"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for (id, session_id) in [ - (11, "sess-11"), - (15, "sess-15"), - (19, "sess-19"), - (30, "sess-30"), - ] { - insert_diff_trace_row_with_id(&db, id, session_id); - } - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_diff_traces_after(10, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![11, 15, 19, 30] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_limit_truncates_and_follow_up_continues() { - let db_path = unique_test_db_path("diff-traces-limit"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for id in 1..=10 { - insert_diff_trace_row_with_id(&db, id, &format!("sess-{id}")); - } - - let reader = AgentTraceExportReader::new(&db); - let first_batch = reader - .read_diff_traces_after(0, 3) - .expect("first limited read should succeed"); - assert_eq!( - first_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![1, 2, 3] - ); - - let next_cursor = first_batch - .last() - .expect("first batch non-empty") - .source_row_id; - let second_batch = reader - .read_diff_traces_after(next_cursor, 3) - .expect("follow-up read should succeed"); - assert_eq!( - second_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![4, 5, 6] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_returns_empty_at_or_beyond_max_id() { - let db_path = unique_test_db_path("diff-traces-empty-tail"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_diff_trace_row_with_id(&db, 1, "sess-1"); - - let reader = AgentTraceExportReader::new(&db); - assert!(reader - .read_diff_traces_after(1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read at max id should succeed") - .is_empty()); - assert!(reader - .read_diff_traces_after(100, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read beyond max id should succeed") - .is_empty()); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_rejects_invalid_cursor_and_limit() { - let db_path = unique_test_db_path("diff-traces-invalid"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - let reader = AgentTraceExportReader::new(&db); - - let cursor_error = reader - .read_diff_traces_after(-1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("negative cursor should error"); - assert!(cursor_error.to_string().contains("cursor")); - - let zero_limit_error = reader - .read_diff_traces_after(0, 0) - .expect_err("zero limit should error"); - assert!(zero_limit_error.to_string().contains("limit")); - - let excess_limit_error = reader - .read_diff_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE + 1) - .expect_err("excess limit should error"); - assert!(excess_limit_error.to_string().contains("limit")); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_rejects_row_above_safe_integer_bound() { - let db_path = unique_test_db_path("diff-traces-unsafe-integer"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.execute( - "INSERT INTO diff_traces (id, time_ms, session_id, patch, model_id, tool_name, tool_version, payload_type) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - ( - 1_i64, - JS_MAX_SAFE_INTEGER + 1, - "sess-1", - "Index: a\n", - Option::<&str>::None, - Option::<&str>::None, - Option::<&str>::None, - PAYLOAD_TYPE_PATCH, - ), - ) - .expect("direct diff_trace insert should succeed"); - - let reader = AgentTraceExportReader::new(&db); - let error = reader - .read_diff_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("row above safe-integer bound should error"); - assert!(error.to_string().contains("JS-safe-integer")); - - remove_test_db(&db_path); - } - - #[test] - fn read_diff_traces_after_performs_no_mutation() { - let db_path = unique_test_db_path("diff-traces-no-mutation"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_diff_trace_row_with_id(&db, 1, "sess-1"); - insert_diff_trace_row_with_id(&db, 2, "sess-2"); - - let diff_traces_before = row_count(&db, "diff_traces"); - let metadata_before = row_count(&db, "repository_metadata"); - - let reader = AgentTraceExportReader::new(&db); - reader - .read_diff_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read should succeed"); - - assert_eq!(row_count(&db, "diff_traces"), diff_traces_before); - assert_eq!(row_count(&db, "repository_metadata"), metadata_before); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_returns_rows_after_cursor_in_order() { - let db_path = unique_test_db_path("agent-traces-basic"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.insert_agent_trace(AgentTraceInsert { - commit_id: "abc123", - commit_time_ms: 1_001, - trace_json: "{\"steps\":[]}", - agent_trace_id: "trace-1", - url: "https://example.com/trace/1", - remote_url: "", - }) - .expect("seed agent_trace should insert"); - db.insert_agent_trace(AgentTraceInsert { - commit_id: "def456", - commit_time_ms: 1_002, - trace_json: "{\"steps\":[1]}", - agent_trace_id: "trace-2", - url: "https://example.com/trace/2", - remote_url: "https://github.com/org/repo/commit/def456", - }) - .expect("seed agent_trace should insert"); - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_agent_traces_after(1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![2] - ); - assert_eq!(rows[0].agent_trace_id, "trace-2"); - assert_eq!(rows[0].commit_id, "def456"); - assert_eq!(rows[0].commit_time_ms, 1_002); - assert_eq!(rows[0].trace_json, "{\"steps\":[1]}"); - assert_eq!(rows[0].url, "https://example.com/trace/2"); - assert_eq!( - rows[0].remote_url, - Some("https://github.com/org/repo/commit/def456".to_string()) - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_preserves_null_remote_url() { - let db_path = unique_test_db_path("agent-traces-remote-url-null"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_agent_trace_row_with_id(&db, 1, "trace-null"); - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_agent_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].remote_url, None); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_returns_non_contiguous_ids() { - let db_path = unique_test_db_path("agent-traces-gap"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for (id, agent_trace_id) in [ - (11, "trace-11"), - (15, "trace-15"), - (19, "trace-19"), - (30, "trace-30"), - ] { - insert_agent_trace_row_with_id(&db, id, agent_trace_id); - } - - let reader = AgentTraceExportReader::new(&db); - let rows = reader - .read_agent_traces_after(10, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read after cursor should succeed"); - - assert_eq!( - rows.iter().map(|row| row.source_row_id).collect::>(), - vec![11, 15, 19, 30] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_limit_truncates_and_follow_up_continues() { - let db_path = unique_test_db_path("agent-traces-limit"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - for id in 1..=10 { - insert_agent_trace_row_with_id(&db, id, &format!("trace-{id}")); - } - - let reader = AgentTraceExportReader::new(&db); - let first_batch = reader - .read_agent_traces_after(0, 3) - .expect("first limited read should succeed"); - assert_eq!( - first_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![1, 2, 3] - ); - - let next_cursor = first_batch - .last() - .expect("first batch non-empty") - .source_row_id; - let second_batch = reader - .read_agent_traces_after(next_cursor, 3) - .expect("follow-up read should succeed"); - assert_eq!( - second_batch - .iter() - .map(|row| row.source_row_id) - .collect::>(), - vec![4, 5, 6] - ); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_returns_empty_at_or_beyond_max_id() { - let db_path = unique_test_db_path("agent-traces-empty-tail"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_agent_trace_row_with_id(&db, 1, "trace-1"); - - let reader = AgentTraceExportReader::new(&db); - assert!(reader - .read_agent_traces_after(1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read at max id should succeed") - .is_empty()); - assert!(reader - .read_agent_traces_after(100, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read beyond max id should succeed") - .is_empty()); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_rejects_invalid_cursor_and_limit() { - let db_path = unique_test_db_path("agent-traces-invalid"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - let reader = AgentTraceExportReader::new(&db); - - let cursor_error = reader - .read_agent_traces_after(-1, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("negative cursor should error"); - assert!(cursor_error.to_string().contains("cursor")); - - let zero_limit_error = reader - .read_agent_traces_after(0, 0) - .expect_err("zero limit should error"); - assert!(zero_limit_error.to_string().contains("limit")); - - let excess_limit_error = reader - .read_agent_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE + 1) - .expect_err("excess limit should error"); - assert!(excess_limit_error.to_string().contains("limit")); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_rejects_row_above_safe_integer_bound() { - let db_path = unique_test_db_path("agent-traces-unsafe-integer"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.execute( - "INSERT INTO agent_traces (id, commit_id, commit_time_ms, trace_json, agent_trace_id, url, remote_url) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - ( - 1_i64, - "abc123", - JS_MAX_SAFE_INTEGER + 1, - "{\"steps\":[]}", - "trace-unsafe", - "https://example.com/trace", - Option::<&str>::None, - ), - ) - .expect("direct agent_trace insert should succeed"); - - let reader = AgentTraceExportReader::new(&db); - let error = reader - .read_agent_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect_err("row above safe-integer bound should error"); - assert!(error.to_string().contains("JS-safe-integer")); - - remove_test_db(&db_path); - } - - #[test] - fn read_agent_traces_after_performs_no_mutation() { - let db_path = unique_test_db_path("agent-traces-no-mutation"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - insert_agent_trace_row_with_id(&db, 1, "trace-1"); - insert_agent_trace_row_with_id(&db, 2, "trace-2"); - - let agent_traces_before = row_count(&db, "agent_traces"); - let metadata_before = row_count(&db, "repository_metadata"); - - let reader = AgentTraceExportReader::new(&db); - reader - .read_agent_traces_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read should succeed"); - - assert_eq!(row_count(&db, "agent_traces"), agent_traces_before); - assert_eq!(row_count(&db, "repository_metadata"), metadata_before); - - remove_test_db(&db_path); - } - - #[test] - fn source_instance_integration() { - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - - let state_root = unique_storage_temp_dir("state"); - let repo_root = - init_git_repo_with_remote("repo", "git@github.com:acme/agent-trace-export-readers.git"); - - let context = AgentTraceStorageContext { - repository_root: &repo_root, - explicit_repository_id: None, - repository_remote: "origin", - }; - - let storage = resolve_agent_trace_storage_at_state_root(&context, &state_root) - .expect("repository-scoped Agent Trace storage should resolve"); - - assert!(!storage.metadata.repository_id.trim().is_empty()); - assert!(!storage.metadata.source_instance_id.trim().is_empty()); - - insert_message_row_with_id(&storage.db, 1, "msg-1"); - - let reader = AgentTraceExportReader::new(&storage.db); - let rows = reader - .read_messages_after(0, AGENT_TRACE_EXPORT_BATCH_SIZE) - .expect("read through storage-resolved db should succeed"); - - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].source_row_id, 1); - assert_eq!(rows[0].message_id, "msg-1"); - - fs::remove_dir_all(&state_root).expect("clean up state root"); - fs::remove_dir_all(&repo_root).expect("clean up repo root"); - } - - fn unique_storage_temp_dir(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - let dir = std::env::temp_dir().join(format!( - "sce-agent-trace-export-storage-{label}-{}-{nonce}", - std::process::id() - )); - fs::create_dir_all(&dir).expect("create temp dir"); - dir - } - - fn init_git_repo_with_remote(label: &str, remote_url: &str) -> PathBuf { - let repo = unique_storage_temp_dir(label); - git(&repo, &["init", "-q"]); - git(&repo, &["remote", "add", "origin", remote_url]); - repo - } - - fn git(repo_root: &std::path::Path, args: &[&str]) { - let output = std::process::Command::new("git") - .args(args) - .current_dir(repo_root) - .output() - .unwrap_or_else(|error| panic!("git {args:?} failed to spawn: {error}")); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } -} diff --git a/cli/src/services/agent_trace_storage/mod.rs b/cli/src/services/agent_trace_storage/mod.rs index 9b775a3cb..9d0a103c8 100644 --- a/cli/src/services/agent_trace_storage/mod.rs +++ b/cli/src/services/agent_trace_storage/mod.rs @@ -1,5 +1,4 @@ use std::path::{Path, PathBuf}; -use std::thread; use std::time::Duration; use anyhow::Result; @@ -48,26 +47,12 @@ pub struct ResolvedAgentTraceStorage { /// Resolves the repository-scoped Agent Trace storage for a checkout using /// the canonical state root from the default-path catalog. -pub fn resolve_agent_trace_storage( +pub async fn resolve_agent_trace_storage( context: &AgentTraceStorageContext<'_>, ) -> Result { let repository_identity = resolve_identity(context)?; let db_path = agent_trace_db_path_for_repository(&repository_identity.identity.repository_id)?; - open_storage(repository_identity, db_path) -} - -/// Resolution core against an explicit state root, so tests can exercise the -/// full path without touching the real user state directory. -pub fn resolve_agent_trace_storage_at_state_root( - context: &AgentTraceStorageContext<'_>, - state_root: &Path, -) -> Result { - let repository_identity = resolve_identity(context)?; - let db_path = agent_trace_db_path_for_repository_at( - state_root, - &repository_identity.identity.repository_id, - )?; - open_storage(repository_identity, db_path) + open_storage(repository_identity, db_path).await } /// Resolves repository-scoped Agent Trace storage for high-frequency hook @@ -78,17 +63,17 @@ pub fn resolve_agent_trace_storage_at_state_root( /// baseline-only (pre-`002`) database fails with the same `sce setup` /// guidance `ensure_schema_ready_for_hooks` already reports, rather than /// silently migrating it from a hook path. -pub fn resolve_agent_trace_storage_for_hook_runtime( +pub async fn resolve_agent_trace_storage_for_hook_runtime( context: &AgentTraceStorageContext<'_>, ) -> Result { let repository_identity = resolve_identity(context)?; let db_path = agent_trace_db_path_for_repository(&repository_identity.identity.repository_id)?; - open_storage_for_hook_runtime(repository_identity, db_path) + open_storage_for_hook_runtime(repository_identity, db_path).await } /// Hook-runtime resolution core against an explicit state root, so tests can /// exercise the full path without touching the real user state directory. -pub fn resolve_agent_trace_storage_for_hook_runtime_at_state_root( +pub async fn resolve_agent_trace_storage_for_hook_runtime_at_state_root( context: &AgentTraceStorageContext<'_>, state_root: &Path, ) -> Result { @@ -97,7 +82,7 @@ pub fn resolve_agent_trace_storage_for_hook_runtime_at_state_root( state_root, &repository_identity.identity.repository_id, )?; - open_storage_for_hook_runtime(repository_identity, db_path) + open_storage_for_hook_runtime(repository_identity, db_path).await } fn resolve_identity(context: &AgentTraceStorageContext<'_>) -> Result { @@ -109,7 +94,7 @@ fn resolve_identity(context: &AgentTraceStorageContext<'_>) -> Result Result { @@ -118,9 +103,10 @@ fn open_storage( db_path, open_repository_db_concurrently_safe, ) + .await } -fn open_storage_for_hook_runtime( +async fn open_storage_for_hook_runtime( repository_identity: ResolvedRepositoryIdentity, db_path: PathBuf, ) -> Result { @@ -129,15 +115,19 @@ fn open_storage_for_hook_runtime( db_path, open_repository_db_for_hook_runtime, ) + .await } -fn open_storage_with( +async fn open_storage_with( repository_identity: ResolvedRepositoryIdentity, db_path: PathBuf, - open_db: impl FnOnce(&Path, &str) -> Result<(RepositoryAgentTraceDb, RepositoryMetadata)>, + open_db: impl std::ops::AsyncFnOnce( + &Path, + &str, + ) -> Result<(RepositoryAgentTraceDb, RepositoryMetadata)>, ) -> Result { let repository_id = &repository_identity.identity.repository_id; - let (db, metadata) = open_db(&db_path, repository_id)?; + let (db, metadata) = open_db(&db_path, repository_id).await?; Ok(ResolvedAgentTraceStorage { repository_identity, @@ -147,27 +137,33 @@ fn open_storage_with( }) } -fn open_repository_db_concurrently_safe( +async fn open_repository_db_concurrently_safe( db_path: &Path, repository_id: &str, ) -> Result<(RepositoryAgentTraceDb, RepositoryMetadata)> { let mut last_error = None; for attempt in 1..=REPOSITORY_DB_INITIALIZATION_ATTEMPTS { - let fast_open = - RepositoryAgentTraceDb::open_without_migrations_at(db_path).and_then(|db| { - if db.ensure_schema_ready_for_hooks().is_err() { - db.repair_missing_repository_schema_migration_metadata()?; - } - let metadata = db.verify_or_initialize_repository_metadata(repository_id)?; - Ok((db, metadata)) - }); + let fast_open = async { + let db = RepositoryAgentTraceDb::open_without_migrations_at(db_path).await?; + if db.ensure_schema_ready_for_hooks().await.is_err() { + db.repair_missing_repository_schema_migration_metadata() + .await?; + } + let metadata = db + .verify_or_initialize_repository_metadata(repository_id) + .await?; + Ok::<_, anyhow::Error>((db, metadata)) + } + .await; match fast_open { Ok(result) => return Ok(result), - Err(fast_error) => match RepositoryAgentTraceDb::new_at(db_path) { + Err(fast_error) => match RepositoryAgentTraceDb::new_at(db_path).await { Ok(db) => { - let metadata = db.verify_or_initialize_repository_metadata(repository_id)?; + let metadata = db + .verify_or_initialize_repository_metadata(repository_id) + .await?; return Ok((db, metadata)); } Err(init_error) => { @@ -178,7 +174,7 @@ fn open_repository_db_concurrently_safe( ) .context(init_error)); if attempt < REPOSITORY_DB_INITIALIZATION_ATTEMPTS { - thread::sleep(REPOSITORY_DB_INITIALIZATION_RETRY_DELAY); + tokio::time::sleep(REPOSITORY_DB_INITIALIZATION_RETRY_DELAY).await; } } }, @@ -196,498 +192,105 @@ fn open_repository_db_concurrently_safe( /// confirmed. Never falls back to running migrations: a missing or /// baseline-only (pre-`002`) database fails with the existing `sce setup` /// guidance instead. -fn open_repository_db_for_hook_runtime( +async fn open_repository_db_for_hook_runtime( db_path: &Path, repository_id: &str, ) -> Result<(RepositoryAgentTraceDb, RepositoryMetadata)> { - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path)?; + let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path).await?; - if db.ensure_schema_ready_for_hooks().is_err() { - db.repair_missing_repository_schema_migration_metadata()?; + if db.ensure_schema_ready_for_hooks().await.is_err() { + db.repair_missing_repository_schema_migration_metadata() + .await?; } - let metadata = db.verify_or_initialize_repository_metadata(repository_id)?; + let metadata = db + .verify_or_initialize_repository_metadata(repository_id) + .await?; Ok((db, metadata)) } -#[cfg(test)] -mod tests { - use super::*; - use std::process::Command; - use std::time::{SystemTime, UNIX_EPOCH}; - - use crate::services::agent_trace_db::{DiffTraceInsert, PAYLOAD_TYPE_PATCH}; - - fn unique_temp_dir(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - let dir = std::env::temp_dir().join(format!( - "sce-agent-trace-storage-{label}-{}-{nonce}", - std::process::id() - )); - std::fs::create_dir_all(&dir).expect("create temp dir"); - dir - } - - fn git(repo_root: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(repo_root) - .output() - .unwrap_or_else(|error| panic!("git {args:?} failed to spawn: {error}")); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - fn init_git_repo_with_remote(label: &str, remote_url: &str) -> PathBuf { - let repo = unique_temp_dir(label); - git(&repo, &["init", "-q"]); - git(&repo, &["remote", "add", "origin", remote_url]); - repo - } - - fn context_for(repository_root: &Path) -> AgentTraceStorageContext<'_> { - AgentTraceStorageContext { - repository_root, - explicit_repository_id: None, - repository_remote: "origin", - } - } - - fn assert_no_legacy_db_paths(state_root: &Path) { - let sce_dir = state_root.join("sce"); - assert!( - !sce_dir.join("agent-trace.db").exists(), - "resolver must not create the legacy global agent-trace.db" - ); - if let Ok(entries) = std::fs::read_dir(&sce_dir) { - for entry in entries { - let name = entry.expect("read sce dir entry").file_name(); - let name = name.to_string_lossy().into_owned(); - let is_db_file = Path::new(&name) - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("db")); - assert!( - !(name.starts_with("agent-trace-") && is_db_file), - "resolver must not create checkout-scoped DB '{name}'" - ); - } - } - } - - #[test] - fn different_repository_identities_use_different_db_paths() { - let state_root = unique_temp_dir("state-separate"); - let repo_a = init_git_repo_with_remote("repo-a", "git@github.com:acme/widgets.git"); - let repo_b = init_git_repo_with_remote("repo-b", "git@github.com:acme/gadgets.git"); - - let storage_a = - resolve_agent_trace_storage_at_state_root(&context_for(&repo_a), &state_root) - .expect("repo A storage should resolve"); - let storage_b = - resolve_agent_trace_storage_at_state_root(&context_for(&repo_b), &state_root) - .expect("repo B storage should resolve"); - - assert_ne!( - storage_a.repository_identity.identity.repository_id, - storage_b.repository_identity.identity.repository_id - ); - assert_ne!(storage_a.db_path, storage_b.db_path); - assert!(storage_a.db_path.is_file()); - assert!(storage_b.db_path.is_file()); - assert_no_legacy_db_paths(&state_root); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo_a).expect("clean up repo A"); - std::fs::remove_dir_all(&repo_b).expect("clean up repo B"); - } - - #[test] - fn clones_of_the_same_repository_share_the_db_path() { - let state_root = unique_temp_dir("state-clones"); - // Equivalent SSH and HTTPS remotes for the same logical repository. - let clone_a = init_git_repo_with_remote("clone-a", "git@github.com:acme/widgets.git"); - let clone_b = init_git_repo_with_remote("clone-b", "https://github.com/acme/widgets.git"); - - let storage_a = - resolve_agent_trace_storage_at_state_root(&context_for(&clone_a), &state_root) - .expect("clone A storage should resolve"); - let storage_b = - resolve_agent_trace_storage_at_state_root(&context_for(&clone_b), &state_root) - .expect("clone B storage should resolve"); - - assert_eq!( - storage_a.repository_identity.identity.repository_id, - storage_b.repository_identity.identity.repository_id - ); - assert_eq!(storage_a.db_path, storage_b.db_path); - assert_no_legacy_db_paths(&state_root); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&clone_a).expect("clean up clone A"); - std::fs::remove_dir_all(&clone_b).expect("clean up clone B"); - } - - #[test] - fn linked_worktree_shares_the_db_path() { - let state_root = unique_temp_dir("state-worktree"); - let repo = init_git_repo_with_remote("worktree-main", "git@github.com:acme/widgets.git"); - git(&repo, &["config", "user.email", "test@example.com"]); - git(&repo, &["config", "user.name", "Test"]); - git(&repo, &["commit", "--allow-empty", "-q", "-m", "init"]); - let worktree = unique_temp_dir("worktree-linked"); - // `git worktree add` refuses to use an existing directory unless empty; - // the helper creates it empty, so add into it directly. - git( - &repo, - &[ - "worktree", - "add", - "-q", - worktree.to_str().expect("utf-8 worktree path"), - ], - ); +#[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" +)] +pub async fn resolve_existing_agent_trace_storage_for_maintenance( + context: &AgentTraceStorageContext<'_>, +) -> Result { + let repository_identity = resolve_identity(context)?; + let db_path = agent_trace_db_path_for_repository(&repository_identity.identity.repository_id)?; + open_existing_storage_for_maintenance(repository_identity, db_path).await +} - let storage_main = - resolve_agent_trace_storage_at_state_root(&context_for(&repo), &state_root) - .expect("main checkout storage should resolve"); - let storage_worktree = - resolve_agent_trace_storage_at_state_root(&context_for(&worktree), &state_root) - .expect("worktree storage should resolve"); +#[allow( + dead_code, + reason = "maintenance wiring lands in later tasks of context/plans/mutation-cursor-ref-reconciliation-wiring.md" +)] +async fn open_existing_storage_for_maintenance( + repository_identity: ResolvedRepositoryIdentity, + db_path: PathBuf, +) -> Result { + let repository_id = &repository_identity.identity.repository_id; + let (db, metadata) = + RepositoryAgentTraceDb::open_verified_existing_at(&db_path, repository_id).await?; - assert_eq!(storage_main.db_path, storage_worktree.db_path); - assert_no_legacy_db_paths(&state_root); + Ok(ResolvedAgentTraceStorage { + repository_identity, + db_path, + db, + metadata, + }) +} - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&worktree).expect("clean up worktree"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } +#[cfg(test)] +pub(crate) async fn resolve_existing_agent_trace_storage_for_maintenance_at_state_root( + context: &AgentTraceStorageContext<'_>, + state_root: &Path, +) -> Result { + let repository_identity = resolve_identity(context)?; + let db_path = agent_trace_db_path_for_repository_at( + state_root, + &repository_identity.identity.repository_id, + )?; + open_existing_storage_for_maintenance(repository_identity, db_path).await +} - #[test] - fn explicit_repository_id_overrides_the_remote() { - let state_root = unique_temp_dir("state-explicit"); - let repo = init_git_repo_with_remote("explicit", "git@github.com:acme/widgets.git"); +#[cfg(test)] +mod tests { + use super::{ + resolve_existing_agent_trace_storage_for_maintenance_at_state_root, + AgentTraceStorageContext, + }; + use crate::services::agent_trace_db::repository::ExistingRepositoryDbError; + + #[tokio::test(flavor = "multi_thread")] + async fn agent_trace_storage_maintenance_open_of_missing_db_creates_nothing() { + let repository = tempfile::tempdir().expect("repository dir"); + let state_root = tempfile::tempdir().expect("state root"); let context = AgentTraceStorageContext { - repository_root: &repo, - explicit_repository_id: Some("acme-monorepo"), + repository_root: repository.path(), + explicit_repository_id: Some("maintenance-open-missing"), repository_remote: "origin", }; - let storage = resolve_agent_trace_storage_at_state_root(&context, &state_root) - .expect("explicit identity storage should resolve"); - assert_eq!( - storage.repository_identity.identity.canonical_identity, - "acme-monorepo" - ); - let expected_path = state_root - .join("sce") - .join("repos") - .join(&storage.repository_identity.identity.repository_id) - .join("agent-trace.db"); - assert_eq!(storage.db_path, expected_path); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn repeated_resolution_is_idempotent() { - let state_root = unique_temp_dir("state-idempotent"); - let repo = init_git_repo_with_remote("idempotent", "git@github.com:acme/widgets.git"); - - let first = resolve_agent_trace_storage_at_state_root(&context_for(&repo), &state_root) - .expect("first resolution should succeed"); - let second = resolve_agent_trace_storage_at_state_root(&context_for(&repo), &state_root) - .expect("second resolution should succeed"); - - assert_eq!(first.db_path, second.db_path); - assert_eq!( - first.repository_identity.identity.repository_id, - second.repository_identity.identity.repository_id - ); - assert_eq!( - first.metadata.repository_id, - first.repository_identity.identity.repository_id - ); - assert_eq!(first.metadata, second.metadata); - assert!(!first.metadata.source_instance_id.trim().is_empty()); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn missing_identity_fails_with_config_guidance_and_creates_nothing() { - let state_root = unique_temp_dir("state-missing"); - let repo = unique_temp_dir("no-remote"); - git(&repo, &["init", "-q"]); - - let Err(error) = - resolve_agent_trace_storage_at_state_root(&context_for(&repo), &state_root) + let Err(error) = resolve_existing_agent_trace_storage_for_maintenance_at_state_root( + &context, + state_root.path(), + ) + .await else { - panic!("repo without identity should fail") + panic!("a missing database must not open"); }; - assert!(error.to_string().contains(".sce/config.json")); - assert!( - !state_root.join("sce").exists(), - "failed resolution must not create state directories" - ); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn empty_repository_id_path_is_rejected() { - let error = agent_trace_db_path_for_repository_at(Path::new("/tmp/state"), " ") - .expect_err("empty repository ID should fail"); - assert!(error.to_string().contains("must not be empty")); - } - - #[test] - fn legacy_checkout_database_files_are_not_selected_or_modified() { - let state_root = unique_temp_dir("state-legacy-untouched"); - let repo = init_git_repo_with_remote("legacy-untouched", "git@github.com:acme/widgets.git"); - let sce_dir = state_root.join("sce"); - std::fs::create_dir_all(&sce_dir).expect("create legacy parent"); - let legacy_db_path = sce_dir.join("agent-trace-legacy-checkout.db"); - let legacy_bytes = b"legacy checkout database bytes must remain untouched"; - std::fs::write(&legacy_db_path, legacy_bytes).expect("write legacy DB fixture"); - - let storage = resolve_agent_trace_storage_at_state_root(&context_for(&repo), &state_root) - .expect("repository storage should resolve"); - - assert_ne!(storage.db_path, legacy_db_path); - assert_eq!( - std::fs::read(&legacy_db_path).expect("legacy DB should still be readable"), - legacy_bytes, - "repository storage resolution must not migrate, rewrite, or delete legacy DB bytes" - ); - assert!(storage.db_path.is_file()); - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn new_repository_database_starts_empty_and_shares_repository_level_rows() { - let state_root = unique_temp_dir("state-repository-rows"); - let clone_a = init_git_repo_with_remote("rows-clone-a", "git@github.com:acme/widgets.git"); - let clone_b = - init_git_repo_with_remote("rows-clone-b", "https://github.com/acme/widgets.git"); - - let storage_a = - resolve_agent_trace_storage_at_state_root(&context_for(&clone_a), &state_root) - .expect("clone A storage should resolve"); - assert!( - storage_a - .db - .recent_diff_trace_patches(0, 10_000) - .expect("empty repository DB should query") - .patches - .is_empty(), - "fresh repository DB should not import legacy or checkout data" - ); - - storage_a - .db - .insert_diff_trace(DiffTraceInsert { - time_ms: 1_234, - session_id: "checkout-a-session", - patch: "Index: src/lib.rs\n===================================================================\n--- src/lib.rs\n+++ src/lib.rs\n@@ -0,0 +1,1 @@\n+shared row\n", - model_id: Some("model-a"), - tool_name: "opencode", - tool_version: Some("1.0.0"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("diff trace insert should succeed"); - - let storage_b = - resolve_agent_trace_storage_at_state_root(&context_for(&clone_b), &state_root) - .expect("clone B storage should resolve"); - assert_eq!(storage_a.db_path, storage_b.db_path); - - let patches = storage_b - .db - .recent_diff_trace_patches(0, 10_000) - .expect("shared repository DB rows should query from clone B"); - assert_eq!(patches.patches.len(), 1); - let parsed = &patches.patches[0]; - assert_eq!(parsed.tool_name.as_deref(), Some("opencode")); - assert_eq!( - parsed.patch.files[0].hunks[0].model_id.as_deref(), - Some("model-a") - ); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&clone_a).expect("clean up clone A"); - std::fs::remove_dir_all(&clone_b).expect("clean up clone B"); - } - - #[test] - fn credential_bearing_remote_is_canonicalized_without_leaking_secrets_to_paths() { - let state_root = unique_temp_dir("state-credential-safe"); - let repo = init_git_repo_with_remote( - "credential-safe", - "https://user:super-secret-token@github.com/acme/widgets.git", - ); - - let storage = resolve_agent_trace_storage_at_state_root(&context_for(&repo), &state_root) - .expect("credential-bearing remote should resolve safely"); - - assert_eq!( - storage.repository_identity.identity.canonical_identity, - "github.com/acme/widgets" - ); - let rendered_path = storage.db_path.display().to_string(); - assert!(!rendered_path.contains("user")); - assert!(!rendered_path.contains("super-secret-token")); - assert!(!storage - .repository_identity - .identity - .repository_id - .contains("super-secret-token")); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn hook_runtime_resolution_fails_before_setup_when_db_is_missing() { - let state_root = unique_temp_dir("state-hook-missing"); - let repo = init_git_repo_with_remote("hook-missing", "git@github.com:acme/widgets.git"); - - let Err(error) = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &context_for(&repo), - &state_root, - ) else { - panic!("hook runtime resolution must fail before sce setup runs"); - }; - assert!( - error.to_string().contains("sce setup"), - "unexpected error: {error}" - ); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn hook_runtime_resolution_fails_before_setup_on_baseline_only_schema_without_recording_migration_002( - ) { - let state_root = unique_temp_dir("state-hook-baseline"); - let repo = init_git_repo_with_remote("hook-baseline", "git@github.com:acme/widgets.git"); - let identity = resolve_repository_identity(&repo, None, "origin") - .expect("identity should resolve for baseline fixture setup"); - let repository_id = identity.identity.repository_id.clone(); - let db_path = agent_trace_db_path_for_repository_at(&state_root, &repository_id) - .expect("db path should resolve for baseline fixture setup"); - - // Simulate a database created before migration 002 existed: build the - // pre-002 `repository_metadata` shape (no `source_instance_id` - // column) and record only migration 001 as applied, mirroring the - // fixture in `repository.rs`'s - // `baseline_only_fixture_migrates_and_gets_a_stable_source_instance_id`. - let baseline_only = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) - .expect("baseline-only repository DB should open"); - baseline_only - .execute( - "CREATE TABLE IF NOT EXISTS __sce_migrations ( - id TEXT PRIMARY KEY, - applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) -)", - (), - ) - .expect("migration metadata table should create"); - baseline_only - .execute( - "CREATE TABLE IF NOT EXISTS repository_metadata ( - id INTEGER PRIMARY KEY CHECK (id = 1), - repository_id TEXT NOT NULL, - created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) -)", - (), - ) - .expect("baseline repository_metadata table should create"); - baseline_only - .execute( - "INSERT INTO __sce_migrations (id) VALUES ('001_repository_schema')", - (), - ) - .expect("baseline migration record should insert"); - baseline_only - .execute( - "INSERT INTO repository_metadata (id, repository_id) VALUES (1, ?1)", - (repository_id.as_str(),), - ) - .expect("baseline metadata row should seed"); - drop(baseline_only); - - let Err(error) = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &context_for(&repo), - &state_root, - ) else { - panic!("hook runtime resolution must fail on a baseline-only (pre-002) schema"); - }; - assert!( - error.to_string().contains("sce setup"), - "unexpected error: {error}" - ); - - let reopened = RepositoryAgentTraceDb::open_without_migrations_at(&db_path) - .expect("baseline DB should still be openable after the failed hook resolution"); - let problems = reopened - .migration_metadata_problems() - .expect("migration metadata problems should be queryable"); + assert!(matches!( + error.downcast_ref::(), + Some(ExistingRepositoryDbError::Missing { .. }) + )); + let entries: Vec<_> = std::fs::read_dir(state_root.path()) + .expect("state root listing") + .collect(); assert!( - problems.iter().any(|problem| problem.contains("002")), - "migration 002 must remain unrecorded: {problems:?}" - ); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn hook_runtime_resolution_after_setup_matches_setup_path_metadata() { - let state_root = unique_temp_dir("state-hook-after-setup"); - let repo = init_git_repo_with_remote("hook-after-setup", "git@github.com:acme/widgets.git"); - - let setup = resolve_agent_trace_storage_at_state_root(&context_for(&repo), &state_root) - .expect("setup/lifecycle resolution should succeed"); - - let hook_runtime = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &context_for(&repo), - &state_root, - ) - .expect("hook runtime resolution should succeed once sce setup has run"); - - assert_eq!(hook_runtime.db_path, setup.db_path); - assert_eq!(hook_runtime.metadata, setup.metadata); - assert_eq!( - hook_runtime.metadata.repository_id, - setup.repository_identity.identity.repository_id + entries.is_empty(), + "state root must stay empty: {entries:?}" ); - assert!(!hook_runtime.metadata.source_instance_id.trim().is_empty()); - - std::fs::remove_dir_all(&state_root).expect("clean up state root"); - std::fs::remove_dir_all(&repo).expect("clean up repo"); - } - - #[test] - fn path_traversal_repository_id_is_rejected() { - for bad in ["../escape", "a/b", "a\\b", ".", ".."] { - let error = agent_trace_db_path_for_repository_at(Path::new("/tmp/state"), bad) - .expect_err("path-unsafe repository ID should be rejected"); - assert!( - error.to_string().contains("not a valid path segment"), - "unexpected error for '{bad}': {error}" - ); - } } } diff --git a/cli/src/services/agent_trace_sync/control_plane.rs b/cli/src/services/agent_trace_sync/control_plane.rs index 6ab29a341..fdd6d95ea 100644 --- a/cli/src/services/agent_trace_sync/control_plane.rs +++ b/cli/src/services/agent_trace_sync/control_plane.rs @@ -6,7 +6,9 @@ //! They perform no HTTP I/O and hold no cursor state themselves. use std::fmt; +use std::future::Future; use std::sync::Arc; +use std::time::Duration; use anyhow::anyhow; use reqwest::StatusCode; @@ -108,10 +110,9 @@ const STATE_PATH: &str = "agent-trace/ingestion/state"; const BATCH_PATH: &str = "agent-trace/ingestion/batch"; const ME_PATH: &str = "me"; -const STATE_RETRY_MAX_ATTEMPTS: u32 = 1; -const STATE_RETRY_TIMEOUT_MS: u64 = 60_000; -const STATE_RETRY_INITIAL_BACKOFF_MS: u64 = 250; -const STATE_RETRY_MAX_BACKOFF_MS: u64 = 2_000; +const BATCH_REQUEST_TIMEOUT: Duration = Duration::from_mins(1); + +const STATE_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(1, 60_000, 250, 2_000); /// Typed failure classification for control-plane HTTP interactions, kept /// separate from `CliError` so the sync engine and CLI wiring can @@ -221,24 +222,24 @@ enum StateAttempt { Terminal(ControlPlaneError), } -/// Seam over `token_storage::{load_tokens, save_tokens}` so tests can assert -/// exactly when a token is (or is not) saved without touching the real, -/// process-wide encrypted auth database. -pub trait CredentialStore: Send + Sync { - fn load(&self) -> Result, ControlPlaneError>; - fn save(&self, token: &TokenResponse) -> Result; +pub trait CredentialStore: Send + Sync + 'static { + fn load(&self) -> impl Future, ControlPlaneError>> + Send; + fn save( + &self, + token: &TokenResponse, + ) -> impl Future> + Send; } /// Production `CredentialStore` backed by the real encrypted auth database. pub struct SystemCredentialStore; impl CredentialStore for SystemCredentialStore { - fn load(&self) -> Result, ControlPlaneError> { - Ok(token_storage::load_tokens()?) + async fn load(&self) -> Result, ControlPlaneError> { + Ok(token_storage::load_tokens().await?) } - fn save(&self, token: &TokenResponse) -> Result { - Ok(token_storage::save_tokens(token)?) + async fn save(&self, token: &TokenResponse) -> Result { + Ok(token_storage::save_tokens(token).await?) } } @@ -247,13 +248,14 @@ impl CredentialStore for SystemCredentialStore { /// Loads/refreshes stored `WorkOS` credentials through the existing /// `auth`/`token_storage` primitives, injects the `Authorization: Bearer` /// header, and retries exactly once on an unexpected `401`. -pub struct AuthenticatedControlPlaneClient { +pub struct AuthenticatedControlPlaneClient { http: reqwest::Client, base_url: String, workos_api_base_url: String, workos_client_id: String, - credential_store: Arc, + credential_store: Arc, refresh_lock: Arc>, + batch_timeout: Duration, } /// Response body for `GET /me`. @@ -301,27 +303,40 @@ impl AuthenticatedControlPlaneClient { base_url, workos_api_base_url, workos_client_id, - Box::new(SystemCredentialStore), + SystemCredentialStore, ) } +} +impl AuthenticatedControlPlaneClient { pub fn with_credential_store( http: reqwest::Client, base_url: impl Into, workos_api_base_url: impl Into, workos_client_id: impl Into, - credential_store: Box, + credential_store: S, ) -> Self { Self { http, base_url: base_url.into(), workos_api_base_url: workos_api_base_url.into(), workos_client_id: workos_client_id.into(), - credential_store: Arc::from(credential_store), + credential_store: Arc::new(credential_store), refresh_lock: Arc::new(tokio::sync::Mutex::new(())), + batch_timeout: BATCH_REQUEST_TIMEOUT, } } + #[cfg(test)] + pub(crate) fn with_batch_timeout(mut self, batch_timeout: Duration) -> Self { + self.batch_timeout = batch_timeout; + self + } + + pub(crate) async fn wait_for_pending_refresh(&self) { + let _guard = self.refresh_lock.lock().await; + } + /// Calls `POST /agent-trace/ingestion/state` with a single attempt and a /// 60-second timeout. `400`/`403`/post-refresh-`401` are terminal and never /// retried. @@ -330,12 +345,7 @@ impl AuthenticatedControlPlaneClient { request: &AgentTraceIngestionStateRequest, ) -> Result { let url = self.endpoint(STATE_PATH); - let policy = RetryPolicy { - max_attempts: STATE_RETRY_MAX_ATTEMPTS, - timeout_ms: STATE_RETRY_TIMEOUT_MS, - initial_backoff_ms: STATE_RETRY_INITIAL_BACKOFF_MS, - max_backoff_ms: STATE_RETRY_MAX_BACKOFF_MS, - }; + let policy = STATE_RETRY_POLICY; let outcome = run_with_retry( policy, @@ -385,6 +395,10 @@ impl AuthenticatedControlPlaneClient { self.post_batch(request).await } + #[allow( + dead_code, + reason = "diff_traces compatibility surface retained per context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md" + )] pub async fn ingest_diff_traces( &self, request: &AgentTraceIngestionBatchRequest, @@ -408,7 +422,13 @@ impl AuthenticatedControlPlaneClient { { let url = self.endpoint(BATCH_PATH); let response = self - .execute_authenticated(|token| self.http.post(&url).bearer_auth(token).json(request)) + .execute_authenticated(|token| { + self.http + .post(&url) + .bearer_auth(token) + .timeout(self.batch_timeout) + .json(request) + }) .await?; classify_response(response).await } @@ -461,10 +481,6 @@ impl AuthenticatedControlPlaneClient { Ok(retried) } - /// Loads the stored token, reusing it as-is when still valid and - /// refreshing (and saving) it only when expired. Makes exactly one - /// expiry decision, so a token cannot be refreshed without also being - /// persisted. async fn resolve_access_token(&self) -> Result { let stored = self .load_credentials() @@ -475,7 +491,7 @@ impl AuthenticatedControlPlaneClient { return Ok(stored.access_token); } - let _refresh_guard = self.refresh_lock.lock().await; + let refresh_guard = Arc::clone(&self.refresh_lock).lock_owned().await; let stored = self .load_credentials() .await? @@ -485,17 +501,14 @@ impl AuthenticatedControlPlaneClient { return Ok(stored.access_token); } - self.refresh_and_save(&stored).await + self.refresh_and_save(refresh_guard, stored).await } - /// Refreshes the stored token while holding the client-wide single-flight - /// guard. Callers that observed the same rejected token can reuse a token - /// saved by an earlier caller instead of issuing another refresh. async fn force_refresh_access_token( &self, rejected_access_token: &str, ) -> Result { - let _refresh_guard = self.refresh_lock.lock().await; + let refresh_guard = Arc::clone(&self.refresh_lock).lock_owned().await; let stored = self .load_credentials() .await? @@ -505,45 +518,41 @@ impl AuthenticatedControlPlaneClient { return Ok(stored.access_token); } - self.refresh_and_save(&stored).await - } - - async fn refresh_and_save(&self, stored: &StoredTokens) -> Result { - let token = auth::renew_stored_token_from_refresh_token( - &self.http, - &self.workos_api_base_url, - &self.workos_client_id, - &stored.refresh_token, - ) - .await?; - self.save_credentials(&token).await?; - Ok(token.access_token) + self.refresh_and_save(refresh_guard, stored).await } - async fn load_credentials(&self) -> Result, ControlPlaneError> { + async fn refresh_and_save( + &self, + refresh_guard: tokio::sync::OwnedMutexGuard<()>, + stored: StoredTokens, + ) -> Result { + let http = self.http.clone(); + let workos_api_base_url = self.workos_api_base_url.clone(); + let workos_client_id = self.workos_client_id.clone(); let credential_store = Arc::clone(&self.credential_store); - tokio::task::spawn_blocking(move || credential_store.load()) - .await - .map_err(|error| { - ControlPlaneError::Storage(format!( - "credential store worker failed while loading credentials: {error}" - )) - })? + + tokio::spawn(async move { + let _refresh_guard = refresh_guard; + let token = auth::renew_stored_token_from_refresh_token( + &http, + &workos_api_base_url, + &workos_client_id, + &stored.refresh_token, + ) + .await?; + credential_store.save(&token).await?; + Ok(token.access_token) + }) + .await + .map_err(|error| { + ControlPlaneError::AuthenticationFailed(format!( + "credential refresh task did not complete: {error}" + )) + })? } - async fn save_credentials( - &self, - token: &TokenResponse, - ) -> Result { - let credential_store = Arc::clone(&self.credential_store); - let token = token.clone(); - tokio::task::spawn_blocking(move || credential_store.save(&token)) - .await - .map_err(|error| { - ControlPlaneError::Storage(format!( - "credential store worker failed while saving credentials: {error}" - )) - })? + async fn load_credentials(&self) -> Result, ControlPlaneError> { + self.credential_store.load().await } fn endpoint(&self, path: &str) -> String { @@ -590,10 +599,13 @@ where { let status = response.status(); if status.is_success() { - return response - .json::() - .await - .map_err(|error| ControlPlaneError::InvalidResponse(error.to_string())); + return response.json::().await.map_err(|error| { + if error.is_timeout() { + ControlPlaneError::Transport(error.to_string()) + } else { + ControlPlaneError::InvalidResponse(error.to_string()) + } + }); } let body = response @@ -636,806 +648,346 @@ where #[cfg(test)] mod tests { + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::TcpListener; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Barrier, Mutex}; + use std::sync::{Arc, Mutex}; use std::thread; - use std::time::{Duration, SystemTime, UNIX_EPOCH}; - - use super::*; - use crate::services::agent_trace_db::MessageRole; - use crate::services::agent_trace_export::{ - AgentTraceAgentTraceExportRow, AgentTraceDiffTraceExportRow, AgentTraceMessageExportRow, - AgentTracePartExportRow, - }; - use crate::services::agent_trace_sync::test_http_server::{CannedResponse, TestHttpServer}; - use serde_json::json; - - #[test] - fn ingestion_stream_serializes_to_exact_snake_case_wire_values() { - assert_eq!( - serde_json::to_value(IngestionStream::Messages).unwrap(), - json!("messages") - ); - assert_eq!( - serde_json::to_value(IngestionStream::Parts).unwrap(), - json!("parts") - ); - assert_eq!( - serde_json::to_value(IngestionStream::DiffTraces).unwrap(), - json!("diff_traces") - ); - assert_eq!( - serde_json::to_value(IngestionStream::AgentTraces).unwrap(), - json!("agent_traces") - ); - } - - #[test] - fn state_request_serializes_to_camel_case_fields() { - let request = AgentTraceIngestionStateRequest { - repository_id: "acme-monorepo".to_string(), - source_instance_id: "src-123".to_string(), - }; - - assert_eq!( - serde_json::to_value(&request).unwrap(), - json!({ - "repositoryId": "acme-monorepo", - "sourceInstanceId": "src-123", - }) - ); - } + use std::time::{SystemTime, UNIX_EPOCH}; - #[test] - fn state_response_deserializes_camel_case_cursor_fields() { - let response: AgentTraceIngestionStateResponse = serde_json::from_value(json!({ - "cursors": { - "messages": 10, - "parts": 20, - "diffTraces": 30, - "agentTraces": 40, - } - })) - .unwrap(); - - assert_eq!( - response.cursors, - AgentTraceCursors { - messages: 10, - parts: 20, - diff_traces: 30, - agent_traces: 40, - } - ); - } - - #[test] - fn agent_trace_cursors_accept_boundary_values() { - for boundary in [0_i64, 1_i64, agent_trace_export::JS_MAX_SAFE_INTEGER] { - let cursors = AgentTraceCursors { - messages: boundary, - parts: boundary, - diff_traces: boundary, - agent_traces: boundary, - }; - assert!( - cursors.validate().is_ok(), - "boundary {boundary} should be valid" - ); - } - } - - #[test] - fn agent_trace_cursors_reject_out_of_range_value_in_each_field() { - let base = AgentTraceCursors { - messages: 0, - parts: 0, - diff_traces: 0, - agent_traces: 0, - }; - - for invalid in [-1_i64, agent_trace_export::JS_MAX_SAFE_INTEGER + 1] { - let cases = [ - AgentTraceCursors { - messages: invalid, - ..base - }, - AgentTraceCursors { - parts: invalid, - ..base - }, - AgentTraceCursors { - diff_traces: invalid, - ..base - }, - AgentTraceCursors { - agent_traces: invalid, - ..base - }, - ]; - - for cursors in cases { - let error = cursors - .validate() - .expect_err(&format!("{invalid} should be rejected in {cursors:?}")); - assert!(matches!(error, ControlPlaneError::InvalidResponse(_))); - } - } - } + use tokio::sync::{Notify, Semaphore}; - #[test] - fn batch_response_deserializes_accepted_and_cursor() { - let response: AgentTraceIngestionBatchResponse = serde_json::from_value(json!({ - "accepted": 3, - "cursor": 13, - })) - .unwrap(); - - assert_eq!( - response, - AgentTraceIngestionBatchResponse { - accepted: 3, - cursor: 13, - } - ); - } - - #[test] - fn batch_request_composes_with_message_export_rows() { - let request = AgentTraceIngestionBatchRequest { - repository_id: "acme-monorepo".to_string(), - source_instance_id: "src-123".to_string(), - stream: IngestionStream::Messages, - expected_cursor: 10, - rows: vec![AgentTraceMessageExportRow { - source_row_id: 11, - session_id: "session-1".to_string(), - message_id: "message-1".to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_700_000_000_000, - }], - }; - - let value = serde_json::to_value(&request).unwrap(); - assert_eq!(value["repositoryId"], json!("acme-monorepo")); - assert_eq!(value["sourceInstanceId"], json!("src-123")); - assert_eq!(value["stream"], json!("messages")); - assert_eq!(value["expectedCursor"], json!(10)); - assert_eq!(value["rows"][0]["sourceRowId"], json!(11)); - } - - #[test] - fn batch_request_composes_with_part_export_rows() { - let request = AgentTraceIngestionBatchRequest { - repository_id: "acme-monorepo".to_string(), - source_instance_id: "src-123".to_string(), - stream: IngestionStream::Parts, - expected_cursor: 0, - rows: vec![AgentTracePartExportRow { - source_row_id: 1, - session_id: "session-1".to_string(), - message_id: "message-1".to_string(), - part_type: "text".to_string(), - text: "hello".to_string(), - generated_at_unix_ms: 1_700_000_000_000, - }], - }; - - let value = serde_json::to_value(&request).unwrap(); - assert_eq!(value["stream"], json!("parts")); - assert_eq!(value["rows"][0]["type"], json!("text")); - } - - #[test] - fn batch_request_composes_with_diff_trace_export_rows() { - let request = AgentTraceIngestionBatchRequest { - repository_id: "acme-monorepo".to_string(), - source_instance_id: "src-123".to_string(), - stream: IngestionStream::DiffTraces, - expected_cursor: 0, - rows: vec![AgentTraceDiffTraceExportRow { - source_row_id: 1, - session_id: "session-1".to_string(), - time_ms: 1_700_000_000_000, - patch: "diff --git a b".to_string(), - model_id: None, - tool_name: None, - tool_version: None, - payload_type: "patch".to_string(), - }], - }; - - let value = serde_json::to_value(&request).unwrap(); - assert_eq!(value["stream"], json!("diff_traces")); - assert_eq!(value["rows"][0]["payloadType"], json!("patch")); - } - - #[test] - fn batch_request_composes_with_agent_trace_export_rows() { - let request = AgentTraceIngestionBatchRequest { - repository_id: "acme-monorepo".to_string(), - source_instance_id: "src-123".to_string(), - stream: IngestionStream::AgentTraces, - expected_cursor: 0, - rows: vec![AgentTraceAgentTraceExportRow { - source_row_id: 1, - agent_trace_id: "agent-trace-1".to_string(), - commit_id: "abc123".to_string(), - commit_time_ms: 1_700_000_000_000, - trace_json: "{}".to_string(), - url: "https://example.com/trace".to_string(), - remote_url: None, - }], - }; - - let value = serde_json::to_value(&request).unwrap(); - assert_eq!(value["stream"], json!("agent_traces")); - assert_eq!(value["rows"][0]["agentTraceId"], json!("agent-trace-1")); - } - - fn block_on(future: F) -> F::Output { - tokio::runtime::Builder::new_current_thread() - .enable_io() - .enable_time() - .build() - .expect("build test tokio runtime") - .block_on(future) - } - - fn now_unix_seconds() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock before unix epoch") - .as_secs() - } - - fn valid_stored_tokens(access_token: &str) -> StoredTokens { - StoredTokens { - access_token: access_token.to_string(), - token_type: "Bearer".to_string(), - expires_in: 3_600, - refresh_token: "refresh-token-1".to_string(), - scope: None, - stored_at_unix_seconds: now_unix_seconds(), - } - } - - fn expired_stored_tokens(access_token: &str) -> StoredTokens { - StoredTokens { - access_token: access_token.to_string(), - token_type: "Bearer".to_string(), - expires_in: 1, - refresh_token: "refresh-token-1".to_string(), - scope: None, - stored_at_unix_seconds: 0, - } - } - - fn token_response_json(access_token: &str) -> serde_json::Value { - json!({ - "access_token": access_token, - "token_type": "Bearer", - "expires_in": 3_600, - "refresh_token": "refresh-token-2", - }) - } - - fn state_response_json() -> serde_json::Value { - json!({ - "cursors": { - "messages": 0, - "parts": 0, - "diffTraces": 0, - "agentTraces": 0, - } - }) - } - - fn sample_state_request() -> AgentTraceIngestionStateRequest { - AgentTraceIngestionStateRequest { - repository_id: "acme-monorepo".to_string(), - source_instance_id: "src-123".to_string(), - } - } - - fn sample_batch_request() -> AgentTraceIngestionBatchRequest { - AgentTraceIngestionBatchRequest { - repository_id: "acme-monorepo".to_string(), - source_instance_id: "src-123".to_string(), - stream: IngestionStream::Messages, - expected_cursor: 0, - rows: vec![AgentTraceMessageExportRow { - source_row_id: 1, - session_id: "session-1".to_string(), - message_id: "message-1".to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_700_000_000_000, - }], - } - } - - #[derive(Clone, Default)] - struct FakeCredentialStore { - tokens: Arc>>, - save_calls: Arc>>, - } - - impl FakeCredentialStore { - fn empty() -> Self { - Self::default() - } + use super::{ + AgentTraceIngestionStateRequest, AgentTraceIngestionStateResponse, + AuthenticatedControlPlaneClient, ControlPlaneError, CredentialStore, + }; + use crate::services::auth::TokenResponse; + use crate::services::token_storage::StoredTokens; - fn with_tokens(tokens: StoredTokens) -> Self { - let store = Self::default(); - *store.tokens.lock().unwrap() = Some(tokens); - store - } + const REPLACEMENT_ACCESS_TOKEN: &str = "replacement-access"; + const REPLACEMENT_REFRESH_TOKEN: &str = "replacement-refresh"; - fn save_call_count(&self) -> usize { - self.save_calls.lock().unwrap().len() - } + struct HeldSaveStore { + stored: Mutex, + saves: AtomicUsize, + save_started: Notify, + release_save: Semaphore, } - impl CredentialStore for FakeCredentialStore { - fn load(&self) -> Result, ControlPlaneError> { - Ok(self.tokens.lock().unwrap().clone()) + impl CredentialStore for HeldSaveStore { + async fn load(&self) -> Result, ControlPlaneError> { + Ok(Some(self.stored.lock().expect("stored lock").clone())) } - fn save(&self, token: &TokenResponse) -> Result { - self.save_calls.lock().unwrap().push(token.clone()); - let stored = StoredTokens { + async fn save(&self, token: &TokenResponse) -> Result { + self.save_started.notify_one(); + self.release_save + .acquire() + .await + .expect("release semaphore open") + .forget(); + let saved = StoredTokens { access_token: token.access_token.clone(), token_type: token.token_type.clone(), expires_in: token.expires_in, refresh_token: token.refresh_token.clone(), scope: token.scope.clone(), - stored_at_unix_seconds: now_unix_seconds(), + stored_at_unix_seconds: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_secs(), }; - *self.tokens.lock().unwrap() = Some(stored.clone()); - Ok(stored) + *self.stored.lock().expect("stored lock") = saved.clone(); + self.saves.fetch_add(1, Ordering::SeqCst); + Ok(saved) } } - #[derive(Clone)] - struct ConcurrentCredentialStore { - tokens: Arc>>, - initial_loads: Arc, - initial_load_barrier: Arc, - save_calls: Arc>>, - } - - impl ConcurrentCredentialStore { - fn with_tokens(tokens: StoredTokens) -> Self { - Self { - tokens: Arc::new(Mutex::new(Some(tokens))), - initial_loads: Arc::new(AtomicUsize::new(0)), - initial_load_barrier: Arc::new(Barrier::new(4)), - save_calls: Arc::new(Mutex::new(Vec::new())), - } - } - - fn save_call_count(&self) -> usize { - self.save_calls.lock().unwrap().len() - } - } - - impl CredentialStore for ConcurrentCredentialStore { - fn load(&self) -> Result, ControlPlaneError> { - let load_number = self.initial_loads.fetch_add(1, Ordering::SeqCst); - if load_number < 4 { - thread::sleep(Duration::from_millis(10)); - self.initial_load_barrier.wait(); + fn start_refresh_server() -> (String, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind refresh server"); + let base_url = format!("http://{}", listener.local_addr().expect("server addr")); + let hits = Arc::new(AtomicUsize::new(0)); + let thread_hits = Arc::clone(&hits); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + continue; + }; + let mut reader = BufReader::new(stream.try_clone().expect("clone stream")); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } + } + let mut body = vec![0u8; content_length]; + let _ = reader.read_exact(&mut body); + thread_hits.fetch_add(1, Ordering::SeqCst); + let payload = format!( + r#"{{"access_token":"{REPLACEMENT_ACCESS_TOKEN}","refresh_token":"{REPLACEMENT_REFRESH_TOKEN}","expires_in":3600}}"# + ); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + payload.len() + ); + let _ = stream.write_all(response.as_bytes()); } - Ok(self.tokens.lock().unwrap().clone()) - } - - fn save(&self, token: &TokenResponse) -> Result { - self.save_calls.lock().unwrap().push(token.clone()); - let stored = StoredTokens { - access_token: token.access_token.clone(), - token_type: token.token_type.clone(), - expires_in: token.expires_in, - refresh_token: token.refresh_token.clone(), - scope: token.scope.clone(), - stored_at_unix_seconds: now_unix_seconds(), - }; - *self.tokens.lock().unwrap() = Some(stored.clone()); - Ok(stored) - } - } - - struct RuntimeCheckingCredentialStore; - - impl CredentialStore for RuntimeCheckingCredentialStore { - fn load(&self) -> Result, ControlPlaneError> { - // This would panic if the blocking credential operation ran on a - // Tokio runtime thread. - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .expect("build nested test runtime"); - runtime.block_on(async {}); - Ok(Some(valid_stored_tokens("valid-access-token"))) - } - - fn save(&self, token: &TokenResponse) -> Result { - let runtime = tokio::runtime::Builder::new_current_thread() + }); + (base_url, hits) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_caller_does_not_lose_rotated_refresh_token() { + let (workos_url, workos_hits) = start_refresh_server(); + let store = HeldSaveStore { + stored: Mutex::new(StoredTokens { + access_token: "expired-access".to_string(), + token_type: "Bearer".to_string(), + expires_in: 0, + refresh_token: "original-refresh".to_string(), + scope: None, + stored_at_unix_seconds: 0, + }), + saves: AtomicUsize::new(0), + save_started: Notify::new(), + release_save: Semaphore::new(0), + }; + let client = Arc::new(AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) .build() - .expect("build nested test runtime"); - runtime.block_on(async {}); - Ok(valid_stored_tokens(&token.access_token)) - } - } - - #[tokio::test(flavor = "multi_thread")] - async fn credential_store_operations_run_outside_the_async_runtime() { - let client = AuthenticatedControlPlaneClient::with_credential_store( - test_http_client(), + .expect("test http client"), "http://127.0.0.1:1", - "http://127.0.0.1:1", - "test-client-id", - Box::new(RuntimeCheckingCredentialStore), - ); + workos_url, + "client", + store, + )); - let access_token = client - .resolve_access_token() - .await - .expect("token should load"); + let first = { + let client = Arc::clone(&client); + tokio::spawn(async move { client.resolve_access_token().await }) + }; + client.credential_store.save_started.notified().await; + let second = { + let client = Arc::clone(&client); + tokio::spawn(async move { client.resolve_access_token().await }) + }; - assert_eq!(access_token, "valid-access-token"); - let token: TokenResponse = serde_json::from_value(token_response_json("saved-token")) - .expect("fixture token should deserialize"); - let saved = client - .save_credentials(&token) + first.abort(); + assert!(first .await - .expect("token should save"); - assert_eq!(saved.access_token, "saved-token"); - } - - /// A plain-`http://` loopback test double never needs TLS root - /// certificates. Skip platform certificate verification so the test - /// suite also passes in sandboxes with no system CA store. - fn test_http_client() -> reqwest::Client { - reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .build() - .expect("build test reqwest client") - } + .expect_err("first caller cancelled") + .is_cancelled()); + assert_eq!(client.credential_store.saves.load(Ordering::SeqCst), 0); - fn client_with( - server: &TestHttpServer, - credential_store: FakeCredentialStore, - ) -> AuthenticatedControlPlaneClient { - AuthenticatedControlPlaneClient::with_credential_store( - test_http_client(), - server.base_url.clone(), - server.base_url.clone(), - "test-client-id", - Box::new(credential_store), - ) - } - - #[test] - fn missing_credentials_fail_before_any_http_call() { - let server = TestHttpServer::start(); - let client = client_with(&server, FakeCredentialStore::empty()); - - let error = block_on(client.ingestion_state(&sample_state_request())).unwrap_err(); - - assert!(matches!(error, ControlPlaneError::MissingCredentials)); - assert!(error.to_string().contains("sce auth login")); - assert_eq!(server.call_count(), 0); - } - - #[test] - fn valid_token_is_reused_without_resave() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response_json())); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let response = block_on(client.ingestion_state(&sample_state_request())).unwrap(); + client.credential_store.release_save.add_permits(1); + let second_token = second + .await + .expect("second caller joined") + .expect("second caller resolved a token"); + let third_token = client + .resolve_access_token() + .await + .expect("third caller resolved a token"); - assert_eq!(response.cursors.messages, 0); - assert_eq!(server.call_count(), 1); - let requests = server.captured_requests(); + assert_eq!(second_token, REPLACEMENT_ACCESS_TOKEN); + assert_eq!(third_token, REPLACEMENT_ACCESS_TOKEN); + assert_eq!(client.credential_store.saves.load(Ordering::SeqCst), 1); + assert_eq!(workos_hits.load(Ordering::SeqCst), 1); assert_eq!( - requests[0].headers.get("authorization").map(String::as_str), - Some("Bearer valid-access-token") + client + .credential_store + .stored + .lock() + .expect("stored lock") + .refresh_token, + REPLACEMENT_REFRESH_TOKEN ); } - #[test] - fn transient_state_failure_fails_after_one_http_attempt() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(503, &json!({"error": "unavailable"}))); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingestion_state(&sample_state_request())).unwrap_err(); - - assert!(matches!(error, ControlPlaneError::ServerError(_))); - assert_eq!(server.call_count(), 1); - } - - #[test] - fn valid_token_is_not_resaved() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response_json())); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let store_handle = store.clone(); - let client = client_with(&server, store); - - block_on(client.ingestion_state(&sample_state_request())).unwrap(); - - assert_eq!(store_handle.save_call_count(), 0); - } - - #[test] - fn expired_token_is_refreshed_and_saved() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json( - 200, - &token_response_json("refreshed-token"), + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn pending_refresh_drain_waits_for_cancelled_callers_save() { + let (workos_url, workos_hits) = start_refresh_server(); + let store = HeldSaveStore { + stored: Mutex::new(StoredTokens { + access_token: "expired-access".to_string(), + token_type: "Bearer".to_string(), + expires_in: 0, + refresh_token: "original-refresh".to_string(), + scope: None, + stored_at_unix_seconds: 0, + }), + saves: AtomicUsize::new(0), + save_started: Notify::new(), + release_save: Semaphore::new(0), + }; + let client = Arc::new(AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + "http://127.0.0.1:1", + workos_url, + "client", + store, )); - server.queue_response(CannedResponse::json(200, &state_response_json())); - let store = FakeCredentialStore::with_tokens(expired_stored_tokens("stale-access-token")); - let store_handle = store.clone(); - let client = client_with(&server, store); - let response = block_on(client.ingestion_state(&sample_state_request())).unwrap(); + let caller = { + let client = Arc::clone(&client); + tokio::spawn(async move { client.resolve_access_token().await }) + }; + client.credential_store.save_started.notified().await; + caller.abort(); + assert!(caller.await.expect_err("caller cancelled").is_cancelled()); + + let mut drain = std::pin::pin!(client.wait_for_pending_refresh()); + let probe = + tokio::time::timeout(std::time::Duration::from_millis(100), drain.as_mut()).await; + assert!(probe.is_err(), "drain returned before the save finished"); + assert_eq!(client.credential_store.saves.load(Ordering::SeqCst), 0); + + client.credential_store.release_save.add_permits(1); + tokio::time::timeout(std::time::Duration::from_secs(10), drain) + .await + .expect("drain completes after the save is released"); - assert_eq!(response.cursors.messages, 0); - assert_eq!(server.call_count(), 2); - assert_eq!(store_handle.save_call_count(), 1); - let requests = server.captured_requests(); + assert_eq!(client.credential_store.saves.load(Ordering::SeqCst), 1); + assert_eq!(workos_hits.load(Ordering::SeqCst), 1); assert_eq!( - requests[1].headers.get("authorization").map(String::as_str), - Some("Bearer refreshed-token") + client + .credential_store + .stored + .lock() + .expect("stored lock") + .refresh_token, + REPLACEMENT_REFRESH_TOKEN ); } - #[test] - fn concurrent_expired_tokens_share_one_refresh_and_save() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json( - 200, - &token_response_json("refreshed-token"), - )); - for _ in 0..4 { - server.queue_response(CannedResponse::json(200, &state_response_json())); + struct FixedStore; + + impl CredentialStore for FixedStore { + async fn load(&self) -> Result, ControlPlaneError> { + Ok(Some(StoredTokens { + access_token: "access".to_string(), + token_type: "Bearer".to_string(), + expires_in: 3600, + refresh_token: "refresh".to_string(), + scope: None, + stored_at_unix_seconds: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_secs(), + })) } - let store = ConcurrentCredentialStore::with_tokens(expired_stored_tokens("stale-token")); - let store_handle = store.clone(); - let client = Arc::new(AuthenticatedControlPlaneClient::with_credential_store( - test_http_client(), - server.base_url.clone(), - server.base_url.clone(), - "test-client-id", - Box::new(store), - )); - let results = block_on(async { - let handles = (0..4) - .map(|_| { - let client = Arc::clone(&client); - tokio::spawn( - async move { client.ingestion_state(&sample_state_request()).await }, - ) - }) - .collect::>(); - let mut results = Vec::new(); - for handle in handles { - results.push(handle.await.expect("concurrent state request task")); - } - results - }); - - assert!(results.iter().all(Result::is_ok)); - assert_eq!(store_handle.save_call_count(), 1); - let requests = server.captured_requests(); - assert_eq!(requests.len(), 5); - assert_eq!( - requests - .iter() - .filter(|request| request.path == "/oauth2/token") - .count(), - 1 - ); - let state_requests = requests - .iter() - .filter(|request| request.path == "/agent-trace/ingestion/state") - .collect::>(); - assert_eq!(state_requests.len(), 4); - assert!(state_requests.iter().all(|request| { - request.headers.get("authorization").map(String::as_str) - == Some("Bearer refreshed-token") - })); - } - - #[test] - fn unexpected_401_refreshes_once_and_retries_once_on_success() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(401, &json!({"error": "unauthorized"}))); - server.queue_response(CannedResponse::json( - 200, - &token_response_json("refreshed-token"), - )); - server.queue_response(CannedResponse::json(200, &state_response_json())); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("stale-but-unexpired")); - let store_handle = store.clone(); - let client = client_with(&server, store); - - let response = block_on(client.ingestion_state(&sample_state_request())).unwrap(); - assert_eq!(response.cursors.messages, 0); - assert_eq!(server.call_count(), 3); - assert_eq!(store_handle.save_call_count(), 1); - let requests = server.captured_requests(); - assert_eq!( - requests[2].headers.get("authorization").map(String::as_str), - Some("Bearer refreshed-token") - ); + async fn save(&self, _token: &TokenResponse) -> Result { + unreachable!("fixed credentials never refresh") + } } - #[test] - fn unexpected_401_twice_fails_without_a_third_attempt() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(401, &json!({"error": "unauthorized"}))); - server.queue_response(CannedResponse::json( - 200, - &token_response_json("refreshed-token"), - )); - server.queue_response(CannedResponse::json(401, &json!({"error": "unauthorized"}))); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("stale-but-unexpired")); - let client = client_with(&server, store); - - let error = block_on(client.ingestion_state(&sample_state_request())).unwrap_err(); - - assert!(matches!(error, ControlPlaneError::AuthenticationFailed(_))); - assert_eq!(server.call_count(), 3); + fn start_state_server(payload: &str) -> String { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind state server"); + let base_url = format!("http://{}", listener.local_addr().expect("server addr")); + let payload = payload.to_string(); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + continue; + }; + let mut reader = BufReader::new(stream.try_clone().expect("clone stream")); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } + } + let mut body = vec![0u8; content_length]; + let _ = reader.read_exact(&mut body); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + payload.len() + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + base_url } - #[test] - fn state_request_body_has_exact_shape_and_call_count() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response_json())); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - block_on(client.ingestion_state(&sample_state_request())).unwrap(); - - assert_eq!(server.call_count(), 1); - let requests = server.captured_requests(); - assert_eq!(requests[0].path, "/agent-trace/ingestion/state"); - let body: serde_json::Value = serde_json::from_str(&requests[0].body).unwrap(); - assert_eq!( - body, - json!({ - "repositoryId": "acme-monorepo", - "sourceInstanceId": "src-123", - }) + async fn request_state( + payload: &str, + ) -> Result { + let client = AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + start_state_server(payload), + "http://127.0.0.1:1", + "client", + FixedStore, ); + client + .ingestion_state(&AgentTraceIngestionStateRequest { + repository_id: "repo".to_string(), + source_instance_id: "source".to_string(), + }) + .await } - #[test] - fn batch_classifies_400_as_bad_request() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(400, &json!({"error": "invalid"}))); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); - - assert!(matches!(error, ControlPlaneError::BadRequest(_))); - } - - #[test] - fn batch_classifies_403_as_forbidden() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(403, &json!({"error": "forbidden"}))); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); - - assert!(matches!(error, ControlPlaneError::Forbidden(_))); - } - - #[test] - fn batch_classifies_409_as_conflict() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(409, &json!({"error": "conflict"}))); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); - - assert!(matches!(error, ControlPlaneError::Conflict(_))); - } - - #[test] - fn batch_classifies_5xx_as_server_error() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(500, &json!({"error": "boom"}))); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); - - assert!(matches!(error, ControlPlaneError::ServerError(_))); - } - - #[test] - fn server_error_body_is_not_leaked() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::text( - 500, - "SQLITE_ERROR: no such table: agent_trace_messages at /var/lib/sce/db.sqlite3", - )); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); - - let rendered = error.to_string(); - assert!(matches!(error, ControlPlaneError::ServerError(_))); - assert!(!rendered.contains("SQLITE_ERROR")); - assert!(!rendered.contains("agent_trace_messages")); - assert!(!rendered.contains("/var/lib/sce")); - assert!(rendered.contains("control plane encountered an internal error")); - } - - #[test] - fn known_safe_error_payload_is_surfaced() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json( - 400, - &json!({"message": "repositoryId is required"}), - )); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); + #[tokio::test] + async fn state_response_decodes_required_diff_traces_cursor() { + let state = + request_state(r#"{"cursors":{"messages":1,"parts":2,"diffTraces":7,"agentTraces":4}}"#) + .await + .expect("valid state"); - match error { - ControlPlaneError::BadRequest(message) => { - assert_eq!(message, "repositoryId is required"); - } - other => panic!("expected BadRequest, got {other:?}"), - } + assert_eq!(state.cursors.diff_traces, 7); } - #[test] - fn html_error_body_falls_back_to_generic_message() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::text( - 500, - "

500 Internal Server Error

", - )); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); + #[tokio::test] + async fn state_response_rejects_missing_diff_traces_cursor() { + let error = request_state(r#"{"cursors":{"messages":1,"parts":2,"agentTraces":4}}"#) + .await + .expect_err("missing diffTraces"); - let rendered = error.to_string(); - assert!(!rendered.contains("")); - assert!(rendered.contains("control plane encountered an internal error")); + assert!( + matches!(&error, ControlPlaneError::InvalidResponse(_)), + "{error:?}" + ); } - #[test] - fn batch_classifies_404_as_protocol_error() { - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(404, &json!({"error": "not found"}))); - let store = FakeCredentialStore::with_tokens(valid_stored_tokens("valid-access-token")); - let client = client_with(&server, store); - - let error = block_on(client.ingest_messages(&sample_batch_request())).unwrap_err(); + #[tokio::test] + async fn state_response_rejects_out_of_range_diff_traces_cursor() { + let error = request_state( + r#"{"cursors":{"messages":1,"parts":2,"diffTraces":-1,"agentTraces":4}}"#, + ) + .await + .expect_err("negative diffTraces"); - match error { - ControlPlaneError::Protocol { status, message } => { - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(message, "not found"); - } - other => panic!("expected Protocol, got {other:?}"), - } + assert!( + matches!(&error, ControlPlaneError::InvalidResponse(reason) if reason.contains("cursors.diffTraces")), + "{error:?}" + ); } } diff --git a/cli/src/services/agent_trace_sync/mod.rs b/cli/src/services/agent_trace_sync/mod.rs index 6f51fcdf8..05729efac 100644 --- a/cli/src/services/agent_trace_sync/mod.rs +++ b/cli/src/services/agent_trace_sync/mod.rs @@ -3,12 +3,7 @@ pub mod control_plane; -#[cfg(test)] -pub(crate) mod test_http_server; - use std::fmt; -use std::future::Future; -use std::pin::Pin; use crate::services::agent_trace_export::{ AgentTraceAgentTraceExportRow, AgentTraceDiffTraceExportRow, AgentTraceMessageExportRow, @@ -18,7 +13,7 @@ use control_plane::ControlPlaneError; /// Bound on consecutive `409`/ambiguous-batch-failure reconciliation attempts /// for one stream, matching the order of magnitude of existing retry -/// constants (`TOKEN_REFRESH_MAX_ATTEMPTS = 3` in `auth.rs`). Exhausting it +/// constants (`TOKEN_REFRESH_RETRY_POLICY` in `auth.rs`, 3 attempts). Exhausting it /// fails the stream rather than looping unboundedly. pub const RECONCILIATION_MAX_ATTEMPTS: u32 = 5; @@ -159,8 +154,6 @@ pub struct StreamSyncOutcome { /// already-accepted rows; if unchanged, the same rows are re-read and /// resent. Both cases share one bounded reconciliation counter. A `Terminal` /// outcome stops immediately without calling `refresh_cursor`. -pub type SyncFuture<'a, Output> = Pin + 'a>>; - pub async fn sync_stream<'a, T, ReadFn, IngestFn, RefreshFn>( initial_cursor: i64, batch_limit: usize, @@ -170,9 +163,9 @@ pub async fn sync_stream<'a, T, ReadFn, IngestFn, RefreshFn>( ) -> Result where T: AgentTraceExportRow + 'a, - ReadFn: FnMut(i64, usize) -> SyncFuture<'a, Result, StreamSyncError>>, - IngestFn: for<'rows> FnMut(i64, &'rows [T]) -> SyncFuture<'a, BatchAttemptOutcome>, - RefreshFn: FnMut() -> SyncFuture<'a, Result>, + ReadFn: std::ops::AsyncFnMut(i64, usize) -> Result, StreamSyncError>, + IngestFn: for<'rows> std::ops::AsyncFnMut(i64, &'rows [T]) -> BatchAttemptOutcome, + RefreshFn: std::ops::AsyncFnMut() -> Result, { let mut cursor = initial_cursor; let mut uploaded = 0usize; @@ -228,317 +221,3 @@ where batches, }) } - -#[cfg(test)] -mod tests { - use std::cell::RefCell; - use std::future::Future; - - use super::*; - use crate::services::agent_trace_db::MessageRole; - - fn block_on(future: F) -> F::Output { - tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .expect("build test runtime") - .block_on(future) - } - - fn ready(value: T) -> SyncFuture<'static, T> { - Box::pin(std::future::ready(value)) - } - - fn row(source_row_id: i64) -> AgentTraceMessageExportRow { - AgentTraceMessageExportRow { - source_row_id, - session_id: "session-1".to_string(), - message_id: format!("message-{source_row_id}"), - role: MessageRole::User, - generated_at_unix_ms: 1_700_000_000_000, - } - } - - struct FakeLocalRows { - rows: Vec, - } - - impl FakeLocalRows { - fn new(count: i64) -> Self { - Self { - rows: (1..=count).map(row).collect(), - } - } - - fn after(&self, cursor: i64, limit: usize) -> Vec { - self.rows - .iter() - .filter(|row| row.source_row_id > cursor) - .take(limit) - .cloned() - .collect() - } - } - - #[test] - fn empty_database_makes_no_batch_calls() { - let local = FakeLocalRows::new(0); - let batch_calls = RefCell::new(0usize); - let outcome = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, rows: &[AgentTraceMessageExportRow]| { - *batch_calls.borrow_mut() += 1; - ready(BatchAttemptOutcome::Accepted { - accepted: rows.len(), - cursor: rows.last().map_or(0, |row| row.source_row_id), - }) - }, - || panic!("refresh should not be called when there is nothing to reconcile"), - )) - .unwrap(); - - assert_eq!(*batch_calls.borrow(), 0); - assert_eq!( - outcome, - StreamSyncOutcome { - uploaded: 0, - initial_cursor: 0, - final_cursor: 0, - batches: 0, - } - ); - } - - #[test] - fn one_batch_uploads_all_rows_and_advances_cursor() { - let local = FakeLocalRows::new(3); - let batch_calls = RefCell::new(0usize); - let outcome = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, rows: &[AgentTraceMessageExportRow]| { - *batch_calls.borrow_mut() += 1; - ready(BatchAttemptOutcome::Accepted { - accepted: rows.len(), - cursor: rows.last().unwrap().source_row_id, - }) - }, - || panic!("refresh should not be called on an all-success run"), - )) - .unwrap(); - - assert_eq!(*batch_calls.borrow(), 1); - assert_eq!(outcome.uploaded, 3); - assert_eq!(outcome.final_cursor, 3); - assert_eq!(outcome.batches, 1); - } - - #[test] - fn more_than_five_hundred_rows_span_multiple_bounded_batches() { - let local = FakeLocalRows::new(1_100); - let batch_sizes = RefCell::new(Vec::new()); - let outcome = block_on(sync_stream( - 0, - 500, - |cursor, limit| { - assert!(limit <= 500); - ready(Ok(local.after(cursor, limit))) - }, - |_cursor, rows: &[AgentTraceMessageExportRow]| { - batch_sizes.borrow_mut().push(rows.len()); - ready(BatchAttemptOutcome::Accepted { - accepted: rows.len(), - cursor: rows.last().unwrap().source_row_id, - }) - }, - || panic!("refresh should not be called on an all-success run"), - )) - .unwrap(); - - assert_eq!(*batch_sizes.borrow(), vec![500, 500, 100]); - assert_eq!(outcome.uploaded, 1_100); - assert_eq!(outcome.final_cursor, 1_100); - assert_eq!(outcome.batches, 3); - } - - #[test] - fn gapped_source_ids_advance_cursor_from_last_id_not_row_count() { - let local = FakeLocalRows { - rows: vec![row(2), row(5), row(9)], - }; - let outcome = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, rows: &[AgentTraceMessageExportRow]| { - ready(BatchAttemptOutcome::Accepted { - accepted: rows.len(), - cursor: rows.last().unwrap().source_row_id, - }) - }, - || panic!("refresh should not be called on an all-success run"), - )) - .unwrap(); - - assert_eq!(outcome.uploaded, 3); - assert_eq!(outcome.final_cursor, 9); - assert_ne!(outcome.final_cursor, outcome.initial_cursor + 3); - } - - #[test] - fn conflict_resends_only_the_unsent_tail() { - let local = FakeLocalRows { - rows: vec![row(11), row(12), row(13)], - }; - let attempts: RefCell)>> = RefCell::new(Vec::new()); - let outcome = block_on(sync_stream( - 10, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |cursor, rows: &[AgentTraceMessageExportRow]| { - let ids = rows.iter().map(|row| row.source_row_id).collect(); - attempts.borrow_mut().push((cursor, ids)); - let result = if attempts.borrow().len() == 1 { - BatchAttemptOutcome::Conflict - } else { - BatchAttemptOutcome::Accepted { - accepted: rows.len(), - cursor: rows.last().unwrap().source_row_id, - } - }; - ready(result) - }, - || ready(Ok(12)), - )) - .unwrap(); - - assert_eq!( - attempts.into_inner(), - vec![(10, vec![11, 12, 13]), (12, vec![13])] - ); - assert_eq!(outcome.uploaded, 1); - assert_eq!(outcome.final_cursor, 13); - } - - #[test] - fn ambiguous_failure_with_advanced_refresh_does_not_resend() { - let local = FakeLocalRows::new(3); - let attempts: RefCell>> = RefCell::new(Vec::new()); - let outcome = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, rows: &[AgentTraceMessageExportRow]| { - let ids = rows.iter().map(|row| row.source_row_id).collect(); - let first = attempts.borrow().is_empty(); - attempts.borrow_mut().push(ids); - ready(if first { - BatchAttemptOutcome::Ambiguous - } else { - BatchAttemptOutcome::Accepted { - accepted: rows.len(), - cursor: rows.last().unwrap().source_row_id, - } - }) - }, - || ready(Ok(3)), - )) - .unwrap(); - - assert_eq!(attempts.into_inner(), vec![vec![1, 2, 3]]); - assert_eq!(outcome.uploaded, 0); - assert_eq!(outcome.final_cursor, 3); - } - - #[test] - fn terminal_failure_does_not_call_refresh() { - let local = FakeLocalRows::new(3); - let refresh_calls = RefCell::new(0usize); - let result = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, _rows: &[AgentTraceMessageExportRow]| { - ready(BatchAttemptOutcome::Terminal( - ControlPlaneError::BadRequest("batch route is not supported".to_string()), - )) - }, - || { - *refresh_calls.borrow_mut() += 1; - ready(Ok(0)) - }, - )); - - assert!(matches!( - result, - Err(StreamSyncError::Terminal(ControlPlaneError::BadRequest(reason))) if reason == "batch route is not supported" - )); - assert_eq!(*refresh_calls.borrow(), 0); - } - - #[test] - fn ambiguous_failure_with_unchanged_refresh_resends_once() { - let local = FakeLocalRows::new(3); - let attempts: RefCell>> = RefCell::new(Vec::new()); - let outcome = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, rows: &[AgentTraceMessageExportRow]| { - let ids = rows.iter().map(|row| row.source_row_id).collect(); - let first = attempts.borrow().is_empty(); - attempts.borrow_mut().push(ids); - ready(if first { - BatchAttemptOutcome::Ambiguous - } else { - BatchAttemptOutcome::Accepted { - accepted: rows.len(), - cursor: rows.last().unwrap().source_row_id, - } - }) - }, - || ready(Ok(0)), - )) - .unwrap(); - - assert_eq!(attempts.into_inner(), vec![vec![1, 2, 3], vec![1, 2, 3]]); - assert_eq!(outcome.uploaded, 3); - assert_eq!(outcome.final_cursor, 3); - } - - #[test] - fn reconciliation_bound_fails_with_did_not_converge() { - let local = FakeLocalRows::new(3); - let result = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, _rows: &[AgentTraceMessageExportRow]| ready(BatchAttemptOutcome::Ambiguous), - || ready(Ok(0)), - )); - - assert!(matches!(result, Err(StreamSyncError::DidNotConverge))); - } - - #[test] - fn invalid_response_rejects_mismatched_accepted_and_cursor() { - let local = FakeLocalRows::new(3); - let result = block_on(sync_stream( - 0, - 500, - |cursor, limit| ready(Ok(local.after(cursor, limit))), - |_cursor, _rows: &[AgentTraceMessageExportRow]| { - ready(BatchAttemptOutcome::Accepted { - accepted: 2, - cursor: 99, - }) - }, - || panic!("refresh should not be called on an invalid response"), - )); - - assert!(matches!(result, Err(StreamSyncError::InvalidResponse(_)))); - } -} diff --git a/cli/src/services/agent_trace_sync/test_http_server.rs b/cli/src/services/agent_trace_sync/test_http_server.rs deleted file mode 100644 index a5d92729e..000000000 --- a/cli/src/services/agent_trace_sync/test_http_server.rs +++ /dev/null @@ -1,462 +0,0 @@ -//! A bespoke, in-repo, test-only HTTP/1.1 server used to exercise -//! `AuthenticatedControlPlaneClient` (and later the sync engine and CLI -//! wiring) against canned responses and captured requests, without adding a -//! `wiremock`/`httptest`-style dev-dependency to a crate that currently -//! declares none. - -use std::collections::{HashMap, VecDeque}; -use std::io::{BufRead, BufReader, Read, Write}; -use std::net::TcpListener; -use std::sync::{Arc, Mutex}; -use std::thread; -use std::time::Duration; - -#[derive(Clone, Debug)] -pub struct CapturedRequest { - pub method: String, - pub path: String, - pub headers: HashMap, - pub body: String, -} - -#[derive(Clone, Debug)] -pub struct CannedResponse { - pub status: u16, - pub body: String, -} - -impl CannedResponse { - pub fn json(status: u16, body: &serde_json::Value) -> Self { - Self { - status, - body: body.to_string(), - } - } - - pub fn text(status: u16, body: impl Into) -> Self { - Self { - status, - body: body.into(), - } - } -} - -/// A single-threaded, sequential test HTTP server: each accepted connection -/// is answered with the next queued [`CannedResponse`] (or a `500` marker -/// response when the queue is empty), and every parsed request is recorded -/// for assertions. -pub struct TestHttpServer { - pub base_url: String, - requests: Arc>>, - responses: Arc>>, -} - -impl TestHttpServer { - pub fn start() -> Self { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind test http server"); - let addr = listener.local_addr().expect("read test http server addr"); - - let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); - let responses: Arc>> = Arc::new(Mutex::new(VecDeque::new())); - - let thread_requests = Arc::clone(&requests); - let thread_responses = Arc::clone(&responses); - thread::spawn(move || { - for stream in listener.incoming() { - let Ok(stream) = stream else { - continue; - }; - handle_connection(stream, &thread_requests, &thread_responses); - } - }); - - Self { - base_url: format!("http://{addr}"), - requests, - responses, - } - } - - pub fn queue_response(&self, response: CannedResponse) { - self.responses - .lock() - .expect("test http server responses lock") - .push_back(response); - } - - pub fn captured_requests(&self) -> Vec { - self.requests - .lock() - .expect("test http server requests lock") - .clone() - } - - pub fn call_count(&self) -> usize { - self.requests - .lock() - .expect("test http server requests lock") - .len() - } -} - -/// A concurrent test server for sync regressions. State responses are queued -/// independently from dynamically selected batch responses, allowing each -/// stream's expected cursor to determine the response it receives. -pub struct ConcurrentBatchTestServer { - pub base_url: String, - requests: Arc>>, - state_response: Arc>>, - batch_responses: Arc>>, - metrics: Arc>, -} - -#[derive(Clone, Debug, Default)] -struct ConcurrentBatchMetrics { - state_requests: usize, - in_flight: usize, - max_in_flight: usize, - in_flight_by_stream: HashMap, - max_in_flight_by_stream: HashMap, - expected_cursors_by_stream: HashMap>, -} - -impl ConcurrentBatchTestServer { - pub fn start(batch_delay: Duration) -> Self { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind concurrent test http server"); - let addr = listener - .local_addr() - .expect("read concurrent test http server addr"); - - let requests: Arc>> = Arc::new(Mutex::new(Vec::new())); - let state_response = Arc::new(Mutex::new(None)); - let batch_responses = Arc::new(Mutex::new(HashMap::new())); - let metrics = Arc::new(Mutex::new(ConcurrentBatchMetrics::default())); - - let thread_requests = Arc::clone(&requests); - let thread_state_response = Arc::clone(&state_response); - let thread_batch_responses = Arc::clone(&batch_responses); - let thread_metrics = Arc::clone(&metrics); - thread::spawn(move || { - for stream in listener.incoming() { - let Ok(stream) = stream else { - continue; - }; - let requests = Arc::clone(&thread_requests); - let state_response = Arc::clone(&thread_state_response); - let batch_responses = Arc::clone(&thread_batch_responses); - let metrics = Arc::clone(&thread_metrics); - thread::spawn(move || { - handle_concurrent_connection( - stream, - &requests, - &state_response, - &batch_responses, - &metrics, - batch_delay, - ); - }); - } - }); - - Self { - base_url: format!("http://{addr}"), - requests, - state_response, - batch_responses, - metrics, - } - } - - pub fn queue_state_response(&self, response: CannedResponse) { - *self - .state_response - .lock() - .expect("concurrent test http server state response lock") = Some(response); - } - - pub fn queue_batch_response( - &self, - stream: impl Into, - expected_cursor: i64, - response: CannedResponse, - ) { - self.batch_responses - .lock() - .expect("concurrent test http server batch responses lock") - .insert((stream.into(), expected_cursor), response); - } - - pub fn captured_requests(&self) -> Vec { - self.requests - .lock() - .expect("concurrent test http server requests lock") - .clone() - } - - pub fn state_request_count(&self) -> usize { - self.metrics - .lock() - .expect("concurrent test http server metrics lock") - .state_requests - } - - pub fn max_in_flight(&self) -> usize { - self.metrics - .lock() - .expect("concurrent test http server metrics lock") - .max_in_flight - } - - pub fn max_in_flight_for(&self, stream: &str) -> usize { - self.metrics - .lock() - .expect("concurrent test http server metrics lock") - .max_in_flight_by_stream - .get(stream) - .copied() - .unwrap_or(0) - } - - pub fn expected_cursors_for(&self, stream: &str) -> Vec { - self.metrics - .lock() - .expect("concurrent test http server metrics lock") - .expected_cursors_by_stream - .get(stream) - .cloned() - .unwrap_or_default() - } -} - -fn handle_connection( - stream: std::net::TcpStream, - requests: &Arc>>, - responses: &Arc>>, -) { - let mut reader = BufReader::new(stream.try_clone().expect("clone test http stream")); - - let mut request_line = String::new(); - if reader.read_line(&mut request_line).unwrap_or(0) == 0 { - return; - } - let mut parts = request_line.split_whitespace(); - let method = parts.next().unwrap_or_default().to_string(); - let path = parts.next().unwrap_or_default().to_string(); - - let mut headers = HashMap::new(); - let mut content_length = 0usize; - loop { - let mut line = String::new(); - if reader.read_line(&mut line).unwrap_or(0) == 0 { - break; - } - let trimmed = line.trim_end_matches(['\r', '\n']); - if trimmed.is_empty() { - break; - } - if let Some((name, value)) = trimmed.split_once(':') { - let name = name.trim().to_ascii_lowercase(); - let value = value.trim().to_string(); - if name == "content-length" { - content_length = value.parse().unwrap_or(0); - } - headers.insert(name, value); - } - } - - let mut body_bytes = vec![0u8; content_length]; - if content_length > 0 && reader.read_exact(&mut body_bytes).is_err() { - return; - } - let body = String::from_utf8_lossy(&body_bytes).to_string(); - - requests - .lock() - .expect("test http server requests lock") - .push(CapturedRequest { - method, - path, - headers, - body, - }); - - let canned = responses - .lock() - .expect("test http server responses lock") - .pop_front() - .unwrap_or_else(|| CannedResponse { - status: 500, - body: r#"{"error":"no canned response queued"}"#.to_string(), - }); - - let mut stream = stream; - let response = format!( - "HTTP/1.1 {} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - canned.status, - status_reason(canned.status), - canned.body.len(), - canned.body, - ); - let _ = stream.write_all(response.as_bytes()); - let _ = stream.flush(); -} - -fn handle_concurrent_connection( - stream: std::net::TcpStream, - requests: &Arc>>, - state_response: &Arc>>, - batch_responses: &Arc>>, - metrics: &Arc>, - batch_delay: Duration, -) { - let Some(request) = read_request(&stream) else { - return; - }; - requests - .lock() - .expect("concurrent test http server requests lock") - .push(request.clone()); - - let response = match request.path.as_str() { - "/agent-trace/ingestion/state" => { - metrics - .lock() - .expect("concurrent test http server metrics lock") - .state_requests += 1; - state_response - .lock() - .expect("concurrent test http server state response lock") - .clone() - .unwrap_or_else(|| CannedResponse::text(500, "no state response queued")) - } - "/agent-trace/ingestion/batch" => { - let body: serde_json::Value = - serde_json::from_str(&request.body).unwrap_or_else(|_| serde_json::json!({})); - let stream = body - .get("stream") - .and_then(serde_json::Value::as_str) - .unwrap_or("unknown") - .to_string(); - let expected_cursor = body - .get("expectedCursor") - .and_then(serde_json::Value::as_i64) - .unwrap_or(i64::MIN); - - { - let mut metrics = metrics - .lock() - .expect("concurrent test http server metrics lock"); - metrics.in_flight += 1; - metrics.max_in_flight = metrics.max_in_flight.max(metrics.in_flight); - let stream_in_flight = { - let count = metrics - .in_flight_by_stream - .entry(stream.clone()) - .or_default(); - *count += 1; - *count - }; - let stream_max_in_flight = metrics - .max_in_flight_by_stream - .entry(stream.clone()) - .or_default(); - *stream_max_in_flight = (*stream_max_in_flight).max(stream_in_flight); - metrics - .expected_cursors_by_stream - .entry(stream.clone()) - .or_default() - .push(expected_cursor); - } - - thread::sleep(batch_delay); - let response = batch_responses - .lock() - .expect("concurrent test http server batch responses lock") - .get(&(stream.clone(), expected_cursor)) - .cloned() - .unwrap_or_else(|| CannedResponse::text(500, "no batch response queued")); - let mut metrics = metrics - .lock() - .expect("concurrent test http server metrics lock"); - metrics.in_flight -= 1; - if let Some(stream_in_flight) = metrics.in_flight_by_stream.get_mut(&stream) { - *stream_in_flight -= 1; - } - drop(metrics); - response - } - _ => CannedResponse::text(404, "unknown test route"), - }; - - write_response(stream, &response); -} - -fn read_request(stream: &std::net::TcpStream) -> Option { - let mut reader = BufReader::new(stream.try_clone().ok()?); - - let mut request_line = String::new(); - if reader.read_line(&mut request_line).unwrap_or(0) == 0 { - return None; - } - let mut parts = request_line.split_whitespace(); - let method = parts.next().unwrap_or_default().to_string(); - let path = parts.next().unwrap_or_default().to_string(); - - let mut headers = HashMap::new(); - let mut content_length = 0usize; - loop { - let mut line = String::new(); - if reader.read_line(&mut line).unwrap_or(0) == 0 { - break; - } - let trimmed = line.trim_end_matches(['\r', '\n']); - if trimmed.is_empty() { - break; - } - if let Some((name, value)) = trimmed.split_once(':') { - let name = name.trim().to_ascii_lowercase(); - let value = value.trim().to_string(); - if name == "content-length" { - content_length = value.parse().unwrap_or(0); - } - headers.insert(name, value); - } - } - - let mut body_bytes = vec![0u8; content_length]; - if content_length > 0 && reader.read_exact(&mut body_bytes).is_err() { - return None; - } - - Some(CapturedRequest { - method, - path, - headers, - body: String::from_utf8_lossy(&body_bytes).to_string(), - }) -} - -fn write_response(mut stream: std::net::TcpStream, canned: &CannedResponse) { - let response = format!( - "HTTP/1.1 {} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - canned.status, - status_reason(canned.status), - canned.body.len(), - canned.body, - ); - let _ = stream.write_all(response.as_bytes()); - let _ = stream.flush(); -} - -fn status_reason(status: u16) -> &'static str { - match status { - 200 => "OK", - 400 => "Bad Request", - 401 => "Unauthorized", - 403 => "Forbidden", - 409 => "Conflict", - 500 => "Internal Server Error", - 503 => "Service Unavailable", - _ => "Unknown", - } -} diff --git a/cli/src/services/app_support.rs b/cli/src/services/app_support.rs index 494b22112..98de5056d 100644 --- a/cli/src/services/app_support.rs +++ b/cli/src/services/app_support.rs @@ -107,7 +107,7 @@ pub(crate) async fn execute_command_phase( stderr: &mut W, ) -> Result where - C: HasLogger + ContextWithRepoRoot, + C: HasLogger + ContextWithRepoRoot + crate::app::HasGit + crate::app::HasFs, W: Write, { let command_name = command.name(); @@ -209,146 +209,3 @@ fn write_startup_diagnostic(writer: &mut W, diagnostic: &str) { writeln!(writer, "{}", services::style::error_code(diagnostic)) .expect("writing startup diagnostic to writer should not fail"); } - -#[cfg(test)] -mod tests { - use super::*; - use services::error::UserError; - use std::sync::{Arc, Mutex}; - - #[derive(Clone, Default)] - struct RecordingLogger { - log_cli_error_calls: Arc>>, - } - - impl LoggerTrait for RecordingLogger { - fn info(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn debug(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn warn(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn error(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - - fn log_cli_error(&self, error: &CliError, _session_id: Option<&str>) { - let has_source = match error { - CliError::User { source, .. } => source.is_some(), - CliError::Internal { .. } => true, - }; - self.log_cli_error_calls - .lock() - .expect("recording logger mutex must not be poisoned") - .push((error.code(), has_source)); - } - } - - fn diagnostic_lines(stderr: &str) -> Vec<&str> { - stderr - .lines() - .filter(|line| line.contains("Error") && line.contains("SCE-ERR-")) - .collect() - } - - #[test] - fn user_error_routes_to_friendly_diagnostic_with_empty_stdout_and_exit_four() { - let error = CliError::user_with_source( - UserError::NotAuthenticated, - anyhow::anyhow!("missing credentials"), - ); - let logger = RecordingLogger::default(); - let outcome = RunOutcome { - result: Err(error), - logger: Some(logger), - startup_diagnostic: None, - }; - - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - let exit_code = render_run_outcome(outcome, &mut stdout, &mut stderr); - - assert!(stdout.is_empty(), "stdout must stay empty on failure"); - assert_eq!(exit_code, ExitCode::from(4)); - - let stderr_text = String::from_utf8(stderr).expect("stderr is valid utf8"); - assert_eq!( - stderr_text, - "You are not logged in. Please log in using the `sce auth login` command.\n" - ); - assert!(!stderr_text.contains("Error")); - assert!(!stderr_text.contains("SCE-ERR-")); - assert!(!stderr_text.contains("Try:")); - assert!(!stderr_text.contains("missing credentials")); - assert!(!stderr_text.to_lowercase().contains("control-plane")); - } - - #[test] - fn user_error_preserves_technical_source_for_observability() { - let error = CliError::user_with_source( - UserError::NotAuthenticated, - anyhow::anyhow!("missing credentials"), - ); - let logger = RecordingLogger::default(); - let calls = logger.log_cli_error_calls.clone(); - let outcome = RunOutcome { - result: Err(error), - logger: Some(logger), - startup_diagnostic: None, - }; - - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - render_run_outcome(outcome, &mut stdout, &mut stderr); - - let recorded = calls.lock().expect("mutex must not be poisoned"); - assert_eq!( - recorded.as_slice(), - [("SCE-ERR-RUNTIME", true)], - "observability must log the CliError with its technical source preserved" - ); - } - - #[test] - fn internal_error_still_renders_full_chain_and_exit_four() { - let source = anyhow::anyhow!("root cause").context("failed to sync"); - let error = CliError::runtime(source); - let outcome: RunOutcome = RunOutcome { - result: Err(error), - logger: None, - startup_diagnostic: None, - }; - - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - let exit_code = render_run_outcome(outcome, &mut stdout, &mut stderr); - - assert!(stdout.is_empty()); - assert_eq!(exit_code, ExitCode::from(4)); - let stderr_text = String::from_utf8(stderr).expect("stderr is valid utf8"); - assert_eq!(diagnostic_lines(&stderr_text).len(), 1); - assert!(stderr_text.contains("failed to sync")); - assert!(stderr_text.contains("root cause")); - } - - #[test] - fn redaction_still_applies_to_the_rendered_message() { - let mut stderr = Vec::new(); - let error = CliError::user(UserError::NotAuthenticated); - write_error_diagnostic_with_color_policy(&mut stderr, &error, false); - - let rendered = String::from_utf8(stderr).expect("stderr is valid utf8"); - let redacted_message = - services::security::redact_sensitive_text(UserError::NotAuthenticated.message()); - assert!(rendered.contains(&redacted_message)); - } - - #[test] - fn user_error_diagnostic_is_plain_in_every_color_policy_mode() { - let error = CliError::user(UserError::NotAuthenticated); - let expected = "You are not logged in. Please log in using the `sce auth login` command.\n"; - - for color_enabled in [true, false] { - let mut stderr = Vec::new(); - write_error_diagnostic_with_color_policy(&mut stderr, &error, color_enabled); - let rendered = String::from_utf8(stderr).expect("stderr is valid utf8"); - - assert_eq!(rendered, expected); - } - } -} diff --git a/cli/src/services/auth.rs b/cli/src/services/auth.rs index 037c6dc33..3e89f8e84 100644 --- a/cli/src/services/auth.rs +++ b/cli/src/services/auth.rs @@ -12,10 +12,7 @@ pub const REFRESH_TOKEN_GRANT_TYPE: &str = "refresh_token"; pub const WORKOS_DEFAULT_BASE_URL: &str = "https://api.workos.com"; pub const DEFAULT_DEVICE_POLL_INTERVAL_SECONDS: u64 = 5; const TOKEN_EXPIRY_SKEW_SECONDS: u64 = 30; -const TOKEN_REFRESH_MAX_ATTEMPTS: u32 = 3; -const TOKEN_REFRESH_TIMEOUT_MS: u64 = 10_000; -const TOKEN_REFRESH_INITIAL_BACKOFF_MS: u64 = 250; -const TOKEN_REFRESH_MAX_BACKOFF_MS: u64 = 2_000; +const TOKEN_REFRESH_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(3, 10_000, 250, 2_000); #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct DeviceAuthorizationRequest { @@ -356,12 +353,7 @@ async fn refresh_access_token( refresh_token: refresh_token.to_string(), client_id: client_id.to_string(), }; - let retry_policy = RetryPolicy { - max_attempts: TOKEN_REFRESH_MAX_ATTEMPTS, - timeout_ms: TOKEN_REFRESH_TIMEOUT_MS, - initial_backoff_ms: TOKEN_REFRESH_INITIAL_BACKOFF_MS, - max_backoff_ms: TOKEN_REFRESH_MAX_BACKOFF_MS, - }; + let retry_policy = TOKEN_REFRESH_RETRY_POLICY; let response = run_with_retry( retry_policy, diff --git a/cli/src/services/auth_command/mod.rs b/cli/src/services/auth_command/mod.rs index 007810d2b..8531567f4 100644 --- a/cli/src/services/auth_command/mod.rs +++ b/cli/src/services/auth_command/mod.rs @@ -61,7 +61,9 @@ pub async fn run_login(format: AuthFormat) -> Result { let client = reqwest::Client::new(); let client_id = resolve_login_client_id().map_err(unexpected_auth_command_error)?; - let stored_tokens = run_credential_operation(token_storage::load_tokens).await?; + let stored_tokens = token_storage::load_tokens() + .await + .map_err(auth_storage_error)?; let client = &client; let client_id = client_id.as_str(); @@ -82,13 +84,16 @@ pub async fn run_login(format: AuthFormat) -> Result { } pub async fn run_logout(format: AuthFormat) -> Result { - let deleted = run_credential_operation(token_storage::delete_tokens).await?; + let deleted = token_storage::delete_tokens() + .await + .map_err(auth_storage_error)?; render_logout_result(deleted, format).map_err(unexpected_auth_command_error) } pub async fn run_whoami(format: AuthFormat) -> Result { - if run_credential_operation(token_storage::load_tokens) - .await? + if token_storage::load_tokens() + .await + .map_err(auth_storage_error)? .is_none() { return render_unauthenticated_whoami(format).map_err(unexpected_auth_command_error); @@ -126,8 +131,9 @@ async fn maybe_renew_stored_credentials( ) .await { - Ok(token) => run_credential_operation(move || token_storage::save_tokens(&token)) + Ok(token) => token_storage::save_tokens(&token) .await + .map_err(auth_storage_error) .map(Some), Err(_) => Ok(None), } @@ -172,8 +178,9 @@ async fn run_text_login(client: &reqwest::Client, client_id: &str) -> Result CliError { ) } -async fn run_credential_operation(operation: F) -> Result -where - T: Send + 'static, - F: FnOnce() -> Result + Send + 'static, -{ - tokio::task::spawn_blocking(operation) - .await - .map_err(|error| unexpected_auth_command_error(error.into()))? - .map_err(auth_storage_error) -} - fn auth_storage_error(error: crate::services::token_storage::TokenStorageError) -> CliError { CliError::user_with_source(UserError::AuthStorageUnavailable, error) } @@ -453,278 +450,3 @@ fn auth_storage_error(error: crate::services::token_storage::TokenStorageError) fn unexpected_auth_command_error(error: anyhow::Error) -> CliError { CliError::user_with_source(UserError::UnexpectedFailure, error) } - -#[cfg(test)] -mod tests { - use super::*; - - fn stored_tokens() -> StoredTokens { - StoredTokens { - access_token: "secret-access".into(), - refresh_token: "secret-refresh".into(), - token_type: "Bearer".into(), - scope: None, - expires_in: 3600, - stored_at_unix_seconds: 100, - } - } - - async fn completed_action(format: AuthFormat, name: &str) -> Result { - assert_eq!(format, AuthFormat::Json); - assert_eq!( - tokio::runtime::Handle::current().runtime_flavor(), - tokio::runtime::RuntimeFlavor::MultiThread - ); - tokio::task::yield_now().await; - Ok(name.to_string()) - } - - #[tokio::test(flavor = "multi_thread")] - async fn auth_dispatch_awaits_only_the_selected_action() { - for (subcommand, expected) in [ - ( - AuthSubcommand::Login { - format: AuthFormat::Json, - }, - "login", - ), - ( - AuthSubcommand::Logout { - format: AuthFormat::Json, - }, - "logout", - ), - ( - AuthSubcommand::Whoami { - format: AuthFormat::Json, - }, - "whoami", - ), - ] { - let result = run_auth_subcommand_with( - AuthRequest { subcommand }, - |format| async move { - assert_eq!(expected, "login"); - completed_action(format, "login").await - }, - |format| async move { - assert_eq!(expected, "logout"); - completed_action(format, "logout").await - }, - |format| async move { - assert_eq!(expected, "whoami"); - completed_action(format, "whoami").await - }, - ) - .await - .expect("selected action should complete"); - assert_eq!(result, expected); - } - } - - #[tokio::test(flavor = "multi_thread")] - async fn login_helper_awaits_owned_renewal_and_redacts_credentials() { - let report = run_login_with_stored_credentials( - AuthFormat::Json, - Some(stored_tokens()), - |tokens| async move { - completed_action(AuthFormat::Json, "renew").await?; - Ok(Some(tokens)) - }, - |_| async { panic!("renewed credentials must skip device login") }, - ) - .await - .expect("renewal should complete"); - assert_eq!( - report, - render_login_refresh_result(&stored_tokens(), AuthFormat::Json).unwrap() - ); - assert!(!report.contains("secret-access")); - assert!(!report.contains("secret-refresh")); - } - - #[tokio::test(flavor = "multi_thread")] - async fn login_helper_awaits_device_fallback_after_unsuccessful_renewal() { - let renewal_completed = std::cell::Cell::new(false); - let renewed = &renewal_completed; - let result = run_login_with_stored_credentials( - AuthFormat::Json, - Some(stored_tokens()), - |_| async { - completed_action(AuthFormat::Json, "renew").await?; - renewed.set(true); - Ok(None) - }, - |format| async move { - assert!(renewed.get()); - completed_action(format, "device login completed").await - }, - ) - .await - .expect("fallback should complete"); - assert_eq!(result, "device login completed"); - } - - fn check_blocking_credential_context() { - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .expect("build credential fixture runtime"); - runtime.block_on(async {}); - } - - #[tokio::test(flavor = "multi_thread")] - async fn auth_credential_load_save_delete_are_isolated_and_awaited() { - let credentials = std::sync::Arc::new(std::sync::Mutex::new(Some(stored_tokens()))); - let load_store = credentials.clone(); - let loaded = run_credential_operation(move || { - check_blocking_credential_context(); - Ok(load_store.lock().unwrap().clone()) - }) - .await - .expect("load should complete") - .unwrap(); - let save_store = credentials.clone(); - let saved = run_credential_operation(move || { - check_blocking_credential_context(); - let mut tokens = loaded; - tokens.access_token = "saved-access".into(); - *save_store.lock().unwrap() = Some(tokens.clone()); - Ok(tokens) - }) - .await - .expect("save should complete"); - assert_eq!(saved.access_token, "saved-access"); - assert_eq!( - credentials.lock().unwrap().as_ref().unwrap().access_token, - "saved-access" - ); - let delete_store = credentials.clone(); - let deleted = run_credential_operation(move || { - check_blocking_credential_context(); - Ok(delete_store.lock().unwrap().take().is_some()) - }) - .await - .expect("delete should complete"); - assert!(deleted); - assert!(credentials.lock().unwrap().is_none()); - } - - #[tokio::test(flavor = "multi_thread")] - async fn auth_credential_storage_and_join_failures_keep_typed_sources() { - let storage_error = run_credential_operation::<(), _>(|| { - Err(token_storage::TokenStorageError::Database( - "fixture failure".into(), - )) - }) - .await - .unwrap_err(); - let join_error = - run_credential_operation::<(), _>(|| panic!("credential worker fixture failure")) - .await - .unwrap_err(); - for (mapped, expected) in [ - (storage_error, UserError::AuthStorageUnavailable), - (join_error, UserError::UnexpectedFailure), - ] { - match mapped { - CliError::User { - error, - source: Some(source), - } => { - assert_eq!(error, expected); - assert!(source.to_string().contains("fixture failure")); - } - _ => panic!("credential failure lost its typed source"), - } - } - } - - #[test] - fn logout_text_reports_whether_credentials_were_removed() { - assert_eq!( - render_logout_result_with_color_policy(false, AuthFormat::Text, false) - .expect("logout should render"), - "No user logged in" - ); - assert_eq!( - render_logout_result_with_color_policy(true, AuthFormat::Text, false) - .expect("logout should render"), - "Logged out" - ); - } - - #[test] - fn logout_json_reports_whether_credentials_were_removed() { - let absent: serde_json::Value = serde_json::from_str( - &render_logout_result_with_color_policy(false, AuthFormat::Json, false) - .expect("logout should render"), - ) - .expect("logout JSON should be valid"); - let present: serde_json::Value = serde_json::from_str( - &render_logout_result_with_color_policy(true, AuthFormat::Json, false) - .expect("logout should render"), - ) - .expect("logout JSON should be valid"); - - assert_eq!(absent["status"], "ok"); - assert_eq!(absent["authenticated"], false); - assert_eq!(absent["credentials_removed"], false); - assert_eq!(present["credentials_removed"], true); - } - - #[test] - fn unauthenticated_whoami_renders_text_guidance() { - assert_eq!( - render_unauthenticated_whoami_with_color_policy(AuthFormat::Text, false) - .expect("unauthenticated whoami should render"), - "You are not logged in. Please log in using the sce auth login command." - ); - } - - #[test] - fn unauthenticated_whoami_json_reports_state() { - let report: serde_json::Value = serde_json::from_str( - &render_unauthenticated_whoami_with_color_policy(AuthFormat::Json, false) - .expect("unauthenticated whoami should render"), - ) - .expect("whoami JSON should be valid"); - - assert_eq!(report["status"], "ok"); - assert_eq!(report["command"], "auth"); - assert_eq!(report["subcommand"], "whoami"); - assert_eq!(report["authentication_state"], "unauthenticated"); - assert_eq!(report["has_stored_credentials"], false); - } - - #[test] - fn authenticated_whoami_failures_keep_typed_errors_and_sources() { - let cases = [ - ( - ControlPlaneError::AuthenticationFailed("expired".to_string()), - UserError::NotAuthenticated, - ), - ( - ControlPlaneError::Storage("database unavailable".to_string()), - UserError::AuthStorageUnavailable, - ), - ( - ControlPlaneError::Transport("connection refused".to_string()), - UserError::UnexpectedFailure, - ), - ]; - - for (control_plane_error, expected_user_error) in cases { - let mapped = map_whoami_control_plane_error(&control_plane_error); - match mapped { - CliError::User { - error, - source: Some(source), - } => { - assert_eq!(error, expected_user_error); - assert!(!source.to_string().is_empty()); - } - _ => panic!("authenticated whoami failure lost its typed source"), - } - } - } -} diff --git a/cli/src/services/auth_db/lifecycle.rs b/cli/src/services/auth_db/lifecycle.rs index 6cf4f9f83..79a5e3160 100644 --- a/cli/src/services/auth_db/lifecycle.rs +++ b/cli/src/services/auth_db/lifecycle.rs @@ -5,7 +5,7 @@ use crate::services::db::{bootstrap_db_parent, collect_db_path_health, DbSpec}; use crate::services::default_paths::auth_db_path; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; use super::{AuthDb, AuthDbSpec}; @@ -14,15 +14,15 @@ use super::{AuthDb, AuthDbSpec}; pub struct AuthDbLifecycle; impl ServiceLifecycle for AuthDbLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::AuthDb - } - - fn diagnose(&self, _ctx: &C) -> Vec { + async fn diagnose(&self, _ctx: &C) -> Vec { diagnose_auth_db_health() } - fn fix(&self, _ctx: &C, problems: &[HealthProblem]) -> Vec { + async fn fix( + &self, + _ctx: &C, + problems: &[HealthProblem], + ) -> Vec { let should_bootstrap_parent = problems.iter().any(|problem| { problem.category == HealthCategory::GlobalState && problem.fixability == HealthFixability::AutoFixable @@ -48,8 +48,14 @@ impl ServiceLifecycle for AuthDbLifecycle { } } - fn setup(&self, _ctx: &C) -> Result { - AuthDb::new().context("Auth DB lifecycle setup failed while initializing auth DB")?; + async fn setup(&self, _ctx: &C) -> Result { + if auth_db_path()?.exists() { + return Ok(SetupOutcome::default()); + } + + AuthDb::new() + .await + .context("Auth DB lifecycle setup failed while initializing auth DB")?; Ok(SetupOutcome::default()) } } diff --git a/cli/src/services/auth_db/mod.rs b/cli/src/services/auth_db/mod.rs index e85de8378..5e2934bb0 100644 --- a/cli/src/services/auth_db/mod.rs +++ b/cli/src/services/auth_db/mod.rs @@ -7,7 +7,7 @@ use anyhow::Result; use crate::{ generated_migrations, services::{ - db::{DbSpec, EncryptedTursoDb}, + db::{DatabaseKind, DbSpec, EncryptedTursoDb}, default_paths::auth_db_path, }, }; @@ -28,195 +28,10 @@ impl DbSpec for AuthDbSpec { generated_migrations::AUTH_MIGRATIONS } - fn db_config_key() -> &'static str { - "auth_db" - } + const KIND: DatabaseKind = DatabaseKind::Auth; } /// Encrypted auth Turso database adapter. pub type AuthDb = EncryptedTursoDb; pub mod lifecycle; - -#[cfg(test)] -mod tests { - use std::{ - collections::HashMap, - fs, - path::PathBuf, - sync::OnceLock, - time::{SystemTime, UNIX_EPOCH}, - }; - - use super::*; - use crate::services::{ - db::{DbSpec, TursoDb}, - lifecycle::{lifecycle_providers, LifecycleProviderId}, - }; - use anyhow::Context; - - static TEST_DB_PATH: OnceLock = OnceLock::new(); - - struct TestAuthDbSpec; - - impl DbSpec for TestAuthDbSpec { - fn db_name() -> &'static str { - "test auth DB" - } - - fn db_path() -> Result { - TEST_DB_PATH - .get() - .cloned() - .context("test DB path should be initialized") - } - - fn migrations() -> &'static [(&'static str, &'static str)] { - generated_migrations::AUTH_MIGRATIONS - } - - fn db_config_key() -> &'static str { - "auth_db" - } - } - - fn unique_test_db_path() -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!("sce-auth-db-test-{}-{nonce}", std::process::id())) - .join("auth.db") - } - - fn sqlite_object_exists(db: &TursoDb, object_type: &str, name: &str) -> bool { - let rows: Vec = db - .query_map( - "SELECT name FROM sqlite_master WHERE type = ?1 AND name = ?2", - (object_type, name), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("sqlite_master query should succeed"); - !rows.is_empty() - } - - fn applied_migration_ids(db: &TursoDb) -> Vec { - db.query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("migration metadata query should succeed") - } - - #[test] - fn auth_db_baseline_migration_creates_table_index_and_constraints() { - let db_path = unique_test_db_path(); - TEST_DB_PATH - .set(db_path.clone()) - .expect("test DB path should only be initialized once"); - - let db = TursoDb::::new().expect("test auth DB should open"); - - // Verify table, trigger, and migration IDs - assert!(sqlite_object_exists(&db, "table", "auth_credentials")); - assert!(sqlite_object_exists( - &db, - "trigger", - "auth_credentials_set_updated_at" - )); - - // Verify migration IDs are ordered - let applied_ids = applied_migration_ids(&db); - assert_eq!( - applied_ids.len(), - generated_migrations::AUTH_MIGRATIONS.len(), - "applied migration count should match generated migration count" - ); - assert!( - applied_ids.windows(2).all(|w| w[0] < w[1]), - "applied migration IDs should be sorted ascending: {applied_ids:?}" - ); - for id in &applied_ids { - assert!( - id.len() > 4 - && id.chars().take(3).all(|c| c.is_ascii_digit()) - && id.chars().nth(3) == Some('_'), - "migration ID '{id}' should match NNN_... pattern" - ); - } - - // Verify column NOT NULL constraints via PRAGMA table_info - // Returns: cid, name, type, notnull, dflt_value, pk - let columns: Vec<(String, i32)> = db - .query_map( - "SELECT name, \"notnull\" FROM pragma_table_info('auth_credentials') ORDER BY cid", - (), - |row| { - let name: String = row.get::(0)?; - let notnull: i32 = row.get::(1)?; - Ok((name, notnull)) - }, - ) - .expect("pragma table_info should succeed"); - - let col_map: HashMap = columns.into_iter().collect(); - - // Required columns must be NOT NULL - for col in &[ - "id", - "access_token", - "token_type", - "expires_in", - "refresh_token", - "stored_at_unix_seconds", - "created_at", - "updated_at", - ] { - assert_eq!( - col_map.get(*col), - Some(&1), - "column '{col}' should be NOT NULL" - ); - } - - // scope must allow NULL - assert_eq!( - col_map.get("scope"), - Some(&0), - "column 'scope' should allow NULL" - ); - - if let Some(parent) = db_path.parent() { - fs::remove_dir_all(parent).expect("test DB directory should be removed"); - } - } - - #[test] - fn auth_db_lifecycle_provider_included() { - let providers = lifecycle_providers(false); - - let auth_count = providers - .iter() - .filter(|p| p.id() == LifecycleProviderId::AuthDb) - .count(); - assert_eq!( - auth_count, 1, - "AuthDb lifecycle provider should be registered exactly once" - ); - - // Verify deterministic order: Config -> LocalDb -> AuthDb -> AgentTraceDb - let provider_ids: Vec = providers.iter().map(|p| p.id()).collect(); - assert_eq!( - provider_ids, - vec![ - LifecycleProviderId::Config, - LifecycleProviderId::LocalDb, - LifecycleProviderId::AuthDb, - LifecycleProviderId::AgentTraceDb, - ], - "lifecycle provider order should be deterministic: Config -> LocalDb -> AuthDb -> AgentTraceDb" - ); - } -} diff --git a/cli/src/services/capabilities.rs b/cli/src/services/capabilities.rs index 7d6ab0f0c..ed255b47c 100644 --- a/cli/src/services/capabilities.rs +++ b/cli/src/services/capabilities.rs @@ -1,51 +1,18 @@ -#![allow(dead_code)] - -use std::fs::{self, Metadata}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use anyhow::{anyhow, Context, Result}; -pub trait FsOps: Send + Sync { - fn read_file(&self, path: &Path) -> Result; - - fn write_file(&self, path: &Path, content: &str) -> Result<()>; - - fn metadata(&self, path: &Path) -> Result; - - fn exists(&self, path: &Path) -> bool; -} +pub trait FsOps: Send + Sync {} #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct StdFsOps; -impl FsOps for StdFsOps { - fn read_file(&self, path: &Path) -> Result { - fs::read_to_string(path) - .with_context(|| format!("Failed to read file '{}'", path.display())) - } - - fn write_file(&self, path: &Path, content: &str) -> Result<()> { - fs::write(path, content) - .with_context(|| format!("Failed to write file '{}'", path.display())) - } - - fn metadata(&self, path: &Path) -> Result { - fs::metadata(path).with_context(|| format!("Failed to inspect path '{}'", path.display())) - } - - fn exists(&self, path: &Path) -> bool { - path.exists() - } -} +impl FsOps for StdFsOps {} pub trait GitOps: Send + Sync { fn run_command(&self, repo: &Path, args: &[&str]) -> Result; - fn resolve_repository_root(&self, dir: &Path) -> Result; - - fn resolve_hooks_directory(&self, repo: &Path) -> Result; - fn is_available(&self) -> bool; } @@ -57,22 +24,6 @@ impl GitOps for ProcessGitOps { run_git_command(repo, args) } - fn resolve_repository_root(&self, dir: &Path) -> Result { - let output = run_git_command(dir, &["rev-parse", "--show-toplevel"])?; - Ok(PathBuf::from(output.trim())) - } - - fn resolve_hooks_directory(&self, repo: &Path) -> Result { - let output = run_git_command(repo, &["rev-parse", "--git-path", "hooks"])?; - let hooks_path = PathBuf::from(output.trim()); - - if hooks_path.is_absolute() { - Ok(hooks_path) - } else { - Ok(repo.join(hooks_path)) - } - } - fn is_available(&self) -> bool { Command::new("git") .arg("--version") diff --git a/cli/src/services/codex_hook_config.rs b/cli/src/services/codex_hook_config.rs index 9f434ccd6..e2c36d81f 100644 --- a/cli/src/services/codex_hook_config.rs +++ b/cli/src/services/codex_hook_config.rs @@ -822,1074 +822,3 @@ fn helper_path_token_is_valid(token: &str) -> bool { let token = token.trim_matches(['"', '\'']); token == CODEX_HELPER_PATH || token == CODEX_ROOTED_HELPER_PATH } - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - const CANONICAL_COMMAND: &str = "root=\"$(git rev-parse --show-toplevel 2>/dev/null)\" || exit 0; exec bash \"$root/.codex/hooks/run-sce-or-show-install-guidance.sh\" sce hooks codex"; - const MUTATION_SCOPE_COMMAND: &str = "root=\"$(git rev-parse --show-toplevel 2>/dev/null)\" || exit 0; exec bash \"$root/.codex/hooks/run-sce-or-show-install-guidance.sh\" sce hooks codex-mutation-scope"; - const MUTATION_SCOPE_PRE_TOOL_USE_COMMAND: &str = "sce_deny(){ printf '%s' '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"permissionDecision\":\"deny\",\"permissionDecisionReason\":\"SCE could not establish mutation attribution for this tool execution.\"}}'; exit 0; }; root=\"$(git rev-parse --show-toplevel 2>/dev/null)\" || sce_deny; test -r \"$root/.codex/hooks/run-sce-or-show-install-guidance.sh\" || sce_deny; SCE_CODEX_PRE_TOOL_USE_FAIL_CLOSED=1 exec bash \"$root/.codex/hooks/run-sce-or-show-install-guidance.sh\" sce hooks codex-mutation-scope"; - - fn codex_handler() -> Value { - json!({"type": "command", "command": CANONICAL_COMMAND}) - } - - fn mutation_scope_handler() -> Value { - json!({"type": "command", "command": MUTATION_SCOPE_COMMAND}) - } - - fn mutation_scope_pre_tool_use_handler() -> Value { - json!({"type": "command", "command": MUTATION_SCOPE_PRE_TOOL_USE_COMMAND}) - } - - fn canonical_document() -> Value { - json!({ - "hooks": { - "UserPromptSubmit": [{"hooks": [codex_handler()]}], - "Stop": [{"hooks": [codex_handler()]}, {"hooks": [mutation_scope_handler()]}], - "PreToolUse": [ - {"matcher": "Bash", "hooks": [codex_handler()]}, - {"matcher": CODEX_MUTATION_SCOPE_TOOL_MATCHER, "hooks": [mutation_scope_pre_tool_use_handler()]} - ], - "PostToolUse": [ - {"matcher": "apply_patch", "hooks": [codex_handler()]}, - {"matcher": CODEX_MUTATION_SCOPE_TOOL_MATCHER, "hooks": [mutation_scope_handler()]} - ], - "Interrupt": [{"hooks": [mutation_scope_handler()]}], - "SubagentStop": [{"hooks": [mutation_scope_handler()]}], - "SessionEnd": [{"hooks": [mutation_scope_handler()]}] - } - }) - } - - fn generated() -> Vec { - let mut generated = serde_json::to_string_pretty(&canonical_document()).unwrap(); - generated.push('\n'); - generated.into_bytes() - } - - fn with_mutation_scope_registrations(mut doc: Value) -> Value { - let hooks = doc["hooks"].as_object_mut().expect("hooks object"); - hooks - .entry("PreToolUse".to_string()) - .or_insert_with(|| Value::Array(Vec::new())) - .as_array_mut() - .expect("event array") - .push(json!({ - "matcher": CODEX_MUTATION_SCOPE_TOOL_MATCHER, - "hooks": [mutation_scope_pre_tool_use_handler()] - })); - hooks - .entry("PostToolUse".to_string()) - .or_insert_with(|| Value::Array(Vec::new())) - .as_array_mut() - .expect("event array") - .push(json!({ - "matcher": CODEX_MUTATION_SCOPE_TOOL_MATCHER, - "hooks": [mutation_scope_handler()] - })); - hooks - .entry("Stop".to_string()) - .or_insert_with(|| Value::Array(Vec::new())) - .as_array_mut() - .expect("event array") - .push(json!({"hooks": [mutation_scope_handler()]})); - for event in ["Interrupt", "SubagentStop", "SessionEnd"] { - hooks.insert( - event.to_string(), - json!([{"hooks": [mutation_scope_handler()]}]), - ); - } - doc - } - - #[test] - fn accepts_upstream_defaulted_groups_and_events() { - let existing = json!({ - "description": "user hooks", - "hooks": { - "Stop": [{}], - "PreToolUse": [{"matcher": null, "hooks": []}], - "SessionStart": [{}] - } - }); - merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - ".codex/hooks.json", - ) - .expect("upstream-valid defaulted groups should merge"); - } - - #[test] - fn preserves_valid_user_handlers_and_optional_fields() { - let existing = json!({ - "description": "user hooks", - "hooks": { - "PostToolUse": [{"matcher": "Write", "hooks": [ - {"type": "command", "command": "python3 /tmp/pre.py", "commandWindows": "powershell -File C:\\\\pre.ps1", "timeout": 10, "async": true, "statusMessage": "checking", "additionalContextLimit": 4096}, - {"type": "mcp_tool", "server": "security", "tool": "scan", "input": {"file_path": "${tool_input.file_path}", "include_ignored": false}, "timeout": 30, "statusMessage": "Scanning"}, - {"type": "prompt"}, - {"type": "agent"} - ]}] - } - }); - let merged = merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - ".codex/hooks.json", - ) - .expect("valid handlers should survive"); - let value: Value = serde_json::from_slice(&merged).unwrap(); - assert_eq!(value["description"], "user hooks"); - let post_tool_use = value["hooks"]["PostToolUse"].as_array().unwrap(); - assert_eq!(post_tool_use.len(), 3); - assert_eq!(post_tool_use[0]["matcher"], "Write"); - assert_eq!(post_tool_use[0]["hooks"].as_array().unwrap().len(), 4); - assert_eq!(post_tool_use[1]["matcher"], "apply_patch"); - assert_eq!(post_tool_use[1]["hooks"][0]["command"], CANONICAL_COMMAND); - assert_eq!( - post_tool_use[2]["matcher"], - CODEX_MUTATION_SCOPE_TOOL_MATCHER - ); - assert_eq!( - post_tool_use[2]["hooks"][0]["command"], - MUTATION_SCOPE_COMMAND - ); - } - - #[test] - fn rejects_unknown_top_level_fields() { - let existing = json!({"custom": true}); - assert!(merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - ".codex/hooks.json" - ) - .is_err()); - } - - // Upstream `HookEventsToml`, `MatcherGroup`, and `HookHandlerConfig` variant - // structs do not use `#[serde(deny_unknown_fields)]` (only `HooksFile` does; - // see openai/codex codex-rs/config/src/hook_config.rs), so Codex silently - // ignores unrecognized nested keys instead of rejecting the file. SCE must - // accept and preserve them rather than fail the merge. - #[test] - fn preserves_unknown_nested_events_groups_and_handler_fields_as_codex_does() { - let existing = json!({ - "description": "user hooks", - "hooks": { - "CustomEvent": [{"hooks": []}], - "Stop": [ - { - "customGroupField": "keep", - "hooks": [ - { - "type": "command", - "command": "echo user", - "customHandlerField": "keep" - } - ] - } - ] - } - }); - let merged = merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - ".codex/hooks.json", - ) - .expect("Codex-compatible unknown nested fields must not be rejected"); - let value: Value = serde_json::from_slice(&merged).unwrap(); - - assert_eq!( - value["hooks"]["CustomEvent"], - json!([{"hooks": []}]), - "unknown top-level event name must be preserved" - ); - assert_eq!( - value["hooks"]["Stop"][0]["customGroupField"], "keep", - "unknown matcher group field must be preserved" - ); - let user_handler = value["hooks"]["Stop"][0]["hooks"] - .as_array() - .unwrap() - .iter() - .find(|handler| handler["command"] == "echo user") - .expect("user handler must survive merge"); - assert_eq!(user_handler["customHandlerField"], "keep"); - } - - #[test] - fn rejects_invalid_matcher_and_handler_shapes() { - for existing in [ - json!({"hooks": {"Stop": [{"matcher": 42} ]}}), - json!({"hooks": {"Stop": [{"hooks": [{"nonsense": true}]}]}}), - json!({"hooks": {"Stop": [{"hooks": [{"type": "unknown"}]}]}}), - json!({"hooks": {"Stop": [{"hooks": [{"type": "command"}]}]}}), - json!({"hooks": {"Stop": [{"hooks": [{"type": "command", "command": "echo ok", "timeout": "fast"}]}]}}), - json!({"hooks": {"Stop": [{"hooks": [{"type": "mcp_tool", "server": "s", "tool": "t", "input": {"x": null}}]}]}}), - ] { - assert!(merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - ".codex/hooks.json" - ) - .is_err()); - } - } - - #[test] - fn preserves_defaulted_groups_without_rewriting_optional_fields() { - let existing = json!({"hooks": {"Stop": [{}]}}); - let merged = merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - ".codex/hooks.json", - ) - .unwrap(); - let value: Value = serde_json::from_slice(&merged).unwrap(); - assert!(!value["hooks"]["Stop"][0] - .as_object() - .unwrap() - .contains_key("matcher")); - assert_eq!( - value["hooks"]["Stop"][0]["hooks"].as_array().unwrap().len(), - 1 - ); - } - - #[test] - fn preserves_unrelated_fields_groups_and_handlers() { - let existing = json!({ - "description": "user hooks", - "hooks": { - "UserPromptSubmit": [{"hooks": [ - {"type": "command", "command": "echo user"}, - {"type": "command", "command": "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"} - ]}], - "SessionStart": [{"hooks": [{"type": "command", "command": "echo session"}]}] - } - }); - let merged = merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - ".codex/hooks.json", - ) - .unwrap(); - let value: Value = serde_json::from_slice(&merged).unwrap(); - assert_eq!(value["description"], "user hooks"); - assert_eq!( - value["hooks"]["SessionStart"][0]["hooks"][0]["command"], - "echo session" - ); - assert_eq!( - value["hooks"]["UserPromptSubmit"][0]["hooks"][0]["command"], - "echo user" - ); - } - - #[test] - fn stale_and_duplicate_owned_handlers_become_one_current_handler() { - let existing = json!({ - "hooks": { - "Stop": [{"hooks": [ - {"type": "command", "command": "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"}, - {"type": "command", "command": "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"} - ]}] - } - }); - let merged = merge_or_create( - Some(&serde_json::to_vec(&existing).unwrap()), - &generated(), - "hooks.json", - ) - .unwrap(); - let value: Value = serde_json::from_slice(&merged).unwrap(); - let handlers = value["hooks"]["Stop"][0]["hooks"].as_array().unwrap(); - assert_eq!(handlers.len(), 1); - assert!(handlers[0]["command"] - .as_str() - .unwrap() - .contains("$root/.codex/hooks")); - } - - #[test] - fn repeated_merge_is_idempotent() { - let first = merge_or_create(None, &generated(), "hooks.json").unwrap(); - let second = merge_or_create(Some(&first), &generated(), "hooks.json").unwrap(); - assert_eq!(first, second); - let document_diagnosis = diagnose_document(Some(&first), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert!(diagnoses - .iter() - .all(|diagnosis| diagnosis.state == RegistrationStructuralState::PresentAndCurrent)); - } - - #[test] - fn ownership_requires_a_bounded_helper_invocation_shape() { - assert_eq!( - command_owning_contract( - "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex" - ), - Some(CodexHookCommand::Codex) - ); - assert_eq!( - command_owning_contract( - r#"root="$(git rev-parse --show-toplevel 2>/dev/null)" || exit 0; exec bash "$root/.codex/hooks/run-sce-or-show-install-guidance.sh" sce hooks codex"# - ), - Some(CodexHookCommand::Codex) - ); - assert_eq!( - command_owning_contract( - r#"root="$(git rev-parse --show-toplevel 2>/dev/null)" || exit 0; exec bash "$root/.codex/hooks/run-sce-or-show-install-guidance.sh" sce hooks codex-mutation-scope"# - ), - Some(CodexHookCommand::MutationScope) - ); - assert_eq!( - command_owning_contract(MUTATION_SCOPE_PRE_TOOL_USE_COMMAND), - Some(CodexHookCommand::MutationScope), - "the fail-closed PreToolUse bootstrap is still MutationScope-owned" - ); - assert_eq!( - command_owning_contract( - r#"SCE_CODEX_PRE_TOOL_USE_FAIL_CLOSED=1 exec bash ".codex/hooks/run-sce-or-show-install-guidance.sh" sce hooks codex-mutation-scope"# - ), - Some(CodexHookCommand::MutationScope) - ); - for command in [ - "echo .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex", - "printf '%s' '.codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex'", - "foo='.codex/hooks/run-sce-or-show-install-guidance.sh'; echo sce hooks codex", - "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex && echo user", - "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex-other", - "FOO=1 exec bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex-mutation-scope", - "SCE_CODEX_PRE_TOOL_USE_FAIL_CLOSED=1 SCE_OTHER=1 exec bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex-mutation-scope", - "SCE_CODEX_PRE_TOOL_USE_FAIL_CLOSED=1 bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex-mutation-scope && echo user", - "sce hooks codex-mutation-scope", - ] { - assert!( - command_owning_contract(command).is_none(), - "claimed ownership for {command}" - ); - } - } - - #[test] - fn malformed_existing_json_is_rejected_without_a_replacement() { - let existing = br#"{\"hooks\":{\"Stop\":\"not-an-array\"}}"#; - let error = merge_or_create(Some(existing), &generated(), ".codex/hooks.json").unwrap_err(); - assert!(error.to_string().contains(".codex/hooks.json")); - assert_eq!(existing, br#"{\"hooks\":{\"Stop\":\"not-an-array\"}}"#); - } - - fn registration<'a>( - diagnoses: &'a [RegistrationDiagnosis], - event: &str, - ) -> &'a RegistrationDiagnosis { - diagnoses - .iter() - .find(|diagnosis| diagnosis.event == event) - .unwrap_or_else(|| panic!("no diagnosis for event '{event}'")) - } - - fn registration_of<'a>( - diagnoses: &'a [RegistrationDiagnosis], - command: CodexHookCommand, - event: &str, - ) -> &'a RegistrationDiagnosis { - diagnoses - .iter() - .find(|diagnosis| diagnosis.command == command && diagnosis.event == event) - .unwrap_or_else(|| panic!("no {} diagnosis for event '{event}'", command.label())) - } - - #[test] - fn diagnose_document_reports_absent_for_a_missing_file() { - assert_eq!( - diagnose_document(None, &generated()).unwrap(), - HooksDocumentDiagnosis::Absent - ); - } - - #[test] - fn diagnose_document_reports_malformed_for_invalid_json() { - let document_diagnosis = diagnose_document( - Some(br#"{\"hooks\":{\"Stop\":\"not-an-array\"}}"#), - &generated(), - ) - .unwrap(); - assert!(matches!( - document_diagnosis, - HooksDocumentDiagnosis::Malformed(_) - )); - } - - #[test] - fn diagnose_document_reports_malformed_for_codex_invalid_structure() { - let existing = serde_json::to_vec(&json!({"custom": true})).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing), &generated()).unwrap(); - assert!(matches!( - document_diagnosis, - HooksDocumentDiagnosis::Malformed(_) - )); - } - - #[test] - fn diagnose_document_reports_missing_registrations_for_an_empty_valid_document() { - let existing = serde_json::to_vec(&json!({})).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected a validated document with per-registration diagnoses"); - }; - assert_eq!(diagnoses.len(), REQUIRED_EVENTS.len()); - for diagnosis in &diagnoses { - assert_eq!(diagnosis.state, RegistrationStructuralState::Missing); - assert!(diagnosis.owned_handler.is_none()); - assert!(diagnosis.position.is_none()); - } - assert!(diagnoses.iter().any(|diagnosis| diagnosis.command - == CodexHookCommand::MutationScope - && diagnosis.event == "SessionEnd")); - } - - #[test] - fn diagnose_document_reports_present_and_current_after_a_fresh_merge() { - let installed = merge_or_create(None, &generated(), "hooks.json").unwrap(); - let document_diagnosis = diagnose_document(Some(&installed), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - for diagnosis in &diagnoses { - assert_eq!( - diagnosis.state, - RegistrationStructuralState::PresentAndCurrent - ); - assert!(diagnosis.owned_handler.is_some()); - } - assert_eq!( - registration_of(&diagnoses, CodexHookCommand::Codex, "Stop").position, - Some((0, 0)) - ); - assert_eq!( - registration_of(&diagnoses, CodexHookCommand::MutationScope, "Stop").position, - Some((1, 0)) - ); - assert_eq!( - registration_of(&diagnoses, CodexHookCommand::MutationScope, "PreToolUse").position, - Some((1, 0)) - ); - assert_eq!( - registration_of(&diagnoses, CodexHookCommand::MutationScope, "Interrupt").position, - Some((0, 0)) - ); - } - - #[test] - fn diagnose_document_reports_stale_for_a_legacy_owned_handler() { - let existing = json!({ - "hooks": { - "Stop": [{"hooks": [ - {"type": "command", "command": "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex", "timeout": 30} - ]}] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - let stop = registration(&diagnoses, "Stop"); - assert_eq!(stop.state, RegistrationStructuralState::Stale); - assert_eq!(stop.position, Some((0, 0))); - assert_eq!( - registration(&diagnoses, "UserPromptSubmit").state, - RegistrationStructuralState::Missing - ); - } - - #[test] - fn diagnose_document_reports_stale_for_duplicate_owned_handlers() { - let owned_command = "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"; - let existing = json!({ - "hooks": { - "Stop": [{"hooks": [ - {"type": "command", "command": owned_command}, - {"type": "command", "command": owned_command} - ]}] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert_eq!( - registration(&diagnoses, "Stop").state, - RegistrationStructuralState::Stale - ); - } - - #[test] - fn diagnose_document_treats_an_owned_handler_in_the_wrong_matcher_group_as_stale() { - // Codex still discovers this handler (it just never dispatches for a - // Bash PreToolUse call, since the matcher does not match); doctor - // must not report "nothing is here" when something structurally - // wrong is actually present. - let owned_command = "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"; - let existing = json!({ - "hooks": { - "PreToolUse": [ - {"matcher": "Write", "hooks": [{"type": "command", "command": owned_command}]} - ] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert_eq!( - registration(&diagnoses, "PreToolUse").state, - RegistrationStructuralState::Stale - ); - } - - #[test] - fn diagnose_document_reports_stale_for_a_canonical_handler_duplicated_in_a_second_matcher_group( - ) { - let existing = json!({ - "hooks": { - "PreToolUse": [ - {"matcher": "Bash", "hooks": [{"type": "command", "command": CANONICAL_COMMAND}]}, - {"matcher": "Bash", "hooks": [{"type": "command", "command": CANONICAL_COMMAND}]} - ] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert_eq!( - registration(&diagnoses, "PreToolUse").state, - RegistrationStructuralState::Stale, - "an owned handler duplicated across two matcher groups must not read PresentAndCurrent, \ - since merge_or_create would still collapse it to one handler" - ); - } - - #[test] - fn diagnose_document_finds_the_canonical_handler_in_a_non_first_matcher_group() { - let existing = json!({ - "hooks": { - "PreToolUse": [ - {"matcher": "Bash", "hooks": [{"type": "command", "command": "echo user only"}]}, - {"matcher": "Bash", "hooks": [{"type": "command", "command": CANONICAL_COMMAND}]} - ] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - let pre_tool_use = registration(&diagnoses, "PreToolUse"); - assert_eq!( - pre_tool_use.state, - RegistrationStructuralState::PresentAndCurrent - ); - assert_eq!( - pre_tool_use.position, - Some((1, 0)), - "position must name the second group, not wrongly default to the first" - ); - } - - #[test] - fn diagnose_document_reports_stale_for_a_canonical_handler_plus_a_wrong_matcher_duplicate() { - let owned_command = "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"; - let existing = json!({ - "hooks": { - "PreToolUse": [ - {"matcher": "Bash", "hooks": [{"type": "command", "command": CANONICAL_COMMAND}]}, - {"matcher": "Write", "hooks": [{"type": "command", "command": owned_command}]} - ] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert_eq!( - registration(&diagnoses, "PreToolUse").state, - RegistrationStructuralState::Stale, - "a canonical placement plus any other owned handler anywhere must still read Stale" - ); - } - - #[test] - fn diagnose_document_reports_missing_when_every_group_holds_only_non_owned_handlers() { - let existing = json!({ - "hooks": { - "Stop": [ - {"hooks": [{"type": "command", "command": "echo one"}]}, - {"hooks": [{"type": "command", "command": "echo two"}]} - ] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert_eq!( - registration(&diagnoses, "Stop").state, - RegistrationStructuralState::Missing - ); - } - - #[test] - fn diagnose_document_finds_the_canonical_handler_mixed_with_arbitrary_user_handlers() { - let existing = json!({ - "hooks": { - "Stop": [{"hooks": [ - {"type": "command", "command": "echo user one"}, - {"type": "command", "command": CANONICAL_COMMAND}, - {"type": "command", "command": "echo user two"} - ]}] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - let stop = registration(&diagnoses, "Stop"); - assert_eq!(stop.state, RegistrationStructuralState::PresentAndCurrent); - assert_eq!(stop.position, Some((0, 1))); - } - - #[test] - fn present_and_current_implies_merge_or_create_is_a_semantic_no_op() { - let canonical = merge_or_create(None, &generated(), "hooks.json").unwrap(); - let document_diagnosis = diagnose_document(Some(&canonical), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert!(diagnoses - .iter() - .all(|diagnosis| diagnosis.state == RegistrationStructuralState::PresentAndCurrent)); - - let merged_again = merge_or_create(Some(&canonical), &generated(), "hooks.json").unwrap(); - let canonical_value: Value = serde_json::from_slice(&canonical).unwrap(); - let merged_again_value: Value = serde_json::from_slice(&merged_again).unwrap(); - assert_eq!( - canonical_value, merged_again_value, - "PresentAndCurrent for every registration must imply merge_or_create is a no-op" - ); - } - - /// A matrix proving `merge_or_create` never rewrites a document every - /// registration is diagnosed `PresentAndCurrent` for, including the - /// specific relocation bug: a canonical handler already sitting in a - /// *non-first* matcher group must stay exactly where it is, not be - /// moved into an earlier matcher group merely because one exists. - #[test] - fn merge_or_create_is_a_no_op_for_every_present_and_current_placement() { - let user_prompt_submit = - json!({"hooks": [{"type": "command", "command": CANONICAL_COMMAND}]}); - let stop = json!({"hooks": [{"type": "command", "command": CANONICAL_COMMAND}]}); - let post_tool_use = json!({"matcher": "apply_patch", "hooks": [{"type": "command", "command": CANONICAL_COMMAND}]}); - - let cases: Vec<(&str, Value)> = vec![ - ( - "canonical handler in the only (first) matching group", - json!({ - "hooks": { - "UserPromptSubmit": [user_prompt_submit.clone()], - "Stop": [stop.clone()], - "PreToolUse": [ - {"matcher": "Bash", "hooks": [{"type": "command", "command": CANONICAL_COMMAND}]} - ], - "PostToolUse": [post_tool_use.clone()] - } - }), - ), - ( - "canonical handler in a second matching group, behind a user-only first group", - json!({ - "hooks": { - "UserPromptSubmit": [user_prompt_submit.clone()], - "Stop": [stop.clone()], - "PreToolUse": [ - {"matcher": "Bash", "hooks": [{"type": "command", "command": "echo user only"}]}, - {"matcher": "Bash", "hooks": [{"type": "command", "command": CANONICAL_COMMAND}]} - ], - "PostToolUse": [post_tool_use.clone()] - } - }), - ), - ( - "canonical handler mixed with arbitrary user handlers in the same group", - json!({ - "hooks": { - "UserPromptSubmit": [user_prompt_submit.clone()], - "Stop": [stop.clone()], - "PreToolUse": [{ - "matcher": "Bash", - "hooks": [ - {"type": "command", "command": "echo user one"}, - {"type": "command", "command": CANONICAL_COMMAND}, - {"type": "command", "command": "echo user two"} - ] - }], - "PostToolUse": [post_tool_use.clone()] - } - }), - ), - ]; - - for (label, existing) in cases { - let existing = with_mutation_scope_registrations(existing); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let document_diagnosis = - diagnose_document(Some(&existing_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("case '{label}': expected per-registration diagnoses"); - }; - assert!( - diagnoses - .iter() - .all(|diagnosis| diagnosis.state - == RegistrationStructuralState::PresentAndCurrent), - "case '{label}': every registration must diagnose PresentAndCurrent, got {diagnoses:?}" - ); - - let merged_bytes = - merge_or_create(Some(&existing_bytes), &generated(), "hooks.json").unwrap(); - let existing_value: Value = serde_json::from_slice(&existing_bytes).unwrap(); - let merged_value: Value = serde_json::from_slice(&merged_bytes).unwrap(); - assert_eq!( - existing_value, merged_value, - "case '{label}': merge_or_create must be a semantic no-op when every \ - registration is already PresentAndCurrent" - ); - } - } - - #[test] - fn merge_relocates_a_wrong_matcher_owned_handler_into_the_correct_matcher_group() { - let owned_command = "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"; - let existing = json!({ - "hooks": { - "PreToolUse": [ - {"matcher": "Bash", "hooks": [{"type": "command", "command": "echo user only"}]}, - {"matcher": "Write", "hooks": [{"type": "command", "command": owned_command}]} - ] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - let merged_bytes = - merge_or_create(Some(&existing_bytes), &generated(), "hooks.json").unwrap(); - let merged: Value = serde_json::from_slice(&merged_bytes).unwrap(); - - let bash_group = &merged["hooks"]["PreToolUse"][0]; - assert_eq!(bash_group["matcher"], "Bash"); - assert_eq!(bash_group["hooks"][0]["command"], "echo user only"); - assert_eq!(bash_group["hooks"][1]["command"], CANONICAL_COMMAND); - - let write_group = &merged["hooks"]["PreToolUse"][1]; - assert_eq!(write_group["matcher"], "Write"); - assert_eq!( - write_group["hooks"].as_array().unwrap().len(), - 0, - "the misplaced handler must be removed from the Write group, not left duplicated" - ); - - let document_diagnosis = diagnose_document(Some(&merged_bytes), &generated()).unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = document_diagnosis else { - panic!("expected per-registration diagnoses"); - }; - assert_eq!( - registration(&diagnoses, "PreToolUse").state, - RegistrationStructuralState::PresentAndCurrent - ); - } - - fn canonical_four_document() -> Value { - json!({ - "hooks": { - "UserPromptSubmit": [{"hooks": [codex_handler()]}], - "Stop": [{"hooks": [codex_handler()]}], - "PreToolUse": [{"matcher": "Bash", "hooks": [codex_handler()]}], - "PostToolUse": [{"matcher": "apply_patch", "hooks": [codex_handler()]}] - } - }) - } - - #[test] - fn upgrading_the_canonical_four_document_preserves_existing_trust_identity() { - let mut existing = canonical_four_document(); - existing["hooks"]["Stop"][0]["hooks"] - .as_array_mut() - .unwrap() - .insert(0, json!({"type": "command", "command": "echo user stop"})); - existing["hooks"]["SessionEnd"] = - json!([{"hooks": [{"type": "command", "command": "echo user session end"}]}]); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - - let merged = - merge_or_create(Some(&existing_bytes), &generated(), ".codex/hooks.json").unwrap(); - let value: Value = serde_json::from_slice(&merged).unwrap(); - - let HooksDocumentDiagnosis::Registrations(diagnoses) = - diagnose_document(Some(&merged), &generated()).unwrap() - else { - panic!("expected per-registration diagnoses"); - }; - - for (event, expected_position) in [ - ("UserPromptSubmit", (0usize, 0usize)), - ("Stop", (0, 1)), - ("PreToolUse", (0, 0)), - ("PostToolUse", (0, 0)), - ] { - let registration = registration_of(&diagnoses, CodexHookCommand::Codex, event); - assert_eq!( - registration.state, - RegistrationStructuralState::PresentAndCurrent, - "{event} must stay present and current" - ); - assert_eq!( - registration.position, - Some(expected_position), - "{event} identity tuple (group/handler index) must not move" - ); - assert_eq!( - registration.owned_handler.as_ref().unwrap(), - &codex_handler() - ); - } - - for event in [ - "PreToolUse", - "PostToolUse", - "Stop", - "Interrupt", - "SubagentStop", - "SessionEnd", - ] { - let registration = registration_of(&diagnoses, CodexHookCommand::MutationScope, event); - assert_eq!( - registration.state, - RegistrationStructuralState::PresentAndCurrent, - "mutation-scope {event} must be added" - ); - } - - let mutation_scope_count = value["hooks"] - .as_object() - .unwrap() - .values() - .flat_map(|groups| groups.as_array().unwrap()) - .flat_map(|group| group["hooks"].as_array().unwrap()) - .filter(|handler| { - handler["command"] - .as_str() - .and_then(command_owning_contract) - == Some(CodexHookCommand::MutationScope) - }) - .count(); - assert_eq!( - mutation_scope_count, 6, - "each mutation-scope handler must appear exactly once" - ); - let fail_closed_pre_tool_use_count = value["hooks"]["PreToolUse"] - .as_array() - .unwrap() - .iter() - .flat_map(|group| group["hooks"].as_array().unwrap()) - .filter(|handler| handler["command"] == MUTATION_SCOPE_PRE_TOOL_USE_COMMAND) - .count(); - assert_eq!( - fail_closed_pre_tool_use_count, 1, - "the tracked-mutation PreToolUse registration uses the fail-closed bootstrap" - ); - - assert_eq!( - value["hooks"]["Stop"][0]["hooks"][0]["command"], - "echo user stop" - ); - assert!(value["hooks"]["SessionEnd"] - .as_array() - .unwrap() - .iter() - .flat_map(|group| group["hooks"].as_array().unwrap()) - .any(|handler| handler["command"] == "echo user session end")); - - let merged_again = - merge_or_create(Some(&merged), &generated(), ".codex/hooks.json").unwrap(); - assert_eq!( - merged, merged_again, - "a second merge over the upgraded document must be byte-identical" - ); - } - - #[test] - fn merge_appends_the_mutation_scope_pre_tool_use_group_without_touching_the_bash_group() { - let existing = serde_json::to_vec(&canonical_four_document()).unwrap(); - let merged = merge_or_create(Some(&existing), &generated(), ".codex/hooks.json").unwrap(); - let value: Value = serde_json::from_slice(&merged).unwrap(); - - let pre_tool_use = value["hooks"]["PreToolUse"].as_array().unwrap(); - assert_eq!(pre_tool_use.len(), 2); - assert_eq!(pre_tool_use[0]["matcher"], "Bash"); - assert_eq!(pre_tool_use[0]["hooks"][0]["command"], CANONICAL_COMMAND); - assert_eq!( - pre_tool_use[1]["matcher"], - CODEX_MUTATION_SCOPE_TOOL_MATCHER - ); - assert_eq!( - pre_tool_use[1]["hooks"][0]["command"], - MUTATION_SCOPE_PRE_TOOL_USE_COMMAND - ); - } - - #[test] - fn the_two_unmatched_stop_groups_diagnose_independently_by_command() { - let installed = merge_or_create(None, &generated(), ".codex/hooks.json").unwrap(); - let HooksDocumentDiagnosis::Registrations(diagnoses) = - diagnose_document(Some(&installed), &generated()).unwrap() - else { - panic!("expected per-registration diagnoses"); - }; - let codex_stop = registration_of(&diagnoses, CodexHookCommand::Codex, "Stop"); - let mutation_stop = registration_of(&diagnoses, CodexHookCommand::MutationScope, "Stop"); - assert_eq!( - codex_stop.state, - RegistrationStructuralState::PresentAndCurrent - ); - assert_eq!( - mutation_stop.state, - RegistrationStructuralState::PresentAndCurrent - ); - assert_eq!(codex_stop.position, Some((0, 0))); - assert_eq!(mutation_stop.position, Some((1, 0))); - } - - #[test] - fn mutation_scope_tool_hooks_carry_the_tracked_only_matcher() { - for (command, event, matcher) in required_registrations() { - match (command, event) { - (CodexHookCommand::MutationScope, "PreToolUse" | "PostToolUse") => { - assert_eq!( - matcher, - Some("^(Bash|apply_patch)$"), - "{event} mutation-scope hook must gate on exactly the tracked tools" - ); - } - (CodexHookCommand::MutationScope, _) => assert_eq!( - matcher, None, - "{event} mutation-scope lifecycle hook stays unmatched" - ), - (CodexHookCommand::Codex, _) => {} - } - } - } - - #[test] - fn a_legacy_unmatched_mutation_scope_tool_hook_is_stale_and_repairs_to_the_tracked_matcher() { - let existing = json!({ - "hooks": { - "PreToolUse": [{"hooks": [mutation_scope_pre_tool_use_handler()]}], - "PostToolUse": [{"hooks": [mutation_scope_handler()]}] - } - }); - let existing_bytes = serde_json::to_vec(&existing).unwrap(); - - let HooksDocumentDiagnosis::Registrations(diagnoses) = - diagnose_document(Some(&existing_bytes), &generated()).unwrap() - else { - panic!("expected per-registration diagnoses"); - }; - for event in ["PreToolUse", "PostToolUse"] { - assert_eq!( - registration_of(&diagnoses, CodexHookCommand::MutationScope, event).state, - RegistrationStructuralState::Stale, - "an unmatched {event} mutation-scope hook is stale under the tracked-tool matcher" - ); - } - - let merged = - merge_or_create(Some(&existing_bytes), &generated(), ".codex/hooks.json").unwrap(); - let value: Value = serde_json::from_slice(&merged).unwrap(); - for event in ["PreToolUse", "PostToolUse"] { - let groups = value["hooks"][event].as_array().unwrap(); - let scoped: Vec<&Value> = groups - .iter() - .filter(|group| { - group["hooks"] - .as_array() - .into_iter() - .flatten() - .any(|handler| { - handler["command"] - .as_str() - .and_then(command_owning_contract) - == Some(CodexHookCommand::MutationScope) - }) - }) - .collect(); - assert_eq!( - scoped.len(), - 1, - "{event} keeps exactly one mutation-scope group" - ); - assert_eq!(scoped[0]["matcher"], CODEX_MUTATION_SCOPE_TOOL_MATCHER); - } - - let HooksDocumentDiagnosis::Registrations(after) = - diagnose_document(Some(&merged), &generated()).unwrap() - else { - panic!("expected per-registration diagnoses"); - }; - for event in ["PreToolUse", "PostToolUse"] { - assert_eq!( - registration_of(&after, CodexHookCommand::MutationScope, event).state, - RegistrationStructuralState::PresentAndCurrent - ); - } - assert_eq!( - merge_or_create(Some(&merged), &generated(), ".codex/hooks.json").unwrap(), - merged - ); - } - - #[test] - fn interrupt_event_key_label_matches_upstream() { - assert_eq!(hook_event_key_label("Interrupt"), "interrupt"); - } - - #[test] - fn subagent_stop_event_key_label_matches_upstream() { - assert_eq!(hook_event_key_label("SubagentStop"), "subagent_stop"); - } - - #[test] - fn session_end_event_key_label_matches_upstream() { - assert_eq!(hook_event_key_label("SessionEnd"), "session_end"); - } - - #[test] - fn every_required_event_has_an_upstream_key_label() { - for (_, event, _) in REQUIRED_EVENTS { - let _ = hook_event_key_label(event); - } - } -} diff --git a/cli/src/services/command_registry.rs b/cli/src/services/command_registry.rs index 77dc373dc..bc8ff9023 100644 --- a/cli/src/services/command_registry.rs +++ b/cli/src/services/command_registry.rs @@ -54,22 +54,13 @@ impl RuntimeCommand { } } - #[allow(dead_code)] - pub async fn execute(&self, context: &C) -> Result - where - C: HasLogger + ContextWithRepoRoot, - { - let mut stderr = std::io::sink(); - self.execute_with_stderr(context, &mut stderr).await - } - pub async fn execute_with_stderr( &self, context: &C, stderr: &mut W, ) -> Result where - C: HasLogger + ContextWithRepoRoot, + C: HasLogger + ContextWithRepoRoot + crate::app::HasGit + crate::app::HasFs, W: Write, { match self { @@ -77,32 +68,9 @@ impl RuntimeCommand { Self::HelpText(command) => Ok(command.execute(context)), Self::Auth(command) => command.execute(context).await, Self::Config(command) => command.execute(context), - Self::Setup(command) => { - if command.request.context_only { - command.execute(context) - } else { - tokio::task::block_in_place(|| command.execute(context)) - } - } - Self::Doctor(command) => tokio::task::block_in_place(|| command.execute(context)), - Self::Hooks(command) => match &command.subcommand { - services::hooks::HookSubcommand::PreCommit - | services::hooks::HookSubcommand::PostRewrite { .. } => command.execute(context), - services::hooks::HookSubcommand::CommitMsg { .. } - | services::hooks::HookSubcommand::PostCommit { .. } - | services::hooks::HookSubcommand::DiffTrace - | services::hooks::HookSubcommand::ConversationTrace - | services::hooks::HookSubcommand::Codex - | services::hooks::HookSubcommand::ClaudeModelState - | services::hooks::HookSubcommand::MutationScope - | services::hooks::HookSubcommand::ClaudeMutationScope - | services::hooks::HookSubcommand::CodexMutationScope - | services::hooks::HookSubcommand::OpenCodeMutationScope - | services::hooks::HookSubcommand::PiMutationScope - | services::hooks::HookSubcommand::ExternalMutationGuard => { - tokio::task::block_in_place(|| command.execute(context)) - } - }, + Self::Setup(command) => command.execute(context).await, + Self::Doctor(command) => command.execute(context).await, + Self::Hooks(command) => command.execute(context).await, Self::Policy(command) => command.execute(), Self::Version(command) => command.execute(context), Self::Completion(command) => Ok(command.execute(context)), @@ -123,13 +91,6 @@ impl CommandRegistry { pub fn contains(&self, name: &str) -> bool { self.names.contains(&name) } - - #[cfg(test)] - pub fn command_names(&self) -> Vec<&'static str> { - let mut names = self.names.to_vec(); - names.sort_unstable(); - names - } } impl Default for CommandRegistry { @@ -222,48 +183,3 @@ pub fn default_runtime_command(name: &str) -> Option { _ => None, } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn default_registry_lists_all_commands_deterministically() { - let registry = CommandRegistry::default(); - - assert_eq!( - registry.command_names(), - vec![ - "auth", - "completion", - "config", - "doctor", - "help", - "hooks", - "policy", - "setup", - "sync", - "version" - ] - ); - } - - #[test] - fn default_registry_reports_known_command_names() { - let registry = CommandRegistry::default(); - - for name in DEFAULT_COMMAND_NAMES { - assert!(registry.contains(name)); - } - assert!(registry.contains("sync")); - } - - #[test] - fn default_runtime_commands_have_expected_names() { - for name in DEFAULT_COMMAND_NAMES { - let command = default_runtime_command(name).expect("command should exist"); - assert_eq!(command.name(), *name); - } - assert!(default_runtime_command("sync").is_some()); - } -} diff --git a/cli/src/services/config/lifecycle.rs b/cli/src/services/config/lifecycle.rs index 6c75f8723..b5178488a 100644 --- a/cli/src/services/config/lifecycle.rs +++ b/cli/src/services/config/lifecycle.rs @@ -6,7 +6,7 @@ use crate::app::HasRepoRoot; use crate::services::default_paths::{resolve_sce_default_locations, RepoPaths}; use crate::services::lifecycle::{ HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, - LifecycleProviderId, ServiceLifecycle, SetupOutcome, + ServiceLifecycle, SetupOutcome, }; use crate::services::setup::bootstrap_repo_local_config; @@ -16,11 +16,7 @@ use super::validate_config_file; pub struct ConfigLifecycle; impl ServiceLifecycle for ConfigLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::Config - } - - fn diagnose(&self, ctx: &C) -> Vec { + async fn diagnose(&self, ctx: &C) -> Vec { let repository_root = match ctx.repo_root() { Some(path) => path.to_path_buf(), None => { @@ -41,7 +37,7 @@ impl ServiceLifecycle for ConfigLifecycle { diagnose_config_health(&repository_root) } - fn setup(&self, ctx: &C) -> Result { + async fn setup(&self, ctx: &C) -> Result { let repository_root = ctx .repo_root() .context("Config lifecycle setup requires a resolved repository root")?; diff --git a/cli/src/services/config/policy.rs b/cli/src/services/config/policy.rs index 3a3302c2a..b64ac1904 100644 --- a/cli/src/services/config/policy.rs +++ b/cli/src/services/config/policy.rs @@ -33,7 +33,6 @@ pub(crate) struct BashPolicyConfig { } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) struct RuntimeBashPolicyPreset { pub(crate) id: String, pub(crate) argv_prefixes: Vec>, @@ -121,7 +120,6 @@ fn builtin_bash_policy_catalog() -> &'static BuiltinBashPolicyCatalog { }) } -#[allow(dead_code)] pub(crate) fn runtime_bash_policy_presets() -> Vec { builtin_bash_policy_catalog() .presets diff --git a/cli/src/services/config/render.rs b/cli/src/services/config/render.rs index 3b1ce40cb..1ab4ad00e 100644 --- a/cli/src/services/config/render.rs +++ b/cli/src/services/config/render.rs @@ -403,7 +403,10 @@ fn abbreviate_text_value(value: &str) -> String { fn retry_policy_display(policy: &crate::services::resilience::RetryPolicy) -> String { format!( "{} attempts, {}ms timeout, {}..{}ms backoff", - policy.max_attempts, policy.timeout_ms, policy.initial_backoff_ms, policy.max_backoff_ms + policy.max_attempts(), + policy.timeout_ms(), + policy.initial_backoff_ms(), + policy.max_backoff_ms() ) } @@ -491,10 +494,10 @@ fn format_per_db_retry_json(config: &super::types::PerDbRetryConfig) -> Value { obj.insert( "connection_open".to_string(), json!({ - "max_attempts": policy.max_attempts, - "timeout_ms": policy.timeout_ms, - "initial_backoff_ms": policy.initial_backoff_ms, - "max_backoff_ms": policy.max_backoff_ms, + "max_attempts": policy.max_attempts(), + "timeout_ms": policy.timeout_ms(), + "initial_backoff_ms": policy.initial_backoff_ms(), + "max_backoff_ms": policy.max_backoff_ms(), }), ); } @@ -502,10 +505,10 @@ fn format_per_db_retry_json(config: &super::types::PerDbRetryConfig) -> Value { obj.insert( "query".to_string(), json!({ - "max_attempts": policy.max_attempts, - "timeout_ms": policy.timeout_ms, - "initial_backoff_ms": policy.initial_backoff_ms, - "max_backoff_ms": policy.max_backoff_ms, + "max_attempts": policy.max_attempts(), + "timeout_ms": policy.timeout_ms(), + "initial_backoff_ms": policy.initial_backoff_ms(), + "max_backoff_ms": policy.max_backoff_ms(), }), ); } @@ -551,116 +554,3 @@ fn format_database_retry_json(value: &ResolvedOptionalValue "config_source": value.source.and_then(ValueSource::config_source).map(ConfigPathSource::as_str), }) } - -#[cfg(test)] -mod database_retry_render_tests { - use serde_json::json; - - use super::{format_database_retry_json, format_database_retry_text}; - use crate::services::config::{ - AgentTraceDbRetryConfig, ConfigPathSource, DatabaseRetryConfig, PerDbRetryConfig, - ResolvedOptionalValue, ValueSource, - }; - use crate::services::resilience::RetryPolicy; - - fn query_policy() -> RetryPolicy { - RetryPolicy { - max_attempts: 3, - timeout_ms: 150, - initial_backoff_ms: 10, - max_backoff_ms: 50, - } - } - - fn resolved( - busy_timeout_ms: Option, - contention_deadline_ms: Option, - ) -> ResolvedOptionalValue { - ResolvedOptionalValue { - value: Some(DatabaseRetryConfig { - local_db: Some(PerDbRetryConfig { - connection_open: None, - query: Some(query_policy()), - }), - agent_trace_db: Some(AgentTraceDbRetryConfig { - retry: PerDbRetryConfig { - connection_open: None, - query: Some(query_policy()), - }, - busy_timeout_ms, - contention_deadline_ms, - }), - auth_db: None, - }), - source: Some(ValueSource::ConfigFile(ConfigPathSource::Flag)), - } - } - - #[test] - fn database_retry_json_renders_contention_keys_for_agent_trace_db_only() { - let rendered = format_database_retry_json(&resolved(Some(750), Some(2_000))); - - assert_eq!( - rendered["resolved"], - json!({ - "local_db": { - "query": { - "max_attempts": 3, - "timeout_ms": 150, - "initial_backoff_ms": 10, - "max_backoff_ms": 50, - }, - }, - "agent_trace_db": { - "query": { - "max_attempts": 3, - "timeout_ms": 150, - "initial_backoff_ms": 10, - "max_backoff_ms": 50, - }, - "busy_timeout_ms": 750, - "contention_deadline_ms": 2000, - }, - }) - ); - } - - #[test] - fn database_retry_json_omits_unset_contention_keys() { - let rendered = format_database_retry_json(&resolved(None, None)); - - let agent_trace_db = rendered["resolved"]["agent_trace_db"].as_object().unwrap(); - assert!(!agent_trace_db.contains_key("busy_timeout_ms")); - assert!(!agent_trace_db.contains_key("contention_deadline_ms")); - assert_eq!( - rendered["resolved"]["agent_trace_db"]["query"]["timeout_ms"], - 150 - ); - } - - #[test] - fn database_retry_text_renders_contention_keys_for_agent_trace_db() { - let rendered = format_database_retry_text(&resolved(Some(750), Some(0))); - - let busy_line = rendered - .lines() - .find(|line| line.contains("(busy_timeout_ms)")) - .unwrap(); - assert!(busy_line.contains("agent_trace_db"), "{rendered}"); - assert!(busy_line.contains("750ms"), "{rendered}"); - let deadline_line = rendered - .lines() - .find(|line| line.contains("(contention_deadline_ms)")) - .unwrap(); - assert!(deadline_line.contains("agent_trace_db"), "{rendered}"); - assert!(deadline_line.contains("0ms"), "{rendered}"); - assert_eq!( - rendered - .lines() - .filter(|line| line.contains("3 attempts, 150ms timeout, 10..50ms backoff (query)")) - .count(), - 2, - "{rendered}" - ); - } -} diff --git a/cli/src/services/config/schema.rs b/cli/src/services/config/schema.rs index c646a3807..449bc85e1 100644 --- a/cli/src/services/config/schema.rs +++ b/cli/src/services/config/schema.rs @@ -11,7 +11,7 @@ use std::path::Path; use std::sync::OnceLock; -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use jsonschema::{validator_for, Validator}; use serde::Deserialize; use serde_json::Value; @@ -22,7 +22,7 @@ use super::types::{ IntegrationTargetId, IntegrationsConfig, LogFormat, LogLevel, PerDbRetryConfig, AGENT_TRACE_DB_BUSY_TIMEOUT_MAX_MS, AGENT_TRACE_DB_CONTENTION_DEADLINE_MAX_MS, }; -use crate::services::resilience::RetryPolicy; +use crate::services::resilience::{RetryPolicy, RetryPolicyError}; pub(crate) const SCE_CONFIG_SCHEMA_JSON: &str = include_str!(concat!( env!("OUT_DIR"), @@ -531,31 +531,29 @@ pub(crate) fn map_database_retry_config( ) })?; - if max_attempts == 0 { - bail!( - "Config key '{context}.max_attempts' in '{}' must be >= 1.", - path.display() - ); - } - if timeout_ms == 0 { - bail!( - "Config key '{context}.timeout_ms' in '{}' must be >= 1.", - path.display() - ); - } - if max_backoff_ms < initial_backoff_ms { - bail!( - "Config key '{context}.max_backoff_ms' in '{}' must be >= initial_backoff_ms.", - path.display() - ); - } - - Ok(RetryPolicy { - max_attempts, - timeout_ms, - initial_backoff_ms, - max_backoff_ms, - }) + RetryPolicy::new(max_attempts, timeout_ms, initial_backoff_ms, max_backoff_ms).map_err( + |error| { + let message = match error { + RetryPolicyError::ZeroMaxAttempts => { + format!( + "'{context}.max_attempts' in '{}' must be >= 1.", + path.display() + ) + } + RetryPolicyError::ZeroTimeout => { + format!( + "'{context}.timeout_ms' in '{}' must be >= 1.", + path.display() + ) + } + RetryPolicyError::MaxBackoffBelowInitial => format!( + "'{context}.max_backoff_ms' in '{}' must be >= initial_backoff_ms.", + path.display() + ), + }; + anyhow!("Config key {message}") + }, + ) }; let per_db_object = |db_key: &str, @@ -1139,21 +1137,11 @@ mod database_retry_config_tests { let database_retry = config.database_retry.unwrap().value; assert_eq!( database_retry.local_db.unwrap().query, - Some(RetryPolicy { - max_attempts: 4, - timeout_ms: 300, - initial_backoff_ms: 20, - max_backoff_ms: 80, - }) + Some(RetryPolicy::new(4, 300, 20, 80).unwrap()) ); assert_eq!( database_retry.agent_trace_db.unwrap().retry.query, - Some(RetryPolicy { - max_attempts: 3, - timeout_ms: 150, - initial_backoff_ms: 10, - max_backoff_ms: 50, - }) + Some(RetryPolicy::new(3, 150, 10, 50).unwrap()) ); let error = parse_error( diff --git a/cli/src/services/config/types.rs b/cli/src/services/config/types.rs index 7de321d86..bed656c23 100644 --- a/cli/src/services/config/types.rs +++ b/cli/src/services/config/types.rs @@ -42,16 +42,6 @@ impl LogLevel { } } - pub(crate) fn parse_env(raw: &str, key: &str) -> anyhow::Result { - match raw { - "error" => Ok(Self::Error), - "warn" => Ok(Self::Warn), - "info" => Ok(Self::Info), - "debug" => Ok(Self::Debug), - _ => anyhow::bail!("Invalid {key} '{raw}'. Valid values: error, warn, info, debug."), - } - } - pub(crate) fn as_str(self) -> &'static str { match self { Self::Error => "error", @@ -88,14 +78,6 @@ impl LogFormat { } } - pub(crate) fn parse_env(raw: &str, key: &str) -> anyhow::Result { - match raw { - "text" => Ok(Self::Text), - "json" => Ok(Self::Json), - _ => anyhow::bail!("Invalid {key} '{raw}'. Valid values: text, json."), - } - } - pub(crate) fn as_str(self) -> &'static str { match self { Self::Text => "text", diff --git a/cli/src/services/db/encryption_key.rs b/cli/src/services/db/encryption_key.rs index b2ed59a7b..ef9cba9e1 100644 --- a/cli/src/services/db/encryption_key.rs +++ b/cli/src/services/db/encryption_key.rs @@ -291,47 +291,3 @@ fn hex_encode(bytes: &[u8]) -> String { } hex } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_hex_encode_empty() { - assert_eq!(hex_encode(&[]), ""); - } - - #[test] - fn test_hex_encode_32_bytes() { - let bytes = [0xdeu8; 32]; - let expected = "de".repeat(32); - assert_eq!(hex_encode(&bytes), expected); - } - - #[test] - fn test_hex_encode_mixed() { - let bytes = [0x00u8, 0x01, 0xfe, 0xff, 0xab, 0xcd]; - assert_eq!(hex_encode(&bytes), "0001feffabcd"); - } - - #[test] - fn test_generate_key_length() { - let key = generate_key(); - assert_eq!(key.len(), 64); - } - - #[test] - fn test_generate_key_lowercase() { - let key = generate_key(); - assert!(key - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit())); - } - - #[test] - fn test_generate_key_is_random() { - let key1 = generate_key(); - let key2 = generate_key(); - assert_ne!(key1, key2); - } -} diff --git a/cli/src/services/db/mod.rs b/cli/src/services/db/mod.rs index 4629c29c2..05f9d76d1 100644 --- a/cli/src/services/db/mod.rs +++ b/cli/src/services/db/mod.rs @@ -11,13 +11,13 @@ use std::{ }; use anyhow::{Context, Result}; -use turso::Value as TursoValue; +use rand::SeedableRng; use crate::services::config::{AgentTraceDbRetryConfig, DatabaseRetryConfig}; use crate::services::lifecycle::{ HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, }; -use crate::services::resilience::{run_with_retry_sync, RetryPolicy}; +use crate::services::resilience::{run_with_retry_elapsed, RetryOperation, RetryPolicy}; const MIGRATIONS_TABLE_SQL: &str = "CREATE TABLE IF NOT EXISTS __sce_migrations ( id TEXT PRIMARY KEY, @@ -26,21 +26,10 @@ const MIGRATIONS_TABLE_SQL: &str = "CREATE TABLE IF NOT EXISTS __sce_migrations const SELECT_MIGRATION_SQL: &str = "SELECT id FROM __sce_migrations WHERE id = ?1 LIMIT 1"; const INSERT_MIGRATION_SQL: &str = "INSERT INTO __sce_migrations (id) VALUES (?1)"; const ENCRYPTION_CIPHER_AEGIS256: &str = "aegis256"; -const CONNECTION_OPEN_RETRY_POLICY: RetryPolicy = RetryPolicy { - max_attempts: 3, - timeout_ms: 1_000, - initial_backoff_ms: 25, - max_backoff_ms: 200, -}; +const CONNECTION_OPEN_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(3, 1_000, 25, 200); const CONNECTION_OPEN_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; -const QUERY_RETRY_POLICY: RetryPolicy = RetryPolicy { - max_attempts: 5, - timeout_ms: 200, - initial_backoff_ms: 25, - max_backoff_ms: 100, -}; +const QUERY_RETRY_POLICY: RetryPolicy = RetryPolicy::builtin(5, 200, 25, 100); const QUERY_RETRY_HINT: &str = "retry after the database lock clears; if the issue persists, stop other SCE processes using this database and rerun the command"; -const AGENT_TRACE_DB_CONFIG_KEY: &str = "agent_trace_db"; const AGENT_TRACE_DB_BUSY_TIMEOUT_MS: u64 = 1_000; const AGENT_TRACE_DB_CONTENTION_DEADLINE_MS: u64 = 2_250; const AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS: u32 = 2; @@ -48,6 +37,13 @@ const AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS: u64 = 100; pub mod encryption_key; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DatabaseKind { + Local, + AgentTrace, + Auth, +} + /// Service-specific Turso database configuration. pub trait DbSpec { /// Human-readable database name used in diagnostics. @@ -59,9 +55,8 @@ pub trait DbSpec { /// Ordered embedded migration SQL files as `(id, sql)` pairs. fn migrations() -> &'static [(&'static str, &'static str)]; - /// Config-file lookup key under `policies.database_retry`. - /// One of `"local_db"`, `"agent_trace_db"`, `"auth_db"`. - fn db_config_key() -> &'static str; + /// Closed database identity selecting retry configuration and contention policy. + const KIND: DatabaseKind; } /// Collect common filesystem health problems for a Turso database path. @@ -162,6 +157,12 @@ fn sentence_case(value: &str) -> String { first.to_uppercase().collect::() + chars.as_str() } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OpenMode { + CreateIfMissing, + ExistingReadOnly, +} + fn ensure_db_parent_dir(db_name: &str, db_path: &Path) -> Result<()> { if let Some(parent) = db_path.parent() { std::fs::create_dir_all(parent).with_context(|| { @@ -175,112 +176,61 @@ fn ensure_db_parent_dir(db_name: &str, db_path: &Path) -> Result<()> { Ok(()) } -fn build_current_thread_runtime(db_name: &str) -> Result { - tokio::runtime::Builder::new_current_thread() - .enable_io() - .enable_time() - .build() - .with_context(|| { - format!("failed to create {db_name} tokio runtime. Try: rerun the command; if the issue persists, verify the local Tokio runtime environment.") - }) -} - -/// Drives `fut` to completion on `runtime`, isolating it on a dedicated -/// thread when the calling thread already has an active Tokio runtime -/// context. -/// -/// `Runtime::block_on` panics ("Cannot start a runtime from within a -/// runtime") if invoked directly from a thread that is already driving -/// another runtime, which happens when async callers (for example, Agent -/// Trace sync's control-plane client) reach into a `TursoDb`/`EncryptedTursoDb` -/// synchronously. Tokio's "already in a runtime" check is thread-local, so -/// running `block_on` on a fresh scoped thread sidesteps it safely. -fn block_on_isolated(runtime: &tokio::runtime::Runtime, fut: F) -> T -where - F: std::future::Future + Send, - T: Send, -{ - if tokio::runtime::Handle::try_current().is_ok() { - std::thread::scope(|scope| scope.spawn(|| runtime.block_on(fut)).join()) - .unwrap_or_else(|payload| std::panic::resume_unwind(payload)) - } else { - runtime.block_on(fut) - } -} - -fn run_embedded_migrations( +async fn run_embedded_migrations( conn: &turso::Connection, - runtime: &tokio::runtime::Runtime, db_name: &str, migrations: &[(&str, &str)], ) -> Result<()> { - ensure_migrations_table(conn, runtime, db_name)?; + ensure_migrations_table(conn, db_name).await?; for (id, sql) in migrations { - if is_migration_applied(conn, runtime, db_name, id)? { + if is_migration_applied(conn, db_name, id).await? { continue; } - apply_migration(conn, runtime, db_name, id, sql)?; + apply_migration(conn, db_name, id, sql).await?; } Ok(()) } - -fn ensure_migrations_table( - conn: &turso::Connection, - runtime: &tokio::runtime::Runtime, - db_name: &str, -) -> Result<()> { - block_on_isolated(runtime, async { - conn.execute(MIGRATIONS_TABLE_SQL, ()) - .await - .map_err(|e| anyhow::anyhow!("{db_name} migration metadata setup failed: {e}")) - })?; +async fn ensure_migrations_table(conn: &turso::Connection, db_name: &str) -> Result<()> { + conn.execute(MIGRATIONS_TABLE_SQL, ()) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata setup failed: {e}"))?; Ok(()) } -fn is_migration_applied( - conn: &turso::Connection, - runtime: &tokio::runtime::Runtime, - db_name: &str, - id: &str, -) -> Result { - block_on_isolated(runtime, async { - let mut rows = conn.query(SELECT_MIGRATION_SQL, (id,)).await.map_err(|e| { - anyhow::anyhow!("{db_name} migration metadata query failed for {id}: {e}") - })?; +async fn is_migration_applied(conn: &turso::Connection, db_name: &str, id: &str) -> Result { + let mut rows = conn + .query(SELECT_MIGRATION_SQL, (id,)) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata query failed for {id}: {e}"))?; - rows.next().await.map(|row| row.is_some()).map_err(|e| { - anyhow::anyhow!("{db_name} migration metadata row fetch failed for {id}: {e}") - }) - }) + rows.next() + .await + .map(|row| row.is_some()) + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata row fetch failed for {id}: {e}")) } -fn apply_migration( +async fn apply_migration( conn: &turso::Connection, - runtime: &tokio::runtime::Runtime, db_name: &str, id: &str, sql: &str, ) -> Result<()> { - block_on_isolated(runtime, async { - // Migration files may contain multiple statements (the repository - // Agent Trace baseline is one multi-statement schema file), so batch - // execution is required; `execute` would stop after the first - // statement. - conn.execute_batch(sql) - .await - .map_err(|e| anyhow::anyhow!("{db_name} migration {id} failed: {e}"))?; - conn.execute(INSERT_MIGRATION_SQL, (id,)) - .await - .map_err(|e| { - anyhow::anyhow!("{db_name} migration metadata record failed for {id}: {e}") - })?; + // Migration files may contain multiple statements (the repository + // Agent Trace baseline is one multi-statement schema file), so batch + // execution is required; `execute` would stop after the first + // statement. + conn.execute_batch(sql) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration {id} failed: {e}"))?; + conn.execute(INSERT_MIGRATION_SQL, (id,)) + .await + .map_err(|e| anyhow::anyhow!("{db_name} migration metadata record failed for {id}: {e}"))?; - Ok(()) - }) + Ok(()) } /// Body of [`TursoDb::execute_transactional_insert_pair_if_absent`], run @@ -338,7 +288,6 @@ async fn execute_insert_pair_if_absent_body( Ok(true) } -#[allow(dead_code)] pub struct TransactionStatement<'a> { sql: &'a str, params: turso::params::Params, @@ -346,7 +295,6 @@ pub struct TransactionStatement<'a> { } impl<'a> TransactionStatement<'a> { - #[allow(dead_code)] pub fn new(sql: &'a str, params: impl turso::params::IntoParams) -> Result { let params = turso::params::IntoParams::into_params(params) .map_err(|e| anyhow::anyhow!("parameter conversion failed: {sql}: {e}"))?; @@ -358,7 +306,6 @@ impl<'a> TransactionStatement<'a> { }) } - #[allow(dead_code)] pub fn expect_rows_affected(mut self, expected: u64) -> Self { self.expected_rows_affected = Some(expected); self @@ -374,6 +321,124 @@ enum WriteAttemptFailure { Deterministic(anyhow::Error), } +trait WriteAttempt { + async fn run(&mut self, attempt: u32) -> std::result::Result; +} + +impl WriteAttempt for F +where + F: FnMut(u32) -> Fut, + Fut: std::future::Future>, +{ + async fn run(&mut self, attempt: u32) -> std::result::Result { + self(attempt).await + } +} + +struct PlainWriteAttempt(A); + +impl> RetryOperation for PlainWriteAttempt { + async fn run(&mut self, attempt: u32) -> Result { + self.0 + .run(attempt) + .await + .map_err(WriteAttemptFailure::into_error) + } +} + +struct CasPlainAttempt(A); + +impl> RetryOperation for CasPlainAttempt { + async fn run(&mut self, attempt: u32) -> Result { + match self.0.run(attempt).await { + Ok(applied) => Ok(CasBatchAttemptOutcome::Settled(applied)), + Err(failure) => cas_batch_failure_into_attempt_result(failure), + } + } +} + +struct InsertPairAttempt<'a> { + conn: &'a mut turso::Connection, + db_name: &'a str, + exists_sql: &'a str, + exists_params: turso::params::Params, + first_sql: &'a str, + first_params: turso::params::Params, + second_sql: &'a str, + second_params: turso::params::Params, + fail_before_second: bool, +} + +impl WriteAttempt for InsertPairAttempt<'_> { + async fn run(&mut self, _attempt: u32) -> std::result::Result { + let db_name = self.db_name; + let tx = turso::transaction::Transaction::new( + &mut *self.conn, + turso::transaction::TransactionBehavior::Immediate, + ) + .await + .map_err(|e| classify_turso_error(db_name, "failed to begin transaction", &e))?; + + let outcome = execute_insert_pair_if_absent_body( + &tx, + db_name, + self.exists_sql, + self.exists_params.clone(), + self.first_sql, + self.first_params.clone(), + self.second_sql, + self.second_params.clone(), + self.fail_before_second, + ) + .await; + + match outcome { + Ok(inserted) => { + tx.commit().await.map_err(|e| { + classify_turso_error(db_name, "failed to commit transaction", &e) + })?; + Ok(inserted) + } + Err(failure) => { + let _ = tx.rollback().await; + Err(failure) + } + } + } +} + +struct CasBatchAttempt<'a, 'b> { + conn: &'a mut turso::Connection, + db_name: &'a str, + guard: &'a TransactionStatement<'b>, + statements: &'a [TransactionStatement<'b>], +} + +impl WriteAttempt for CasBatchAttempt<'_, '_> { + async fn run(&mut self, _attempt: u32) -> std::result::Result { + let db_name = self.db_name; + let tx = turso::transaction::Transaction::new( + &mut *self.conn, + turso::transaction::TransactionBehavior::Immediate, + ) + .await + .map_err(|e| classify_turso_error(db_name, "failed to begin transaction", &e))?; + + match execute_cas_batch_body(&tx, db_name, self.guard, self.statements).await { + Ok(applied) => { + tx.commit().await.map_err(|e| { + classify_turso_error(db_name, "failed to commit transaction", &e) + })?; + Ok(applied) + } + Err(failure) => { + let _ = tx.rollback().await; + Err(failure) + } + } + } +} + impl WriteAttemptFailure { fn into_error(self) -> anyhow::Error { match self { @@ -392,13 +457,11 @@ fn classify_turso_error(db_name: &str, action: &str, error: &turso::Error) -> Wr } } -#[allow(dead_code)] enum CasBatchAttemptOutcome { Settled(bool), Deterministic(anyhow::Error), } -#[allow(dead_code)] fn cas_batch_failure_into_attempt_result( failure: WriteAttemptFailure, ) -> Result { @@ -408,7 +471,6 @@ fn cas_batch_failure_into_attempt_result( } } -#[allow(dead_code)] async fn execute_cas_batch_body( tx: &turso::transaction::Transaction<'_>, db_name: &str, @@ -456,31 +518,28 @@ async fn execute_cas_batch_body( struct TursoConnectionCore { conn: turso::Connection, - runtime: tokio::runtime::Runtime, spec: PhantomData M>, } impl TursoConnectionCore { - fn new(conn: turso::Connection, runtime: tokio::runtime::Runtime) -> Self { + fn new(conn: turso::Connection) -> Self { Self { conn, - runtime, spec: PhantomData, } } - fn run_migrations(&self) -> Result<()> { - run_embedded_migrations(&self.conn, &self.runtime, M::db_name(), M::migrations()) + async fn run_migrations(&self) -> Result<()> { + run_embedded_migrations(&self.conn, M::db_name(), M::migrations()).await } } fn resolve_connection_open_retry_policy() -> RetryPolicy { if let Some(config) = crate::services::config::get_database_retry_config() { - let per_db = match M::db_config_key() { - "local_db" => config.local_db.as_ref(), - "agent_trace_db" => config.agent_trace_db.as_ref().map(|db| &db.retry), - "auth_db" => config.auth_db.as_ref(), - _ => None, + let per_db = match M::KIND { + DatabaseKind::Local => config.local_db.as_ref(), + DatabaseKind::AgentTrace => config.agent_trace_db.as_ref().map(|db| &db.retry), + DatabaseKind::Auth => config.auth_db.as_ref(), }; if let Some(per_db) = per_db { if let Some(policy) = per_db.connection_open { @@ -493,11 +552,10 @@ fn resolve_connection_open_retry_policy() -> RetryPolicy { fn resolve_query_retry_policy() -> RetryPolicy { if let Some(config) = crate::services::config::get_database_retry_config() { - let per_db = match M::db_config_key() { - "local_db" => config.local_db.as_ref(), - "agent_trace_db" => config.agent_trace_db.as_ref().map(|db| &db.retry), - "auth_db" => config.auth_db.as_ref(), - _ => None, + let per_db = match M::KIND { + DatabaseKind::Local => config.local_db.as_ref(), + DatabaseKind::AgentTrace => config.agent_trace_db.as_ref().map(|db| &db.retry), + DatabaseKind::Auth => config.auth_db.as_ref(), }; if let Some(per_db) = per_db { if let Some(policy) = per_db.query { @@ -514,7 +572,7 @@ fn resolve_query_retry_policy() -> RetryPolicy { /// not wait for a contended lock. The busy timeout is Turso's own wait policy /// for a `Busy` result: while another process holds the write lock, Turso /// sleeps in short phases until the lock clears or the timeout elapses, and -/// only then returns `Busy`. Because `Transaction::new_unchecked(.., Immediate)` +/// only then returns `Busy`. Because `Transaction::new(.., Immediate)` /// runs `BEGIN IMMEDIATE` through `Connection::execute` on the same connection, /// the wait also covers writer-lock acquisition. The timeout is connection-wide. /// @@ -561,7 +619,7 @@ fn agent_trace_db_millis( select: impl Fn(&AgentTraceDbRetryConfig) -> Option, default_ms: u64, ) -> std::time::Duration { - if M::db_config_key() != AGENT_TRACE_DB_CONFIG_KEY { + if M::KIND != DatabaseKind::AgentTrace { return std::time::Duration::ZERO; } let configured = config @@ -580,7 +638,7 @@ struct WriteContentionPolicy { } fn write_contention_policy() -> Option { - if M::db_config_key() != AGENT_TRACE_DB_CONFIG_KEY { + if M::KIND != DatabaseKind::AgentTrace { return None; } Some(WriteContentionPolicy { @@ -607,10 +665,7 @@ fn write_contention_retry_may_sleep( elapsed: std::time::Duration, backoff: std::time::Duration, ) -> bool { - let Some(remaining) = policy.contention_deadline.checked_sub(elapsed) else { - return false; - }; - !remaining.is_zero() && remaining >= backoff.saturating_add(policy.busy_timeout) + elapsed.saturating_add(backoff) < policy.contention_deadline } fn write_contention_retry_may_start_now( @@ -620,46 +675,41 @@ fn write_contention_retry_may_start_now( write_contention_retry_may_sleep(policy, elapsed, std::time::Duration::ZERO) } -fn run_with_write_contention_retry( +async fn run_with_write_contention_retry( policy: WriteContentionPolicy, operation_name: &str, retry_hint: &str, - attempt: impl FnMut(u32) -> std::result::Result, + attempt: impl WriteAttempt, ) -> Result { - let mut jitter = rand::thread_rng(); + let mut jitter = rand::rngs::StdRng::from_entropy(); let started_at = std::time::Instant::now(); run_with_write_contention_retry_using( policy, &mut || write_contention_backoff(&mut jitter, policy.backoff_cap), - &mut std::thread::sleep, + &mut tokio::time::sleep, &mut || started_at.elapsed(), operation_name, retry_hint, attempt, ) + .await } -fn run_with_write_contention_retry_using( +async fn run_with_write_contention_retry_using>( policy: WriteContentionPolicy, draw_backoff: &mut impl FnMut() -> std::time::Duration, - sleep: &mut impl FnMut(std::time::Duration), + sleep: &mut impl FnMut(std::time::Duration) -> Sleep, elapsed: &mut impl FnMut() -> std::time::Duration, operation_name: &str, retry_hint: &str, - mut attempt: impl FnMut(u32) -> std::result::Result, + mut attempt: impl WriteAttempt, ) -> Result { let mut attempt_number = 0; loop { attempt_number += 1; - #[cfg(test)] - note_write_contention(|counts| counts.attempts += 1); - #[cfg(test)] - note_write_contention_timeline(WriteContentionTimelineEvent::AttemptStart); - let outcome = attempt(attempt_number); - #[cfg(test)] - note_write_contention_timeline(WriteContentionTimelineEvent::AttemptEnd); + let outcome = attempt.run(attempt_number).await; let error = match outcome { Ok(value) => return Ok(value), @@ -670,16 +720,9 @@ fn run_with_write_contention_retry_using( if attempt_number < policy.max_attempts { let backoff = draw_backoff(); if write_contention_retry_may_sleep(policy, elapsed(), backoff) { - #[cfg(test)] - note_write_contention_timeline(WriteContentionTimelineEvent::BackoffRequested( - backoff, - )); - sleep(backoff); - #[cfg(test)] - note_write_contention_timeline(WriteContentionTimelineEvent::BackoffSlept); + sleep(backoff).await; + if write_contention_retry_may_start_now(policy, elapsed()) { - #[cfg(test)] - note_write_contention(|counts| counts.outer_retries += 1); continue; } } @@ -712,9 +755,6 @@ fn contention_exhausted_error( u64::try_from(policy.contention_deadline.as_millis()).unwrap_or(u64::MAX); let elapsed_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX); - #[cfg(test)] - note_write_contention(|counts| counts.exhaustions += 1); - tracing::warn!( target: "sce", event_id = CONTENTION_EXHAUSTED_EVENT_ID, @@ -748,135 +788,10 @@ fn apply_busy_timeout( .map_err(|e| anyhow::anyhow!("failed to set {db_name} database busy timeout: {e}")) } -#[cfg(test)] -thread_local! { - static READ_STATEMENTS_ISSUED: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// Record that one [`TursoDb`] read statement was issued on this thread. -#[cfg(test)] -fn note_read_statement_issued() { - READ_STATEMENTS_ISSUED.with(|count| count.set(count.get() + 1)); -} - -/// Run `body`, returning its result together with the number of [`TursoDb`] -/// read statements ([`TursoDb::query`], [`TursoDb::query_values`], -/// [`TursoDb::query_map`]) it issued on the current thread. -/// -/// Each read method bumps the counter once in its synchronous prelude, before -/// the retry wrapper, so a transient retry never inflates the count and the -/// number reflects *logical* read statements, not connection round-trips. -/// Lets a deterministic single-threaded test assert that an operation which -/// must observe one coherent database snapshot — for example -/// `MutationTraceStore::load_all_tree_roots`, a single `UNION` statement — -/// issues exactly one, and fail if it is ever reimplemented as several -/// independent `SELECT`s unioned in Rust. Not shared across threads. -#[cfg(test)] -pub(crate) fn count_read_statements(body: impl FnOnce() -> T) -> (T, usize) { - READ_STATEMENTS_ISSUED.with(|count| count.set(0)); - let result = body(); - let issued = READ_STATEMENTS_ISSUED.with(std::cell::Cell::get); - (result, issued) -} - -#[cfg(test)] -thread_local! { - static WRITE_STATEMENTS_ISSUED: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -#[cfg(test)] -fn note_write_statement_issued() { - WRITE_STATEMENTS_ISSUED.with(|count| count.set(count.get() + 1)); -} - -#[cfg(test)] -pub(crate) fn count_write_statements(body: impl FnOnce() -> T) -> (T, usize) { - WRITE_STATEMENTS_ISSUED.with(|count| count.set(0)); - let result = body(); - let issued = WRITE_STATEMENTS_ISSUED.with(std::cell::Cell::get); - (result, issued) -} - -#[cfg(test)] -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub(crate) struct WriteContentionCounts { - pub(crate) attempts: u32, - pub(crate) outer_retries: u32, - pub(crate) exhaustions: u32, -} - -#[cfg(test)] -thread_local! { - static WRITE_CONTENTION_COUNTS: std::cell::Cell = - const { std::cell::Cell::new(WriteContentionCounts { attempts: 0, outer_retries: 0, exhaustions: 0 }) }; -} - -#[cfg(test)] -fn note_write_contention(update: impl FnOnce(&mut WriteContentionCounts)) { - WRITE_CONTENTION_COUNTS.with(|cell| { - let mut counts = cell.get(); - update(&mut counts); - cell.set(counts); - }); -} - -#[cfg(test)] -pub(crate) fn count_write_contention(body: impl FnOnce() -> T) -> (T, WriteContentionCounts) { - WRITE_CONTENTION_COUNTS.with(|cell| cell.set(WriteContentionCounts::default())); - let result = body(); - let counts = WRITE_CONTENTION_COUNTS.with(std::cell::Cell::get); - (result, counts) -} - -#[cfg(test)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum WriteContentionTimelineEvent { - AttemptStart, - AttemptEnd, - BackoffRequested(std::time::Duration), - BackoffSlept, -} - -#[cfg(test)] -thread_local! { - static WRITE_CONTENTION_TIMELINE: std::cell::RefCell> = - const { std::cell::RefCell::new(Vec::new()) }; -} - -#[cfg(test)] -fn note_write_contention_timeline(event: WriteContentionTimelineEvent) { - let now = std::time::Instant::now(); - WRITE_CONTENTION_TIMELINE.with(|timeline| timeline.borrow_mut().push((now, event))); -} - -#[cfg(test)] -pub(crate) fn record_write_contention_timeline( - body: impl FnOnce() -> T, -) -> (T, Vec<(std::time::Instant, WriteContentionTimelineEvent)>) { - WRITE_CONTENTION_TIMELINE.with(|timeline| timeline.borrow_mut().clear()); - let result = body(); - let timeline = WRITE_CONTENTION_TIMELINE.with(std::cell::RefCell::take); - (result, timeline) -} - -/// Generic Turso database adapter. -/// -/// Wraps a Turso connection with a tokio current-thread runtime so callers can -/// use synchronous `execute`/`query` methods while the underlying Turso API -/// remains async. pub struct TursoDb { core: TursoConnectionCore, } -/// Fully fetched SQL query result for deterministic rendering outside the -/// async Turso row iterator lifetime. -#[derive(Clone, Debug, PartialEq)] -#[allow(dead_code)] -pub struct QueryRows { - pub columns: Vec, - pub rows: Vec>, -} - /// Generic encrypted Turso database adapter. /// /// Mirrors the structural seams of [`TursoDb`] while reserving encrypted local @@ -890,10 +805,11 @@ impl TursoDb { /// /// Parent directories are created automatically. Migrations are run after /// the database connection is established. - pub fn new() -> Result { - let db = Self::open_without_migrations()?; + pub async fn new() -> Result { + let db = Self::open_without_migrations().await?; db.run_migrations() + .await .with_context(|| format!("failed to run {} migrations", M::db_name()))?; Ok(db) @@ -904,10 +820,11 @@ impl TursoDb { /// Parent directories are created automatically. Migrations are run after /// the database connection is established. The service-specific retry and /// migration configuration still comes from `M`. - pub fn new_at(db_path: impl AsRef) -> Result { - let db = Self::open_without_migrations_at(db_path)?; + pub async fn new_at(db_path: impl AsRef) -> Result { + let db = Self::open_without_migrations_at(db_path).await?; db.run_migrations() + .await .with_context(|| format!("failed to run {} migrations", M::db_name()))?; Ok(db) @@ -919,41 +836,46 @@ impl TursoDb { /// Parent directories are created automatically and the connection-open /// retry policy is preserved. Runtime callers that use this path are /// responsible for verifying schema readiness before query/write work. - pub fn open_without_migrations() -> Result { + pub async fn open_without_migrations() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; - Self::open_without_migrations_at(db_path) + Self::open_without_migrations_at(db_path).await } - /// Open or create the database at an explicit path without running embedded - /// migrations. - /// - /// Parent directories are created automatically and the connection-open - /// retry policy is preserved. Runtime callers that use this path are - /// responsible for verifying schema readiness before query/write work. - pub fn open_without_migrations_at(db_path: impl AsRef) -> Result { + pub async fn open_without_migrations_at(db_path: impl AsRef) -> Result { + Self::open_at_path(db_path.as_ref(), OpenMode::CreateIfMissing).await + } + + pub async fn open_existing_without_migrations_at(db_path: impl AsRef) -> Result { + Self::open_at_path(db_path.as_ref(), OpenMode::ExistingReadOnly).await + } + + async fn open_at_path(db_path: &Path, mode: OpenMode) -> Result { let db_name = M::db_name(); - let db_path = db_path.as_ref().to_path_buf(); + let db_path = db_path.to_path_buf(); - ensure_db_parent_dir(db_name, &db_path)?; + if mode == OpenMode::CreateIfMissing { + ensure_db_parent_dir(db_name, &db_path)?; + } + let read_only = mode == OpenMode::ExistingReadOnly; - let runtime = build_current_thread_runtime(db_name)?; let retry_policy = resolve_connection_open_retry_policy::(); let busy_timeout = resolve_busy_timeout::(); let operation_name = format!("open {db_name} database connection"); - let conn = run_with_retry_sync( + let conn = run_with_retry_elapsed( retry_policy, &operation_name, CONNECTION_OPEN_RETRY_HINT, - |_| { - block_on_isolated(&runtime, async { + async |_| { + async { let path_str = db_path.to_str().ok_or_else(|| { anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) })?; let db = turso::Builder::new_local(path_str) .experimental_multiprocess_wal(true) + .read_only(read_only) .build() .await .map_err(|e| { @@ -967,12 +889,14 @@ impl TursoDb { })?; apply_busy_timeout(&conn, db_name, busy_timeout)?; Ok(conn) - }) + } + .await }, - )?; + ) + .await?; Ok(Self { - core: TursoConnectionCore::new(conn, runtime), + core: TursoConnectionCore::new(conn), }) } @@ -984,49 +908,45 @@ impl TursoDb { /// /// # Returns /// Number of rows affected. - pub fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { + pub async fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { let params = turso::params::IntoParams::into_params(params).map_err(|e| { anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) })?; let operation_name = format!("execute {} database query", M::db_name()); - #[cfg(test)] - note_write_statement_issued(); - - run_with_retry_sync( + run_with_retry_elapsed( resolve_query_retry_policy::(), &operation_name, QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { + async |_| { + async { self.core .conn .execute(sql, params.clone()) .await .map_err(|e| anyhow::anyhow!("{} execute failed: {sql}: {e}", M::db_name())) - }) + } + .await }, ) + .await } - pub fn execute_idempotent_write( + pub async fn execute_idempotent_write( &self, sql: &str, params: impl turso::params::IntoParams, ) -> Result { let Some(policy) = write_contention_policy::() else { - return self.execute(sql, params); + return self.execute(sql, params).await; }; let db_name = M::db_name(); let params = turso::params::IntoParams::into_params(params) .map_err(|e| anyhow::anyhow!("{db_name} parameter conversion failed: {sql}: {e}"))?; let operation_name = format!("execute {db_name} database query"); - #[cfg(test)] - note_write_statement_issued(); - - run_with_write_contention_retry(policy, &operation_name, QUERY_RETRY_HINT, |_| { - block_on_isolated(&self.core.runtime, async { + run_with_write_contention_retry(policy, &operation_name, QUERY_RETRY_HINT, async |_| { + async { self.core .conn .execute(sql, params.clone()) @@ -1034,96 +954,10 @@ impl TursoDb { .map_err(|e| { classify_turso_error(db_name, &format!("execute failed: {sql}"), &e) }) - }) + } + .await }) - } - - /// Execute a SQL query that returns rows. - /// - /// # Arguments - /// * `sql` - SQL query, which may contain `?` placeholders. - /// * `params` - Parameter values implementing `IntoParams`. - /// - /// # Returns - /// A `turso::Rows` iterator over the result set. - #[allow(dead_code)] - pub fn query(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - let params = turso::params::IntoParams::into_params(params).map_err(|e| { - anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) - })?; - let operation_name = format!("query {} database", M::db_name()); - - #[cfg(test)] - note_read_statement_issued(); - - run_with_retry_sync( - resolve_query_retry_policy::(), - &operation_name, - QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { - self.core - .conn - .query(sql, params.clone()) - .await - .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name())) - }) - }, - ) - } - - /// Execute a SQL query and synchronously fetch column names plus raw values. - #[allow(dead_code)] - pub fn query_values( - &self, - sql: &str, - params: impl turso::params::IntoParams, - ) -> Result { - let params = turso::params::IntoParams::into_params(params).map_err(|e| { - anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) - })?; - let operation_name = format!("query and fetch {} database values", M::db_name()); - - #[cfg(test)] - note_read_statement_issued(); - - run_with_retry_sync( - resolve_query_retry_policy::(), - &operation_name, - QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { - let mut rows = - self.core - .conn - .query(sql, params.clone()) - .await - .map_err(|e| { - anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name()) - })?; - let columns = rows.column_names(); - let column_count = rows.column_count(); - let mut fetched_rows = Vec::new(); - - while let Some(row) = rows.next().await.map_err(|e| { - anyhow::anyhow!("{} row fetch failed: {sql}: {e}", M::db_name()) - })? { - let mut values = Vec::with_capacity(column_count); - for column_index in 0..column_count { - values.push(row.get_value(column_index).map_err(|e| { - anyhow::anyhow!("{} value fetch failed: {sql}: {e}", M::db_name()) - })?); - } - fetched_rows.push(values); - } - - Ok(QueryRows { - columns, - rows: fetched_rows, - }) - }) - }, - ) + .await } /// Run an "insert row pair if absent" write transaction. @@ -1142,9 +976,15 @@ impl TursoDb { /// runs or the transaction commits, so callers can prove the whole /// transaction — including the already-executed `first_sql` — rolls /// back together. + /// + /// Takes `&mut self` and begins the transaction with the checked + /// `Transaction::new(&mut Connection, ..)`, so the exclusive borrow spans + /// the whole transaction and no other operation on this adapter can run + /// inside it. Dropping the future uncommitted schedules a rollback that + /// runs on the connection's next use rather than immediately. #[allow(clippy::too_many_arguments)] - pub fn execute_transactional_insert_pair_if_absent( - &self, + pub async fn execute_transactional_insert_pair_if_absent( + &mut self, operation_name: &str, retry_hint: &str, exists_sql: &str, @@ -1166,62 +1006,34 @@ impl TursoDb { anyhow::anyhow!("{db_name} parameter conversion failed: {second_sql}: {e}") })?; - #[cfg(test)] - note_write_statement_issued(); - - let run_attempt = || { - block_on_isolated(&self.core.runtime, async { - let tx = turso::transaction::Transaction::new_unchecked( - &self.core.conn, - turso::transaction::TransactionBehavior::Immediate, - ) - .await - .map_err(|e| classify_turso_error(db_name, "failed to begin transaction", &e))?; - - let outcome = execute_insert_pair_if_absent_body( - &tx, - db_name, - exists_sql, - exists_params.clone(), - first_sql, - first_params.clone(), - second_sql, - second_params.clone(), - fail_before_second, - ) - .await; - - match outcome { - Ok(inserted) => { - tx.commit().await.map_err(|e| { - classify_turso_error(db_name, "failed to commit transaction", &e) - })?; - Ok(inserted) - } - Err(failure) => { - let _ = tx.rollback().await; - Err(failure) - } - } - }) + let attempt = InsertPairAttempt { + conn: &mut self.core.conn, + db_name, + exists_sql, + exists_params, + first_sql, + first_params, + second_sql, + second_params, + fail_before_second, }; if let Some(policy) = write_contention_policy::() { - return run_with_write_contention_retry(policy, operation_name, retry_hint, |_| { - run_attempt() - }); + return run_with_write_contention_retry(policy, operation_name, retry_hint, attempt) + .await; } - run_with_retry_sync( + run_with_retry_elapsed( resolve_query_retry_policy::(), operation_name, retry_hint, - |_| run_attempt().map_err(WriteAttemptFailure::into_error), + PlainWriteAttempt(attempt), ) + .await } /// Execute a SQL query and synchronously map all returned rows. - pub fn query_map( + pub async fn query_map( &self, sql: &str, params: impl turso::params::IntoParams, @@ -1235,15 +1047,12 @@ impl TursoDb { })?; let operation_name = format!("query and fetch {} database rows", M::db_name()); - #[cfg(test)] - note_read_statement_issued(); - - let rows = run_with_retry_sync( + let rows = run_with_retry_elapsed( resolve_query_retry_policy::(), &operation_name, QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { + async |_| { + async { let mut rows = self.core .conn @@ -1261,9 +1070,11 @@ impl TursoDb { } Ok(fetched_rows) - }) + } + .await }, - )?; + ) + .await?; let mut results = Vec::new(); @@ -1277,9 +1088,16 @@ impl TursoDb { Ok(results) } - #[allow(dead_code)] - pub fn execute_transactional_cas_batch( - &self, + /// Run a compare-and-swap guard plus ordered statements in one + /// `BEGIN IMMEDIATE` transaction. + /// + /// Takes `&mut self` and begins the transaction with the checked + /// `Transaction::new(&mut Connection, ..)`, so the exclusive borrow spans + /// the whole transaction and no other operation on this adapter can run + /// inside it. Dropping the future uncommitted schedules a rollback that + /// runs on the connection's next use rather than immediately. + pub async fn execute_transactional_cas_batch( + &mut self, operation_name: &str, retry_hint: &str, guard: &TransactionStatement<'_>, @@ -1287,48 +1105,25 @@ impl TursoDb { ) -> Result { let db_name = M::db_name(); - #[cfg(test)] - note_write_statement_issued(); - - let run_attempt = || { - block_on_isolated(&self.core.runtime, async { - let tx = turso::transaction::Transaction::new_unchecked( - &self.core.conn, - turso::transaction::TransactionBehavior::Immediate, - ) - .await - .map_err(|e| classify_turso_error(db_name, "failed to begin transaction", &e))?; - - match execute_cas_batch_body(&tx, db_name, guard, statements).await { - Ok(applied) => { - tx.commit().await.map_err(|e| { - classify_turso_error(db_name, "failed to commit transaction", &e) - })?; - Ok(applied) - } - Err(failure) => { - let _ = tx.rollback().await; - Err(failure) - } - } - }) + let attempt = CasBatchAttempt { + conn: &mut self.core.conn, + db_name, + guard, + statements, }; if let Some(policy) = write_contention_policy::() { - return run_with_write_contention_retry(policy, operation_name, retry_hint, |_| { - run_attempt() - }); + return run_with_write_contention_retry(policy, operation_name, retry_hint, attempt) + .await; } - let outcome = run_with_retry_sync( + let outcome = run_with_retry_elapsed( resolve_query_retry_policy::(), operation_name, retry_hint, - |_| match run_attempt() { - Ok(applied) => Ok(CasBatchAttemptOutcome::Settled(applied)), - Err(failure) => cas_batch_failure_into_attempt_result(failure), - }, - )?; + CasPlainAttempt(attempt), + ) + .await?; match outcome { CasBatchAttemptOutcome::Settled(applied) => Ok(applied), @@ -1342,8 +1137,8 @@ impl TursoDb { /// initializations apply only migrations that were not already recorded. /// Existing databases without migration metadata are brought forward by /// re-applying the current idempotent migration set and recording each ID. - pub fn run_migrations(&self) -> Result<()> { - self.core.run_migrations() + pub async fn run_migrations(&self) -> Result<()> { + self.core.run_migrations().await } /// Run a passive WAL checkpoint (`PRAGMA wal_checkpoint(PASSIVE)`). @@ -1352,15 +1147,15 @@ impl TursoDb { /// into the main database file and never blocks on active readers or /// writers, so it does not guarantee WAL truncation. Safe to call /// repeatedly. Routine maintenance only; not a durability boundary. - pub fn passive_checkpoint(&self) -> Result<()> { + pub async fn passive_checkpoint(&self) -> Result<()> { let operation_name = format!("checkpoint {} database WAL", M::db_name()); - run_with_retry_sync( + run_with_retry_elapsed( resolve_query_retry_policy::(), &operation_name, QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { + async |_| { + async { let mut rows = self .core .conn @@ -1380,9 +1175,11 @@ impl TursoDb { {} Ok(()) - }) + } + .await }, ) + .await } /// Check migration metadata for problems that would prevent safe hook @@ -1391,22 +1188,24 @@ impl TursoDb { /// Returns a list of problems: missing migration metadata table, /// incomplete applied migrations, or unexpected extra migrations. /// An empty list means the schema is ready. - pub fn migration_metadata_problems(&self) -> Result> { + pub async fn migration_metadata_problems(&self) -> Result> { let migration_table_exists = self.query_map( "SELECT name FROM sqlite_master WHERE type = 'table' AND name = '__sce_migrations' LIMIT 1", (), |row| row.get::(0).map_err(Into::into), - )?; + ).await?; if migration_table_exists.is_empty() { return Ok(vec![String::from("missing migration metadata table")]); } - let applied_ids = self.query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - )?; + let applied_ids = self + .query_map( + "SELECT id FROM __sce_migrations ORDER BY id ASC", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await?; let expected_ids = M::migrations() .iter() .map(|(id, _)| *id) @@ -1451,8 +1250,8 @@ impl TursoDb { /// This check is intentionally non-mutating. Missing or incomplete schema /// is reported with the provided setup guidance instead of running /// migrations from a high-frequency hook path. - pub fn ensure_schema_ready(&self, setup_guidance: &str) -> Result<()> { - let problems = self.migration_metadata_problems()?; + pub async fn ensure_schema_ready(&self, setup_guidance: &str) -> Result<()> { + let problems = self.migration_metadata_problems().await?; if problems.is_empty() { return Ok(()); @@ -1472,23 +1271,27 @@ impl EncryptedTursoDb { /// /// This constructor is the encrypted counterpart to [`TursoDb::new`] and /// uses a strict encrypted local-builder path. - pub fn new() -> Result { + pub async fn new() -> Result { let db_name = M::db_name(); let db_path = M::db_path().with_context(|| format!("failed to resolve {db_name} path"))?; - let encryption_key = encryption_key::get_or_create_encryption_key(&db_path, db_name)?; + let key_path = db_path.clone(); + let key_name = db_name.to_owned(); + let encryption_key = tokio::task::spawn_blocking(move || { + encryption_key::get_or_create_encryption_key(&key_path, &key_name) + }) + .await + .context("encryption-key resolution worker failed")??; ensure_db_parent_dir(db_name, &db_path)?; - let runtime = build_current_thread_runtime(db_name)?; let retry_policy = resolve_connection_open_retry_policy::(); let operation_name = format!("open encrypted {db_name} database connection"); - let conn = run_with_retry_sync( + let conn = run_with_retry_elapsed( retry_policy, &operation_name, CONNECTION_OPEN_RETRY_HINT, - |_| { - block_on_isolated(&runtime, async { + |_| async { let path_str = db_path.to_str().ok_or_else(|| { anyhow::anyhow!("invalid UTF-8 in database path: {}", db_path.display()) })?; @@ -1513,15 +1316,15 @@ impl EncryptedTursoDb { db.connect().map_err(|e| { anyhow::anyhow!("failed to connect to encrypted {db_name} database: {e}") }) - }) }, - )?; + ).await?; let db = Self { - core: TursoConnectionCore::new(conn, runtime), + core: TursoConnectionCore::new(conn), }; db.run_migrations() + .await .with_context(|| format!("failed to run {db_name} migrations"))?; Ok(db) @@ -1535,61 +1338,32 @@ impl EncryptedTursoDb { /// /// # Returns /// Number of rows affected. - pub fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { + pub async fn execute(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { let params = turso::params::IntoParams::into_params(params).map_err(|e| { anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) })?; let operation_name = format!("execute encrypted {} database query", M::db_name()); - run_with_retry_sync( + run_with_retry_elapsed( resolve_query_retry_policy::(), &operation_name, QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { + async |_| { + async { self.core .conn .execute(sql, params.clone()) .await .map_err(|e| anyhow::anyhow!("{} execute failed: {sql}: {e}", M::db_name())) - }) - }, - ) - } - - /// Execute a SQL query that returns rows. - /// - /// # Arguments - /// * `sql` - SQL query, which may contain `?` placeholders. - /// * `params` - Parameter values implementing `IntoParams`. - /// - /// # Returns - /// A `turso::Rows` iterator over the result set. - #[allow(dead_code)] - pub fn query(&self, sql: &str, params: impl turso::params::IntoParams) -> Result { - let params = turso::params::IntoParams::into_params(params).map_err(|e| { - anyhow::anyhow!("{} parameter conversion failed: {sql}: {e}", M::db_name()) - })?; - let operation_name = format!("query encrypted {} database", M::db_name()); - - run_with_retry_sync( - resolve_query_retry_policy::(), - &operation_name, - QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { - self.core - .conn - .query(sql, params.clone()) - .await - .map_err(|e| anyhow::anyhow!("{} query failed: {sql}: {e}", M::db_name())) - }) + } + .await }, ) + .await } /// Execute a SQL query and synchronously map all returned rows. - pub fn query_map( + pub async fn query_map( &self, sql: &str, params: impl turso::params::IntoParams, @@ -1603,12 +1377,12 @@ impl EncryptedTursoDb { })?; let operation_name = format!("query and fetch encrypted {} database rows", M::db_name()); - let rows = run_with_retry_sync( + let rows = run_with_retry_elapsed( resolve_query_retry_policy::(), &operation_name, QUERY_RETRY_HINT, - |_| { - block_on_isolated(&self.core.runtime, async { + async |_| { + async { let mut rows = self.core .conn @@ -1626,9 +1400,11 @@ impl EncryptedTursoDb { } Ok(fetched_rows) - }) + } + .await }, - )?; + ) + .await?; let mut results = Vec::new(); @@ -1648,1329 +1424,89 @@ impl EncryptedTursoDb { /// initializations apply only migrations that were not already recorded. /// Existing databases without migration metadata are brought forward by /// re-applying the current idempotent migration set and recording each ID. - pub fn run_migrations(&self) -> Result<()> { - self.core.run_migrations() + pub async fn run_migrations(&self) -> Result<()> { + self.core.run_migrations().await } } #[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::thread; - use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; - - use rand::SeedableRng; - - use super::*; - - const QUERY_RETRY_FAILURE_BUDGET_MS: u64 = 2_000; - - struct TestDbSpec; - - impl DbSpec for TestDbSpec { - fn db_name() -> &'static str { - "test" - } - - fn db_path() -> Result { - unreachable!("tests always open via TursoDb::new_at with an explicit path") - } +mod transaction_cancellation_tests; - fn migrations() -> &'static [(&'static str, &'static str)] { - &[] - } - - fn db_config_key() -> &'static str { - "test_db" - } - } - - struct AgentTraceTestDbSpec; - - impl DbSpec for AgentTraceTestDbSpec { - fn db_name() -> &'static str { - "agent trace test" - } - - fn db_path() -> Result { - unreachable!("tests always open via TursoDb::new_at with an explicit path") - } - - fn migrations() -> &'static [(&'static str, &'static str)] { - &[] - } - - fn db_config_key() -> &'static str { - AGENT_TRACE_DB_CONFIG_KEY - } - } - - const BUSY_TIMEOUT_LOCK_HOLD_MS: u64 = 300; - - fn begin_immediate_while_write_lock_is_held( - db_path: &Path, - ) -> (std::result::Result, Duration) { - let hold = Duration::from_millis(BUSY_TIMEOUT_LOCK_HOLD_MS); - drop(TursoDb::::new_at(db_path).expect("test DB should be created up front")); - let contender = - TursoDb::::open_without_migrations_at(db_path).expect("contender DB should open"); - let lock_acquired = std::sync::Arc::new(std::sync::Barrier::new(2)); - let holder = { - let db_path = db_path.to_path_buf(); - let lock_acquired = std::sync::Arc::clone(&lock_acquired); - thread::spawn(move || { - let holder = TursoDb::::open_without_migrations_at(&db_path) - .expect("holder DB should open"); - holder - .execute("BEGIN IMMEDIATE", ()) - .expect("holder should acquire the write lock"); - lock_acquired.wait(); - thread::sleep(hold); - holder - .execute("COMMIT", ()) - .expect("holder should release the write lock"); - }) - }; - - lock_acquired.wait(); - let started_at = Instant::now(); - let outcome = block_on_isolated(&contender.core.runtime, async { - contender.core.conn.execute("BEGIN IMMEDIATE", ()).await - }); - let elapsed = started_at.elapsed(); - if outcome.is_ok() { - contender - .execute("ROLLBACK", ()) - .expect("contender should release the write lock"); - } - holder.join().expect("holder thread should not panic"); - drop(contender); - - (outcome, elapsed) - } - - #[test] - fn busy_timeout_resolves_default_for_agent_trace_db_and_zero_for_other_dbs() { - assert_eq!( - resolve_busy_timeout::(), - Duration::from_millis(AGENT_TRACE_DB_BUSY_TIMEOUT_MS) - ); - assert_eq!(resolve_busy_timeout::(), Duration::ZERO); - } +#[cfg(test)] +mod database_kind_tests { + use std::time::Duration; - fn agent_trace_retry_config( - busy_timeout_ms: Option, - contention_deadline_ms: Option, - ) -> DatabaseRetryConfig { - DatabaseRetryConfig { - local_db: None, - agent_trace_db: Some(AgentTraceDbRetryConfig { - retry: crate::services::config::PerDbRetryConfig { - connection_open: None, - query: None, - }, - busy_timeout_ms, - contention_deadline_ms, - }), - auth_db: None, - } - } + use super::{busy_timeout_from_config, write_contention_policy, DatabaseKind, DbSpec}; + use crate::services::{ + agent_trace_db::repository::RepositoryAgentTraceDbSpec, auth_db::AuthDbSpec, + local_db::LocalDbSpec, + }; #[test] - fn database_retry_configured_busy_timeout_overrides_agent_trace_default() { - let config = agent_trace_retry_config(Some(1_500), None); - assert_eq!( - busy_timeout_from_config::(Some(&config)), - Duration::from_millis(1_500) - ); - assert_eq!( - busy_timeout_from_config::(Some(&config)), - Duration::ZERO - ); - - let disabled = agent_trace_retry_config(Some(0), None); - assert_eq!( - busy_timeout_from_config::(Some(&disabled)), - Duration::ZERO - ); - - let unset = agent_trace_retry_config(None, None); - assert_eq!( - busy_timeout_from_config::(Some(&unset)), - Duration::from_millis(AGENT_TRACE_DB_BUSY_TIMEOUT_MS) - ); + fn database_kinds_match_their_specs() { + assert_eq!(LocalDbSpec::KIND, DatabaseKind::Local); + assert_eq!(RepositoryAgentTraceDbSpec::KIND, DatabaseKind::AgentTrace); + assert_eq!(AuthDbSpec::KIND, DatabaseKind::Auth); } #[test] - fn database_retry_contention_deadline_resolves_default_and_configured_value() { - assert_eq!( - resolve_contention_deadline::(), - Duration::from_millis(AGENT_TRACE_DB_CONTENTION_DEADLINE_MS) - ); + fn contention_policy_is_agent_trace_only() { + assert!(write_contention_policy::().is_some()); + assert!(write_contention_policy::().is_none()); + assert!(write_contention_policy::().is_none()); assert_eq!( - AGENT_TRACE_DB_CONTENTION_DEADLINE_MS, 2_250, - "contention deadline default" - ); - assert_eq!(resolve_contention_deadline::(), Duration::ZERO); - - let config = agent_trace_retry_config(None, Some(3_500)); - assert_eq!( - contention_deadline_from_config::(Some(&config)), - Duration::from_millis(3_500) - ); - assert_eq!( - contention_deadline_from_config::(Some(&config)), + busy_timeout_from_config::(None), Duration::ZERO ); - - let zero = agent_trace_retry_config(None, Some(0)); + assert_eq!(busy_timeout_from_config::(None), Duration::ZERO); assert_eq!( - contention_deadline_from_config::(Some(&zero)), - Duration::ZERO - ); - } - - #[test] - fn busy_timeout_unset_connection_returns_busy_promptly_on_begin_immediate() { - let db_path = unique_test_db_path(); - - let (outcome, elapsed) = begin_immediate_while_write_lock_is_held::(&db_path); - - assert!( - matches!(outcome, Err(turso::Error::Busy(_))), - "a connection without a busy handler should get Busy, got {outcome:?}" - ); - assert!( - elapsed < Duration::from_millis(BUSY_TIMEOUT_LOCK_HOLD_MS / 2), - "Busy should be returned without waiting for the holder, took {elapsed:?}" - ); - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); - } - } - - #[test] - fn busy_timeout_agent_trace_connection_waits_for_begin_immediate_holder() { - let db_path = unique_test_db_path(); - - let (outcome, elapsed) = - begin_immediate_while_write_lock_is_held::(&db_path); - - assert!( - outcome.is_ok(), - "Turso's busy handler should wait out the holder, got {outcome:?}" - ); - assert!( - elapsed >= Duration::from_millis(BUSY_TIMEOUT_LOCK_HOLD_MS / 2), - "BEGIN IMMEDIATE should have waited for the holder, took {elapsed:?}" - ); - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); - } - } - - fn unique_test_db_path() -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!("sce-db-mod-test-{}-{nonce}", std::process::id())) - .join("test.db") - } - - fn open_test_db() -> (TursoDb, PathBuf) { - let db_path = unique_test_db_path(); - let db = TursoDb::::new_at(&db_path).expect("test DB should open"); - db.execute( - "CREATE TABLE IF NOT EXISTS checkpoint_probe (value TEXT NOT NULL)", - (), - ) - .expect("test table creation should succeed"); - - (db, db_path) - } - - fn cleanup_test_db(db: TursoDb, db_path: &Path) { - drop(db); - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); - } - } - - fn open_cas_test_db() -> (TursoDb, PathBuf) { - let db_path = unique_test_db_path(); - let db = TursoDb::::new_at(&db_path).expect("test DB should open"); - db.execute( - "CREATE TABLE IF NOT EXISTS cas_target (id INTEGER PRIMARY KEY, revision INTEGER NOT NULL)", - (), - ) - .expect("cas_target table creation should succeed"); - db.execute( - "CREATE TABLE IF NOT EXISTS cas_effect (name TEXT PRIMARY KEY)", - (), - ) - .expect("cas_effect table creation should succeed"); - db.execute("INSERT INTO cas_target (id, revision) VALUES (1, 0)", ()) - .expect("cas_target seed row should insert"); - - (db, db_path) - } - - fn cas_target_revision(db: &TursoDb, id: i64) -> i64 { - db.query_map( - "SELECT revision FROM cas_target WHERE id = ?1", - (id,), - |row| row.get::(0).map_err(Into::into), - ) - .expect("cas_target revision read should succeed") - .into_iter() - .next() - .expect("cas_target seed row should exist") - } - - fn cas_effect_names(db: &TursoDb) -> Vec { - db.query_map("SELECT name FROM cas_effect ORDER BY name", (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("cas_effect read should succeed") - } - - #[test] - fn execute_transactional_cas_batch_returns_false_and_runs_nothing_when_guard_matches_no_rows() { - let (db, db_path) = open_cas_test_db(); - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 999", - (), - ) - .expect("guard statement should build"); - let statements = - [ - TransactionStatement::new("INSERT INTO cas_effect (name) VALUES ('applied')", ()) - .expect("effect statement should build"), - ]; - - let applied = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect("no-op CAS batch should succeed"); - - assert!(!applied); - assert_eq!(cas_target_revision(&db, 1), 0); - assert!(cas_effect_names(&db).is_empty()); - - cleanup_test_db(db, &db_path); - } - - #[test] - fn execute_transactional_cas_batch_returns_true_and_runs_every_statement_when_guard_matches_one_row( - ) { - let (db, db_path) = open_cas_test_db(); - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 0", - (), - ) - .expect("guard statement should build"); - let statements = - [ - TransactionStatement::new("INSERT INTO cas_effect (name) VALUES ('applied')", ()) - .expect("effect statement should build"), - ]; - - let applied = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect("applied CAS batch should succeed"); - - assert!(applied); - assert_eq!(cas_target_revision(&db, 1), 1); - assert_eq!(cas_effect_names(&db), vec![String::from("applied")]); - - cleanup_test_db(db, &db_path); - } - - #[test] - fn execute_transactional_cas_batch_rolls_back_and_fails_after_one_attempt_on_deterministic_failure( - ) { - let (db, db_path) = open_cas_test_db(); - db.execute("INSERT INTO cas_effect (name) VALUES ('applied')", ()) - .expect("pre-existing conflicting row should insert"); - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 0", - (), - ) - .expect("guard statement should build"); - let statements = - [ - TransactionStatement::new("INSERT INTO cas_effect (name) VALUES ('applied')", ()) - .expect("effect statement should build"), - ]; - - let started_at = Instant::now(); - let error = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect_err("duplicate insert should fail deterministically"); - let elapsed = started_at.elapsed(); - - assert!( - elapsed < Duration::from_millis(150), - "deterministic failure appears to have been retried instead of failing after one attempt: {elapsed:?}" + busy_timeout_from_config::(None), + Duration::from_secs(1) ); - assert!(error.to_string().contains("execute failed")); - assert_eq!(cas_target_revision(&db, 1), 0); - assert_eq!(cas_effect_names(&db), vec![String::from("applied")]); - - cleanup_test_db(db, &db_path); } +} - #[test] - fn execute_transactional_cas_batch_rejects_a_guard_matching_more_than_one_row_without_retrying() - { - let (db, db_path) = open_cas_test_db(); - db.execute("INSERT INTO cas_target (id, revision) VALUES (2, 0)", ()) - .expect("second cas_target row should insert"); - let guard = - TransactionStatement::new("UPDATE cas_target SET revision = 1 WHERE revision = 0", ()) - .expect("guard statement should build"); - let statements = - [ - TransactionStatement::new("INSERT INTO cas_effect (name) VALUES ('applied')", ()) - .expect("effect statement should build"), - ]; - - let started_at = Instant::now(); - let error = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect_err("a guard matching more than one row should fail deterministically"); - let elapsed = started_at.elapsed(); - - assert!( - elapsed < Duration::from_millis(150), - "guard over-match appears to have been retried instead of failing after one attempt: {elapsed:?}" - ); - assert!(error.to_string().contains("expected 0 or 1")); - assert_eq!(cas_target_revision(&db, 1), 0); - assert_eq!(cas_target_revision(&db, 2), 0); - assert!(cas_effect_names(&db).is_empty()); - - cleanup_test_db(db, &db_path); - } +#[cfg(test)] +mod write_contention_eligibility_tests { + use std::time::Duration; - #[test] - fn execute_transactional_cas_batch_applies_a_statement_whose_expected_rows_affected_matches() { - let (db, db_path) = open_cas_test_db(); - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 0", - (), - ) - .expect("guard statement should build"); - let statements = - [ - TransactionStatement::new("INSERT INTO cas_effect (name) VALUES ('applied')", ()) - .expect("effect statement should build") - .expect_rows_affected(1), - ]; - - let applied = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect("a statement matching its row expectation should succeed"); - - assert!(applied); - assert_eq!(cas_target_revision(&db, 1), 1); - assert_eq!(cas_effect_names(&db), vec![String::from("applied")]); - - cleanup_test_db(db, &db_path); - } + use super::{ + write_contention_retry_may_sleep, write_contention_retry_may_start_now, + WriteContentionPolicy, + }; #[test] - fn execute_transactional_cas_batch_rejects_a_statement_affecting_fewer_rows_than_expected() { - let (db, db_path) = open_cas_test_db(); - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 0", - (), - ) - .expect("guard statement should build"); - let statements = [TransactionStatement::new( - "UPDATE cas_effect SET name = 'applied' WHERE name = 'missing'", - (), - ) - .expect("effect statement should build") - .expect_rows_affected(1)]; - - let error = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect_err("a statement affecting zero rows should fail its row expectation"); - - assert!(error.to_string().contains("affected 0 rows")); - assert!(error.to_string().contains("expected 1")); - assert_eq!(cas_target_revision(&db, 1), 0); - - cleanup_test_db(db, &db_path); - } + fn retry_eligibility_depends_only_on_backoff_ending_before_deadline() { + let policy = WriteContentionPolicy { + db_name: "agent_trace_db", + max_attempts: 3, + backoff_cap: Duration::from_millis(100), + busy_timeout: Duration::from_secs(1), + contention_deadline: Duration::from_millis(2250), + }; - #[test] - fn execute_transactional_cas_batch_rejects_a_statement_affecting_more_rows_than_expected() { - let (db, db_path) = open_cas_test_db(); - db.execute("INSERT INTO cas_effect (name) VALUES ('a')", ()) - .expect("first pre-existing effect row should insert"); - db.execute("INSERT INTO cas_effect (name) VALUES ('b')", ()) - .expect("second pre-existing effect row should insert"); - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 0", - (), - ) - .expect("guard statement should build"); - let statements = - [ - TransactionStatement::new("DELETE FROM cas_effect WHERE name IN ('a', 'b')", ()) - .expect("effect statement should build") - .expect_rows_affected(1), - ]; - - let error = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect_err( - "a statement affecting more rows than expected should fail its row expectation", + for (elapsed_ms, backoff_ms, expected) in [ + (1185, 100, true), + (2200, 100, false), + (2250, 0, false), + (2249, 0, true), + ] { + let elapsed = Duration::from_millis(elapsed_ms); + let backoff = Duration::from_millis(backoff_ms); + assert_eq!( + write_contention_retry_may_sleep(policy, elapsed, backoff), + expected, + "elapsed={elapsed_ms}ms backoff={backoff_ms}ms" ); - - assert!(error.to_string().contains("affected 2 rows")); - assert!(error.to_string().contains("expected 1")); - assert_eq!(cas_target_revision(&db, 1), 0); - assert_eq!( - cas_effect_names(&db), - vec![String::from("a"), String::from("b")] - ); - - cleanup_test_db(db, &db_path); - } - - #[test] - fn execute_transactional_cas_batch_allows_a_statement_with_no_row_expectation_to_affect_zero_rows( - ) { - let (db, db_path) = open_cas_test_db(); - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 0", - (), - ) - .expect("guard statement should build"); - let statements = [TransactionStatement::new( - "UPDATE cas_effect SET name = 'applied' WHERE name = 'missing'", - (), - ) - .expect("effect statement should build")]; - - let applied = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect("a statement with no row expectation should not enforce a row count"); - - assert!(applied); - assert_eq!(cas_target_revision(&db, 1), 1); - - cleanup_test_db(db, &db_path); - } - - #[test] - fn is_retryable_turso_error_classifies_busy_and_busy_snapshot_as_retryable() { - assert!(is_retryable_turso_error(&turso::Error::Busy(String::from( - "database is locked" - )))); - assert!(is_retryable_turso_error(&turso::Error::BusySnapshot( - String::from("snapshot is busy") - ))); - } - - #[test] - fn is_retryable_turso_error_classifies_every_other_variant_as_deterministic() { - assert!(!is_retryable_turso_error(&turso::Error::Constraint( - String::from("UNIQUE constraint failed") - ))); - assert!(!is_retryable_turso_error(&turso::Error::Misuse( - String::from("misuse") - ))); - assert!(!is_retryable_turso_error(&turso::Error::Corrupt( - String::from("corrupt") - ))); - assert!(!is_retryable_turso_error(&turso::Error::NotAdb( - String::from("not a database") - ))); - assert!(!is_retryable_turso_error(&turso::Error::DatabaseFull( - String::from("database full") - ))); - assert!(!is_retryable_turso_error(&turso::Error::Readonly( - String::from("readonly") - ))); - assert!(!is_retryable_turso_error(&turso::Error::Error( - String::from("generic error") - ))); - assert!(!is_retryable_turso_error(&turso::Error::IoError( - std::io::ErrorKind::Other, - "io" - ))); - } - - #[test] - fn classify_turso_error_wraps_busy_as_retryable_with_the_supplied_action_context() { - let failure = classify_turso_error( - "test", - "failed to begin transaction", - &turso::Error::Busy(String::from("database is locked")), - ); - - match failure { - WriteAttemptFailure::Retryable(err) => { - let message = err.to_string(); - assert!(message.contains("failed to begin transaction")); - assert!(message.contains("database is locked")); - } - WriteAttemptFailure::Deterministic(err) => { - panic!("Busy should classify as retryable, got deterministic: {err}") - } } - } - - #[test] - fn classify_turso_error_wraps_constraint_violations_as_deterministic_with_the_supplied_action_context( - ) { - let failure = classify_turso_error( - "test", - "failed to commit transaction", - &turso::Error::Constraint(String::from("UNIQUE constraint failed")), - ); - - match failure { - WriteAttemptFailure::Deterministic(err) => { - let message = err.to_string(); - assert!(message.contains("failed to commit transaction")); - assert!(message.contains("UNIQUE constraint failed")); - } - WriteAttemptFailure::Retryable(err) => { - panic!("Constraint should classify as deterministic, got retryable: {err}") - } - } - } - - #[test] - fn execute_transactional_cas_batch_retries_a_begin_immediate_busy_error_and_then_succeeds() { - const LOCK_HOLD_MS: u64 = 60; - - let (db, db_path) = open_cas_test_db(); - let lock_holder = - TursoDb::::new_at(&db_path).expect("second handle should open"); - lock_holder - .execute("BEGIN IMMEDIATE", ()) - .expect("lock holder should acquire the write lock before any guard or statement runs"); - - let hold_handle = thread::spawn(move || { - thread::sleep(Duration::from_millis(LOCK_HOLD_MS)); - lock_holder - .execute("COMMIT", ()) - .expect("lock holder should release the write lock"); - }); - - let guard = TransactionStatement::new( - "UPDATE cas_target SET revision = 1 WHERE id = 1 AND revision = 0", - (), - ) - .expect("guard statement should build"); - let statements = - [ - TransactionStatement::new("INSERT INTO cas_effect (name) VALUES ('applied')", ()) - .expect("effect statement should build"), - ]; - - let started_at = Instant::now(); - let applied = db - .execute_transactional_cas_batch("cas test", "retry the operation", &guard, &statements) - .expect( - "CAS batch should retry BEGIN IMMEDIATE through the transient lock and succeed", - ); - let elapsed = started_at.elapsed(); - - hold_handle - .join() - .expect("lock holder thread should finish"); - - assert!( - elapsed >= Duration::from_millis(LOCK_HOLD_MS / 2), - "success arrived before the lock holder could plausibly have released the write lock, meaning BEGIN IMMEDIATE contention was not actually retried: {elapsed:?}" - ); - assert!(applied); - assert_eq!(cas_target_revision(&db, 1), 1); - assert_eq!(cas_effect_names(&db), vec![String::from("applied")]); - - cleanup_test_db(db, &db_path); - } - - #[test] - fn passive_checkpoint_keeps_previously_written_data_readable() { - let (db, db_path) = open_test_db(); - - db.execute( - "INSERT INTO checkpoint_probe (value) VALUES (?1)", - ("hello",), - ) - .expect("insert should succeed"); - - db.passive_checkpoint() - .expect("passive checkpoint should succeed"); - - let values = db - .query_map("SELECT value FROM checkpoint_probe", (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("post-checkpoint read should succeed"); - - assert_eq!(values, vec![String::from("hello")]); - - cleanup_test_db(db, &db_path); - } - - #[test] - fn passive_checkpoint_is_safe_to_call_repeatedly() { - let (db, db_path) = open_test_db(); - - db.passive_checkpoint() - .expect("first passive checkpoint should succeed"); - db.passive_checkpoint() - .expect("second passive checkpoint should succeed"); - - cleanup_test_db(db, &db_path); - } - - fn worst_case_retry_failure_budget_ms(policy: RetryPolicy) -> u64 { - let attempt_timeouts = policy - .timeout_ms - .saturating_mul(u64::from(policy.max_attempts)); - let retry_backoffs = (2..=policy.max_attempts) - .map(|attempt| retry_backoff_ms(policy, attempt)) - .fold(0_u64, u64::saturating_add); - - attempt_timeouts.saturating_add(retry_backoffs) - } - - fn retry_backoff_ms(policy: RetryPolicy, attempt: u32) -> u64 { - if attempt <= 1 { - return 0; - } - - let exponent = (attempt - 2).min(20); - let multiplier = 1_u64 << exponent; - - policy - .initial_backoff_ms - .saturating_mul(multiplier) - .min(policy.max_backoff_ms) - } - - #[test] - fn default_query_retry_policy_stays_within_two_second_failure_budget() { - let budget_ms = worst_case_retry_failure_budget_ms(QUERY_RETRY_POLICY); - - assert!( - budget_ms <= QUERY_RETRY_FAILURE_BUDGET_MS, - "default query retry failure budget was {budget_ms}ms; expected <= {QUERY_RETRY_FAILURE_BUDGET_MS}ms" - ); - } - - fn write_contention_test_policy( - busy_timeout_ms: u64, - contention_deadline_ms: u64, - ) -> WriteContentionPolicy { - WriteContentionPolicy { - db_name: "agent trace test", - max_attempts: AGENT_TRACE_DB_WRITE_CONTENTION_MAX_ATTEMPTS, - backoff_cap: Duration::from_millis(AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS), - busy_timeout: Duration::from_millis(busy_timeout_ms), - contention_deadline: Duration::from_millis(contention_deadline_ms), - } - } - - fn busy_failure() -> WriteAttemptFailure { - classify_turso_error( - "agent trace test", - "failed to begin transaction", - &turso::Error::Busy(String::from("database is locked")), - ) - } - - fn run_write_contention_test( - policy: WriteContentionPolicy, - attempt: impl FnMut(u32) -> std::result::Result, - ) -> (Result, Vec, WriteContentionCounts) { - let mut jitter = rand::rngs::StdRng::seed_from_u64(7); - let clock = std::cell::Cell::new(Duration::ZERO); - let mut sleeps = Vec::new(); - let (result, counts) = count_write_contention(|| { - run_with_write_contention_retry_using( - policy, - &mut || write_contention_backoff(&mut jitter, policy.backoff_cap), - &mut |backoff| { - sleeps.push(backoff); - clock.set(clock.get() + backoff); - }, - &mut || clock.get(), - "write contention test", - "retry the operation", - attempt, - ) - }); - (result, sleeps, counts) - } - - struct OversleepOutcome { - result: Result<&'static str>, - attempt_calls: u32, - counts: WriteContentionCounts, - } - - fn run_oversleep_scenario( - busy_at: Duration, - backoff: Duration, - oversleep: Duration, - ) -> OversleepOutcome { - let policy = write_contention_test_policy(500, 1_250); - let clock = std::cell::Cell::new(Duration::ZERO); - let mut attempt_calls = 0; - let (result, counts) = count_write_contention(|| { - run_with_write_contention_retry_using( - policy, - &mut || backoff, - &mut |requested| clock.set(clock.get() + requested + oversleep), - &mut || clock.get(), - "write contention test", - "retry the operation", - |_| { - attempt_calls += 1; - if attempt_calls == 1 { - clock.set(busy_at); - Err(busy_failure()) - } else { - Ok("written") - } - }, - ) - }); - OversleepOutcome { - result, - attempt_calls, - counts, - } - } - - #[test] - fn agent_trace_db_write_contention_retry_seeded_jitter_is_reproducible_and_bounded() { - let cap = Duration::from_millis(AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS); - let draw = |seed| { - let mut jitter = rand::rngs::StdRng::seed_from_u64(seed); - (0..64) - .map(|_| write_contention_backoff(&mut jitter, cap)) - .collect::>() - }; - - let first = draw(42); - assert_eq!( - first, - draw(42), - "a seeded jitter source must be reproducible" - ); - assert!(first.iter().all(|backoff| *backoff <= cap)); - assert!( - first.iter().any(|backoff| *backoff != first[0]), - "full jitter should vary across draws" - ); - assert_eq!( - write_contention_backoff(&mut rand::rngs::StdRng::seed_from_u64(1), Duration::ZERO), - Duration::ZERO - ); - } - - #[test] - fn agent_trace_db_write_contention_retry_start_rule_boundaries() { - let policy = write_contention_test_policy(500, 1_250); - let backoff = Duration::from_millis(50); - - assert!(write_contention_retry_may_sleep( - policy, - Duration::from_millis(699), - backoff - )); - assert!(write_contention_retry_may_sleep( - policy, - Duration::from_millis(700), - backoff - )); - assert!(!write_contention_retry_may_sleep( - policy, - Duration::from_millis(701), - backoff - )); - assert!(!write_contention_retry_may_sleep( - policy, - Duration::from_millis(1_250), - backoff - )); - assert!(!write_contention_retry_may_sleep( - policy, - Duration::from_secs(5), - backoff - )); - - let no_wait = write_contention_test_policy(0, 1_250); - assert!(write_contention_retry_may_sleep( - no_wait, - Duration::from_millis(1_249), - Duration::ZERO - )); - assert!( - !write_contention_retry_may_sleep( - no_wait, - Duration::from_millis(1_250), - Duration::ZERO - ), - "nothing starts once the deadline has expired, even with zero backoff and busy timeout" - ); - assert!( - !write_contention_retry_may_sleep( - write_contention_test_policy(0, 0), - Duration::ZERO, - Duration::ZERO - ), - "a zero contention deadline never starts an outer retry" - ); - } - - #[test] - fn agent_trace_db_write_contention_retry_post_sleep_admission_requires_a_full_busy_timeout() { - let policy = write_contention_test_policy(500, 1_250); assert!(write_contention_retry_may_start_now( policy, - Duration::from_millis(749) - )); - assert!( - write_contention_retry_may_start_now(policy, Duration::from_millis(750)), - "remaining == busy_timeout must still admit the next attempt" - ); - assert!(!write_contention_retry_may_start_now( - policy, - Duration::from_millis(751) + Duration::from_millis(2249) )); assert!(!write_contention_retry_may_start_now( policy, - Duration::from_millis(1_250) + Duration::from_millis(2250) )); - assert!(!write_contention_retry_may_start_now( - write_contention_test_policy(0, 1_250), - Duration::from_millis(1_250) - )); - } - - #[test] - fn agent_trace_db_write_contention_retry_rejects_a_retry_after_the_backoff_sleep_oversleeps() { - let outcome = run_oversleep_scenario( - Duration::from_millis(690), - Duration::from_millis(50), - Duration::from_millis(70), - ); - - let message = outcome - .result - .expect_err("an oversleep past the admission budget must exhaust") - .to_string(); - assert!( - message.contains("failed after 1 attempt(s) under write contention"), - "{message}" - ); - assert!(message.contains("elapsed_ms=810"), "{message}"); - assert!(message.contains("database is locked"), "{message}"); - assert_eq!(outcome.attempt_calls, 1, "attempt 2 must not run"); - assert_eq!( - outcome.counts, - WriteContentionCounts { - attempts: 1, - outer_retries: 0, - exhaustions: 1 - } - ); - } - - #[derive(Clone, Debug, Default)] - struct CapturedEvent { - target: String, - level: String, - fields: BTreeMap, - } - - struct CapturedEventVisitor<'a>(&'a mut BTreeMap); - - impl tracing::field::Visit for CapturedEventVisitor<'_> { - fn record_str(&mut self, field: &tracing::field::Field, value: &str) { - self.0.insert(field.name().to_string(), value.to_string()); - } - - fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { - self.0.insert(field.name().to_string(), value.to_string()); - } - - fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { - self.0 - .insert(field.name().to_string(), format!("{value:?}")); - } - } - - #[derive(Clone, Default)] - struct CapturingSubscriber { - events: std::sync::Arc>>, - } - - impl tracing::Subscriber for CapturingSubscriber { - fn enabled(&self, _metadata: &tracing::Metadata<'_>) -> bool { - true - } - - fn new_span(&self, _span: &tracing::span::Attributes<'_>) -> tracing::span::Id { - tracing::span::Id::from_u64(1) - } - - fn record(&self, _span: &tracing::span::Id, _values: &tracing::span::Record<'_>) {} - - fn record_follows_from(&self, _span: &tracing::span::Id, _follows: &tracing::span::Id) {} - - fn event(&self, event: &tracing::Event<'_>) { - let mut fields = BTreeMap::new(); - event.record(&mut CapturedEventVisitor(&mut fields)); - self.events - .lock() - .expect("captured events mutex should not be poisoned") - .push(CapturedEvent { - target: event.metadata().target().to_string(), - level: event.metadata().level().to_string(), - fields, - }); - } - - fn enter(&self, _span: &tracing::span::Id) {} - - fn exit(&self, _span: &tracing::span::Id) {} - } - - fn capture_tracing_events(body: impl FnOnce() -> T) -> (T, Vec) { - let subscriber = CapturingSubscriber::default(); - let events = std::sync::Arc::clone(&subscriber.events); - let result = tracing::subscriber::with_default(subscriber, body); - let events = events - .lock() - .expect("captured events mutex should not be poisoned") - .clone(); - (result, events) - } - - fn contention_exhausted_events(events: &[CapturedEvent]) -> Vec<&CapturedEvent> { - events - .iter() - .filter(|event| { - event.fields.get("event_id").map(String::as_str) - == Some(CONTENTION_EXHAUSTED_EVENT_ID) - }) - .collect() - } - - #[test] - fn agent_trace_db_contention_exhausted_error_and_event_carry_every_field() { - let (outcome, events) = capture_tracing_events(|| { - run_oversleep_scenario( - Duration::from_millis(690), - Duration::from_millis(50), - Duration::from_millis(70), - ) - }); - - let message = outcome - .result - .expect_err("an oversleep past the admission budget must exhaust") - .to_string(); - assert_eq!( - message, - "Operation 'write contention test' failed after 1 attempt(s) under write contention \ - (db_name=agent trace test, operation=write contention test, attempts=1, \ - busy_timeout_ms=500, contention_deadline_ms=1250 [no retry is scheduled past this cutoff], \ - elapsed_ms=810, cause=database busy (busy timeout exhausted)). \ - Last error: agent trace test failed to begin transaction: database is locked. \ - Try: retry the operation" - ); - assert_eq!(outcome.counts.exhaustions, 1); - - let exhausted = contention_exhausted_events(&events); - assert_eq!( - exhausted.len(), - 1, - "exactly one exhaustion event: {events:?}" - ); - let event = exhausted[0]; - assert_eq!(event.target, "sce"); - assert_eq!(event.level, "WARN"); - let expected = [ - ("db_name", "agent trace test"), - ("operation", "write contention test"), - ("attempts", "1"), - ("busy_timeout_ms", "500"), - ("contention_deadline_ms", "1250"), - ("elapsed_ms", "810"), - ("cause", "database busy (busy timeout exhausted)"), - ( - "last_error", - "agent trace test failed to begin transaction: database is locked", - ), - ]; - for (key, value) in expected { - assert_eq!( - event.fields.get(key).map(String::as_str), - Some(value), - "field {key}: {event:?}" - ); - } - } - - #[test] - fn agent_trace_db_contention_exhausted_event_is_emitted_once_after_the_attempt_cap() { - let (outcome, events) = capture_tracing_events(|| { - run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { - Err::<(), _>(busy_failure()) - }) - }); - let (result, _sleeps, counts) = outcome; - - let message = result - .expect_err("persistent Busy must exhaust") - .to_string(); - assert!(message.contains("attempts=2"), "{message}"); - assert_eq!(counts.exhaustions, 1); - - let exhausted = contention_exhausted_events(&events); - assert_eq!(exhausted.len(), 1, "{events:?}"); - assert_eq!( - exhausted[0].fields.get("attempts").map(String::as_str), - Some("2") - ); - } - - #[test] - fn agent_trace_db_contention_exhausted_event_is_not_emitted_for_success_or_deterministic_errors( - ) { - let ((), events) = capture_tracing_events(|| { - let mut calls = 0; - let _ = run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { - calls += 1; - if calls == 1 { - Err(busy_failure()) - } else { - Ok(()) - } - }); - let _ = run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { - Err::<(), _>(WriteAttemptFailure::Deterministic(anyhow::anyhow!( - "constraint failed" - ))) - }); - }); - - assert!( - contention_exhausted_events(&events).is_empty(), - "{events:?}" - ); - } - - #[test] - fn agent_trace_db_write_contention_retry_admits_a_retry_when_post_sleep_remaining_equals_busy_timeout( - ) { - let outcome = run_oversleep_scenario( - Duration::from_millis(690), - Duration::from_millis(50), - Duration::from_millis(10), - ); - - assert_eq!( - outcome - .result - .expect("remaining == busy_timeout must admit attempt 2"), - "written" - ); - assert_eq!(outcome.attempt_calls, 2); - assert_eq!( - outcome.counts, - WriteContentionCounts { - attempts: 2, - outer_retries: 1, - exhaustions: 0 - } - ); - } - - #[test] - fn agent_trace_db_write_contention_retry_retries_busy_and_busy_snapshot_once() { - for failure in [ - turso::Error::Busy(String::from("database is locked")), - turso::Error::BusySnapshot(String::from("snapshot is stale")), - ] { - let mut failure = Some(failure); - let (result, sleeps, counts) = run_write_contention_test( - write_contention_test_policy(0, 30_000), - |_| match failure.take() { - Some(error) => Err(classify_turso_error("agent trace test", "write", &error)), - None => Ok("written"), - }, - ); - - assert_eq!(result.expect("the retry should succeed"), "written"); - assert_eq!(sleeps.len(), 1); - assert!( - sleeps[0] <= Duration::from_millis(AGENT_TRACE_DB_WRITE_CONTENTION_BACKOFF_CAP_MS) - ); - assert_eq!( - counts, - WriteContentionCounts { - attempts: 2, - outer_retries: 1, - exhaustions: 0 - } - ); - } - } - - #[test] - fn agent_trace_db_write_contention_retry_fails_deterministic_errors_once_without_sleeping() { - for error in [ - turso::Error::Constraint(String::from("UNIQUE constraint failed")), - turso::Error::Misuse(String::from("misuse")), - turso::Error::Readonly(String::from("readonly")), - ] { - let mut error = Some(error); - let (result, sleeps, counts) = - run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { - Err::<(), _>(classify_turso_error( - "agent trace test", - "write", - &error - .take() - .expect("a deterministic error must not be retried"), - )) - }); - - assert!(result.is_err()); - assert!(sleeps.is_empty(), "a deterministic error must not sleep"); - assert_eq!( - counts, - WriteContentionCounts { - attempts: 1, - outer_retries: 0, - exhaustions: 0 - } - ); - } - } - - #[test] - fn agent_trace_db_write_contention_retry_caps_outer_attempts_at_two() { - let (result, sleeps, counts) = - run_write_contention_test(write_contention_test_policy(0, 30_000), |_| { - Err::<(), _>(busy_failure()) - }); - - let message = result - .expect_err("persistent Busy must exhaust") - .to_string(); - assert!(message.contains("failed after 2 attempt(s)"), "{message}"); - assert!(message.contains("database is locked"), "{message}"); - assert!(message.contains("Try: retry the operation"), "{message}"); - assert_eq!(sleeps.len(), 1); - assert_eq!( - counts, - WriteContentionCounts { - attempts: 2, - outer_retries: 1, - exhaustions: 1 - } - ); - } - - #[test] - fn agent_trace_db_write_contention_retry_starts_no_retry_the_rule_disallows() { - for policy in [ - write_contention_test_policy(0, 0), - write_contention_test_policy(1_251, 1_250), - ] { - let (result, sleeps, counts) = - run_write_contention_test(policy, |_| Err::<(), _>(busy_failure())); - - let message = result.expect_err("Busy must fail").to_string(); - assert!(message.contains("failed after 1 attempt(s)"), "{message}"); - assert!(sleeps.is_empty()); - assert_eq!( - counts, - WriteContentionCounts { - attempts: 1, - outer_retries: 0, - exhaustions: 1 - } - ); - } - } - - #[test] - fn agent_trace_db_write_contention_retry_policy_applies_only_to_agent_trace_db() { - assert_eq!( - write_contention_policy::(), - Some(write_contention_test_policy( - AGENT_TRACE_DB_BUSY_TIMEOUT_MS, - AGENT_TRACE_DB_CONTENTION_DEADLINE_MS - )) - ); - assert_eq!(write_contention_policy::(), None); - } - - #[test] - fn agent_trace_db_write_contention_retry_leaves_reads_and_other_writes_on_the_generic_policy() { - assert_eq!( - resolve_query_retry_policy::(), - QUERY_RETRY_POLICY - ); - - let db_path = unique_test_db_path(); - let db = TursoDb::::new_at(&db_path).expect("test DB should open"); - db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)", ()) - .expect("table should be created"); - - let ((), generic) = count_write_contention(|| { - db.execute("INSERT INTO t (id, v) VALUES (1, 'a')", ()) - .expect("generic execute should succeed"); - db.query("SELECT v FROM t", ()) - .expect("query should succeed"); - db.query_values("SELECT v FROM t", ()) - .expect("query_values should succeed"); - db.query_map("SELECT v FROM t", (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("query_map should succeed"); - db.passive_checkpoint() - .expect("passive checkpoint should succeed"); - }); - assert_eq!( - generic, - WriteContentionCounts::default(), - "reads, passive_checkpoint, and generic execute must not use the write-contention policy" - ); - - let (affected, opted_in) = count_write_contention(|| { - db.execute_idempotent_write( - "INSERT INTO t (id, v) VALUES (1, 'b') ON CONFLICT (id) DO NOTHING", - (), - ) - .expect("idempotent write should succeed") - }); - assert_eq!(affected, 0); - assert_eq!(opted_in.attempts, 1); - - drop(db); - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); - } } } diff --git a/cli/src/services/db/transaction_cancellation_tests.rs b/cli/src/services/db/transaction_cancellation_tests.rs new file mode 100644 index 000000000..6846390b4 --- /dev/null +++ b/cli/src/services/db/transaction_cancellation_tests.rs @@ -0,0 +1,45 @@ +use crate::services::agent_trace_db::transaction_tests::{message, open, pair_counts, part}; +use crate::services::agent_trace_db::INSERT_MESSAGE_SQL; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn abandoned_transaction_commits_nothing_and_rolls_back_on_next_use_of_its_connection() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("agent-trace.db"); + let mut writer = open(&path).await; + let mut observer = open(&path).await; + + { + let tx = turso::transaction::Transaction::new( + &mut writer.core.conn, + turso::transaction::TransactionBehavior::Immediate, + ) + .await + .unwrap(); + tx.execute(INSERT_MESSAGE_SQL, ("session", "abandoned", "user", 1_i64)) + .await + .unwrap(); + } + + assert_eq!(pair_counts(&observer, "abandoned").await, (0, 0)); + + let blocked = observer + .insert_conversation_text_event(message("blocked"), part("blocked")) + .await + .unwrap_err() + .to_string(); + assert!(blocked.contains("under write contention"), "{blocked}"); + + assert!(writer + .insert_conversation_text_event(message("later"), part("later")) + .await + .unwrap()); + + assert!(observer + .insert_conversation_text_event(message("unblocked"), part("unblocked")) + .await + .unwrap()); + + assert_eq!(pair_counts(&observer, "abandoned").await, (0, 0)); + assert_eq!(pair_counts(&observer, "later").await, (1, 1)); + assert_eq!(pair_counts(&writer, "unblocked").await, (1, 1)); +} diff --git a/cli/src/services/default_paths.rs b/cli/src/services/default_paths.rs index 7155b947c..86ec44208 100644 --- a/cli/src/services/default_paths.rs +++ b/cli/src/services/default_paths.rs @@ -7,8 +7,11 @@ mod roots { use anyhow::{anyhow, Result}; - #[allow(dead_code)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] + #[allow( + dead_code, + reason = "constructed only under cfg(target_os) on non-Linux platforms" + )] pub(crate) enum PlatformFamily { Linux, Macos, @@ -56,11 +59,6 @@ mod roots { pub(crate) fn state_root(&self) -> &Path { &self.state_root } - - #[allow(dead_code)] - pub(crate) fn cache_root(&self) -> &Path { - &self.cache_root - } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -76,15 +74,6 @@ mod roots { pub(crate) fn global_config_file(&self) -> PathBuf { self.roots.config_root().join("sce").join("config.json") } - - #[allow(dead_code)] - pub(crate) fn persisted_artifact_locations(&self) -> Vec { - vec![super::PersistedArtifactLocation { - id: super::PersistedArtifactId::GlobalConfig, - root_kind: super::PersistedArtifactRootKind::Config, - path: self.global_config_file(), - }] - } } pub(crate) fn resolve_state_data_root() -> Result { @@ -322,62 +311,26 @@ fn sce_state_dir() -> anyhow::Result { .join("sce")) } -#[allow(dead_code)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum PersistedArtifactRootKind { - Config, - State, - Cache, -} - -#[allow(dead_code)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum PersistedArtifactId { - GlobalConfig, -} - -#[allow(dead_code)] -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct PersistedArtifactLocation { - pub id: PersistedArtifactId, - pub root_kind: PersistedArtifactRootKind, - pub path: PathBuf, -} - pub(crate) mod repo_dir { pub const SCE: &str = ".sce"; pub const OPENCODE: &str = ".opencode"; pub const CLAUDE: &str = ".claude"; pub const PI: &str = ".pi"; pub const CODEX: &str = ".codex"; - pub const GIT: &str = ".git"; } pub(crate) mod repo_file { pub const SCE_CONFIG: &str = "config.json"; pub const OPENCODE_MANIFEST: &str = "opencode.json"; - pub const GIT_COMMIT_EDITMSG: &str = "COMMIT_EDITMSG"; -} - -pub(crate) mod hook_dir { - pub const HOOKS: &str = "hooks"; - pub const PRE_COMMIT: &str = "pre-commit"; - pub const COMMIT_MSG: &str = "commit-msg"; - pub const POST_COMMIT: &str = "post-commit"; } -#[allow(dead_code)] pub(crate) mod opencode_asset { - pub const OPENCODE_DIR: &str = "opencode"; pub const PLUGINS_DIR: &str = "plugins"; - pub const PLUGIN_FILE: &str = "sce-bash-policy.ts"; - pub const PLUGIN_MANIFEST_ENTRY: &str = "./plugins/sce-bash-policy.ts"; pub const LIB_DIR: &str = "lib"; pub const PRESET_CATALOG: &str = "bash-policy-presets.json"; pub const OPENCODE_AGENT_DIR: &str = "agent"; pub const OPENCODE_COMMAND_DIR: &str = "command"; pub const SKILLS_DIR: &str = "skills"; - pub const AGENTS_DIR: &str = "agents"; } pub(crate) mod claude_asset { @@ -415,7 +368,6 @@ pub(crate) mod context_file { pub const PATTERNS: &str = "patterns.md"; pub const CONTEXT_MAP: &str = "context-map.md"; pub const TMP_GITIGNORE: &str = ".gitignore"; - pub const SKILL_DEFINITION: &str = "SKILL.md"; } pub(crate) mod schema { @@ -428,7 +380,6 @@ pub(crate) struct RepoPaths { root: PathBuf, } -#[allow(dead_code)] impl RepoPaths { pub(crate) fn new(root: impl Into) -> Self { Self { root: root.into() } @@ -462,22 +413,6 @@ impl RepoPaths { self.root.join(repo_dir::CODEX) } - pub(crate) fn git_dir(&self) -> PathBuf { - self.root.join(repo_dir::GIT) - } - - pub(crate) fn git_hooks_dir(&self) -> PathBuf { - self.git_dir().join(hook_dir::HOOKS) - } - - pub(crate) fn git_hook_file(&self, hook_name: &str) -> PathBuf { - self.git_hooks_dir().join(hook_name) - } - - pub(crate) fn git_commit_editmsg(&self) -> PathBuf { - self.git_dir().join(repo_file::GIT_COMMIT_EDITMSG) - } - pub(crate) fn context_dir(&self) -> PathBuf { self.root.join(context_dir::CONTEXT_ROOT) } @@ -528,7 +463,6 @@ pub(crate) struct InstallTargetPaths { repo_root: PathBuf, } -#[allow(dead_code)] impl InstallTargetPaths { pub(crate) fn new(repo_root: impl Into) -> Self { Self { @@ -555,45 +489,9 @@ impl InstallTargetPaths { self.repo_root.clone() } - pub(crate) fn opencode_plugin_target(&self) -> PathBuf { - self.opencode_target_dir() - .join(opencode_asset::PLUGINS_DIR) - .join(opencode_asset::PLUGIN_FILE) - } - pub(crate) fn opencode_preset_catalog_target(&self) -> PathBuf { self.opencode_target_dir() .join(opencode_asset::LIB_DIR) .join(opencode_asset::PRESET_CATALOG) } - - pub(crate) fn skill_tile_relative_path(skill_name: &str) -> String { - format!( - "{}/{}/{}", - opencode_asset::SKILLS_DIR, - skill_name, - context_file::SKILL_DEFINITION - ) - } - - pub(crate) fn pre_commit_hook_path(&self) -> PathBuf { - self.repo_root - .join(repo_dir::GIT) - .join(hook_dir::HOOKS) - .join(hook_dir::PRE_COMMIT) - } - - pub(crate) fn commit_msg_hook_path(&self) -> PathBuf { - self.repo_root - .join(repo_dir::GIT) - .join(hook_dir::HOOKS) - .join(hook_dir::COMMIT_MSG) - } - - pub(crate) fn post_commit_hook_path(&self) -> PathBuf { - self.repo_root - .join(repo_dir::GIT) - .join(hook_dir::HOOKS) - .join(hook_dir::POST_COMMIT) - } } diff --git a/cli/src/services/doctor/command.rs b/cli/src/services/doctor/command.rs index a8b8f40ca..50f2fc933 100644 --- a/cli/src/services/doctor/command.rs +++ b/cli/src/services/doctor/command.rs @@ -7,8 +7,12 @@ pub struct DoctorCommand { } impl DoctorCommand { - pub fn execute(&self, context: &C) -> Result { + pub async fn execute( + &self, + context: &C, + ) -> Result { doctor::run_doctor_with_context(self.request, context) + .await .map_err(|source| CliError::user_with_source(UserError::UnexpectedFailure, source)) } } diff --git a/cli/src/services/doctor/inspect.rs b/cli/src/services/doctor/inspect.rs index 095f326d7..0ea03100d 100644 --- a/cli/src/services/doctor/inspect.rs +++ b/cli/src/services/doctor/inspect.rs @@ -1,3 +1,4 @@ +use crate::services::capabilities::GitOps; use std::fs; use std::path::{Path, PathBuf}; @@ -21,14 +22,13 @@ use crate::services::hooks::{ pi_mutation_scope, }; use crate::services::mutation_trace::runtime::resolve_git_dir; -use crate::services::observability::traits::Logger; use crate::services::repository_identity::resolve::{ resolve_repository_identity, RepositoryIdentitySource, }; use crate::services::setup::{ config_merge, hook_merge, iter_embedded_assets_for_setup_target_with_selection, iter_required_hook_assets, persisted_optional_workflows, repair_merge_target_asset, - EmbeddedAsset, SetupTarget, + ConcreteSetupTarget, EmbeddedAsset, }; use super::render::integration_target_label; @@ -42,10 +42,16 @@ use super::types::{ }; use super::{is_executable, DoctorDependencies, DoctorMode, REQUIRED_HOOKS}; -pub(super) fn build_report_with_lifecycle_problems( +pub(super) async fn build_report_with_lifecycle_problems( mode: DoctorMode, repository_root: &Path, - dependencies: &DoctorDependencies<'_>, + dependencies: &DoctorDependencies< + '_, + impl crate::services::capabilities::GitOps, + impl Fn() -> anyhow::Result, + impl Fn() -> anyhow::Result, + impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, + >, lifecycle_problems: Vec, codex_policy_readiness: &CodexHookPolicyReadiness, ) -> HookDoctorReport { @@ -55,73 +61,70 @@ pub(super) fn build_report_with_lifecycle_problems( dependencies, lifecycle_problems, codex_policy_readiness, - ); - report.agent_trace_db = collect_agent_trace_db_health(repository_root, &mut report.problems); + ) + .await; + report.agent_trace_db = + collect_agent_trace_db_health(repository_root, &mut report.problems).await; report.readiness = compute_readiness(&report.problems); report } -fn build_report_without_service_owned_problem_checks( +async fn build_report_without_service_owned_problem_checks( mode: DoctorMode, repository_root: &Path, - dependencies: &DoctorDependencies<'_>, + dependencies: &DoctorDependencies< + '_, + impl crate::services::capabilities::GitOps, + impl Fn() -> anyhow::Result, + impl Fn() -> anyhow::Result, + impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, + >, mut problems: Vec, codex_policy_readiness: &CodexHookPolicyReadiness, ) -> HookDoctorReport { let global_state = collect_global_state_locations(repository_root, dependencies); - let agent_trace_db = collect_agent_trace_db_health(repository_root, &mut problems); - let git_available = (dependencies.check_git_available)(); + let agent_trace_db = collect_agent_trace_db_health(repository_root, &mut problems).await; + let git_available = dependencies.git.is_available(); let detected_repository_root = if git_available { - (dependencies.run_git_command)(repository_root, &["rev-parse", "--show-toplevel"]) - .map(PathBuf::from) - } else { - None - }; - - let bare_repository = if git_available { - (dependencies.run_git_command)(repository_root, &["rev-parse", "--is-bare-repository"]) - .is_some_and(|value| value == "true") - } else { - false - }; - - let local_hooks_path = if git_available { - (dependencies.run_git_command)( + doctor_git_output( + dependencies.git, repository_root, - &["config", "--local", "--get", "core.hooksPath"], + &["rev-parse", "--show-toplevel"], ) + .map(PathBuf::from) } else { None }; - let global_hooks_path = if git_available { - (dependencies.run_git_command)( + + let bare_repository = if git_available { + doctor_git_output( + dependencies.git, repository_root, - &["config", "--global", "--get", "core.hooksPath"], + &["rev-parse", "--is-bare-repository"], ) + .is_some_and(|value| value == "true") } else { - None + false }; - let hook_path_source = if local_hooks_path.is_some() { - HookPathSource::LocalConfig - } else if global_hooks_path.is_some() { - HookPathSource::GlobalConfig - } else { - HookPathSource::Default - }; + let hook_path_source = + detect_hook_path_source(dependencies.git, git_available, repository_root); let hooks_directory = detected_repository_root.as_ref().and_then(|resolved_root| { - (dependencies.run_git_command)(resolved_root, &["rev-parse", "--git-path", "hooks"]).map( - |value| { - let path = PathBuf::from(value); - if path.is_absolute() { - path - } else { - resolved_root.join(path) - } - }, + doctor_git_output( + dependencies.git, + resolved_root, + &["rev-parse", "--git-path", "hooks"], ) + .map(|value| { + let path = PathBuf::from(value); + if path.is_absolute() { + path + } else { + resolved_root.join(path) + } + }) }); let hooks = if git_available && !bare_repository && detected_repository_root.is_some() { @@ -156,7 +159,8 @@ fn build_report_without_service_owned_problem_checks( bare_repository, detected_repository_root.as_deref(), &mut problems, - ); + ) + .await; HookDoctorReport { mode, @@ -176,7 +180,40 @@ fn build_report_without_service_owned_problem_checks( } } -fn inspect_mutation_scope_health( +fn detect_hook_path_source( + git: &impl GitOps, + git_available: bool, + repository_root: &Path, +) -> HookPathSource { + let local_hooks_path = if git_available { + doctor_git_output( + git, + repository_root, + &["config", "--local", "--get", "core.hooksPath"], + ) + } else { + None + }; + let global_hooks_path = if git_available { + doctor_git_output( + git, + repository_root, + &["config", "--global", "--get", "core.hooksPath"], + ) + } else { + None + }; + + if local_hooks_path.is_some() { + HookPathSource::LocalConfig + } else if global_hooks_path.is_some() { + HookPathSource::GlobalConfig + } else { + HookPathSource::Default + } +} + +async fn inspect_mutation_scope_health( git_available: bool, bare_repository: bool, detected_repository_root: Option<&Path>, @@ -192,119 +229,131 @@ fn inspect_mutation_scope_health( if targets.is_empty() { return Vec::new(); } - let Ok(git_dir) = resolve_git_dir(resolved_root) else { + let Ok(git_dir) = resolve_git_dir(resolved_root).await else { return Vec::new(); }; - targets - .into_iter() - .map(|target_id| { - let (target, health) = match target_id { - IntegrationTargetId::Claude => ( - IntegrationTarget::ClaudeCode, - claude_mutation_scope::health::classify_health(&git_dir), - ), - IntegrationTargetId::Codex => ( - IntegrationTarget::Codex, - codex_mutation_scope::health::classify_health(&git_dir), - ), - IntegrationTargetId::Opencode => ( - IntegrationTarget::OpenCode, - opencode_mutation_scope::health::classify_health(&git_dir), - ), - IntegrationTargetId::Pi => ( - IntegrationTarget::Pi, - pi_mutation_scope::health::classify_health(&git_dir), - ), - }; - let remediation = - push_mutation_scope_health_problem(target, &health, &git_dir, problems); - MutationScopeHealthRow { - target, - status: health.status, - reason: health.reason, - detail: health.detail, - remediation, - } - }) - .collect() + let mut rows = Vec::with_capacity(targets.len()); + for target_id in targets { + let (target, health) = match target_id { + IntegrationTargetId::Claude => ( + IntegrationTarget::ClaudeCode, + claude_mutation_scope::health::classify_health(&git_dir).await, + ), + IntegrationTargetId::Codex => ( + IntegrationTarget::Codex, + codex_mutation_scope::health::classify_health(&git_dir).await, + ), + IntegrationTargetId::Opencode => ( + IntegrationTarget::OpenCode, + opencode_mutation_scope::health::classify_health(&git_dir).await, + ), + IntegrationTargetId::Pi => ( + IntegrationTarget::Pi, + pi_mutation_scope::health::classify_health(&git_dir).await, + ), + }; + let remediation = + push_mutation_scope_health_problem(target, &health, &git_dir, problems).await; + rows.push(MutationScopeHealthRow { + target, + status: health.status, + reason: health.reason, + detail: health.detail, + remediation, + }); + } + rows } -pub(super) type MutationScopeRepairSeam<'a> = - &'a dyn Fn(&Path, &str, Option<&dyn Logger>) -> anyhow::Result; - -pub(super) fn repair_blocked_mutation_scope_targets_with_seam( +pub(super) async fn repair_blocked_mutation_scope_targets_with_seam( initial_report: &HookDoctorReport, - seam: MutationScopeRepairSeam<'_>, + seam: &impl std::ops::AsyncFn( + &std::path::Path, + &str, + Option<&crate::services::observability::traits::NoopLogger>, + ) -> anyhow::Result, ) -> Vec { let Some(repository_root) = initial_report.repository_root.as_deref() else { return Vec::new(); }; - let Ok(git_dir) = resolve_git_dir(repository_root) else { + let Ok(git_dir) = resolve_git_dir(repository_root).await else { return Vec::new(); }; - initial_report - .mutation_scope_health - .iter() - .filter(|row| row.status == MutationScopeHealthStatus::Blocked) - .filter_map(|row| { - repair_blocked_mutation_scope_target(row.target, &git_dir, repository_root, seam) - }) - .collect() + let mut repaired = Vec::new(); + for row in &initial_report.mutation_scope_health { + if row.status == MutationScopeHealthStatus::Blocked { + if let Some(target) = + repair_blocked_mutation_scope_target(row.target, &git_dir, repository_root, seam) + .await + { + repaired.push(target); + } + } + } + repaired } -pub(super) fn mutation_scope_repair_seam( +pub(super) async fn mutation_scope_repair_seam< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> anyhow::Result { - mutation_scope::run_mutation_scope_from_payload(repository_root, payload, logger) + mutation_scope::run_mutation_scope_from_payload(repository_root, payload, logger).await } -fn repair_blocked_mutation_scope_target( +async fn repair_blocked_mutation_scope_target( target: IntegrationTarget, git_dir: &Path, repository_root: &Path, - seam: MutationScopeRepairSeam<'_>, + seam: &impl std::ops::AsyncFn( + &std::path::Path, + &str, + Option<&crate::services::observability::traits::NoopLogger>, + ) -> anyhow::Result, ) -> Option { match target { IntegrationTarget::ClaudeCode => { - if claude_repairability(git_dir) != Repairability::AutoFixable { + if claude_repairability(git_dir).await != Repairability::AutoFixable { return None; } - let _ = claude_mutation_scope::repair_blocked(git_dir, repository_root, None, seam); + let _ = + claude_mutation_scope::repair_blocked(git_dir, repository_root, None, seam).await; Some(target) } IntegrationTarget::OpenCode => { - if opencode_repairability(git_dir) != Repairability::AutoFixable { + if opencode_repairability(git_dir).await != Repairability::AutoFixable { return None; } - let _ = opencode_mutation_scope::repair_blocked(git_dir, repository_root, None, seam); + let _ = + opencode_mutation_scope::repair_blocked(git_dir, repository_root, None, seam).await; Some(target) } IntegrationTarget::Pi | IntegrationTarget::Codex => None, } } -fn claude_repairability(git_dir: &Path) -> Repairability { - match claude_mutation_scope::assess_repairability(git_dir) { +async fn claude_repairability(git_dir: &Path) -> Repairability { + match claude_mutation_scope::assess_repairability(git_dir).await { claude_mutation_scope::Repairability::AutoFixable => Repairability::AutoFixable, claude_mutation_scope::Repairability::ManualOnly => Repairability::ManualOnly, } } -fn opencode_repairability(git_dir: &Path) -> Repairability { - match opencode_mutation_scope::assess_repairability(git_dir) { +async fn opencode_repairability(git_dir: &Path) -> Repairability { + match opencode_mutation_scope::assess_repairability(git_dir).await { opencode_mutation_scope::Repairability::AutoFixable => Repairability::AutoFixable, opencode_mutation_scope::Repairability::ManualOnly => Repairability::ManualOnly, } } -fn mutation_scope_repairability(target: IntegrationTarget, git_dir: &Path) -> Repairability { +async fn mutation_scope_repairability(target: IntegrationTarget, git_dir: &Path) -> Repairability { match target { - IntegrationTarget::ClaudeCode => claude_repairability(git_dir), - IntegrationTarget::OpenCode => opencode_repairability(git_dir), + IntegrationTarget::ClaudeCode => claude_repairability(git_dir).await, + IntegrationTarget::OpenCode => opencode_repairability(git_dir).await, IntegrationTarget::Pi | IntegrationTarget::Codex => Repairability::ManualOnly, } } @@ -366,7 +415,7 @@ fn mutation_scope_state_path(target: IntegrationTarget, git_dir: &Path) -> PathB } } -fn push_mutation_scope_health_problem( +async fn push_mutation_scope_health_problem( target: IntegrationTarget, health: &MutationScopeAdapterHealth, git_dir: &Path, @@ -387,36 +436,38 @@ fn push_mutation_scope_health_problem( state remains recovering unexpectedly.", ), ), - MutationScopeHealthStatus::Blocked => match mutation_scope_repairability(target, git_dir) { - Repairability::AutoFixable => ( - ProblemKind::MutationScopeHealthBlocked, - ProblemSeverity::Error, - ProblemFixability::AutoFixable, - "doctor_fix", - format!( - "Run 'sce doctor --fix' to recover this state: the owning process for \ + MutationScopeHealthStatus::Blocked => { + match mutation_scope_repairability(target, git_dir).await { + Repairability::AutoFixable => ( + ProblemKind::MutationScopeHealthBlocked, + ProblemSeverity::Error, + ProblemFixability::AutoFixable, + "doctor_fix", + format!( + "Run 'sce doctor --fix' to recover this state: the owning process for \ the blocking attempt(s) has been positively proven dead, so automatic \ recovery is safe. The persisted state is at '{}'.", - mutation_scope_state_path(target, git_dir).display() + mutation_scope_state_path(target, git_dir).display() + ), ), - ), - Repairability::ManualOnly => ( - ProblemKind::MutationScopeHealthBlocked, - ProblemSeverity::Error, - ProblemFixability::ManualOnly, - "manual_steps", - format!( - "Agent tracing remains blocked. Inspect '{}'. 'sce doctor --fix' will \ + Repairability::ManualOnly => ( + ProblemKind::MutationScopeHealthBlocked, + ProblemSeverity::Error, + ProblemFixability::ManualOnly, + "manual_steps", + format!( + "Agent tracing remains blocked. Inspect '{}'. 'sce doctor --fix' will \ not modify persisted mutation-scope recovery state automatically: \ clearing it could silently discard unresolved mutation-scope \ lifecycle or recovery evidence. No safe generic recovery command \ exists yet for this case; preserve the persisted state while \ reviewing this adapter's recovery model directly before taking \ manual action.", - mutation_scope_state_path(target, git_dir).display() + mutation_scope_state_path(target, git_dir).display() + ), ), - ), - }, + } + } MutationScopeHealthStatus::Invalid => ( ProblemKind::MutationScopeHealthInvalid, ProblemSeverity::Error, @@ -508,7 +559,13 @@ fn post_commit_auto_sync_state( fn collect_global_state_locations( repository_root: &Path, - dependencies: &DoctorDependencies<'_>, + dependencies: &DoctorDependencies< + '_, + impl crate::services::capabilities::GitOps, + impl Fn() -> anyhow::Result, + impl Fn() -> anyhow::Result, + impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, + >, ) -> GlobalStateHealth { let state_root = (dependencies.resolve_state_root)() @@ -553,11 +610,11 @@ fn collect_global_state_locations( } } -fn collect_agent_trace_db_health( +async fn collect_agent_trace_db_health( repository_root: &Path, problems: &mut Vec, ) -> Option { - let agent_trace_problems = diagnose_agent_trace_db_health(Some(repository_root)); + let agent_trace_problems = diagnose_agent_trace_db_health(Some(repository_root)).await; let mut agent_trace_db = None; for problem in &agent_trace_problems { @@ -684,81 +741,6 @@ fn hook_managed_block_content_state( } } -#[allow(dead_code)] -fn inspect_repository_hooks( - repository_root: &Path, - git_available: bool, - bare_repository: bool, - detected_repository_root: Option<&Path>, - hooks_directory: Option<&Path>, - problems: &mut Vec, -) -> Vec { - if !git_available { - problems.push(DoctorProblem { - kind: ProblemKind::GitUnavailable, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from("Git is not available on this machine."), - remediation: String::from("Install an accessible 'git' binary and ensure it is on PATH before rerunning 'sce doctor'."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - return Vec::new(); - } - - if bare_repository { - problems.push(DoctorProblem { - kind: ProblemKind::BareRepository, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from( - "The current repository is bare and does not support local SCE hook rollout.", - ), - remediation: String::from("Run 'sce doctor' from a non-bare working tree clone to inspect repo-scoped SCE hook health."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - return Vec::new(); - } - - if detected_repository_root.is_none() { - problems.push(DoctorProblem { - kind: ProblemKind::NotInsideGitRepository, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from("The current directory is not inside a git repository."), - remediation: String::from("Run 'sce doctor' from inside the target repository working tree to inspect repo-scoped SCE hook health."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - return Vec::new(); - } - - if let Some(directory) = hooks_directory { - return collect_hook_health(directory, problems); - } - - let _ = repository_root; - problems.push(DoctorProblem { - kind: ProblemKind::UnableToResolveGitHooksDirectory, - category: ProblemCategory::RepositoryTargeting, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: String::from("Unable to resolve git hooks directory."), - remediation: String::from("Verify that git repository inspection succeeds and rerun 'sce doctor' inside a non-bare git repository."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - Vec::new() -} - /// Returns `true` when the doctor was able to check for integration targets /// and found none (neither configured in `.sce/config.json` nor detected /// via repo-root `.opencode/` / `.claude/` / `.pi/` directories). @@ -920,7 +902,7 @@ pub(super) fn repair_merge_target_configs( collect_claude_integration_groups(repository_root, &selected_optional_workflows); if let Some(result) = repair_merge_target_if_mismatched( repository_root, - SetupTarget::Claude, + ConcreteSetupTarget::Claude, claude_asset::SETTINGS_FILE, &claude_groups, ) { @@ -933,7 +915,7 @@ pub(super) fn repair_merge_target_configs( collect_opencode_integration_groups(repository_root, &selected_optional_workflows); if let Some(result) = repair_merge_target_if_mismatched( repository_root, - SetupTarget::OpenCode, + ConcreteSetupTarget::OpenCode, OPENCODE_CONFIG_RELATIVE_PATH, &opencode_groups, ) { @@ -992,7 +974,7 @@ fn repair_codex_hooks_json_if_structurally_unhealthy( Some( match repair_merge_target_asset( repository_root, - SetupTarget::Codex, + ConcreteSetupTarget::Codex, CODEX_HOOKS_JSON_RELATIVE_PATH, ) { Ok(()) => DoctorFixResultRecord { @@ -1016,7 +998,7 @@ fn repair_codex_hooks_json_if_structurally_unhealthy( fn repair_merge_target_if_mismatched( repository_root: &Path, - target: SetupTarget, + target: ConcreteSetupTarget, relative_path: &str, groups: &[IntegrationGroupHealth], ) -> Option { @@ -1049,228 +1031,6 @@ fn repair_merge_target_if_mismatched( ) } -#[allow(dead_code)] -fn collect_global_state_health( - repository_root: &Path, - problems: &mut Vec, - dependencies: &DoctorDependencies<'_>, -) -> GlobalStateHealth { - let mut state_root_health = None; - let mut config_locations = Vec::new(); - - match (dependencies.resolve_state_root)() { - Ok(state_root) => { - state_root_health = Some(FileLocationHealth { - label: "State root", - state: if state_root.exists() { "present" } else { "expected" }, - path: state_root.clone(), - }); - } - Err(error) => problems.push(DoctorProblem { - kind: ProblemKind::UnableToResolveStateRoot, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!("Unable to resolve expected state root: {error}"), - remediation: String::from("Verify that the current platform exposes a writable SCE state directory before rerunning 'sce doctor'."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }), - } - - match (dependencies.resolve_global_config_path)() { - Ok(global_path) => { - if global_path.exists() { - if let Err(error) = (dependencies.validate_config_file)(&global_path) { - problems.push(DoctorProblem { - kind: ProblemKind::GlobalConfigValidationFailed, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!( - "Global config file '{}' failed validation: {error}", - global_path.display() - ), - remediation: format!( - "Repair or remove the invalid global config file at '{}' and rerun 'sce doctor'.", - global_path.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - } - } - config_locations.push(FileLocationHealth { - label: "Global config", - state: if global_path.exists() { "present" } else { "expected" }, - path: global_path, - }); - } - Err(error) => problems.push(DoctorProblem { - kind: ProblemKind::UnableToResolveGlobalConfigPath, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!("Unable to resolve expected global config path: {error}"), - remediation: String::from("Verify that the current platform exposes a writable SCE config directory before rerunning 'sce doctor'."), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }), - } - - let local_path = RepoPaths::new(repository_root).sce_config_file(); - if local_path.exists() { - if let Err(error) = (dependencies.validate_config_file)(&local_path) { - problems.push(DoctorProblem { - kind: ProblemKind::LocalConfigValidationFailed, - category: ProblemCategory::GlobalState, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!( - "Local config file '{}' failed validation: {error}", - local_path.display() - ), - remediation: format!( - "Repair or remove the invalid local config file at '{}' and rerun 'sce doctor'.", - local_path.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - } - } - config_locations.push(FileLocationHealth { - label: "Local config", - state: if local_path.exists() { - "present" - } else { - "expected" - }, - path: local_path, - }); - - GlobalStateHealth { - state_root: state_root_health, - config_locations, - } -} - -#[allow(dead_code)] -#[allow(clippy::too_many_lines)] -fn collect_hook_health(directory: &Path, problems: &mut Vec) -> Vec { - if !directory.exists() { - problems.push(DoctorProblem { - kind: ProblemKind::HooksDirectoryMissing, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!("Hooks directory '{}' does not exist.", directory.display()), - remediation: format!( - "Run 'sce doctor --fix' to install the canonical SCE-managed hooks into '{}', or run 'sce setup --hooks' directly.", - directory.display() - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } else if !directory.is_dir() { - problems.push(DoctorProblem { - kind: ProblemKind::HooksPathNotDirectory, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!("Hooks path '{}' is not a directory.", directory.display()), - remediation: format!( - "Replace '{}' with a writable hooks directory, then rerun 'sce doctor' or 'sce setup --hooks'.", - directory.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - } - - REQUIRED_HOOKS - .iter() - .map(|hook_name| { - let hook_path = directory.join(hook_name); - let metadata = fs::metadata(&hook_path).ok(); - let exists = metadata.is_some(); - let executable = metadata - .as_ref() - .is_some_and(|entry| entry.is_file() && is_executable(entry)); - let content_state = inspect_hook_content_state(hook_name, &hook_path, exists, problems); - - if !exists { - problems.push(DoctorProblem { - kind: ProblemKind::RequiredHookMissing, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!( - "Missing required hook '{}' at '{}'.", - hook_name, - hook_path.display() - ), - remediation: format!( - "Run 'sce doctor --fix' to install the canonical '{hook_name}' hook, or run 'sce setup --hooks' directly." - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } else if !executable { - problems.push(DoctorProblem { - kind: ProblemKind::HookNotExecutable, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!("Hook '{hook_name}' exists but is not executable."), - remediation: format!( - "Run 'sce doctor --fix' to restore the canonical executable hook, or run 'sce setup --hooks' / 'chmod +x {}' manually.", - hook_path.display() - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } - - if content_state == HookContentState::Stale { - problems.push(DoctorProblem { - kind: ProblemKind::HookContentStale, - category: ProblemCategory::HookRollout, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::AutoFixable, - summary: format!( - "Hook '{}' at '{}' differs from the canonical SCE-managed content.", - hook_name, - hook_path.display() - ), - remediation: format!( - "Run 'sce doctor --fix' to reinstall the canonical '{hook_name}' hook content, or run 'sce setup --hooks' directly." - ), - next_action: "doctor_fix", - scope: None, - mutation_scope_target: None, - }); - } - - HookFileHealth { - name: hook_name, - path: hook_path, - exists, - executable, - content_state, - } - }) - .collect() -} - fn inspect_opencode_integration_health( repository_root: &Path, integration_groups: &[IntegrationGroupHealth], @@ -2104,7 +1864,7 @@ fn collect_opencode_integration_groups( let opencode_root = repo_paths.opencode_dir(); let manifest_path = repo_paths.opencode_manifest_file(); let embedded_assets = iter_embedded_assets_for_setup_target_with_selection( - SetupTarget::OpenCode, + ConcreteSetupTarget::OpenCode, selected_optional_workflows, ) .collect::>(); @@ -2192,7 +1952,7 @@ fn collect_claude_integration_groups( let repo_paths = RepoPaths::new(repository_root); let claude_root = repo_paths.claude_dir(); let embedded_assets = iter_embedded_assets_for_setup_target_with_selection( - SetupTarget::Claude, + ConcreteSetupTarget::Claude, selected_optional_workflows, ) .collect::>(); @@ -2254,7 +2014,7 @@ fn collect_pi_integration_groups( let repo_paths = RepoPaths::new(repository_root); let pi_root = repo_paths.pi_dir(); let embedded_assets = iter_embedded_assets_for_setup_target_with_selection( - SetupTarget::Pi, + ConcreteSetupTarget::Pi, selected_optional_workflows, ) .collect::>(); @@ -2314,7 +2074,7 @@ fn collect_codex_integration_groups( ) -> Vec { let codex_root = InstallTargetPaths::new(repository_root).codex_target_dir(); let embedded_assets = iter_embedded_assets_for_setup_target_with_selection( - SetupTarget::Codex, + ConcreteSetupTarget::Codex, selected_optional_workflows, ) .collect::>(); @@ -2641,2956 +2401,8 @@ fn path_is_file(path: &Path) -> bool { fs::metadata(path).is_ok_and(|metadata| metadata.is_file()) } -#[allow(dead_code)] -fn inspect_hook_content_state( - hook_name: &str, - hook_path: &Path, - exists: bool, - problems: &mut Vec, -) -> HookContentState { - if !exists { - return HookContentState::Missing; - } - - let Some(expected_hook) = - iter_required_hook_assets().find(|asset| asset.relative_path == hook_name) - else { - return HookContentState::Unknown; - }; - - match fs::read(hook_path) { - Ok(bytes) => hook_managed_block_content_state(hook_name, &bytes, expected_hook.bytes), - Err(error) => { - problems.push(DoctorProblem { - kind: ProblemKind::HookReadFailed, - category: ProblemCategory::FilesystemPermissions, - severity: ProblemSeverity::Error, - fixability: ProblemFixability::ManualOnly, - summary: format!( - "Unable to read hook '{}' at '{}': {error}", - hook_name, - hook_path.display() - ), - remediation: format!( - "Verify that '{}' is readable before rerunning 'sce doctor'.", - hook_path.display() - ), - next_action: "manual_steps", - scope: None, - mutation_scope_target: None, - }); - HookContentState::Unknown - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use super::super::fixes::build_manual_fix_results; - use super::{ - claude_mutation_scope, codex_hook_config, codex_hook_registration_child, codex_hook_trust, - collect_claude_integration_groups, collect_codex_integration_groups, - collect_hook_file_health, collect_opencode_integration_groups, - collect_pi_integration_groups, compute_readiness, inspect_claude_integration_health, - inspect_codex_integration_health, inspect_mutation_scope_health, - inspect_opencode_plugin_ordering_health, resolve_doctor_integration_targets, - CodexHookPolicyReadiness, DoctorFixResultRecord, DoctorMode, DoctorProblem, FixResult, - HookContentState, HookDoctorReport, HookPathSource, IntegrationArea, - IntegrationContentState, IntegrationGroupHealth, IntegrationGroupKey, IntegrationTarget, - MutationScopeHealthRow, MutationScopeHealthStatus, PostCommitAutoSyncHealth, - PostCommitAutoSyncState, ProblemCategory, ProblemFixability, ProblemKind, ProblemSeverity, - Readiness, - }; - use crate::services::config::IntegrationTargetId; - use crate::services::hooks::claude_mutation_scope::state::{ - AdapterAttempt, AdapterState, AttemptPhase, - }; - use crate::services::setup::{SetupTarget, OPTIONAL_WORKFLOWS}; - - /// The collectors only read file state, so a non-existent root is enough to - /// observe which children they expect. - fn absent_repository_root() -> PathBuf { - PathBuf::from("/nonexistent-sce-doctor-optional-workflow-fixture") - } - - fn brownfield_slugs() -> (&'static str, &'static str) { - let workflow = OPTIONAL_WORKFLOWS - .iter() - .find(|workflow| workflow.id == "brownfield") - .expect("brownfield is an optional workflow in the embedded catalog"); - (workflow.command_slug, workflow.skill_slug) - } - - fn child_paths(groups: &[IntegrationGroupHealth]) -> Vec { - groups - .iter() - .flat_map(|group| group.children.iter()) - .map(|child| child.relative_path.clone()) - .collect() - } - - #[test] - fn integration_children_exclude_unselected_optional_workflow() { - let root = absent_repository_root(); - let (command_slug, skill_slug) = brownfield_slugs(); - - let cases = [ - ( - child_paths(&collect_opencode_integration_groups(&root, &[])), - format!("command/{command_slug}.md"), - ), - ( - child_paths(&collect_claude_integration_groups(&root, &[])), - format!("commands/{command_slug}.md"), - ), - ( - child_paths(&collect_pi_integration_groups(&root, &[])), - format!("prompts/{command_slug}.md"), - ), - ]; - - let skill_prefix = format!("skills/{skill_slug}/"); - for (paths, command_path) in cases { - assert!( - !paths.contains(&command_path), - "{command_path} still expected" - ); - assert!( - !paths.iter().any(|path| path.starts_with(&skill_prefix)), - "{skill_prefix} assets still expected" - ); - assert!( - paths.iter().any(|path| path.ends_with("validate.md")), - "core workflow assets were dropped" - ); - } - } - - #[test] - fn integration_children_include_selected_optional_workflow() { - let root = absent_repository_root(); - let (command_slug, skill_slug) = brownfield_slugs(); - let selection = vec![String::from("brownfield")]; - - let cases = [ - ( - child_paths(&collect_opencode_integration_groups(&root, &selection)), - format!("command/{command_slug}.md"), - ), - ( - child_paths(&collect_claude_integration_groups(&root, &selection)), - format!("commands/{command_slug}.md"), - ), - ( - child_paths(&collect_pi_integration_groups(&root, &selection)), - format!("prompts/{command_slug}.md"), - ), - ]; - - let skill_prefix = format!("skills/{skill_slug}/"); - for (paths, command_path) in cases { - assert!(paths.contains(&command_path), "{command_path} not expected"); - assert!( - paths.iter().any(|path| path.starts_with(&skill_prefix)), - "{skill_prefix} assets not expected" - ); - } - } - - #[test] - fn missing_file_problems_follow_the_optional_workflow_selection() { - let root = absent_repository_root(); - let (command_slug, _) = brownfield_slugs(); - let command_path = format!("commands/{command_slug}.md"); - - let unselected_groups = collect_claude_integration_groups(&root, &[]); - let mut unselected_problems = Vec::new(); - inspect_claude_integration_health(&unselected_groups, &mut unselected_problems); - assert!( - !unselected_problems - .iter() - .any(|problem| problem.summary.contains(command_slug)), - "an unselected optional workflow produced a problem" - ); - assert!( - !unselected_problems.is_empty(), - "core assets are absent, so missing-file problems are still expected" - ); - - let selection = vec![String::from("brownfield")]; - let selected_groups = collect_claude_integration_groups(&root, &selection); - assert!(selected_groups.iter().any(|group| group - .children - .iter() - .any(|child| child.relative_path == command_path - && matches!(child.content_state, IntegrationContentState::Missing)))); - - let mut selected_problems = Vec::new(); - inspect_claude_integration_health(&selected_groups, &mut selected_problems); - assert!( - selected_problems - .iter() - .any(|problem| problem.summary.contains(&command_path)), - "a selected optional workflow's missing file was not reported" - ); - } - - /// A `TrustContext` pointed at a codex-home directory with no - /// `config.toml`, so tests never depend on the real `$CODEX_HOME` or - /// `~/.codex` of the machine running them: every registration diagnosed - /// as structurally current resolves deterministically to `Untrusted`. - fn deterministic_untrusted_context(label: &str) -> codex_hook_trust::TrustContext { - codex_hook_trust::TrustContext { - codex_home: Some(unique_temp_repository_root(&format!( - "{label}-codex-home-absent" - ))), - } - } - - /// Deterministic "policy allows project hooks" reading, so existing - /// trust-focused tests keep exercising only the trust dimension. - fn allowed_policy() -> CodexHookPolicyReadiness { - CodexHookPolicyReadiness::ProjectHooksAllowed - } - - fn unique_temp_repository_root(label: &str) -> PathBuf { - let nonce = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - let dir = std::env::temp_dir().join(format!( - "sce-doctor-merge-target-{label}-{}-{nonce}", - std::process::id() - )); - std::fs::create_dir_all(&dir).expect("create temp repository root"); - dir - } - - fn embedded_claude_settings_bytes() -> &'static [u8] { - crate::services::setup::iter_embedded_assets_for_setup_target_with_selection( - crate::services::setup::SetupTarget::Claude, - &[] as &[String], - ) - .find(|asset| { - asset.relative_path == crate::services::default_paths::claude_asset::SETTINGS_FILE - }) - .expect("embedded Claude catalog carries settings.json") - .bytes - } - - fn legacy_claude_agent_trace_hook(event: &str) -> serde_json::Value { - serde_json::json!({ - "hooks": [{ - "type": "command", - "command": "bun", - "args": [".claude/plugins/sce-agent-trace.ts", event] - }] - }) - } - - fn embedded_opencode_config_bytes() -> &'static [u8] { - crate::services::setup::iter_embedded_assets_for_setup_target_with_selection( - crate::services::setup::SetupTarget::OpenCode, - &[] as &[String], - ) - .find(|asset| asset.relative_path == "opencode.json") - .expect("embedded OpenCode catalog carries opencode.json") - .bytes - } - - fn embedded_codex_asset_bytes(relative_path: &str) -> &'static [u8] { - crate::services::setup::iter_embedded_assets_for_setup_target_with_selection( - crate::services::setup::SetupTarget::Codex, - &[] as &[String], - ) - .find(|asset| asset.relative_path == relative_path) - .unwrap_or_else(|| panic!("embedded Codex catalog carries {relative_path}")) - .bytes - } - - #[test] - fn codex_integration_groups_split_into_skills_and_hooks_areas() { - let root = absent_repository_root(); - let groups = collect_codex_integration_groups( - &root, - &[], - &deterministic_untrusted_context("split-areas"), - &allowed_policy(), - ); - - let skills_group = groups - .iter() - .find(|group| group.key.area == IntegrationArea::Skills) - .expect("Codex skills group present"); - assert_eq!(skills_group.key.target, IntegrationTarget::Codex); - assert!( - skills_group - .children - .iter() - .all(|child| child.relative_path.starts_with(".agents/skills/")), - "Codex skills group children should all live under .agents/skills/" - ); - assert!(!skills_group.children.is_empty()); - - let hooks_group = groups - .iter() - .find(|group| group.key.area == IntegrationArea::Hooks) - .expect("Codex hooks group present"); - for suffix in [ - "UserPromptSubmit", - "Stop", - "PreToolUse(Bash)", - "PostToolUse(apply_patch)", - ] { - assert!( - hooks_group - .children - .iter() - .any(|child| child.relative_path == format!(".codex/hooks.json#{suffix}")), - "Codex hooks group should include a .codex/hooks.json#{suffix} registration" - ); - } - assert!( - hooks_group - .children - .iter() - .any(|child| child.relative_path - == ".codex/hooks/run-sce-or-show-install-guidance.sh"), - "Codex hooks group should include the hook helper script" - ); - - assert!( - groups - .iter() - .flat_map(|group| &group.children) - .all(|child| matches!(child.content_state, IntegrationContentState::Missing)), - "an absent repository root should report every Codex asset as missing" - ); - } - - #[test] - fn codex_hooks_json_reports_present_and_current_but_untrusted_then_missing() { - let root = unique_temp_repository_root("codex-hooks"); - let codex_hooks_dir = root.join(".codex"); - std::fs::create_dir_all(&codex_hooks_dir).unwrap(); - std::fs::write( - codex_hooks_dir.join("hooks.json"), - embedded_codex_asset_bytes(".codex/hooks.json"), - ) - .unwrap(); - - let trust_context = deterministic_untrusted_context("codex-hooks-match"); - let groups = - collect_codex_integration_groups(&root, &[], &trust_context, &allowed_policy()); - let registration_children = groups - .iter() - .flat_map(|group| &group.children) - .filter(|child| child.relative_path.starts_with(".codex/hooks.json#")) - .collect::>(); - assert_eq!( - registration_children.len(), - codex_hook_config::required_registrations().len() - ); - for child in ®istration_children { - assert_eq!( - child.content_state, - IntegrationContentState::NotTrusted("untrusted".to_string()), - "a current-but-never-trusted registration ('{}') should report not-trusted, \ - not a bare content mismatch", - child.relative_path - ); - } - - let mut trust_problems = Vec::new(); - inspect_codex_integration_health(&groups, &mut trust_problems); - let trust_problem = trust_problems - .iter() - .find(|problem| problem.kind == ProblemKind::CodexHookRegistrationNotTrusted) - .expect("a not-trusted problem was reported for current registrations"); - assert!( - trust_problem.remediation.contains("trust"), - "not-trusted remediation should mention the project hook trust/review requirement: {}", - trust_problem.remediation - ); - - std::fs::remove_file(codex_hooks_dir.join("hooks.json")).unwrap(); - - let groups_after_delete = - collect_codex_integration_groups(&root, &[], &trust_context, &allowed_policy()); - let mut problems = Vec::new(); - inspect_codex_integration_health(&groups_after_delete, &mut problems); - - let hooks_scope = - IntegrationGroupKey::new(IntegrationTarget::Codex, IntegrationArea::Hooks); - let hooks_problem = problems - .iter() - .find(|problem| { - problem.kind == ProblemKind::CodexIntegrationFilesMissing - && problem.scope == Some(hooks_scope) - }) - .expect("a missing Codex hook registration problem was reported"); - assert!( - hooks_problem.remediation.contains("trust"), - "Codex hooks remediation should mention the project hook trust/review requirement: {}", - hooks_problem.remediation - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn codex_hooks_json_stale_registration_is_repaired_preserving_unrelated_user_content() { - let root = unique_temp_repository_root("codex-hooks-fix"); - let codex_hooks_dir = root.join(".codex"); - std::fs::create_dir_all(&codex_hooks_dir).unwrap(); - - let existing = stale_stop_registration_fixture(); - let hooks_json_path = codex_hooks_dir.join("hooks.json"); - std::fs::write( - &hooks_json_path, - serde_json::to_vec_pretty(&existing).unwrap(), - ) - .unwrap(); - - let trust_context = deterministic_untrusted_context("codex-hooks-fix"); - let groups = - collect_codex_integration_groups(&root, &[], &trust_context, &allowed_policy()); - let stop_child = groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == ".codex/hooks.json#Stop") - .expect(".codex/hooks.json#Stop child present"); - assert_eq!(stop_child.content_state, IntegrationContentState::Stale); - - let fix_results = super::repair_merge_target_configs(&root, &allowed_policy()); - assert!( - fix_results - .iter() - .any(|result| matches!(result.outcome, super::FixResult::Fixed)), - "expected the stale Codex Stop registration to be repaired: {fix_results:?}" - ); - - let repaired: serde_json::Value = - serde_json::from_slice(&std::fs::read(&hooks_json_path).unwrap()).unwrap(); - assert_eq!(repaired["description"], "user hooks"); - assert_eq!( - repaired["hooks"]["SessionStart"][0]["hooks"][0]["command"], "echo user session hook", - "unrelated user hooks must survive the repair" - ); - - let groups_after_fix = - collect_codex_integration_groups(&root, &[], &trust_context, &allowed_policy()); - for suffix in [ - "UserPromptSubmit", - "Stop", - "PreToolUse(Bash)", - "PostToolUse(apply_patch)", - ] { - let child = groups_after_fix - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == format!(".codex/hooks.json#{suffix}")) - .unwrap_or_else(|| panic!("expected a .codex/hooks.json#{suffix} child")); - assert_eq!( - child.content_state, - IntegrationContentState::NotTrusted("untrusted".to_string()), - "repair only fixes structure; a never-trusted registration stays not-trusted \ - rather than becoming falsely healthy" - ); - } - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn codex_hooks_json_repair_never_runs_for_not_trusted_only_drift() { - let root = unique_temp_repository_root("codex-hooks-no-fix"); - let codex_hooks_dir = root.join(".codex"); - std::fs::create_dir_all(&codex_hooks_dir).unwrap(); - std::fs::write( - codex_hooks_dir.join("hooks.json"), - embedded_codex_asset_bytes(".codex/hooks.json"), - ) - .unwrap(); - - let fix_results = super::repair_merge_target_configs(&root, &allowed_policy()); - assert!( - fix_results.is_empty(), - "a structurally current but never-trusted Codex hooks.json must never trigger a \ - repair attempt: {fix_results:?}" - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn resolve_doctor_integration_targets_detects_codex_directory() { - let root = unique_temp_repository_root("codex-detect"); - std::fs::create_dir_all(root.join(".codex")).unwrap(); - - let targets = resolve_doctor_integration_targets(&root); - assert!( - targets.contains(&IntegrationTargetId::Codex), - "a repo-root .codex/ directory should be detected without a configured target" - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn claude_settings_reports_match_despite_extra_user_permissions() { - let root = unique_temp_repository_root("claude-pass"); - let claude_dir = root.join(".claude"); - std::fs::create_dir_all(&claude_dir).unwrap(); - - let generated_bytes = embedded_claude_settings_bytes(); - let installed_bytes = - crate::services::setup::config_merge::merge_or_create_claude_settings( - None, - generated_bytes, - "settings.json", - ) - .unwrap(); - let mut installed: serde_json::Value = serde_json::from_slice(&installed_bytes).unwrap(); - installed["permissions"] = serde_json::json!({"allow": ["Bash(git *)"]}); - std::fs::write( - claude_dir.join("settings.json"), - serde_json::to_vec_pretty(&installed).unwrap(), - ) - .unwrap(); - - let groups = collect_claude_integration_groups(&root, &[]); - let settings_child = groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "settings.json") - .expect("settings.json child present"); - assert!(matches!( - settings_child.content_state, - IntegrationContentState::Match - )); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn claude_settings_reports_mismatch_when_sce_hook_entry_deleted_then_fix_repairs_it() { - let root = unique_temp_repository_root("claude-fix"); - let claude_dir = root.join(".claude"); - std::fs::create_dir_all(&claude_dir).unwrap(); - - let generated_bytes = embedded_claude_settings_bytes(); - let installed_bytes = - crate::services::setup::config_merge::merge_or_create_claude_settings( - None, - generated_bytes, - "settings.json", - ) - .unwrap(); - let mut drifted: serde_json::Value = serde_json::from_slice(&installed_bytes).unwrap(); - drifted["permissions"] = serde_json::json!({"allow": ["Bash(git *)"]}); - // Drop every hook event's entries to simulate a deleted SCE hook entry. - for (_, entries) in drifted["hooks"].as_object_mut().unwrap() { - *entries = serde_json::json!([]); - } - let settings_path = claude_dir.join("settings.json"); - std::fs::write(&settings_path, serde_json::to_vec_pretty(&drifted).unwrap()).unwrap(); - - let groups = collect_claude_integration_groups(&root, &[]); - let settings_child = groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "settings.json") - .expect("settings.json child present"); - assert!(matches!( - settings_child.content_state, - IntegrationContentState::Mismatch - )); - - let fix_results = super::repair_merge_target_configs(&root, &allowed_policy()); - assert!( - fix_results - .iter() - .any(|result| matches!(result.outcome, super::FixResult::Fixed)), - "expected the drifted settings.json to be repaired" - ); - - let repaired: serde_json::Value = - serde_json::from_slice(&std::fs::read(&settings_path).unwrap()).unwrap(); - assert_eq!(repaired["permissions"]["allow"][0], "Bash(git *)"); - - let groups_after_fix = collect_claude_integration_groups(&root, &[]); - let settings_child_after_fix = groups_after_fix - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "settings.json") - .expect("settings.json child present"); - assert!(matches!( - settings_child_after_fix.content_state, - IntegrationContentState::Match - )); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn claude_settings_doctor_repairs_historical_bun_hooks_through_merge_path() { - let root = unique_temp_repository_root("claude-legacy-bun-fix"); - let claude_dir = root.join(".claude"); - std::fs::create_dir_all(&claude_dir).unwrap(); - - let existing = serde_json::json!({ - "$schema": "https://old.example/schema.json", - "permissions": {"allow": ["Bash(git *)"]}, - "hooks": { - "SessionStart": [ - { - "hooks": [{ - "type": "command", - "command": "bun", - "args": [".claude/plugins/my-company-hook.ts"] - }] - }, - legacy_claude_agent_trace_hook("SessionStart") - ], - "UserPromptSubmit": [legacy_claude_agent_trace_hook("UserPromptSubmit")], - "PostToolUse": [legacy_claude_agent_trace_hook("PostToolUse")], - "Stop": [legacy_claude_agent_trace_hook("Stop")] - } - }); - let settings_path = claude_dir.join("settings.json"); - std::fs::write( - &settings_path, - serde_json::to_vec_pretty(&existing).unwrap(), - ) - .unwrap(); - - let before = collect_claude_integration_groups(&root, &[]); - let before_child = before - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "settings.json") - .expect("settings.json child present before repair"); - assert_eq!( - before_child.content_state, - IntegrationContentState::Mismatch - ); - - let fix_results = super::repair_merge_target_configs(&root, &allowed_policy()); - assert!( - fix_results - .iter() - .any(|result| matches!(result.outcome, super::FixResult::Fixed)), - "doctor --fix should repair historical SCE hooks: {fix_results:?}" - ); - - let repaired: serde_json::Value = - serde_json::from_slice(&std::fs::read(&settings_path).unwrap()).unwrap(); - assert_eq!(repaired["permissions"]["allow"][0], "Bash(git *)"); - assert_eq!( - repaired["hooks"]["SessionStart"][0]["hooks"][0]["args"][0], - ".claude/plugins/my-company-hook.ts" - ); - assert!(!repaired - .to_string() - .contains(".claude/plugins/sce-agent-trace.ts")); - assert_eq!( - repaired["hooks"]["SessionStart"] - .as_array() - .unwrap() - .iter() - .filter(|entry| { - entry["hooks"][0]["command"] - .as_str() - .is_some_and(|command| command.contains("sce hooks claude-model-state")) - }) - .count(), - 1 - ); - - let after = collect_claude_integration_groups(&root, &[]); - let after_child = after - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "settings.json") - .expect("settings.json child present after repair"); - assert_eq!(after_child.content_state, IntegrationContentState::Match); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn claude_settings_reports_missing_lifecycle_hook_individually_then_fix_repairs_it() { - for missing_event in ["SessionStart", "PostModelSwitch"] { - let root = unique_temp_repository_root(&format!("claude-lifecycle-{missing_event}")); - let claude_dir = root.join(".claude"); - std::fs::create_dir_all(&claude_dir).unwrap(); - - let generated_bytes = embedded_claude_settings_bytes(); - let installed_bytes = - crate::services::setup::config_merge::merge_or_create_claude_settings( - None, - generated_bytes, - "settings.json", - ) - .unwrap(); - let mut drifted: serde_json::Value = serde_json::from_slice(&installed_bytes).unwrap(); - drifted["hooks"][missing_event] = serde_json::json!([]); - let settings_path = claude_dir.join("settings.json"); - std::fs::write(&settings_path, serde_json::to_vec_pretty(&drifted).unwrap()).unwrap(); - - let groups = collect_claude_integration_groups(&root, &[]); - let settings_child = groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "settings.json") - .expect("settings.json child present"); - assert_eq!( - settings_child.content_state, - IntegrationContentState::Mismatch, - "doctor should report missing {missing_event} as drift" - ); - - let fix_results = super::repair_merge_target_configs(&root, &allowed_policy()); - assert!( - fix_results - .iter() - .any(|result| matches!(result.outcome, super::FixResult::Fixed)), - "doctor --fix should repair missing {missing_event}: {fix_results:?}" - ); - - let repaired: serde_json::Value = - serde_json::from_slice(&std::fs::read(&settings_path).unwrap()).unwrap(); - assert_eq!( - repaired["hooks"][missing_event][0]["hooks"][0]["command"], - r#"bash "$CLAUDE_PROJECT_DIR/.claude/hooks/run-sce-or-show-install-guidance.sh" sce hooks claude-model-state"# - ); - assert!(collect_claude_integration_groups(&root, &[]) - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "settings.json") - .is_some_and(|child| { child.content_state == IntegrationContentState::Match })); - - std::fs::remove_dir_all(&root).ok(); - } - } - - #[test] - fn opencode_config_reports_match_despite_extra_user_plugin_then_drift_and_fix() { - let root = unique_temp_repository_root("opencode-fix"); - let opencode_dir = root.join(".opencode"); - std::fs::create_dir_all(&opencode_dir).unwrap(); - - let generated_bytes = embedded_opencode_config_bytes(); - let installed_bytes = - crate::services::setup::config_merge::merge_or_create_opencode_config( - None, - generated_bytes, - "opencode.json", - ) - .unwrap(); - let mut installed: serde_json::Value = serde_json::from_slice(&installed_bytes).unwrap(); - installed["model"] = serde_json::json!("anthropic/claude"); - installed["plugin"] - .as_array_mut() - .unwrap() - .insert(0, serde_json::json!("./plugins/my-plugin.ts")); - let manifest_path = opencode_dir.join("opencode.json"); - std::fs::write( - &manifest_path, - serde_json::to_vec_pretty(&installed).unwrap(), - ) - .unwrap(); - - let groups = collect_opencode_integration_groups(&root, &[]); - let manifest_child = groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "opencode.json") - .expect("opencode.json child present"); - assert!(matches!( - manifest_child.content_state, - IntegrationContentState::Match - )); - - // Drop the plugin array entirely to simulate a stale/removed SCE registration. - installed["plugin"] = serde_json::json!(["./plugins/my-plugin.ts"]); - std::fs::write( - &manifest_path, - serde_json::to_vec_pretty(&installed).unwrap(), - ) - .unwrap(); - - let drifted_groups = collect_opencode_integration_groups(&root, &[]); - let drifted_child = drifted_groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == "opencode.json") - .expect("opencode.json child present"); - assert!(matches!( - drifted_child.content_state, - IntegrationContentState::Mismatch - )); - - let fix_results = super::repair_merge_target_configs(&root, &allowed_policy()); - assert!( - fix_results - .iter() - .any(|result| matches!(result.outcome, super::FixResult::Fixed)), - "expected the drifted opencode.json to be repaired" - ); - - let repaired: serde_json::Value = - serde_json::from_slice(&std::fs::read(&manifest_path).unwrap()).unwrap(); - assert_eq!(repaired["model"], "anthropic/claude"); - let plugin = repaired["plugin"].as_array().unwrap(); - assert!(plugin.contains(&serde_json::json!("./plugins/my-plugin.ts"))); - assert!(plugin.contains(&serde_json::json!("./plugins/sce-bash-policy.ts"))); - assert!(plugin.contains(&serde_json::json!("./plugins/sce-agent-trace.ts"))); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn opencode_plugin_ordering_health_flags_mutation_scope_not_last() { - let root = unique_temp_repository_root("opencode-plugin-order"); - let opencode_dir = root.join(".opencode"); - std::fs::create_dir_all(&opencode_dir).unwrap(); - let manifest_path = opencode_dir.join("opencode.json"); - - let ordered = serde_json::json!({ - "$schema": "https://opencode.ai/config.json", - "plugin": [ - "./plugins/my-plugin.ts", - "./plugins/sce-bash-policy.ts", - "./plugins/sce-agent-trace.ts", - "./plugins/sce-mutation-scope.ts" - ] - }); - std::fs::write(&manifest_path, serde_json::to_vec_pretty(&ordered).unwrap()).unwrap(); - let mut problems = Vec::new(); - inspect_opencode_plugin_ordering_health(&root, &mut problems); - assert!( - problems.is_empty(), - "mutation-scope last is a valid ordering: {problems:?}" - ); - - let misordered = serde_json::json!({ - "$schema": "https://opencode.ai/config.json", - "plugin": [ - "./plugins/sce-bash-policy.ts", - "./plugins/sce-mutation-scope.ts", - "./plugins/sce-agent-trace.ts", - "./plugins/my-plugin.ts" - ] - }); - std::fs::write( - &manifest_path, - serde_json::to_vec_pretty(&misordered).unwrap(), - ) - .unwrap(); - let mut problems = Vec::new(); - inspect_opencode_plugin_ordering_health(&root, &mut problems); - assert_eq!(problems.len(), 1, "misordered registry must be flagged"); - assert!(matches!( - problems[0].kind, - ProblemKind::OpenCodePluginRegistryInvalid - )); - assert!(problems[0].summary.contains("sce-agent-trace.ts")); - - std::fs::remove_dir_all(&root).ok(); - } - - fn canonical_pre_commit_bytes() -> &'static [u8] { - crate::services::setup::iter_required_hook_assets() - .find(|asset| asset.relative_path == "pre-commit") - .expect("embedded catalog carries pre-commit") - .bytes - } - - #[cfg(unix)] - fn mark_executable(path: &std::path::Path) { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)) - .expect("mark hook executable"); - } - - #[test] - fn hook_with_foreign_content_and_current_block_reports_current() { - let dir = unique_temp_repository_root("hook-foreign-current"); - let foreign_prefix = b"#!/bin/sh\necho husky-style-guard\n".to_vec(); - let merge = crate::services::setup::hook_merge::merge_or_create_hook( - Some(&foreign_prefix), - canonical_pre_commit_bytes(), - "pre-commit", - ) - .expect("merge over foreign hook should succeed"); - - let hook_path = dir.join("pre-commit"); - std::fs::write(&hook_path, &merge.bytes).expect("write foreign-plus-block hook"); - #[cfg(unix)] - mark_executable(&hook_path); - - let health = collect_hook_file_health(&dir); - let pre_commit = health - .iter() - .find(|hook| hook.name == "pre-commit") - .expect("pre-commit health present"); - assert_eq!(pre_commit.content_state, HookContentState::Current); - - std::fs::remove_dir_all(&dir).ok(); - } - - #[test] - fn hook_with_drifted_managed_block_reports_stale() { - let dir = unique_temp_repository_root("hook-drifted-stale"); - let canonical_text = - String::from_utf8(canonical_pre_commit_bytes().to_vec()).expect("hook is utf8"); - let drifted_text = canonical_text.replace( - "sce hooks pre-commit \"$@\"", - "sce hooks pre-commit \"$@\" # drifted", - ); - assert_ne!( - drifted_text, canonical_text, - "drift fixture should actually differ from canonical" - ); - - let hook_path = dir.join("pre-commit"); - std::fs::write(&hook_path, drifted_text.as_bytes()).expect("write drifted hook"); - #[cfg(unix)] - mark_executable(&hook_path); - - let health = collect_hook_file_health(&dir); - let pre_commit = health - .iter() - .find(|hook| hook.name == "pre-commit") - .expect("pre-commit health present"); - assert_eq!(pre_commit.content_state, HookContentState::Stale); - - std::fs::remove_dir_all(&dir).ok(); - } - - fn init_git_repo(label: &str) -> PathBuf { - let repo = unique_temp_repository_root(label); - let output = std::process::Command::new("git") - .args(["init", "-q"]) - .current_dir(&repo) - .output() - .expect("git init should spawn"); - assert!( - output.status.success(), - "git init failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - repo - } - - #[test] - fn fix_repairs_drifted_hook_content_while_preserving_foreign_content() { - let repo = init_git_repo("hook-fix-repair"); - - let initial_outcome = crate::services::setup::install_required_git_hooks(&repo) - .expect("initial hook install should succeed"); - let pre_commit_path = initial_outcome - .hook_results - .iter() - .find(|result| result.hook_name == "pre-commit") - .expect("pre-commit hook installed") - .hook_path - .clone(); - let hooks_directory = pre_commit_path - .parent() - .expect("hook path has a parent directory") - .to_path_buf(); - - let foreign_prefix = b"#!/bin/sh\necho husky-style-guard\n".to_vec(); - let foreign_plus_block = crate::services::setup::hook_merge::merge_or_create_hook( - Some(&foreign_prefix), - canonical_pre_commit_bytes(), - "pre-commit", - ) - .expect("merge over foreign hook should succeed"); - let drifted_text = String::from_utf8(foreign_plus_block.bytes.clone()) - .expect("hook is utf8") - .replace( - "sce hooks pre-commit \"$@\"", - "sce hooks pre-commit \"$@\" # drifted", - ); - std::fs::write(&pre_commit_path, drifted_text.as_bytes()) - .expect("seed foreign-plus-drifted-block hook"); - #[cfg(unix)] - mark_executable(&pre_commit_path); - - let health_before = collect_hook_file_health(&hooks_directory); - let pre_commit_before = health_before - .iter() - .find(|hook| hook.name == "pre-commit") - .expect("pre-commit health present"); - assert_eq!(pre_commit_before.content_state, HookContentState::Stale); - - crate::services::setup::install_required_git_hooks(&repo) - .expect("'--fix' repair reuses the canonical setup hook installation"); - - let repaired_bytes = std::fs::read(&pre_commit_path).expect("read repaired hook"); - assert!( - repaired_bytes.starts_with(&foreign_prefix), - "foreign content should survive the repair" - ); - - let health_after = collect_hook_file_health(&hooks_directory); - let pre_commit_after = health_after - .iter() - .find(|hook| hook.name == "pre-commit") - .expect("pre-commit health present"); - assert_eq!(pre_commit_after.content_state, HookContentState::Current); - - std::fs::remove_dir_all(&repo).ok(); - } - - // -- Codex hook-discovery *policy* readiness (T22/AC28 repair) ---------- - - fn bare_command_handler_json() -> serde_json::Value { - serde_json::json!({ - "type": "command", - "command": "true" - }) - } - - /// A `Stop` registration whose command identifies it as SCE-owned but - /// whose JSON shape does not match the canonical generated handler - /// (extra `timeout`, no `async`), so structural diagnosis reports - /// `Stale`. Reused everywhere a deterministic stale fixture is needed. - fn stale_stop_registration_fixture() -> serde_json::Value { - serde_json::json!({ - "description": "user hooks", - "hooks": { - "Stop": [{"hooks": [ - { - "type": "command", - "command": "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex", - "timeout": 30 - } - ]}], - "SessionStart": [{"hooks": [{"type": "command", "command": "echo user session hook"}]}] - } - }) - } - - /// Writes `[hooks.state.""]` plus - /// `body` verbatim into a fresh `$CODEX_HOME/config.toml`, mirroring - /// `codex_hook_trust::tests::write_state_toml`, so doctor-level tests can - /// drive `codex_hook_registration_child`'s trust branch deterministically - /// without depending on the real `$CODEX_HOME`. - fn write_trust_state( - dir: &std::path::Path, - label: &str, - hooks_json_path: &std::path::Path, - event: &str, - position: (usize, usize), - body: &str, - ) -> codex_hook_trust::TrustContext { - let absolute = std::fs::canonicalize(hooks_json_path).unwrap(); - let key = format!( - "{}:{}:{}:{}", - absolute.display(), - codex_hook_config::hook_event_key_label(event), - position.0, - position.1 - ); - let escaped_key = key.replace('\\', "\\\\").replace('"', "\\\""); - let codex_home = dir.join(format!("codex-home-{label}")); - std::fs::create_dir_all(&codex_home).unwrap(); - std::fs::write( - codex_home.join("config.toml"), - format!("[hooks.state.\"{escaped_key}\"]\n{body}\n"), - ) - .unwrap(); - codex_hook_trust::TrustContext { - codex_home: Some(codex_home), - } - } - - fn present_and_current_diagnosis( - event: &'static str, - matcher: Option<&'static str>, - handler: serde_json::Value, - ) -> codex_hook_config::RegistrationDiagnosis { - codex_hook_config::RegistrationDiagnosis { - command: codex_hook_config::CodexHookCommand::Codex, - event, - matcher, - state: codex_hook_config::RegistrationStructuralState::PresentAndCurrent, - owned_handler: Some(handler), - position: Some((0, 0)), - } - } - - fn mutation_scope_diagnosis( - event: &'static str, - state: codex_hook_config::RegistrationStructuralState, - handler: Option, - position: Option<(usize, usize)>, - ) -> codex_hook_config::RegistrationDiagnosis { - codex_hook_config::RegistrationDiagnosis { - command: codex_hook_config::CodexHookCommand::MutationScope, - event, - matcher: None, - state, - owned_handler: handler, - position, - } - } - - #[test] - fn registration_child_is_match_when_trusted_and_policy_allows_project_hooks() { - let root = unique_temp_repository_root("policy-trusted-match"); - let hooks_json_path = root.join("hooks.json"); - std::fs::write(&hooks_json_path, "{}").unwrap(); - let handler = bare_command_handler_json(); - let hash = codex_hook_trust::hash_command_handler("Stop", None, &handler).unwrap(); - let trust_context = write_trust_state( - &root, - "trusted", - &hooks_json_path, - "Stop", - (0, 0), - &format!("trusted_hash = \"{hash}\""), - ); - let diagnosis = present_and_current_diagnosis("Stop", None, handler); - - let child = codex_hook_registration_child( - &hooks_json_path, - &diagnosis, - &trust_context, - &CodexHookPolicyReadiness::ProjectHooksAllowed, - ); - assert_eq!(child.content_state, IntegrationContentState::Match); - - std::fs::remove_dir_all(&root).ok(); - } - - /// The main regression test for this repair: a structurally current, - /// fully trusted registration must still report `PolicyBlocked` (not - /// `Match`) when Codex's effective `allow_managed_hooks_only` policy - /// excludes project hooks, and the resulting problem must name policy, - /// never trust, as the cause. - #[test] - fn registration_child_is_policy_blocked_even_when_structurally_current_and_trusted() { - let root = unique_temp_repository_root("policy-blocked-trusted"); - let hooks_json_path = root.join("hooks.json"); - std::fs::write(&hooks_json_path, "{}").unwrap(); - let handler = bare_command_handler_json(); - let hash = codex_hook_trust::hash_command_handler("Stop", None, &handler).unwrap(); - let trust_context = write_trust_state( - &root, - "blocked-but-trusted", - &hooks_json_path, - "Stop", - (0, 0), - &format!("trusted_hash = \"{hash}\""), - ); - let diagnosis = present_and_current_diagnosis("Stop", None, handler); - - let child = codex_hook_registration_child( - &hooks_json_path, - &diagnosis, - &trust_context, - &CodexHookPolicyReadiness::PolicyBlocked, - ); - assert!( - matches!( - child.content_state, - IntegrationContentState::PolicyBlocked(_) - ), - "a trusted handler must still report PolicyBlocked when Codex's effective policy \ - excludes project hooks: {:?}", - child.content_state - ); - assert_ne!( - child.content_state, - IntegrationContentState::Match, - "trust alone must never make a policy-blocked registration report healthy" - ); - - let groups = vec![IntegrationGroupHealth::new( - IntegrationGroupKey::new(IntegrationTarget::Codex, IntegrationArea::Hooks), - vec![child], - )]; - let mut problems = Vec::new(); - inspect_codex_integration_health(&groups, &mut problems); - let policy_problem = problems - .iter() - .find(|problem| problem.kind == ProblemKind::CodexHookRegistrationPolicyBlocked) - .expect("a policy-blocked problem was reported"); - assert_eq!(policy_problem.severity, ProblemSeverity::Error); - assert!( - !problems - .iter() - .any(|problem| problem.kind == ProblemKind::CodexHookRegistrationNotTrusted), - "policy blocking must short-circuit before trust is ever reported as the cause" - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn registration_child_modified_trust_is_unaffected_by_allowed_policy() { - let root = unique_temp_repository_root("policy-allowed-modified"); - let hooks_json_path = root.join("hooks.json"); - std::fs::write(&hooks_json_path, "{}").unwrap(); - let handler = bare_command_handler_json(); - let trust_context = write_trust_state( - &root, - "modified", - &hooks_json_path, - "Stop", - (0, 0), - "trusted_hash = \"sha256:stale\"", - ); - let diagnosis = present_and_current_diagnosis("Stop", None, handler); - - let child = codex_hook_registration_child( - &hooks_json_path, - &diagnosis, - &trust_context, - &CodexHookPolicyReadiness::ProjectHooksAllowed, - ); - assert_eq!( - child.content_state, - IntegrationContentState::NotTrusted("modified".to_string()) - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn registration_child_disabled_trust_is_unaffected_by_allowed_policy() { - let root = unique_temp_repository_root("policy-allowed-disabled"); - let hooks_json_path = root.join("hooks.json"); - std::fs::write(&hooks_json_path, "{}").unwrap(); - let handler = bare_command_handler_json(); - let hash = codex_hook_trust::hash_command_handler("Stop", None, &handler).unwrap(); - let trust_context = write_trust_state( - &root, - "disabled", - &hooks_json_path, - "Stop", - (0, 0), - &format!("trusted_hash = \"{hash}\"\nenabled = false"), - ); - let diagnosis = present_and_current_diagnosis("Stop", None, handler); - - let child = codex_hook_registration_child( - &hooks_json_path, - &diagnosis, - &trust_context, - &CodexHookPolicyReadiness::ProjectHooksAllowed, - ); - assert_eq!( - child.content_state, - IntegrationContentState::NotTrusted("disabled".to_string()) - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn registration_child_is_policy_unknown_when_the_probe_could_not_determine_policy() { - let root = unique_temp_repository_root("policy-unknown"); - let hooks_json_path = root.join("hooks.json"); - std::fs::write(&hooks_json_path, "{}").unwrap(); - let handler = bare_command_handler_json(); - let hash = codex_hook_trust::hash_command_handler("Stop", None, &handler).unwrap(); - let trust_context = write_trust_state( - &root, - "unknown-policy", - &hooks_json_path, - "Stop", - (0, 0), - &format!("trusted_hash = \"{hash}\""), - ); - let diagnosis = present_and_current_diagnosis("Stop", None, handler); - - let child = codex_hook_registration_child( - &hooks_json_path, - &diagnosis, - &trust_context, - &CodexHookPolicyReadiness::Unknown("codex executable not found".to_string()), - ); - match &child.content_state { - IntegrationContentState::PolicyUnknown(reason) => { - assert!(reason.contains("codex executable not found")); - } - other => panic!("expected PolicyUnknown, got {other:?}"), - } - assert_ne!(child.content_state, IntegrationContentState::Match); - - let groups = vec![IntegrationGroupHealth::new( - IntegrationGroupKey::new(IntegrationTarget::Codex, IntegrationArea::Hooks), - vec![child], - )]; - let mut problems = Vec::new(); - inspect_codex_integration_health(&groups, &mut problems); - let unknown_problem = problems - .iter() - .find(|problem| problem.kind == ProblemKind::CodexHookRegistrationPolicyUnknown) - .expect("a policy-unknown problem was reported"); - assert_eq!(unknown_problem.severity, ProblemSeverity::Warning); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn a_mutation_scope_registration_gets_the_full_three_dimension_health_model() { - let root = unique_temp_repository_root("codex-mutation-scope-3d"); - let hooks_json_path = root.join("hooks.json"); - std::fs::write(&hooks_json_path, "{}").unwrap(); - let handler = bare_command_handler_json(); - - let missing = mutation_scope_diagnosis( - "SessionEnd", - codex_hook_config::RegistrationStructuralState::Missing, - None, - None, - ); - assert_eq!( - codex_hook_registration_child( - &hooks_json_path, - &missing, - &deterministic_untrusted_context("ms-missing"), - &allowed_policy(), - ) - .content_state, - IntegrationContentState::Missing - ); - - let stale = mutation_scope_diagnosis( - "SessionEnd", - codex_hook_config::RegistrationStructuralState::Stale, - Some(handler.clone()), - Some((0, 1)), - ); - assert_eq!( - codex_hook_registration_child( - &hooks_json_path, - &stale, - &deterministic_untrusted_context("ms-stale"), - &allowed_policy(), - ) - .content_state, - IntegrationContentState::Stale - ); - - let present = mutation_scope_diagnosis( - "SessionEnd", - codex_hook_config::RegistrationStructuralState::PresentAndCurrent, - Some(handler.clone()), - Some((0, 0)), - ); - - let untrusted = codex_hook_registration_child( - &hooks_json_path, - &present, - &deterministic_untrusted_context("ms-untrusted"), - &allowed_policy(), - ); - assert_eq!( - untrusted.content_state, - IntegrationContentState::NotTrusted("untrusted".to_string()) - ); - assert_eq!( - untrusted.relative_path, - ".codex/hooks.json#SessionEnd(mutation-scope)" - ); - - let policy_blocked = codex_hook_registration_child( - &hooks_json_path, - &present, - &deterministic_untrusted_context("ms-blocked"), - &CodexHookPolicyReadiness::PolicyBlocked, - ); - assert!(matches!( - policy_blocked.content_state, - IntegrationContentState::PolicyBlocked(_) - )); - - let policy_unknown = codex_hook_registration_child( - &hooks_json_path, - &present, - &deterministic_untrusted_context("ms-unknown"), - &CodexHookPolicyReadiness::Unknown("codex executable not found".to_string()), - ); - assert!(matches!( - policy_unknown.content_state, - IntegrationContentState::PolicyUnknown(_) - )); - - let hash = codex_hook_trust::hash_command_handler("SessionEnd", None, &handler).unwrap(); - let trust_context = write_trust_state( - &root, - "ms-trusted", - &hooks_json_path, - "SessionEnd", - (0, 0), - &format!("trusted_hash = \"{hash}\""), - ); - assert_eq!( - codex_hook_registration_child( - &hooks_json_path, - &present, - &trust_context, - &allowed_policy(), - ) - .content_state, - IntegrationContentState::Match - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn mutation_scope_and_sce_hooks_codex_registrations_report_distinct_readiness() { - let root = unique_temp_repository_root("codex-mutation-scope-distinct"); - let codex_dir = root.join(".codex"); - std::fs::create_dir_all(&codex_dir).unwrap(); - let hooks_json_path = codex_dir.join("hooks.json"); - std::fs::write( - &hooks_json_path, - embedded_codex_asset_bytes(".codex/hooks.json"), - ) - .unwrap(); - - let generated = embedded_codex_asset_bytes(".codex/hooks.json"); - let codex_stop_handler: serde_json::Value = { - let document: serde_json::Value = serde_json::from_slice(generated).unwrap(); - document["hooks"]["Stop"][0]["hooks"][0].clone() - }; - let hash = - codex_hook_trust::hash_command_handler("Stop", None, &codex_stop_handler).unwrap(); - let trust_context = write_trust_state( - &root, - "codex-stop-trusted", - &hooks_json_path, - "Stop", - (0, 0), - &format!("trusted_hash = \"{hash}\""), - ); - - let groups = - collect_codex_integration_groups(&root, &[], &trust_context, &allowed_policy()); - let child = |suffix: &str| { - groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == format!(".codex/hooks.json#{suffix}")) - .unwrap_or_else(|| panic!("expected a .codex/hooks.json#{suffix} child")) - .content_state - .clone() - }; - - assert_eq!(child("Stop"), IntegrationContentState::Match); - assert_eq!( - child("Stop(mutation-scope)"), - IntegrationContentState::NotTrusted("untrusted".to_string()) - ); - assert_eq!( - child("Interrupt(mutation-scope)"), - IntegrationContentState::NotTrusted("untrusted".to_string()) - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn structural_missing_state_wins_over_a_blocked_or_unknown_policy() { - let root = absent_repository_root(); - for policy in [ - CodexHookPolicyReadiness::PolicyBlocked, - CodexHookPolicyReadiness::Unknown("probe failed".to_string()), - ] { - let groups = collect_codex_integration_groups( - &root, - &[], - &deterministic_untrusted_context("structural-missing-wins"), - &policy, - ); - let registration_children = groups - .iter() - .flat_map(|group| &group.children) - .filter(|child| child.relative_path.starts_with(".codex/hooks.json#")) - .collect::>(); - assert_eq!( - registration_children.len(), - codex_hook_config::required_registrations().len() - ); - for child in registration_children { - assert_eq!( - child.content_state, - IntegrationContentState::Missing, - "a missing registration must stay Missing regardless of policy: {policy:?}" - ); - } - } - } - - #[test] - fn structural_stale_state_wins_over_a_blocked_policy() { - let root = unique_temp_repository_root("codex-hooks-stale-policy"); - let codex_hooks_dir = root.join(".codex"); - std::fs::create_dir_all(&codex_hooks_dir).unwrap(); - std::fs::write( - codex_hooks_dir.join("hooks.json"), - serde_json::to_vec_pretty(&stale_stop_registration_fixture()).unwrap(), - ) - .unwrap(); - - let trust_context = deterministic_untrusted_context("codex-hooks-stale-policy"); - let groups = collect_codex_integration_groups( - &root, - &[], - &trust_context, - &CodexHookPolicyReadiness::PolicyBlocked, - ); - let stop_child = groups - .iter() - .flat_map(|group| &group.children) - .find(|child| child.relative_path == ".codex/hooks.json#Stop") - .expect(".codex/hooks.json#Stop child present"); - assert_eq!( - stop_child.content_state, - IntegrationContentState::Stale, - "a stale registration must not be reclassified as PolicyBlocked" - ); - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn policy_readiness_is_reused_unchanged_across_all_four_registrations() { - let root = unique_temp_repository_root("codex-hooks-policy-reused"); - let codex_hooks_dir = root.join(".codex"); - std::fs::create_dir_all(&codex_hooks_dir).unwrap(); - std::fs::write( - codex_hooks_dir.join("hooks.json"), - embedded_codex_asset_bytes(".codex/hooks.json"), - ) - .unwrap(); - - let trust_context = deterministic_untrusted_context("codex-hooks-policy-reused"); - let groups = collect_codex_integration_groups( - &root, - &[], - &trust_context, - &CodexHookPolicyReadiness::PolicyBlocked, - ); - let registration_children = groups - .iter() - .flat_map(|group| &group.children) - .filter(|child| child.relative_path.starts_with(".codex/hooks.json#")) - .collect::>(); - assert_eq!( - registration_children.len(), - codex_hook_config::required_registrations().len() - ); - for child in registration_children { - assert!( - matches!( - child.content_state, - IntegrationContentState::PolicyBlocked(_) - ), - "the single probed policy value must apply identically to every \ - registration, not be re-probed per registration: '{}' was {:?}", - child.relative_path, - child.content_state - ); - } - - std::fs::remove_dir_all(&root).ok(); - } - - #[test] - fn fix_never_modifies_policy_or_trust_state_for_a_structurally_current_document() { - let root = unique_temp_repository_root("codex-hooks-fix-policy-noop"); - let codex_hooks_dir = root.join(".codex"); - std::fs::create_dir_all(&codex_hooks_dir).unwrap(); - std::fs::write( - codex_hooks_dir.join("hooks.json"), - embedded_codex_asset_bytes(".codex/hooks.json"), - ) - .unwrap(); - - for policy in [ - CodexHookPolicyReadiness::PolicyBlocked, - CodexHookPolicyReadiness::Unknown("probe failed".to_string()), - ] { - let fix_results = super::repair_merge_target_configs(&root, &policy); - assert!( - fix_results.is_empty(), - "'--fix' must never attempt to repair a structurally current document merely \ - because policy is blocked/unknown ({policy:?}): {fix_results:?}" - ); - } - - std::fs::remove_dir_all(&root).ok(); - } - - fn init_git_repo_with_claude_target(label: &str) -> PathBuf { - let repo = init_git_repo(label); - std::fs::create_dir_all(repo.join(".claude")).expect("create .claude directory"); - repo - } - - fn claude_state_path(repo: &std::path::Path) -> PathBuf { - claude_mutation_scope::state::state_path( - &super::resolve_git_dir(repo).expect("resolve git dir"), - ) - } - - fn claude_manual_only_blocked_state() -> AdapterState { - AdapterState { - version: 1, - next_attempt_seq: 2, - recovery_pending: true, - attempts: vec![AdapterAttempt { - attempt_seq: 1, - scope_id: "claude|s=1:a|c=1:b".to_string(), - session_id: "session-a".to_string(), - agent_id: None, - tool_use_id: "tool-use-a".to_string(), - tool_name: "Edit".to_string(), - phase: AttemptPhase::Active, - }], - } - } - - fn claude_autofixable_blocked_state() -> AdapterState { - let key = crate::services::hooks::claude_mutation_scope::AttemptKey { - session_id: "session-a".to_string(), - agent_id: None, - tool_use_id: "tool-use-a".to_string(), - }; - let scope_id = - crate::services::hooks::claude_mutation_scope::format_claude_scope_id(1, &key); - AdapterState { - version: 1, - next_attempt_seq: 2, - recovery_pending: true, - attempts: vec![AdapterAttempt { - attempt_seq: 1, - scope_id, - session_id: key.session_id, - agent_id: key.agent_id, - tool_use_id: key.tool_use_id, - tool_name: "Edit".to_string(), - phase: AttemptPhase::PendingAbandon, - }], - } - } - - fn claude_recovering_state() -> AdapterState { - AdapterState { - version: 1, - next_attempt_seq: 1, - recovery_pending: true, - attempts: Vec::new(), - } - } - - #[derive(Clone, Copy, Debug)] - enum ClaudeStateFixture { - Absent, - Recovering, - BlockedAutoFixable, - BlockedManualOnly, - Invalid, - } - - fn seed_claude_state(repo: &std::path::Path, fixture: ClaudeStateFixture) { - let path = claude_state_path(repo); - let state = match fixture { - ClaudeStateFixture::Absent => { - std::fs::remove_file(&path).ok(); - return; - } - ClaudeStateFixture::Recovering => claude_recovering_state(), - ClaudeStateFixture::BlockedAutoFixable => claude_autofixable_blocked_state(), - ClaudeStateFixture::BlockedManualOnly => claude_manual_only_blocked_state(), - ClaudeStateFixture::Invalid => { - write_state_bytes(&path, b"not json"); - return; - } - }; - write_state_bytes( - &path, - &serde_json::to_vec(&state).expect("serialize adapter state"), - ); - } - - fn write_state_bytes(path: &std::path::Path, bytes: &[u8]) { - std::fs::create_dir_all(path.parent().expect("state path has a parent")) - .expect("create sce state directory"); - std::fs::write(path, bytes).expect("write adapter state file"); - } - - fn assert_manual_only_remediation( - remediation: &str, - state_path: &std::path::Path, - context: &str, - ) { - assert!( - remediation.contains(&state_path.display().to_string()), - "{context}: manual-only remediation must name the real state file path: {remediation}" - ); - let lowered = remediation.to_ascii_lowercase(); - assert!( - !lowered.contains("delete"), - "{context}: manual-only remediation must never contain deletion wording: {remediation}" - ); - assert!( - !lowered.contains("run 'sce doctor --fix'"), - "{context}: manual-only remediation must never recommend 'sce doctor --fix': {remediation}" - ); - } - - #[test] - fn mutation_scope_health_maps_adapter_health_to_doctor_problem_and_readiness() { - let cases = [ - ( - ClaudeStateFixture::Absent, - MutationScopeHealthStatus::Healthy, - None, - Readiness::Ready, - ), - ( - ClaudeStateFixture::Recovering, - MutationScopeHealthStatus::Recovering, - Some(( - ProblemKind::MutationScopeHealthRecovering, - ProblemSeverity::Warning, - ProblemFixability::NoActionRequired, - "no_action_required", - )), - Readiness::Ready, - ), - ( - ClaudeStateFixture::BlockedAutoFixable, - MutationScopeHealthStatus::Blocked, - Some(( - ProblemKind::MutationScopeHealthBlocked, - ProblemSeverity::Error, - ProblemFixability::AutoFixable, - "doctor_fix", - )), - Readiness::NotReady, - ), - ( - ClaudeStateFixture::BlockedManualOnly, - MutationScopeHealthStatus::Blocked, - Some(( - ProblemKind::MutationScopeHealthBlocked, - ProblemSeverity::Error, - ProblemFixability::ManualOnly, - "manual_steps", - )), - Readiness::NotReady, - ), - ( - ClaudeStateFixture::Invalid, - MutationScopeHealthStatus::Invalid, - Some(( - ProblemKind::MutationScopeHealthInvalid, - ProblemSeverity::Error, - ProblemFixability::ManualOnly, - "manual_steps", - )), - Readiness::NotReady, - ), - ]; - - let repo = init_git_repo_with_claude_target("mutation-scope-health-mapping"); - let state_path = claude_state_path(&repo); - - for (fixture, status, expected_problem, readiness) in cases { - let context = format!("{fixture:?}"); - seed_claude_state(&repo, fixture); - - let mut problems = Vec::new(); - let rows = inspect_mutation_scope_health(true, false, Some(&repo), &mut problems); - - assert_eq!(rows.len(), 1, "{context}: {rows:?}"); - assert_eq!(rows[0].target, IntegrationTarget::ClaudeCode, "{context}"); - assert_eq!(rows[0].status, status, "{context}"); - assert_eq!(compute_readiness(&problems), readiness, "{context}"); - - let Some((kind, severity, fixability, next_action)) = expected_problem else { - assert!(problems.is_empty(), "{context}: {problems:?}"); - assert_eq!(rows[0].remediation, None, "{context}"); - continue; - }; - - assert_eq!(problems.len(), 1, "{context}: {problems:?}"); - let problem = &problems[0]; - assert_eq!(problem.kind, kind, "{context}"); - assert_eq!( - problem.category, - ProblemCategory::MutationScopeHealth, - "{context}" - ); - assert_eq!(problem.severity, severity, "{context}"); - assert_eq!(problem.fixability, fixability, "{context}"); - assert_eq!(problem.next_action, next_action, "{context}"); - assert_eq!( - problem.mutation_scope_target, - Some(IntegrationTarget::ClaudeCode), - "{context}" - ); - assert_eq!( - rows[0].remediation.as_deref(), - Some(problem.remediation.as_str()), - "{context}: the health row and the problem must carry the same remediation" - ); - - assert_remediation_matches_fixability(problem, &state_path, &context); - } - - std::fs::remove_dir_all(&repo).ok(); - } - - fn assert_remediation_matches_fixability( - problem: &DoctorProblem, - state_path: &std::path::Path, - context: &str, - ) { - match problem.fixability { - ProblemFixability::AutoFixable => assert!( - problem.remediation.contains("Run 'sce doctor --fix'"), - "{context}: auto-fixable remediation must recommend 'sce doctor --fix': {}", - problem.remediation - ), - ProblemFixability::ManualOnly => { - assert_manual_only_remediation(&problem.remediation, state_path, context); - } - ProblemFixability::NoActionRequired => assert!( - !problem.remediation.contains("--fix"), - "{context}: a no-action-required remediation must not suggest '--fix': {}", - problem.remediation - ), - } - } - - #[test] - fn recovering_remediation_never_promises_a_specific_guaranteed_next_event() { - let repo = init_git_repo_with_claude_target("mutation-scope-health-recovering-wording"); - seed_claude_state(&repo, ClaudeStateFixture::Recovering); - - let mut problems = Vec::new(); - inspect_mutation_scope_health(true, false, Some(&repo), &mut problems); - - assert_eq!(problems.len(), 1); - let remediation = problems[0].remediation.to_ascii_lowercase(); - assert!( - !remediation.contains("next mutation-capable tool call"), - "generic Recovering remediation must not promise that the next \ - mutation-capable tool call clears recovery: {remediation}" - ); - assert!( - !remediation.contains("the next tool call"), - "generic Recovering remediation must not name a specific guaranteed next \ - event: {remediation}" - ); - assert!( - remediation.contains("no manual action is required"), - "generic Recovering remediation must say no manual action is required: {remediation}" - ); - - std::fs::remove_dir_all(&repo).ok(); - } - - #[test] - fn mutation_scope_health_is_absent_for_an_unconfigured_undetected_target() { - let repo = init_git_repo("mutation-scope-health-no-targets"); - - let mut problems = Vec::new(); - let rows = inspect_mutation_scope_health(true, false, Some(&repo), &mut problems); - - assert!( - rows.is_empty(), - "no integration target is configured/detected, so no health row is expected: {rows:?}" - ); - assert!(problems.is_empty()); - - std::fs::remove_dir_all(&repo).ok(); - } - - #[test] - fn mutation_scope_health_dispatches_all_four_configured_targets() { - let repo = init_git_repo("mutation-scope-health-all-targets"); - for dir in [".claude", ".codex", ".opencode", ".pi"] { - std::fs::create_dir_all(repo.join(dir)).expect("create integration directory"); - } - let git_dir = super::resolve_git_dir(&repo).expect("resolve git dir"); - let targets = [ - ( - IntegrationTarget::ClaudeCode, - claude_mutation_scope::state::state_path(&git_dir), - ), - ( - IntegrationTarget::Codex, - super::codex_mutation_scope::state::state_path(&git_dir), - ), - ( - IntegrationTarget::OpenCode, - super::opencode_mutation_scope::state::state_path(&git_dir), - ), - ( - IntegrationTarget::Pi, - super::pi_mutation_scope::state::state_path(&git_dir), - ), - ]; - - let mut problems = Vec::new(); - let rows = inspect_mutation_scope_health(true, false, Some(&repo), &mut problems); - - assert_eq!( - rows.len(), - 4, - "expected exactly one mutation-scope health row per resolved target: {rows:?}" - ); - for (target, _) in &targets { - let matching = rows - .iter() - .filter(|row| row.target == *target) - .collect::>(); - assert_eq!( - matching.len(), - 1, - "expected exactly one row for {target:?}: {rows:?}" - ); - assert_eq!(matching[0].status, MutationScopeHealthStatus::Healthy); - } - assert!( - problems.is_empty(), - "four healthy adapters must not produce any doctor problem: {problems:?}" - ); - - for (target, state_path) in &targets { - write_state_bytes(state_path, b"not json"); - - let mut problems = Vec::new(); - let rows = inspect_mutation_scope_health(true, false, Some(&repo), &mut problems); - - let unhealthy_targets = rows - .iter() - .filter(|row| row.status != MutationScopeHealthStatus::Healthy) - .map(|row| row.target) - .collect::>(); - assert_eq!( - unhealthy_targets, - vec![*target], - "only the adapter whose own state file is unreadable may be reported unhealthy: \ - {rows:?}" - ); - assert_eq!(problems.len(), 1, "{target:?}: {problems:?}"); - assert_eq!(problems[0].mutation_scope_target, Some(*target)); - assert!( - problems[0] - .remediation - .contains(&state_path.display().to_string()), - "{target:?} remediation must name that adapter's own state path: {}", - problems[0].remediation - ); - - std::fs::remove_file(state_path).expect("remove unreadable state file"); - } - - std::fs::remove_dir_all(&repo).ok(); - } - - static FULL_REPORT_ENV_GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - fn with_isolated_global_state(f: impl FnOnce() -> R) -> R { - let guard = FULL_REPORT_ENV_GUARD - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - let sandbox = unique_temp_repository_root("full-report-xdg-sandbox"); - let state_home = sandbox.join("state"); - let config_home = sandbox.join("config"); - let cache_home = sandbox.join("cache"); - let home = sandbox.join("home"); - std::fs::create_dir_all(&home).expect("create sandbox home"); - std::fs::create_dir_all(&state_home).expect("create sandbox state home"); - std::fs::create_dir_all(&config_home).expect("create sandbox config home"); - std::fs::create_dir_all(&cache_home).expect("create sandbox cache home"); - - let prior_state = std::env::var_os("XDG_STATE_HOME"); - let prior_config = std::env::var_os("XDG_CONFIG_HOME"); - let prior_cache = std::env::var_os("XDG_CACHE_HOME"); - let prior_no_color = std::env::var_os("NO_COLOR"); - let prior_home = std::env::var_os("HOME"); - - unsafe { - std::env::set_var("HOME", &home); - std::env::set_var("XDG_STATE_HOME", &state_home); - std::env::set_var("XDG_CONFIG_HOME", &config_home); - std::env::set_var("XDG_CACHE_HOME", &cache_home); - std::env::set_var("NO_COLOR", "1"); - } - - let result = f(); - - unsafe { - match prior_state { - Some(value) => std::env::set_var("XDG_STATE_HOME", value), - None => std::env::remove_var("XDG_STATE_HOME"), - } - match prior_config { - Some(value) => std::env::set_var("XDG_CONFIG_HOME", value), - None => std::env::remove_var("XDG_CONFIG_HOME"), - } - match prior_cache { - Some(value) => std::env::set_var("XDG_CACHE_HOME", value), - None => std::env::remove_var("XDG_CACHE_HOME"), - } - match prior_no_color { - Some(value) => std::env::set_var("NO_COLOR", value), - None => std::env::remove_var("NO_COLOR"), - } - match prior_home { - Some(value) => std::env::set_var("HOME", value), - None => std::env::remove_var("HOME"), - } - } - - std::fs::remove_dir_all(&sandbox).ok(); - drop(guard); - result - } - - fn init_git_repo_with_healthy_targets(label: &str, targets: &[SetupTarget]) -> PathBuf { - let repo = init_git_repo(label); - let remote_output = std::process::Command::new("git") - .args([ - "remote", - "add", - "origin", - &format!("https://example.invalid/{label}.git"), - ]) - .current_dir(&repo) - .output() - .expect("git remote add should spawn"); - assert!( - remote_output.status.success(), - "git remote add failed: {}", - String::from_utf8_lossy(&remote_output.stderr) - ); - crate::services::setup::install_required_git_hooks(&repo) - .expect("install canonical git hooks"); - for target in targets { - crate::services::setup::run_setup_for_mode( - &repo, - crate::services::setup::SetupMode::NonInteractive(*target), - None, - None, - None, - ) - .expect("install canonical integration assets"); - } - run_full_doctor_report(&repo, super::DoctorMode::Fix); - repo - } - - struct FullReportRepoContext { - repo_root: PathBuf, - } - - impl crate::app::HasRepoRoot for FullReportRepoContext { - fn repo_root(&self) -> Option<&std::path::Path> { - Some(&self.repo_root) - } - } - - fn run_full_doctor_report( - repo: &std::path::Path, - mode: super::DoctorMode, - ) -> super::super::DoctorExecution { - run_full_doctor_report_with_seam(repo, mode, &super::mutation_scope_repair_seam) - } - - fn run_full_doctor_report_with_seam( - repo: &std::path::Path, - mode: super::DoctorMode, - mutation_scope_seam: super::MutationScopeRepairSeam<'_>, - ) -> super::super::DoctorExecution { - let context = FullReportRepoContext { - repo_root: repo.to_path_buf(), - }; - super::super::execute_doctor_with_context( - super::super::DoctorRequest { - mode, - format: super::super::DoctorFormat::Text, - }, - repo, - &context, - mutation_scope_seam, - ) - } - - fn render_text(execution: &super::super::DoctorExecution) -> String { - super::super::render_report( - super::super::DoctorRequest { - mode: execution.report.mode, - format: super::super::DoctorFormat::Text, - }, - execution, - ) - .expect("text rendering should succeed") - } - - fn render_json(execution: &super::super::DoctorExecution) -> serde_json::Value { - let rendered = super::super::render_report( - super::super::DoctorRequest { - mode: execution.report.mode, - format: super::super::DoctorFormat::Json, - }, - execution, - ) - .expect("json rendering should succeed"); - serde_json::from_str(&rendered).expect("doctor JSON output should parse") - } - - fn mutation_scope_fix_results( - execution: &super::super::DoctorExecution, - ) -> Vec<&DoctorFixResultRecord> { - execution - .fix_results - .iter() - .filter(|result| result.category == ProblemCategory::MutationScopeHealth) - .collect() - } - - fn no_op_repair_seam() -> super::MutationScopeRepairSeam<'static> { - &|_root, _payload, _logger| Ok(String::new()) - } - - fn failing_repair_seam() -> super::MutationScopeRepairSeam<'static> { - &|_root, _payload, _logger| Err(anyhow::anyhow!("simulated seam failure")) - } - - fn counting_repair_seam( - calls: &std::cell::Cell, - ) -> impl Fn(&std::path::Path, &str, Option<&dyn super::Logger>) -> anyhow::Result + '_ - { - move |_root, _payload, _logger| { - calls.set(calls.get() + 1); - Ok(String::new()) - } - } - - struct DiagnoseSurfaceCase { - fixture: ClaudeStateFixture, - readiness: Readiness, - json_readiness: &'static str, - text_row: &'static str, - text_summary: &'static str, - json_status: &'static str, - json_severity: &'static str, - json_fixability: &'static str, - } - - const DIAGNOSE_SURFACE_CASES: [DiagnoseSurfaceCase; 3] = [ - DiagnoseSurfaceCase { - fixture: ClaudeStateFixture::Recovering, - readiness: Readiness::Ready, - json_readiness: "ready", - text_row: "[WARN] Agent tracing", - text_summary: "Summary: 0 blocking problem(s), 1 warning(s)", - json_status: "recovering", - json_severity: "warning", - json_fixability: "no_action_required", - }, - DiagnoseSurfaceCase { - fixture: ClaudeStateFixture::BlockedAutoFixable, - readiness: Readiness::NotReady, - json_readiness: "not_ready", - text_row: "[FAIL] Agent tracing", - text_summary: "Summary: 1 blocking problem(s), 0 warning(s)", - json_status: "blocked", - json_severity: "error", - json_fixability: "auto_fixable", - }, - DiagnoseSurfaceCase { - fixture: ClaudeStateFixture::BlockedManualOnly, - readiness: Readiness::NotReady, - json_readiness: "not_ready", - text_row: "[FAIL] Agent tracing", - text_summary: "Summary: 1 blocking problem(s), 0 warning(s)", - json_status: "blocked", - json_severity: "error", - json_fixability: "manual_only", - }, - ]; - - #[test] - fn full_report_diagnose_is_consistent_across_surfaces_and_never_mutates_state() { - with_isolated_global_state(|| { - let repo = - init_git_repo_with_healthy_targets("full-report-diagnose", &[SetupTarget::Claude]); - let state_path = claude_state_path(&repo); - - for case in &DIAGNOSE_SURFACE_CASES { - let context = format!("{:?}", case.fixture); - seed_claude_state(&repo, case.fixture); - let seeded_bytes = std::fs::read(&state_path).expect("read seeded state"); - - let execution = run_full_doctor_report(&repo, super::DoctorMode::Diagnose); - - assert!(execution.fix_results.is_empty(), "{context}"); - assert_eq!( - std::fs::read(&state_path).expect("read state after diagnose"), - seeded_bytes, - "{context}: diagnose mode must never mutate mutation-scope state" - ); - - assert_eq!(execution.report.readiness, case.readiness, "{context}"); - assert_eq!( - execution.report.problems.len(), - 1, - "{context}: {:#?}", - execution.report.problems - ); - let problem = &execution.report.problems[0]; - assert!( - problem - .summary - .contains(&format!("agent tracing is {}", case.json_status)), - "{context}: problem summary must use human-facing 'agent tracing' wording: {}", - problem.summary - ); - assert!( - !problem.summary.contains("mutation-scope health"), - "{context}: problem summary must not expose internal 'mutation-scope health' \ - wording: {}", - problem.summary - ); - - let text = render_text(&execution); - assert!(text.contains(case.text_row), "{context}: {text}"); - assert!(text.contains(case.text_summary), "{context}: {text}"); - assert!( - text.contains(&problem.remediation), - "{context}: human text must show the problem remediation: {text}" - ); - - let json = render_json(&execution); - assert_eq!(json["readiness"], case.json_readiness, "{context}"); - let health_rows = json["mutation_scope_health"] - .as_array() - .expect("mutation_scope_health is an array"); - assert_eq!(health_rows.len(), 1, "{context}"); - assert_eq!(health_rows[0]["status"], case.json_status, "{context}"); - assert!(health_rows[0]["reason"].is_string(), "{context}"); - - let json_problems = json["problems"].as_array().expect("problems is an array"); - assert_eq!(json_problems.len(), 1, "{context}: {json_problems:?}"); - let json_problem = &json_problems[0]; - assert_eq!(json_problem["category"], "mutation_scope_health"); - assert_eq!(json_problem["severity"], case.json_severity, "{context}"); - assert_eq!( - json_problem["fixability"], case.json_fixability, - "{context}" - ); - assert_eq!( - json_problem["remediation"]["next_action"], problem.next_action, - "{context}" - ); - assert_eq!( - json_problem["remediation"]["text"], problem.remediation, - "{context}" - ); - } - - std::fs::remove_dir_all(&repo).ok(); - }); - } - - fn assert_doctor_repairs_autofixable_target( - repo: &std::path::Path, - target: IntegrationTarget, - state_path: &std::path::Path, - fixed_detail_prefix: &str, - ) { - let git_dir = super::resolve_git_dir(repo).expect("resolve git dir"); - - let mut initial_problems = Vec::new(); - inspect_mutation_scope_health(true, false, Some(repo), &mut initial_problems); - assert_eq!(initial_problems.len(), 1, "{initial_problems:?}"); - assert_eq!(initial_problems[0].mutation_scope_target, Some(target)); - assert_eq!( - initial_problems[0].fixability, - ProblemFixability::AutoFixable - ); - assert_eq!(initial_problems[0].next_action, "doctor_fix"); - let blocked_bytes = std::fs::read(state_path).expect("read blocked state"); - - let attempted = super::repair_blocked_mutation_scope_target( - target, - &git_dir, - repo, - no_op_repair_seam(), - ); - assert_eq!( - attempted, - Some(target), - "an autofixable blocked target must have a repair attempted" - ); - assert_ne!( - std::fs::read(state_path).expect("read repaired state"), - blocked_bytes, - "the adapter repair must have rewritten the persisted state" - ); - - let mut final_problems = Vec::new(); - let final_rows = - inspect_mutation_scope_health(true, false, Some(repo), &mut final_problems); - assert_eq!(final_rows.len(), 1, "{final_rows:?}"); - assert_eq!(final_rows[0].target, target); - assert!( - !matches!( - final_rows[0].status, - MutationScopeHealthStatus::Blocked | MutationScopeHealthStatus::Invalid - ), - "a repaired target must not remain blocked or invalid: {final_rows:?}" - ); - - let records = super::finalize_mutation_scope_repair_results(&[target], &final_rows); - assert_eq!(records.len(), 1, "{records:?}"); - assert_eq!(records[0].category, ProblemCategory::MutationScopeHealth); - assert_eq!(records[0].outcome, FixResult::Fixed); - assert!( - records[0].detail.starts_with(fixed_detail_prefix), - "fix-result detail must match the documented '[fixed] Recovered Agent \ - tracing (...)' contract: {}", - records[0].detail - ); - } - - #[test] - fn repair_blocked_mutation_scope_target_repairs_an_autofixable_claude_state() { - let repo = init_git_repo_with_claude_target("repair-target-claude-auto"); - seed_claude_state(&repo, ClaudeStateFixture::BlockedAutoFixable); - - assert_doctor_repairs_autofixable_target( - &repo, - IntegrationTarget::ClaudeCode, - &claude_state_path(&repo), - "Recovered Claude Code Agent tracing (", - ); - - std::fs::remove_dir_all(&repo).ok(); - } - - fn init_git_repo_with_opencode_target(label: &str) -> PathBuf { - let repo = init_git_repo(label); - std::fs::create_dir_all(repo.join(".opencode")).expect("create .opencode directory"); - repo - } - - fn opencode_dead_owner() -> crate::services::hooks::mutation_scope_owner::ProcessOwner { - let mut child = std::process::Command::new("true") - .spawn() - .expect("spawning 'true' should succeed"); - let pid = i32::try_from(child.id()).expect("pid fits in i32"); - child.wait().expect("child should exit and be reaped"); - crate::services::hooks::mutation_scope_owner::ProcessOwner { - pid, - instance_token: None, - } - } - - fn seed_opencode_manual_only_blocked_state( - git_dir: &std::path::Path, - ) -> crate::services::hooks::opencode_mutation_scope::state::AdapterAttempt { - crate::services::hooks::opencode_mutation_scope::state::seed_attempt_for_tests( - git_dir, - &crate::services::hooks::opencode_mutation_scope::AttemptKey { - session_id: "ses-main".to_string(), - call_id: "call-1".to_string(), - }, - "write", - crate::services::hooks::opencode_mutation_scope::state::AttemptPhase::PendingStart, - ) - } - - #[test] - fn repair_blocked_mutation_scope_target_repairs_an_autofixable_opencode_state() { - let repo = init_git_repo_with_opencode_target("repair-target-opencode-auto"); - let git_dir = super::resolve_git_dir(&repo).expect("resolve git dir"); - let attempt = seed_opencode_manual_only_blocked_state(&git_dir); - crate::services::hooks::opencode_mutation_scope::state::set_attempt_owner_for_tests( - &git_dir, - &attempt.scope_id, - Some(opencode_dead_owner()), - ); - - assert_doctor_repairs_autofixable_target( - &repo, - IntegrationTarget::OpenCode, - &super::opencode_mutation_scope::state::state_path(&git_dir), - "Recovered OpenCode Agent tracing (", - ); - - std::fs::remove_dir_all(&repo).ok(); - } - - #[test] - fn repair_blocked_mutation_scope_target_never_attempts_or_mutates_a_manual_only_state() { - let repo = init_git_repo("repair-target-manual-only"); - for dir in [".claude", ".opencode"] { - std::fs::create_dir_all(repo.join(dir)).expect("create integration directory"); - } - let git_dir = super::resolve_git_dir(&repo).expect("resolve git dir"); - seed_claude_state(&repo, ClaudeStateFixture::BlockedManualOnly); - seed_opencode_manual_only_blocked_state(&git_dir); - - let seam_calls = std::cell::Cell::new(0); - let seam = counting_repair_seam(&seam_calls); - - for (target, state_path) in [ - ( - IntegrationTarget::ClaudeCode, - claude_mutation_scope::state::state_path(&git_dir), - ), - ( - IntegrationTarget::OpenCode, - super::opencode_mutation_scope::state::state_path(&git_dir), - ), - ] { - let blocked_bytes = std::fs::read(&state_path).expect("read blocked state"); - - let attempted = - super::repair_blocked_mutation_scope_target(target, &git_dir, &repo, &seam); - - assert_eq!( - attempted, None, - "a ManualOnly blocked {target:?} state must never have a repair attempted" - ); - assert_eq!( - std::fs::read(&state_path).expect("read state after dispatch"), - blocked_bytes, - "a ManualOnly blocked {target:?} state must never be mutated by the repair dispatch" - ); - } - assert_eq!( - seam_calls.get(), - 0, - "a ManualOnly blocked state must never reach the adapter repair seam" - ); - - std::fs::remove_dir_all(&repo).ok(); - } - - fn minimal_report( - problems: Vec, - mutation_scope_health: Vec, - ) -> HookDoctorReport { - HookDoctorReport { - mode: DoctorMode::Fix, - readiness: compute_readiness(&problems), - state_root: None, - agent_trace_db: None, - repository_root: None, - hook_path_source: HookPathSource::Default, - hooks_directory: None, - post_commit_auto_sync: PostCommitAutoSyncHealth { - state: PostCommitAutoSyncState::NotApplicable, - enabled: false, - source: "test", - config_source: None, - }, - config_locations: Vec::new(), - hooks: Vec::new(), - integration_groups: Vec::new(), - integration_targets_absent: false, - mutation_scope_health, - problems, - } - } - - fn combined_mutation_scope_fix_results( - report: &HookDoctorReport, - attempted_targets: &[IntegrationTarget], - ) -> Vec { - let mut results = super::finalize_mutation_scope_repair_results( - attempted_targets, - &report.mutation_scope_health, - ); - results.extend(build_manual_fix_results(report, attempted_targets)); - results - .into_iter() - .filter(|result| result.category == ProblemCategory::MutationScopeHealth) - .collect() - } - - #[derive(Clone, Copy, Debug)] - enum RepairSeamBehavior { - Succeeds, - Fails, - } - - struct FinalizationCase { - label: &'static str, - initial: ClaudeStateFixture, - seam: RepairSeamBehavior, - attempted: bool, - status_after_repair: MutationScopeHealthStatus, - state_before_final_inspection: Option, - final_status: MutationScopeHealthStatus, - outcome: FixResult, - } - - const FINALIZATION_CASES: [FinalizationCase; 6] = [ - FinalizationCase { - label: "repair succeeded and the final inspection is healthy", - initial: ClaudeStateFixture::BlockedAutoFixable, - seam: RepairSeamBehavior::Succeeds, - attempted: true, - status_after_repair: MutationScopeHealthStatus::Healthy, - state_before_final_inspection: None, - final_status: MutationScopeHealthStatus::Healthy, - outcome: FixResult::Fixed, - }, - FinalizationCase { - label: "repair succeeded and the final inspection is recovering", - initial: ClaudeStateFixture::BlockedAutoFixable, - seam: RepairSeamBehavior::Succeeds, - attempted: true, - status_after_repair: MutationScopeHealthStatus::Healthy, - state_before_final_inspection: Some(ClaudeStateFixture::Recovering), - final_status: MutationScopeHealthStatus::Recovering, - outcome: FixResult::Fixed, - }, - FinalizationCase { - label: "repair succeeded but the state regressed to manual-only blocked before \ - the final inspection", - initial: ClaudeStateFixture::BlockedAutoFixable, - seam: RepairSeamBehavior::Succeeds, - attempted: true, - status_after_repair: MutationScopeHealthStatus::Healthy, - state_before_final_inspection: Some(ClaudeStateFixture::BlockedManualOnly), - final_status: MutationScopeHealthStatus::Blocked, - outcome: FixResult::Manual, - }, - FinalizationCase { - label: "repair succeeded but the state became invalid before the final inspection", - initial: ClaudeStateFixture::BlockedAutoFixable, - seam: RepairSeamBehavior::Succeeds, - attempted: true, - status_after_repair: MutationScopeHealthStatus::Healthy, - state_before_final_inspection: Some(ClaudeStateFixture::Invalid), - final_status: MutationScopeHealthStatus::Invalid, - outcome: FixResult::Manual, - }, - FinalizationCase { - label: "repair was attempted but failed and the state stays blocked", - initial: ClaudeStateFixture::BlockedAutoFixable, - seam: RepairSeamBehavior::Fails, - attempted: true, - status_after_repair: MutationScopeHealthStatus::Blocked, - state_before_final_inspection: None, - final_status: MutationScopeHealthStatus::Blocked, - outcome: FixResult::Manual, - }, - FinalizationCase { - label: "manual-only blocked state is never attempted", - initial: ClaudeStateFixture::BlockedManualOnly, - seam: RepairSeamBehavior::Succeeds, - attempted: false, - status_after_repair: MutationScopeHealthStatus::Blocked, - state_before_final_inspection: None, - final_status: MutationScopeHealthStatus::Blocked, - outcome: FixResult::Manual, - }, - ]; - - #[test] - fn mutation_scope_fix_result_is_decided_only_by_the_final_inspection() { - for (index, case) in FINALIZATION_CASES.iter().enumerate() { - let label = case.label; - let repo = init_git_repo_with_claude_target(&format!("finalization-case-{index}")); - let git_dir = super::resolve_git_dir(&repo).expect("resolve git dir"); - let state_path = claude_state_path(&repo); - seed_claude_state(&repo, case.initial); - - let attempted_targets = super::repair_blocked_mutation_scope_target( - IntegrationTarget::ClaudeCode, - &git_dir, - &repo, - match case.seam { - RepairSeamBehavior::Succeeds => no_op_repair_seam(), - RepairSeamBehavior::Fails => failing_repair_seam(), - }, - ) - .into_iter() - .collect::>(); - assert_eq!( - attempted_targets, - if case.attempted { - vec![IntegrationTarget::ClaudeCode] - } else { - Vec::new() - }, - "{label}" - ); - assert_eq!( - claude_mutation_scope::health::classify_health(&git_dir).status, - case.status_after_repair, - "{label}: status read immediately after the repair dispatch" - ); - - if let Some(fixture) = case.state_before_final_inspection { - seed_claude_state(&repo, fixture); - } - - let mut problems = Vec::new(); - let final_rows = inspect_mutation_scope_health(true, false, Some(&repo), &mut problems); - assert_eq!(final_rows.len(), 1, "{label}: {final_rows:?}"); - assert_eq!(final_rows[0].status, case.final_status, "{label}"); - - let report = minimal_report(problems, final_rows); - let results = combined_mutation_scope_fix_results(&report, &attempted_targets); - - assert_eq!( - results.len(), - 1, - "{label}: exactly one mutation-scope fix result is expected per adapter: \ - {results:?}" - ); - assert_eq!(results[0].outcome, case.outcome, "{label}: {results:?}"); - match case.outcome { - FixResult::Fixed => assert!( - results[0] - .detail - .starts_with("Recovered Claude Code Agent tracing ("), - "{label}: {}", - results[0].detail - ), - FixResult::Manual => { - assert!( - results[0] - .detail - .contains(&state_path.display().to_string()), - "{label}: manual detail must name the exact persisted state path: {}", - results[0].detail - ); - assert!( - !results[0].detail.to_ascii_lowercase().contains("delete"), - "{label}: manual detail must not contain deletion wording: {}", - results[0].detail - ); - } - other => panic!("{label}: unexpected expected outcome {other:?}"), - } - - std::fs::remove_dir_all(&repo).ok(); - } - } - - #[test] - fn full_report_fix_mode_leaves_a_manual_only_claude_blocked_state_untouched() { - with_isolated_global_state(|| { - let repo = init_git_repo_with_healthy_targets( - "full-report-fix-claude-manual", - &[SetupTarget::Claude], - ); - seed_claude_state(&repo, ClaudeStateFixture::BlockedManualOnly); - let state_path = claude_state_path(&repo); - let blocked_bytes = std::fs::read(&state_path).expect("read blocked state"); - - let seam_calls = std::cell::Cell::new(0); - let execution = run_full_doctor_report_with_seam( - &repo, - super::DoctorMode::Fix, - &counting_repair_seam(&seam_calls), - ); - - assert_eq!( - seam_calls.get(), - 0, - "'sce doctor --fix' must never attempt to repair a ManualOnly blocked state" - ); - assert_eq!( - std::fs::read(&state_path).expect("read state after fix"), - blocked_bytes, - "'sce doctor --fix' must leave a ManualOnly blocked state completely untouched" - ); - - let results = mutation_scope_fix_results(&execution); - assert_eq!(results.len(), 1, "{results:?}"); - assert_eq!(results[0].outcome, FixResult::Manual); - assert_manual_only_remediation(&results[0].detail, &state_path, "manual fix result"); - assert_eq!(execution.report.readiness, Readiness::NotReady); - - let text = render_text(&execution); - assert!( - text.contains("[manual] Agent tracing remains blocked. Inspect '"), - "human fix results must show the '[manual] Agent tracing remains blocked. \ - Inspect ...' line: {text}" - ); - - std::fs::remove_dir_all(&repo).ok(); - }); - } - - #[test] - fn full_report_fix_mode_has_no_mutation_scope_effect_when_nothing_is_blocked() { - with_isolated_global_state(|| { - let repo = init_git_repo_with_healthy_targets( - "full-report-fix-nothing-blocked", - &[SetupTarget::Claude], - ); - let state_path = claude_state_path(&repo); - - for fixture in [ClaudeStateFixture::Absent, ClaudeStateFixture::Recovering] { - seed_claude_state(&repo, fixture); - let seeded_bytes = std::fs::read(&state_path).ok(); - - let seam_calls = std::cell::Cell::new(0); - let execution = run_full_doctor_report_with_seam( - &repo, - super::DoctorMode::Fix, - &counting_repair_seam(&seam_calls), - ); - - assert_eq!( - seam_calls.get(), - 0, - "{fixture:?}: 'sce doctor --fix' must not attempt a mutation-scope repair \ - when nothing is blocked" - ); - assert_eq!( - std::fs::read(&state_path).ok(), - seeded_bytes, - "{fixture:?}: 'sce doctor --fix' must not mutate non-blocked mutation-scope \ - state" - ); - assert!( - mutation_scope_fix_results(&execution).is_empty(), - "{fixture:?}: no mutation-scope fix result is expected when nothing is \ - blocked: {:?}", - execution.fix_results - ); - assert_eq!(execution.report.readiness, Readiness::Ready, "{fixture:?}"); - } - - std::fs::remove_dir_all(&repo).ok(); - }); - } - - #[test] - fn full_report_multi_adapter_diagnose_names_doctor_fix_for_one_target_and_the_real_path_for_the_other( - ) { - with_isolated_global_state(|| { - let repo = init_git_repo_with_healthy_targets( - "full-report-multi-adapter-diagnose", - &[SetupTarget::Claude, SetupTarget::OpenCode], - ); - seed_claude_state(&repo, ClaudeStateFixture::BlockedAutoFixable); - let git_dir = super::resolve_git_dir(&repo).expect("resolve git dir"); - seed_opencode_manual_only_blocked_state(&git_dir); - - let execution = run_full_doctor_report(&repo, super::DoctorMode::Diagnose); - assert_eq!(execution.report.readiness, Readiness::NotReady); - let mutation_scope_problems: Vec<_> = execution - .report - .problems - .iter() - .filter(|problem| problem.category == ProblemCategory::MutationScopeHealth) - .collect(); - assert_eq!( - mutation_scope_problems.len(), - 2, - "{mutation_scope_problems:#?}" - ); - - let claude_problem = mutation_scope_problems - .iter() - .find(|problem| { - problem.mutation_scope_target == Some(IntegrationTarget::ClaudeCode) - }) - .expect("claude mutation-scope problem present"); - assert_eq!(claude_problem.fixability, ProblemFixability::AutoFixable); - assert_eq!(claude_problem.next_action, "doctor_fix"); - assert!(claude_problem - .remediation - .contains("Run 'sce doctor --fix'")); - - let opencode_problem = mutation_scope_problems - .iter() - .find(|problem| problem.mutation_scope_target == Some(IntegrationTarget::OpenCode)) - .expect("opencode mutation-scope problem present"); - assert_eq!(opencode_problem.fixability, ProblemFixability::ManualOnly); - assert_eq!(opencode_problem.next_action, "manual_steps"); - let opencode_state_path = super::opencode_mutation_scope::state::state_path(&git_dir); - assert_manual_only_remediation( - &opencode_problem.remediation, - &opencode_state_path, - "opencode", - ); - - let text = render_text(&execution); - assert!( - text.contains("Run 'sce doctor --fix'"), - "human text must name 'sce doctor --fix' for the autofixable claude row: {text}" - ); - assert!( - text.contains(&opencode_state_path.display().to_string()), - "human text must name the real opencode state path: {text}" - ); - - let json = render_json(&execution); - assert_eq!(json["readiness"], "not_ready"); - let json_problems = json["problems"].as_array().expect("problems is an array"); - let json_mutation_scope_problems: Vec<_> = json_problems - .iter() - .filter(|problem| problem["category"] == "mutation_scope_health") - .collect(); - assert_eq!( - json_mutation_scope_problems.len(), - 2, - "{json_mutation_scope_problems:?}" - ); - let json_autofixable = json_mutation_scope_problems - .iter() - .find(|problem| problem["fixability"] == "auto_fixable") - .expect("json autofixable problem present"); - assert_eq!(json_autofixable["remediation"]["next_action"], "doctor_fix"); - assert_eq!( - json_autofixable["remediation"]["text"], - claude_problem.remediation - ); - let json_manual_only = json_mutation_scope_problems - .iter() - .find(|problem| problem["fixability"] == "manual_only") - .expect("json manual-only problem present"); - assert_eq!( - json_manual_only["remediation"]["next_action"], - "manual_steps" - ); - assert_eq!( - json_manual_only["remediation"]["text"], - opencode_problem.remediation - ); - - std::fs::remove_dir_all(&repo).ok(); - }); - } - - #[test] - fn full_report_multi_adapter_fix_mode_resolves_one_target_and_leaves_the_other_manual() { - with_isolated_global_state(|| { - let repo = init_git_repo_with_healthy_targets( - "full-report-multi-adapter-fix", - &[SetupTarget::Claude, SetupTarget::OpenCode], - ); - seed_claude_state(&repo, ClaudeStateFixture::BlockedAutoFixable); - let git_dir = super::resolve_git_dir(&repo).expect("resolve git dir"); - seed_opencode_manual_only_blocked_state(&git_dir); - let opencode_state_path = super::opencode_mutation_scope::state::state_path(&git_dir); - let opencode_blocked_bytes = - std::fs::read(&opencode_state_path).expect("read blocked opencode state"); - - let execution = run_full_doctor_report_with_seam( - &repo, - super::DoctorMode::Fix, - no_op_repair_seam(), - ); - - let results = mutation_scope_fix_results(&execution); - assert_eq!(results.len(), 2, "{results:?}"); - let fixed = results - .iter() - .find(|result| result.outcome == FixResult::Fixed) - .expect("one of the two results is fixed"); - assert!( - fixed - .detail - .starts_with("Recovered Claude Code Agent tracing ("), - "expected the fixed result to belong to claude: {results:?}" - ); - let manual = results - .iter() - .find(|result| result.outcome == FixResult::Manual) - .expect("one of the two results is manual"); - assert_manual_only_remediation( - &manual.detail, - &opencode_state_path, - "opencode manual fix result", - ); - - assert_eq!(execution.report.readiness, Readiness::NotReady); - let final_mutation_scope_problems: Vec<_> = execution - .report - .problems - .iter() - .filter(|problem| problem.category == ProblemCategory::MutationScopeHealth) - .collect(); - assert_eq!( - final_mutation_scope_problems.len(), - 1, - "the repaired claude target must no longer appear as a problem in the final \ - report, leaving only the still-blocked opencode target: {final_mutation_scope_problems:#?}" - ); - assert_eq!( - final_mutation_scope_problems[0].mutation_scope_target, - Some(IntegrationTarget::OpenCode) - ); - assert_eq!( - final_mutation_scope_problems[0].fixability, - ProblemFixability::ManualOnly - ); - - let text = render_text(&execution); - assert!( - text.contains("[fixed] Recovered Claude Code Agent tracing ("), - "human fix results must show the claude '[fixed] Recovered ...' line: {text}" - ); - assert!( - text.contains("[manual] Agent tracing remains blocked. Inspect '"), - "human fix results must show the opencode '[manual] Agent tracing remains \ - blocked ...' line: {text}" - ); - - assert_eq!( - std::fs::read(&opencode_state_path).expect("read opencode state after fix"), - opencode_blocked_bytes, - "a ManualOnly opencode target must remain completely untouched" - ); - let final_status = |target| { - execution - .report - .mutation_scope_health - .iter() - .find(|row| row.target == target) - .map(|row| row.status) - }; - assert_eq!( - final_status(IntegrationTarget::OpenCode), - Some(MutationScopeHealthStatus::Blocked) - ); - assert!( - matches!( - final_status(IntegrationTarget::ClaudeCode), - Some( - MutationScopeHealthStatus::Healthy | MutationScopeHealthStatus::Recovering - ) - ), - "a reported fixed claude target must never remain blocked or invalid" - ); - - std::fs::remove_dir_all(&repo).ok(); - }); - } +fn doctor_git_output(git: &impl GitOps, repository_root: &Path, args: &[&str]) -> Option { + let output = git.run_command(repository_root, args).ok()?; + let trimmed = output.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) } diff --git a/cli/src/services/doctor/mod.rs b/cli/src/services/doctor/mod.rs index 4771c33eb..b42906607 100644 --- a/cli/src/services/doctor/mod.rs +++ b/cli/src/services/doctor/mod.rs @@ -1,6 +1,5 @@ use std::fs; use std::path::{Path, PathBuf}; -use std::process::Command; use anyhow::{Context, Result}; @@ -11,6 +10,10 @@ use crate::services::lifecycle::{ lifecycle_providers, FixOutcome, HealthCategory, HealthFixability, HealthProblem, HealthProblemKind, HealthSeverity, LifecycleProvider, LifecycleProviderId, }; +use crate::services::mutation_trace::runtime::{ + inspect_reconciliation_recommendation, run_explicit_reconciliation, ReconciliationFix, + ReconciliationRecommendation, +}; use crate::services::output_format::OutputFormat; use crate::services::setup; @@ -25,7 +28,7 @@ use fixes::build_manual_fix_results; use inspect::{ build_report_with_lifecycle_problems, finalize_mutation_scope_repair_results, mutation_scope_repair_seam, repair_blocked_mutation_scope_targets_with_seam, - repair_merge_target_configs, MutationScopeRepairSeam, + repair_merge_target_configs, }; use render::render_report; use types::{ @@ -51,24 +54,18 @@ pub struct DoctorRequest { pub format: DoctorFormat, } -struct DoctorDependencies<'a> { - run_git_command: &'a dyn Fn(&Path, &[&str]) -> Option, - check_git_available: &'a dyn Fn() -> bool, - resolve_state_root: &'a dyn Fn() -> Result, - resolve_global_config_path: &'a dyn Fn() -> Result, - validate_config_file: &'a dyn Fn(&Path) -> Result<()>, - /// Probes Codex's effective hook-discovery policy - /// (`allow_managed_hooks_only`). Invoked exactly once per doctor - /// invocation (see `execute_doctor_with_lifecycle_providers`) and reused - /// for every Codex integration inspection within that invocation — - /// initial report, `--fix`, and final report alike — never once per - /// registration. - probe_codex_hook_policy: &'a dyn Fn() -> codex_hook_policy::CodexHookPolicyReadiness, +struct DoctorDependencies<'a, G, S, C, P> { + git: &'a G, + resolve_state_root: &'a S, + resolve_global_config_path: &'a C, + probe_codex_hook_policy: &'a P, } struct DoctorExecution { report: HookDoctorReport, fix_results: Vec, + ref_reconciliation: Option, + ref_reconciliation_fix: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -77,9 +74,9 @@ struct ProviderDoctorProblem { problem: DoctorProblem, } -pub fn run_doctor_with_context(request: DoctorRequest, context: &C) -> Result +pub async fn run_doctor_with_context(request: DoctorRequest, context: &C) -> Result where - C: ContextWithRepoRoot, + C: ContextWithRepoRoot + crate::app::HasGit, { let repository_root = if let Some(path) = context.repo_root() { path.to_path_buf() @@ -93,49 +90,68 @@ where request, &repository_root, &scoped_context, - &mutation_scope_repair_seam, - ); + &mutation_scope_repair_seam::, + &run_explicit_reconciliation, + &inspect_reconciliation_recommendation, + ) + .await; render_report(request, &execution) } -fn execute_doctor_with_context( +async fn execute_doctor_with_context( request: DoctorRequest, repository_root: &Path, - context: &impl HasRepoRoot, - mutation_scope_seam: MutationScopeRepairSeam<'_>, + context: &(impl HasRepoRoot + crate::app::HasGit), + mutation_scope_seam: &impl std::ops::AsyncFn( + &Path, + &str, + Option<&crate::services::observability::traits::NoopLogger>, + ) -> anyhow::Result, + reconcile_seam: &impl std::ops::AsyncFn(&Path) -> ReconciliationFix, + recommendation_seam: &impl Fn(&Path) -> Option, ) -> DoctorExecution { execute_doctor_with_lifecycle_providers( request, repository_root, context, &DoctorDependencies { - run_git_command: &run_git_command, - check_git_available: &is_git_available, + git: context.git(), resolve_state_root: &resolve_state_data_root, resolve_global_config_path: &|| { Ok(resolve_sce_default_locations()?.global_config_file()) }, - validate_config_file: &crate::services::config::validate_config_file, probe_codex_hook_policy: &codex_hook_policy::probe_default, }, mutation_scope_seam, + reconcile_seam, + recommendation_seam, ) + .await } -fn execute_doctor_with_lifecycle_providers( +async fn execute_doctor_with_lifecycle_providers( request: DoctorRequest, repository_root: &Path, context: &impl HasRepoRoot, - dependencies: &DoctorDependencies<'_>, - mutation_scope_seam: MutationScopeRepairSeam<'_>, + dependencies: &DoctorDependencies< + '_, + impl crate::services::capabilities::GitOps, + impl Fn() -> Result, + impl Fn() -> Result, + impl Fn() -> crate::services::codex_hook_policy::CodexHookPolicyReadiness, + >, + mutation_scope_seam: &impl std::ops::AsyncFn( + &Path, + &str, + Option<&crate::services::observability::traits::NoopLogger>, + ) -> anyhow::Result, + reconcile_seam: &impl std::ops::AsyncFn(&Path) -> ReconciliationFix, + recommendation_seam: &impl Fn(&Path) -> Option, ) -> DoctorExecution { - // Probed exactly once per doctor invocation, then reused for every - // Codex integration inspection below (initial report, `--fix`, and final - // report alike) instead of once per registration or once per report. let policy_readiness = (dependencies.probe_codex_hook_policy)(); let providers = lifecycle_providers(true); - let initial_problems = diagnose_lifecycle_providers(context, &providers); + let initial_problems = diagnose_lifecycle_providers(context, &providers).await; let initial_doctor_problems = initial_problems .iter() .map(|problem| problem.problem.clone()) @@ -146,23 +162,26 @@ fn execute_doctor_with_lifecycle_providers( dependencies, initial_doctor_problems, &policy_readiness, - ); + ) + .await; if request.mode != DoctorMode::Fix { return DoctorExecution { report: initial_report, fix_results: Vec::new(), + ref_reconciliation: recommendation_seam(repository_root), + ref_reconciliation_fix: None, }; } - let mut fix_results = fix_lifecycle_providers(context, &providers, &initial_problems); + let mut fix_results = fix_lifecycle_providers(context, &providers, &initial_problems).await; fix_results.extend(repair_merge_target_configs( repository_root, &policy_readiness, )); let mutation_scope_repairs = - repair_blocked_mutation_scope_targets_with_seam(&initial_report, mutation_scope_seam); - let final_problems = diagnose_lifecycle_providers(context, &providers); + repair_blocked_mutation_scope_targets_with_seam(&initial_report, mutation_scope_seam).await; + let final_problems = diagnose_lifecycle_providers(context, &providers).await; let final_doctor_problems = final_problems .into_iter() .map(|problem| problem.problem) @@ -173,7 +192,8 @@ fn execute_doctor_with_lifecycle_providers( dependencies, final_doctor_problems, &policy_readiness, - ); + ) + .await; fix_results.extend(finalize_mutation_scope_repair_results( &mutation_scope_repairs, &final_report.mutation_scope_health, @@ -183,48 +203,94 @@ fn execute_doctor_with_lifecycle_providers( &mutation_scope_repairs, )); + let reconciliation_fix = reconcile_seam(repository_root).await; + fix_results.push(reconciliation_fix_result(&reconciliation_fix)); + DoctorExecution { report: final_report, fix_results, + ref_reconciliation: None, + ref_reconciliation_fix: Some(reconciliation_fix), + } +} + +fn reconciliation_fix_result(fix: &ReconciliationFix) -> DoctorFixResultRecord { + let (outcome, detail) = match fix { + ReconciliationFix::Completed(counts) => ( + FixResult::Fixed, + format!( + "Snapshot ref reconciliation completed: deleted {}, retained {}, locally required {}.", + counts.deleted, counts.retained, counts.local_required + ), + ), + ReconciliationFix::CompletedStatePersistFailed { counts, warning } => ( + FixResult::Fixed, + format!( + "Snapshot ref reconciliation completed: deleted {}, retained {}, locally required {}. Warning: {warning}", + counts.deleted, counts.retained, counts.local_required + ), + ), + ReconciliationFix::Failed { + kind, + message, + state_warning, + } => { + let mut detail = format!("Snapshot ref reconciliation failed ({kind}): {message}"); + if let Some(warning) = state_warning { + detail = format!("{detail} Warning: {warning}"); + } + (FixResult::Failed, detail) + } + ReconciliationFix::Busy => ( + FixResult::Skipped, + "Snapshot ref reconciliation skipped: another mutation boundary or maintenance pass holds the worktree lock. Retry `sce doctor --fix`.".to_string(), + ), + }; + DoctorFixResultRecord { + category: ProblemCategory::MutationScopeHealth, + outcome, + detail, } } -fn diagnose_lifecycle_providers( +async fn diagnose_lifecycle_providers( context: &impl HasRepoRoot, providers: &[LifecycleProvider], ) -> Vec { - providers - .iter() - .flat_map(|provider| { - let provider_id = provider.id(); - provider - .diagnose(context) - .into_iter() - .map(move |problem| ProviderDoctorProblem { - provider_id, - problem: doctor_problem_from_health(problem), - }) - }) - .collect() + let mut problems = Vec::new(); + for provider in providers { + let provider_id = provider.id(); + problems.extend(provider.diagnose(context).await.into_iter().map(|problem| { + ProviderDoctorProblem { + provider_id, + problem: doctor_problem_from_health(problem), + } + })); + } + problems } -fn fix_lifecycle_providers( +async fn fix_lifecycle_providers( context: &impl HasRepoRoot, providers: &[LifecycleProvider], problems: &[ProviderDoctorProblem], ) -> Vec { - providers - .iter() - .flat_map(|provider| { - let health_problems = problems - .iter() - .filter(|problem| problem.provider_id == provider.id()) - .map(|problem| health_problem_from_doctor(problem.problem.clone())) - .collect::>(); - provider.fix(context, &health_problems) - }) - .map(doctor_fix_result_from_lifecycle) - .collect() + let mut results = Vec::new(); + for provider in providers { + let health_problems = problems + .iter() + .filter(|problem| problem.provider_id == provider.id()) + .map(|problem| health_problem_from_doctor(problem.problem.clone())) + .collect::>(); + results.extend( + provider + .fix(context, &health_problems) + .await + .into_iter() + .map(doctor_fix_result_from_lifecycle), + ); + } + results } fn doctor_problem_from_health(problem: HealthProblem) -> DoctorProblem { @@ -477,13 +543,6 @@ fn health_problem_kind(kind: ProblemKind) -> HealthProblemKind { } } -fn is_git_available() -> bool { - Command::new("git") - .arg("--version") - .output() - .is_ok_and(|output| output.status.success()) -} - #[cfg(unix)] fn is_executable(metadata: &fs::Metadata) -> bool { use std::os::unix::fs::PermissionsExt; @@ -496,21 +555,132 @@ fn is_executable(metadata: &fs::Metadata) -> bool { metadata.is_file() } -fn run_git_command(repository_root: &Path, args: &[&str]) -> Option { - let output = Command::new("git") - .args(args) - .current_dir(repository_root) - .output() - .ok()?; - if !output.status.success() { - return None; +#[cfg(test)] +mod reconciliation_fix_tests { + use super::{reconciliation_fix_result, FixResult, ReconciliationFix}; + use crate::services::mutation_trace::runtime::{ReconciliationCounts, StateWarning}; + + const COUNTS: ReconciliationCounts = ReconciliationCounts { + deleted: 3, + retained: 2, + local_required: 1, + }; + + fn not_applied() -> StateWarning { + StateWarning::NotApplied { + phase: "rename", + cause: "disk full".to_string(), + } } - let stdout = String::from_utf8(output.stdout).ok()?; - let trimmed = stdout.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed.to_string()) + fn durability_uncertain() -> StateWarning { + StateWarning::DurabilityUncertain { + phase: "sync_parent_directory", + cause: "io error".to_string(), + } + } + + #[test] + fn reconciliation_fix_results_render_expected_text() { + struct Case { + label: &'static str, + fix: ReconciliationFix, + outcome: FixResult, + required: Vec<&'static str>, + forbidden: Vec<&'static str>, + } + let cases = [ + Case { + label: "completed", + fix: ReconciliationFix::Completed(COUNTS), + outcome: FixResult::Fixed, + required: vec![ + "Snapshot ref reconciliation completed: deleted 3, retained 2, locally required 1.", + ], + forbidden: vec!["Warning", "failed"], + }, + Case { + label: "pre-rename persistence failure", + fix: ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: not_applied(), + }, + outcome: FixResult::Fixed, + required: vec![ + "deleted 3", + "Maintenance-state update was not applied.", + "disk full", + ], + forbidden: vec!["were not recorded", "failed"], + }, + Case { + label: "post-rename persistence failure", + fix: ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: durability_uncertain(), + }, + outcome: FixResult::Fixed, + required: vec![ + "deleted 3", + "Maintenance-state durability could not be confirmed.", + "io error", + ], + forbidden: vec!["not applied", "were not recorded", "failed"], + }, + Case { + label: "unreadable previous state", + fix: ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::PreviousStateUnreadable { + cause: "permission denied".to_string(), + }, + }, + outcome: FixResult::Fixed, + required: vec!["deleted 3", "could not be read", "permission denied"], + forbidden: vec!["were not recorded", "failed"], + }, + Case { + label: "failed pass", + fix: ReconciliationFix::Failed { + kind: "agent_trace_db_missing", + message: "db missing".to_string(), + state_warning: Some(not_applied()), + }, + outcome: FixResult::Failed, + required: vec![ + "agent_trace_db_missing", + "db missing", + "Maintenance-state update was not applied.", + ], + forbidden: vec!["completed"], + }, + Case { + label: "busy pass", + fix: ReconciliationFix::Busy, + outcome: FixResult::Skipped, + required: vec!["skipped"], + forbidden: vec!["completed", "deleted"], + }, + ]; + for case in cases { + let record = reconciliation_fix_result(&case.fix); + assert_eq!(record.outcome, case.outcome, "{}", case.label); + for fragment in case.required { + assert!( + record.detail.contains(fragment), + "{}: missing {fragment:?} in {:?}", + case.label, + record.detail + ); + } + for fragment in case.forbidden { + assert!( + !record.detail.contains(fragment), + "{}: unexpected {fragment:?} in {:?}", + case.label, + record.detail + ); + } + } } } diff --git a/cli/src/services/doctor/render.rs b/cli/src/services/doctor/render.rs index 65fee4c25..587eea855 100644 --- a/cli/src/services/doctor/render.rs +++ b/cli/src/services/doctor/render.rs @@ -13,6 +13,9 @@ use super::types::{ PostCommitAutoSyncState, ProblemKind, ProblemSeverity, Readiness, }; use super::{DoctorExecution, DoctorFormat, DoctorMode, DoctorRequest, NAME}; +use crate::services::mutation_trace::runtime::{ + ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, StateWarning, +}; /// Guidance message rendered in the Integrations section when no integration /// targets are configured, detected, or both. @@ -48,9 +51,58 @@ fn format_execution(execution: &DoctorExecution) -> String { } } + if let Some(recommendation) = &execution.ref_reconciliation { + lines.push(format!("\n{}:", heading("Snapshot ref reconciliation"))); + lines.push(format!( + " [{}] {}", + value("recommended"), + value(&recommendation_text(recommendation)) + )); + } + lines.join("\n") } +fn recommendation_text(recommendation: &ReconciliationRecommendation) -> String { + let mut parts = vec!["Reconciliation is recommended. Run `sce doctor --fix`.".to_string()]; + if recommendation.last_attempt_failed { + parts.push(format!( + "The latest reconciliation attempt failed ({} consecutive failure(s)).", + recommendation.consecutive_failures + )); + } + if let Some(age) = recommendation.last_attempt_age_ms { + parts.push(format!( + "Last reconciliation attempt {} ago.", + format_age(age) + )); + } + if recommendation.state_unusable { + parts.push("Maintenance state is unreadable or invalid.".to_string()); + } + if recommendation.invalid_timestamps { + parts.push("Maintenance state has invalid timestamps.".to_string()); + } + if let Some(age) = recommendation.last_success_age_ms { + parts.push(format!("Last success {} ago.", format_age(age))); + } + if let Some(age) = recommendation.last_advised_age_ms { + parts.push(format!("Last reminder {} ago.", format_age(age))); + } + parts.join(" ") +} + +fn format_age(age_ms: i64) -> String { + let minutes = age_ms / 60_000; + if minutes < 60 { + format!("{minutes}m") + } else if minutes < 60 * 24 { + format!("{}h", minutes / 60) + } else { + format!("{}d", minutes / (60 * 24)) + } +} + fn format_report(report: &HookDoctorReport) -> String { format_report_with_color_policy(report, supports_color()) } @@ -887,6 +939,8 @@ fn render_report_json(execution: &DoctorExecution) -> Result { } else { Vec::new() }, + "ref_reconciliation": execution.ref_reconciliation.as_ref().map(ref_reconciliation_json), + "ref_reconciliation_fix": execution.ref_reconciliation_fix.as_ref().map(ref_reconciliation_fix_json), }); serde_json::to_string_pretty(&payload).context("failed to serialize doctor report to JSON") @@ -922,168 +976,251 @@ fn hook_content_state(state: HookContentState) -> &'static str { } } -#[cfg(test)] -mod tests { - use super::{ - mutation_scope_health_node, mutation_scope_health_status, mutation_scope_target_id, - render_display_node, DoctorDisplayStatus, IntegrationTarget, MutationScopeHealthRow, - MutationScopeHealthStatus, - }; - - fn row( - target: IntegrationTarget, - status: MutationScopeHealthStatus, - reason: &str, - detail: Option<&str>, - ) -> MutationScopeHealthRow { - row_with_remediation(target, status, reason, detail, None) - } - - fn row_with_remediation( - target: IntegrationTarget, - status: MutationScopeHealthStatus, - reason: &str, - detail: Option<&str>, - remediation: Option<&str>, - ) -> MutationScopeHealthRow { - MutationScopeHealthRow { - target, - status, - reason: reason.to_string(), - detail: detail.map(str::to_string), - remediation: remediation.map(str::to_string), - } - } +fn ref_reconciliation_json(recommendation: &ReconciliationRecommendation) -> serde_json::Value { + json!({ + "recommended": true, + "last_attempt_failed": recommendation.last_attempt_failed, + "consecutive_failures": recommendation.consecutive_failures, + "state_unusable": recommendation.state_unusable, + "invalid_timestamps": recommendation.invalid_timestamps, + "last_success_age_ms": recommendation.last_success_age_ms, + "last_attempt_age_ms": recommendation.last_attempt_age_ms, + "last_advised_age_ms": recommendation.last_advised_age_ms, + "text": recommendation_text(recommendation), + }) +} - fn rendered_lines(row: &MutationScopeHealthRow) -> Vec { - let node = mutation_scope_health_node(row); - let mut lines = Vec::new(); - render_display_node(&mut lines, &node, false, 4, true); - lines +fn ref_reconciliation_fix_json(fix: &ReconciliationFix) -> serde_json::Value { + match fix { + ReconciliationFix::Completed(counts) => json!({ + "outcome": "completed", + "counts": reconciliation_counts_json(counts), + "failure": null, + "state_warning": null, + }), + ReconciliationFix::CompletedStatePersistFailed { counts, warning } => json!({ + "outcome": "completed_state_persist_failed", + "counts": reconciliation_counts_json(counts), + "failure": null, + "state_warning": state_warning_json(warning), + }), + ReconciliationFix::Failed { + kind, + message, + state_warning, + } => json!({ + "outcome": "failed", + "counts": null, + "failure": { "kind": kind, "message": message }, + "state_warning": state_warning.as_ref().map(state_warning_json), + }), + ReconciliationFix::Busy => json!({ + "outcome": "busy", + "counts": null, + "failure": null, + "state_warning": null, + }), } +} - #[test] - fn healthy_row_collapses_to_a_single_pass_line() { - let row = row( - IntegrationTarget::ClaudeCode, - MutationScopeHealthStatus::Healthy, - "no persisted recovery problem", - None, - ); +fn reconciliation_counts_json(counts: &ReconciliationCounts) -> serde_json::Value { + json!({ + "deleted": counts.deleted, + "retained": counts.retained, + "local_required": counts.local_required, + }) +} - assert_eq!( - rendered_lines(&row), - vec![" [PASS] Agent tracing".to_string()] - ); - } +fn state_warning_json(warning: &StateWarning) -> serde_json::Value { + json!({ + "kind": warning.kind(), + "phase": warning.phase(), + "message": warning.message(), + "cause": warning.cause(), + }) +} - #[test] - fn recovering_row_expands_with_warn_and_reason() { - let row = row( - IntegrationTarget::Codex, - MutationScopeHealthStatus::Recovering, - "pending recovery with no unresolved attempts", - None, - ); - let lines = rendered_lines(&row); +#[cfg(test)] +mod reconciliation_render_tests { + use serde_json::json; - assert_eq!(lines[0], " [WARN] Agent tracing"); - assert!(lines - .iter() - .any(|line| line.contains("Reason: pending recovery with no unresolved attempts"))); - } + use super::{recommendation_text, ref_reconciliation_fix_json}; + use crate::services::mutation_trace::runtime::{ + ReconciliationCounts, ReconciliationFix, ReconciliationRecommendation, StateWarning, + }; - #[test] - fn blocked_row_expands_with_fail_reason_and_detail() { - let row = row( - IntegrationTarget::ClaudeCode, - MutationScopeHealthStatus::Blocked, - "stale attempts remain after a failed abandon", - Some("2 stale attempts"), - ); - let lines = rendered_lines(&row); + const COUNTS: ReconciliationCounts = ReconciliationCounts { + deleted: 3, + retained: 2, + local_required: 1, + }; - assert_eq!(lines[0], " [FAIL] Agent tracing"); - assert!(lines - .iter() - .any(|line| line.contains("Reason: stale attempts remain after a failed abandon"))); - assert!(lines - .iter() - .any(|line| line.contains("Detail: 2 stale attempts"))); + fn recommendation(last_attempt_age_ms: Option) -> ReconciliationRecommendation { + ReconciliationRecommendation { + invalid_timestamps: false, + state_unusable: false, + last_success_age_ms: Some(3_600_000 * 30), + last_attempt_age_ms, + last_advised_age_ms: Some(120_000), + last_attempt_failed: last_attempt_age_ms.is_some(), + consecutive_failures: 2, + } } #[test] - fn blocked_row_renders_remediation_when_present() { - let row = row_with_remediation( - IntegrationTarget::ClaudeCode, - MutationScopeHealthStatus::Blocked, - "stale attempts remain after a failed abandon", - None, - Some("Run 'sce doctor --fix' to recover this state."), + fn recommendation_shows_last_attempt_age_for_failed_recent_attempt() { + let text = recommendation_text(&recommendation(Some(12 * 60_000))); + assert!(text.contains("Last reconciliation attempt 12m ago.")); + assert!( + text.contains("The latest reconciliation attempt failed (2 consecutive failure(s)).") ); - let lines = rendered_lines(&row); - - assert!(lines.iter().any( - |line| line.contains("Remediation: Run 'sce doctor --fix' to recover this state.") - )); + assert!(text.contains("Last success 1d ago.")); + assert!(text.contains("Last reminder 2m ago.")); } #[test] - fn blocked_row_omits_remediation_line_when_absent() { - let row = row( - IntegrationTarget::ClaudeCode, - MutationScopeHealthStatus::Blocked, - "stale attempts remain after a failed abandon", - None, - ); - let lines = rendered_lines(&row); - - assert!(!lines.iter().any(|line| line.contains("Remediation:"))); + fn recommendation_omits_last_attempt_age_when_no_attempt_exists() { + let text = recommendation_text(&recommendation(None)); + assert!(!text.contains("Last reconciliation attempt")); + assert!(text.contains("Reconciliation is recommended.")); } #[test] - fn invalid_row_maps_to_fail() { - let row = row( - IntegrationTarget::Pi, - MutationScopeHealthStatus::Invalid, - "state file failed to parse", - Some("unexpected EOF"), - ); - - assert_eq!( - mutation_scope_health_node(&row).status, - DoctorDisplayStatus::Fail - ); + fn json_ref_reconciliation_fix_results_match_contract() { + let counts = json!({ "deleted": 3, "retained": 2, "local_required": 1 }); + let cases = [ + ( + "completed", + ReconciliationFix::Completed(COUNTS), + json!({ + "outcome": "completed", + "counts": counts, + "failure": null, + "state_warning": null, + }), + ), + ( + "failed without warning", + ReconciliationFix::Failed { + kind: "agent_trace_db_missing", + message: "db missing".to_string(), + state_warning: None, + }, + json!({ + "outcome": "failed", + "counts": null, + "failure": { "kind": "agent_trace_db_missing", "message": "db missing" }, + "state_warning": null, + }), + ), + ( + "failed with warning", + ReconciliationFix::Failed { + kind: "missing_required_pins", + message: "pins".to_string(), + state_warning: Some(StateWarning::NotApplied { + phase: "write_staging", + cause: "full".to_string(), + }), + }, + json!({ + "outcome": "failed", + "counts": null, + "failure": { "kind": "missing_required_pins", "message": "pins" }, + "state_warning": { + "kind": "not_applied", + "phase": "write_staging", + "message": "Maintenance-state update was not applied.", + "cause": "full", + }, + }), + ), + ( + "busy", + ReconciliationFix::Busy, + json!({ + "outcome": "busy", + "counts": null, + "failure": null, + "state_warning": null, + }), + ), + ]; + for (label, fix, expected) in cases { + assert_eq!(ref_reconciliation_fix_json(&fix), expected, "{label}"); + } } #[test] - fn json_slugs_are_stable() { - assert_eq!( - mutation_scope_target_id(IntegrationTarget::ClaudeCode), - "claude" - ); - assert_eq!( - mutation_scope_target_id(IntegrationTarget::OpenCode), - "opencode" - ); - assert_eq!(mutation_scope_target_id(IntegrationTarget::Pi), "pi"); - assert_eq!(mutation_scope_target_id(IntegrationTarget::Codex), "codex"); - - assert_eq!( - mutation_scope_health_status(MutationScopeHealthStatus::Healthy), - "healthy" - ); - assert_eq!( - mutation_scope_health_status(MutationScopeHealthStatus::Recovering), - "recovering" - ); - assert_eq!( - mutation_scope_health_status(MutationScopeHealthStatus::Blocked), - "blocked" - ); - assert_eq!( - mutation_scope_health_status(MutationScopeHealthStatus::Invalid), - "invalid" - ); + fn json_completed_state_persist_failed_results_match_contract() { + let counts = json!({ "deleted": 3, "retained": 2, "local_required": 1 }); + let cases = [ + ( + "completed + not applied", + ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::NotApplied { + phase: "rename", + cause: "denied".to_string(), + }, + }, + json!({ + "outcome": "completed_state_persist_failed", + "counts": counts, + "failure": null, + "state_warning": { + "kind": "not_applied", + "phase": "rename", + "message": "Maintenance-state update was not applied.", + "cause": "denied", + }, + }), + ), + ( + "completed + durability uncertain", + ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::DurabilityUncertain { + phase: "sync_parent_directory", + cause: "eio".to_string(), + }, + }, + json!({ + "outcome": "completed_state_persist_failed", + "counts": counts, + "failure": null, + "state_warning": { + "kind": "durability_uncertain", + "phase": "sync_parent_directory", + "message": "Maintenance-state durability could not be confirmed.", + "cause": "eio", + }, + }), + ), + ( + "completed + previous state unreadable", + ReconciliationFix::CompletedStatePersistFailed { + counts: COUNTS, + warning: StateWarning::PreviousStateUnreadable { + cause: "eacces".to_string(), + }, + }, + json!({ + "outcome": "completed_state_persist_failed", + "counts": counts, + "failure": null, + "state_warning": { + "kind": "previous_state_unreadable", + "phase": null, + "message": + "Maintenance state could not be read; the update was not attempted.", + "cause": "eacces", + }, + }), + ), + ]; + for (label, fix, expected) in cases { + assert_eq!(ref_reconciliation_fix_json(&fix), expected, "{label}"); + } } } diff --git a/cli/src/services/doctor/types.rs b/cli/src/services/doctor/types.rs index 317b2e2f9..f03489545 100644 --- a/cli/src/services/doctor/types.rs +++ b/cli/src/services/doctor/types.rs @@ -179,31 +179,6 @@ impl IntegrationGroupHealth { pub(super) const fn display_label(&self) -> &'static str { self.key.display_label() } - - #[allow(dead_code)] - pub(super) fn display_node(&self) -> DoctorDisplayNode { - let children = self - .children - .iter() - .map(IntegrationChildHealth::display_node) - .collect::>(); - let status = children - .iter() - .fold(DoctorDisplayStatus::Pass, |status, child| { - status.worst(if child.status == DoctorDisplayStatus::Miss { - DoctorDisplayStatus::Fail - } else { - child.status - }) - }); - DoctorDisplayNode::branch_with_status( - DoctorDisplayNodeKind::Group, - self.display_label(), - status, - Vec::new(), - children, - ) - } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -247,7 +222,6 @@ pub(super) enum IntegrationContentState { } #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] -#[allow(dead_code)] pub(super) enum DoctorDisplayStatus { Pass, Warn, @@ -256,7 +230,6 @@ pub(super) enum DoctorDisplayStatus { } impl DoctorDisplayStatus { - #[allow(dead_code)] pub(super) const fn worst(self, other: Self) -> Self { match (self, other) { (Self::Fail, _) | (_, Self::Fail) => Self::Fail, @@ -268,7 +241,6 @@ impl DoctorDisplayStatus { } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(super) enum DoctorDisplayDetail { MissingPath(PathBuf), ContentMismatch { @@ -309,7 +281,6 @@ pub(super) enum DoctorDisplayDetail { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(super) enum DoctorDisplayNodeKind { Domain, Group, @@ -317,7 +288,6 @@ pub(super) enum DoctorDisplayNodeKind { } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(super) struct DoctorDisplayNode { pub(super) kind: DoctorDisplayNodeKind, pub(super) label: String, @@ -327,17 +297,6 @@ pub(super) struct DoctorDisplayNode { } impl DoctorDisplayNode { - #[allow(dead_code)] - pub(super) fn domain(label: impl Into, children: Vec) -> Self { - Self::branch(DoctorDisplayNodeKind::Domain, label, children) - } - - #[allow(dead_code)] - pub(super) fn group(label: impl Into, children: Vec) -> Self { - Self::branch(DoctorDisplayNodeKind::Group, label, children) - } - - #[allow(dead_code)] fn asset( label: impl Into, status: DoctorDisplayStatus, @@ -352,11 +311,6 @@ impl DoctorDisplayNode { } } - #[allow(dead_code)] - fn branch(kind: DoctorDisplayNodeKind, label: impl Into, children: Vec) -> Self { - Self::branch_with_status(kind, label, DoctorDisplayStatus::Pass, Vec::new(), children) - } - pub(super) fn branch_with_status( kind: DoctorDisplayNodeKind, label: impl Into, @@ -388,7 +342,6 @@ impl DoctorDisplayNode { } impl IntegrationChildHealth { - #[allow(dead_code)] pub(super) fn display_node(&self) -> DoctorDisplayNode { let (status, detail) = match &self.content_state { IntegrationContentState::Match => (DoctorDisplayStatus::Pass, None), diff --git a/cli/src/services/error.rs b/cli/src/services/error.rs index e97579828..a99303543 100644 --- a/cli/src/services/error.rs +++ b/cli/src/services/error.rs @@ -71,7 +71,6 @@ impl UserError { } } - #[allow(dead_code)] pub fn key(&self) -> &'static str { match self { Self::NotAuthenticated => "auth.not_authenticated", @@ -114,7 +113,6 @@ impl std::fmt::Display for UserError { /// observability and diagnostic rendering. #[derive(Debug)] pub enum CliError { - #[allow(dead_code)] User { error: UserError, source: Option, @@ -126,15 +124,6 @@ pub enum CliError { } impl CliError { - #[allow(dead_code)] - pub fn user(error: UserError) -> Self { - Self::User { - error, - source: None, - } - } - - #[allow(dead_code)] pub fn user_with_source(error: UserError, source: impl Into) -> Self { Self::User { error, @@ -206,7 +195,10 @@ mod tests { #[test] fn user_error_not_authenticated_classifies_as_runtime() { - let error = CliError::user(UserError::NotAuthenticated); + let error = CliError::User { + error: UserError::NotAuthenticated, + source: None, + }; assert_eq!(error.class(), FailureClass::Runtime); assert_eq!(error.code(), "SCE-ERR-RUNTIME"); @@ -216,7 +208,10 @@ mod tests { #[test] fn not_git_repository_has_stable_runtime_catalog_mapping() { - let error = CliError::user(UserError::NotGitRepository); + let error = CliError::User { + error: UserError::NotGitRepository, + source: None, + }; assert_eq!(error.class(), FailureClass::Runtime); assert_eq!(error.code(), "SCE-ERR-RUNTIME"); @@ -229,7 +224,10 @@ mod tests { #[test] fn unexpected_failure_has_stable_runtime_catalog_mapping() { - let error = CliError::user(UserError::UnexpectedFailure); + let error = CliError::User { + error: UserError::UnexpectedFailure, + source: None, + }; assert_eq!(error.class(), FailureClass::Runtime); assert_eq!(error.code(), "SCE-ERR-RUNTIME"); diff --git a/cli/src/services/hooks/claude_bridge_session.rs b/cli/src/services/hooks/claude_bridge_session.rs index 31731a67b..f32e672c4 100644 --- a/cli/src/services/hooks/claude_bridge_session.rs +++ b/cli/src/services/hooks/claude_bridge_session.rs @@ -101,170 +101,3 @@ fn extract_claude_bridge_session_id_from_reader( None } - -#[cfg(test)] -mod tests { - use std::{ - fs, - io::Cursor, - path::{Path, PathBuf}, - time::{SystemTime, UNIX_EPOCH}, - }; - - use super::*; - - fn unique_temp_dir(label: &str) -> PathBuf { - let suffix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock should be after Unix epoch") - .as_nanos(); - let path = std::env::temp_dir().join(format!("sce-claude-bridge-{label}-{suffix}")); - fs::create_dir_all(&path).expect("temporary directory should be created"); - path - } - - fn transcript(bridge_session_id: &str, session_id: &str) -> String { - format!( - concat!( - "{{\"type\":\"file-history-snapshot\",\"messageId\":\"msg-1\"}}\n", - "{{\"type\":\"bridge-session\",\"sessionId\":\"{session_id}\",", - "\"bridgeSessionId\":\"{bridge_session_id}\"}}\n", - "{{\"type\":\"user\",\"sessionId\":\"{session_id}\"}}\n" - ), - bridge_session_id = bridge_session_id, - session_id = session_id, - ) - } - - #[test] - fn extracts_bridge_session_id_from_real_shaped_leading_records() { - let content = transcript("cse_bridge-123", "session-new"); - - assert_eq!( - extract_claude_bridge_session_id_from_reader(Ok(Cursor::new(content))), - Some(String::from("cse_bridge-123")) - ); - } - - #[test] - fn bridge_extraction_fails_open_for_missing_unreadable_or_malformed_records() { - let directory = unique_temp_dir("unreadable"); - let malformed = concat!( - r#"{"type":"bridge-session","sessionId":"session-1","bridgeSessionId":42}"#, - "\n" - ); - - assert_eq!( - extract_claude_bridge_session_id(Path::new("/does/not/exist.jsonl")), - None - ); - assert_eq!( - extract_claude_bridge_session_id_from_reader(Ok(Cursor::new(malformed))), - None - ); - assert_eq!(extract_claude_bridge_session_id(&directory), None); - - fs::remove_dir_all(directory).expect("temporary directory should be removed"); - } - - #[test] - fn bridge_extraction_does_not_scan_beyond_the_leading_record_bound() { - let mut content = String::new(); - for _ in 0..MAX_LEADING_RECORDS { - content.push_str("{\"type\":\"user\"}\n"); - } - content.push_str(&transcript("cse_too-late", "session-late")); - - assert_eq!( - extract_claude_bridge_session_id_from_reader(Ok(Cursor::new(content))), - None - ); - } - - #[test] - fn returns_every_chain_member_session_id_in_deterministic_order() { - let directory = unique_temp_dir("chain-multi"); - let source = directory.join("session-current.jsonl"); - let member_b = directory.join("session-b.jsonl"); - let member_a = directory.join("session-a.jsonl"); - let unrelated = directory.join("session-unrelated.jsonl"); - - fs::write(&source, transcript("cse_shared", "session-current")) - .expect("source transcript should be written"); - fs::write(&member_b, transcript("cse_shared", "session-b")) - .expect("member transcript should be written"); - fs::write(&member_a, transcript("cse_shared", "session-a")) - .expect("member transcript should be written"); - fs::write(&unrelated, transcript("cse_other", "session-unrelated")) - .expect("unrelated transcript should be written"); - - assert_eq!( - find_claude_bridge_chain_session_ids(&source, "cse_shared"), - vec![String::from("session-a"), String::from("session-b")] - ); - - fs::remove_dir_all(directory).expect("temporary directory should be removed"); - } - - #[test] - fn returns_the_single_chain_member_when_only_one_sibling_matches() { - let directory = unique_temp_dir("chain-single"); - let source = directory.join("session-current.jsonl"); - let member = directory.join("session-only.jsonl"); - - fs::write(&source, transcript("cse_shared", "session-current")) - .expect("source transcript should be written"); - fs::write(&member, transcript("cse_shared", "session-only")) - .expect("member transcript should be written"); - - assert_eq!( - find_claude_bridge_chain_session_ids(&source, "cse_shared"), - vec![String::from("session-only")] - ); - - fs::remove_dir_all(directory).expect("temporary directory should be removed"); - } - - #[test] - fn chain_discovery_fails_open_to_an_empty_result() { - let directory = unique_temp_dir("chain-none"); - let source = directory.join("session-current.jsonl"); - let unrelated = directory.join("session-unrelated.jsonl"); - - fs::write(&source, transcript("cse_shared", "session-current")) - .expect("source transcript should be written"); - fs::write(&unrelated, transcript("cse_other", "session-unrelated")) - .expect("unrelated transcript should be written"); - - assert!(find_claude_bridge_chain_session_ids(&source, "cse_shared").is_empty()); - assert!(find_claude_bridge_chain_session_ids(&source, " ").is_empty()); - assert!(find_claude_bridge_chain_session_ids( - Path::new("/does/not/exist/session.jsonl"), - "cse_shared" - ) - .is_empty()); - - fs::remove_dir_all(directory).expect("temporary directory should be removed"); - } - - #[test] - fn chain_discovery_ignores_a_sibling_whose_bridge_record_is_past_the_leading_bound() { - let directory = unique_temp_dir("chain-bounded"); - let source = directory.join("session-current.jsonl"); - let late = directory.join("session-late.jsonl"); - - fs::write(&source, transcript("cse_shared", "session-current")) - .expect("source transcript should be written"); - - let mut late_content = String::new(); - for _ in 0..MAX_LEADING_RECORDS { - late_content.push_str("{\"type\":\"user\"}\n"); - } - late_content.push_str(&transcript("cse_shared", "session-late")); - fs::write(&late, late_content).expect("late transcript should be written"); - - assert!(find_claude_bridge_chain_session_ids(&source, "cse_shared").is_empty()); - - fs::remove_dir_all(directory).expect("temporary directory should be removed"); - } -} diff --git a/cli/src/services/hooks/claude_model_state.rs b/cli/src/services/hooks/claude_model_state.rs index 4ee2aad6f..17a3e4f99 100644 --- a/cli/src/services/hooks/claude_model_state.rs +++ b/cli/src/services/hooks/claude_model_state.rs @@ -5,7 +5,6 @@ use serde_json::Value; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::agent_trace_db::{ClaudeModelStateObservation, ObservationKind}; -use crate::services::observability::traits::Logger; use super::{ current_unix_time_ms, normalize_claude_model_id, prefixed_diff_trace_session_id, @@ -24,9 +23,11 @@ struct BridgeInheritanceCandidate { transcript_path: PathBuf, } -pub(super) fn run_claude_model_state_subcommand( +pub(super) async fn run_claude_model_state_subcommand< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> String { let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, @@ -48,38 +49,13 @@ pub(super) fn run_claude_model_state_subcommand( run_claude_model_state_from_payload(repository_root, &stdin_payload, logger, || { Ok(observed_at_ms) }) + .await } -#[cfg(test)] -pub(super) fn run_claude_model_state_from_payload_at_state_root( +async fn run_claude_model_state_from_payload( repository_root: &Path, - state_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, - observed_at_ms: F, -) -> String -where - F: FnOnce() -> Result, -{ - run_claude_model_state_from_payload_with( - repository_root, - stdin_payload, - logger, - observed_at_ms, - |repository_root, context_message| { - super::open_agent_trace_db_for_hook_runtime_at_state_root( - repository_root, - state_root, - context_message, - ) - }, - ) -} - -fn run_claude_model_state_from_payload( - repository_root: &Path, - stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, observed_at_ms: F, ) -> String where @@ -92,18 +68,23 @@ where observed_at_ms, super::open_agent_trace_db_for_hook_runtime, ) + .await } -fn run_claude_model_state_from_payload_with( +async fn run_claude_model_state_from_payload_with< + L: crate::services::observability::traits::Logger, + F, + O, +>( repository_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, observed_at_ms: F, open_db: O, ) -> String where F: FnOnce() -> Result, - O: FnOnce(&Path, &'static str) -> Result, + O: std::ops::AsyncFnOnce(&Path, &'static str) -> Result, { let session_id = fail_open_session_id(stdin_payload); let observed_at_ms = match observed_at_ms() { @@ -147,7 +128,9 @@ where let db = match open_db( repository_root, "Failed to open Agent Trace DB for Claude model-state persistence.", - ) { + ) + .await + { Ok(db) => db, Err(error) => { log_fail_open( @@ -168,7 +151,8 @@ where } else { let candidate = bridge_candidate .expect("bridge candidate must exist when no direct observation exists"); - let Some(model_id) = newest_bridge_chain_model(&db, &candidate.transcript_path) else { + let Some(model_id) = newest_bridge_chain_model(&db, &candidate.transcript_path).await + else { return String::new(); }; @@ -182,7 +166,7 @@ where } }; - if let Err(error) = persist_claude_model_state(&db, observation) { + if let Err(error) = persist_claude_model_state(&db, observation).await { log_fail_open(logger, DB_WRITE_FAILED_EVENT, &error, session_id.as_deref()); } @@ -229,7 +213,7 @@ fn bridge_inheritance_candidate(stdin_payload: &str) -> Result Option { @@ -243,7 +227,9 @@ pub(super) fn newest_bridge_chain_model( let mut winner: Option<(i64, String, String)> = None; for member in members { let member_session_id = prefixed_diff_trace_session_id(CLAUDE_TOOL_NAME, &member); - let Ok(Some(state)) = db.claude_model_state_by_session_and_agent(&member_session_id, "") + let Ok(Some(state)) = db + .claude_model_state_by_session_and_agent(&member_session_id, "") + .await else { continue; }; @@ -263,11 +249,12 @@ pub(super) fn newest_bridge_chain_model( winner.map(|(_, _, model_id)| model_id) } -fn persist_claude_model_state( +async fn persist_claude_model_state( db: &RepositoryAgentTraceDb, observation: ClaudeModelStateObservation, ) -> Result<()> { db.upsert_claude_model_state(observation) + .await .context("Failed to persist Claude model-state observation.")?; Ok(()) } @@ -413,8 +400,8 @@ fn fail_open_session_id(stdin_payload: &str) -> Option { .map(str::to_string) } -fn log_fail_open( - logger: Option<&dyn Logger>, +fn log_fail_open( + logger: Option<&L>, event_id: &str, error: &anyhow::Error, session_id: Option<&str>, @@ -423,470 +410,3 @@ fn log_fail_open( log.error(event_id, &format!("{error:#}"), &[], session_id); } } - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::{Path, PathBuf}, - process::Command, - sync::{Arc, Mutex}, - time::{SystemTime, UNIX_EPOCH}, - }; - - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - - use super::*; - use crate::services::agent_trace_db::ObservationKind; - use serde_json::json; - - fn parse(payload: &Value, observed_at_ms: i64) -> Option { - parse_claude_model_state_payload(&payload.to_string(), observed_at_ms) - .expect("model-state payload should parse") - } - - #[derive(Clone, Default)] - struct RecordingLogger { - errors: Arc>>, - } - - impl Logger for RecordingLogger { - fn info(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn debug(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn warn(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - - fn error(&self, event_id: &str, message: &str, _: &[(&str, &str)], _: Option<&str>) { - self.errors - .lock() - .expect("recording logger mutex must not be poisoned") - .push((event_id.to_string(), message.to_string())); - } - - fn log_cli_error(&self, _: &crate::services::error::CliError, _: Option<&str>) {} - } - - fn unique_temp_dir(label: &str) -> PathBuf { - let suffix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock should be after Unix epoch") - .as_nanos(); - let path = std::env::temp_dir().join(format!("sce-claude-model-state-{label}-{suffix}")); - fs::create_dir_all(&path).expect("temporary directory should be created"); - path - } - - fn run_git(repo_root: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(repo_root) - .output() - .expect("git should start"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - fn init_git_repo(label: &str) -> PathBuf { - let repo_root = unique_temp_dir(label); - run_git(&repo_root, &["init", "-q"]); - run_git( - &repo_root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - repo_root - } - - #[test] - fn session_start_normalizes_main_session_and_model() { - let observation = parse( - &json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "model": "claude-opus-4-1", - "source": "startup" - }), - 42, - ) - .expect("SessionStart should persist"); - - assert_eq!(observation.session_id, "cc_session-1"); - assert_eq!(observation.agent_id, ""); - assert_eq!(observation.model_id, "claude/claude-opus-4-1"); - assert_eq!(observation.observation_kind, ObservationKind::SessionStart); - assert_eq!(observation.source, "startup"); - assert_eq!(observation.observed_at_ms, 42); - } - - #[test] - fn model_less_session_start_is_a_no_op() { - assert_eq!( - parse( - &json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "source": "resume" - }), - 42, - ), - None - ); - assert_eq!( - parse( - &json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "model": null, - "source": "resume" - }), - 42, - ), - None - ); - assert_eq!( - parse( - &json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "model": " ", - "source": "resume" - }), - 42, - ), - None - ); - } - - #[test] - fn post_model_switch_uses_to_model_and_exact_agent_scope() { - let observation = parse( - &json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "cc_session-1", - "agent_id": "agent-1", - "from_model": "old-model", - "to_model": "claude/new-model", - "source": "picker" - }), - 43, - ) - .expect("PostModelSwitch should persist"); - - assert_eq!(observation.session_id, "cc_session-1"); - assert_eq!(observation.agent_id, "agent-1"); - assert_eq!(observation.model_id, "claude/new-model"); - assert_eq!( - observation.observation_kind, - ObservationKind::PostModelSwitch - ); - assert_eq!(observation.source, "picker"); - } - - #[test] - fn post_model_switch_accepts_current_and_future_sources_unchanged() { - for (source, expected) in [ - ("command", "command"), - ("picker", "picker"), - ("sdk", "sdk"), - ("auto", "auto"), - ("resume", "resume"), - ("future-source", "future-source"), - (" future-source ", "future-source"), - ] { - let observation = parse( - &json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "model-a", - "to_model": "model-b", - "source": source - }), - 43, - ) - .expect("source should produce an observation"); - assert_eq!(observation.source, expected); - } - } - - #[test] - fn malformed_switch_rejects_missing_or_invalid_fields() { - for payload in [ - json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "model-a", - "to_model": "model-b" - }), - json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "model-a", - "to_model": "model-b", - "source": null - }), - json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "model-a", - "to_model": "model-b", - "source": "" - }), - json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "model-a", - "to_model": "model-b", - "source": " " - }), - json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "model-a", - "to_model": "model-b", - "source": 42 - }), - json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "", - "to_model": "model-b", - "source": "command" - }), - ] { - assert!(parse_claude_model_state_payload(&payload.to_string(), 43).is_err()); - } - } - - #[test] - fn agent_id_uses_main_scope_only_when_missing_or_null() { - let base = json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "model": "model-a", - "source": "startup" - }); - - for (agent_id, expected) in [ - (None, ""), - (Some(Value::Null), ""), - (Some(json!("agent-1")), "agent-1"), - (Some(json!(" agent-1 ")), "agent-1"), - ] { - let mut payload = base.clone(); - if let Some(agent_id) = agent_id { - payload["agent_id"] = agent_id; - } - assert_eq!( - parse(&payload, 42).expect("agent ID should parse").agent_id, - expected - ); - } - } - - #[test] - fn agent_id_rejects_present_empty_whitespace_and_non_string_values() { - let base = json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "model": "model-a", - "source": "startup" - }); - - for agent_id in [json!(""), json!(" "), json!(42)] { - let mut payload = base.clone(); - payload["agent_id"] = agent_id; - assert!( - parse_claude_model_state_payload(&payload.to_string(), 42).is_err(), - "present malformed agent_id must be rejected" - ); - } - } - - #[test] - fn present_empty_agent_id_fails_open_without_opening_or_writing_a_database() { - let repo_root = std::env::temp_dir().join(format!( - "sce-claude-model-state-invalid-agent-{}", - std::process::id() - )); - let logger = RecordingLogger::default(); - let output = run_claude_model_state_from_payload( - &repo_root, - &json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "agent_id": " ", - "model": "model-a", - "source": "startup" - }) - .to_string(), - Some(&logger), - || Ok(42), - ); - - assert_eq!(output, ""); - assert!(!repo_root.exists(), "validation failure must not open a DB"); - let errors = logger - .errors - .lock() - .expect("logger mutex must not be poisoned"); - assert!(errors - .iter() - .any(|(_, message)| message.contains("agent_id"))); - } - - #[test] - fn unsupported_event_is_a_no_op() { - assert_eq!( - parse( - &json!({ - "hook_event_name": "Stop", - "session_id": "session-1" - }), - 42, - ), - None - ); - } - - #[test] - fn lifecycle_handler_returns_empty_output_for_intake_failures_and_no_ops() { - assert_eq!( - run_claude_model_state_from_payload(Path::new("/unused"), "not-json", None, || Ok(42),), - "" - ); - assert_eq!( - run_claude_model_state_from_payload( - Path::new("/unused"), - &json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "model": null, - "source": "resume" - }) - .to_string(), - None, - || Ok(42), - ), - "" - ); - } - - #[test] - fn pre_003_hook_runtime_fails_open_without_migrating_or_writing_state() { - let repo_root = init_git_repo("pre-003-repo"); - let state_root = unique_temp_dir("pre-003-state"); - let storage = resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &repo_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("setup path should create the pre-003 fixture"); - let db_path = storage.db_path.clone(); - storage - .db - .execute("DROP TABLE claude_model_state", ()) - .expect("fixture should remove migration 003 table"); - storage - .db - .execute( - "DELETE FROM __sce_migrations WHERE id = '003_claude_model_state'", - (), - ) - .expect("fixture should remove migration 003 metadata"); - drop(storage); - - let logger = RecordingLogger::default(); - let output = run_claude_model_state_from_payload_at_state_root( - &repo_root, - &state_root, - &json!({ - "hook_event_name": "SessionStart", - "session_id": "session-1", - "model": "model-a", - "source": "startup" - }) - .to_string(), - Some(&logger), - || Ok(42), - ); - - assert_eq!(output, "", "hook stdout must remain empty"); - let errors = logger - .errors - .lock() - .expect("logger mutex must not be poisoned"); - assert!( - errors.iter().any(|(event_id, message)| { - event_id == DB_OPEN_FAILED_EVENT && message.contains("Run 'sce setup'.") - }), - "unexpected diagnostics: {errors:?}" - ); - drop(errors); - - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("pre-003 DB should remain openable without migrations"); - let table_exists = db - .query_map( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'claude_model_state'", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("table existence query should succeed"); - assert!( - table_exists.is_empty(), - "hook must not create migration 003 table" - ); - let applied_ids = db - .query_map( - "SELECT id FROM __sce_migrations ORDER BY id ASC", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("migration metadata query should succeed"); - assert!(!applied_ids.iter().any(|id| id == "003_claude_model_state")); - - drop(db); - fs::remove_dir_all(repo_root).expect("repo fixture should be removed"); - fs::remove_dir_all(state_root).expect("state fixture should be removed"); - } - - #[test] - fn lifecycle_observation_is_written_to_the_repository_state_register() { - let suffix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock should be after Unix epoch") - .as_nanos(); - let db_path = std::env::temp_dir().join(format!("sce-claude-model-state-{suffix}.db")); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let observation = parse( - &json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-1", - "from_model": "model-a", - "to_model": "model-b", - "source": "command" - }), - 42, - ) - .expect("switch should produce an observation"); - - persist_claude_model_state(&db, observation).expect("state write should succeed"); - - let state = db - .claude_model_state_by_session_and_agent("cc_session-1", "") - .expect("state lookup should succeed") - .expect("state should be present"); - assert_eq!(state.model_id, "claude/model-b"); - assert_eq!(state.observation_kind, ObservationKind::PostModelSwitch); - assert_eq!(state.observed_at_ms, 42); - - drop(db); - let _ = fs::remove_file(db_path); - } -} diff --git a/cli/src/services/hooks/claude_mutation_scope/events.rs b/cli/src/services/hooks/claude_mutation_scope/events.rs index 226b8cd2e..1191fa1ca 100644 --- a/cli/src/services/hooks/claude_mutation_scope/events.rs +++ b/cli/src/services/hooks/claude_mutation_scope/events.rs @@ -59,10 +59,6 @@ impl ClaudeToolIdentity { tool_use_id: self.tool_use_id.clone(), } } - - pub(crate) fn is_subagent(&self) -> bool { - self.agent_id.is_some() - } } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/cli/src/services/hooks/claude_mutation_scope/health.rs b/cli/src/services/hooks/claude_mutation_scope/health.rs index 88218fe45..bac0b2cde 100644 --- a/cli/src/services/hooks/claude_mutation_scope/health.rs +++ b/cli/src/services/hooks/claude_mutation_scope/health.rs @@ -13,8 +13,8 @@ pub(crate) enum Repairability { ManualOnly, } -pub(crate) fn assess_repairability(git_dir: &Path) -> Repairability { - let Ok(state) = state::read_state(git_dir) else { +pub(crate) async fn assess_repairability(git_dir: &Path) -> Repairability { + let Ok(state) = state::read_state(git_dir).await else { return Repairability::ManualOnly; }; @@ -34,8 +34,8 @@ pub(crate) fn assess_repairability(git_dir: &Path) -> Repairability { } } -pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { - let state = match state::read_state(git_dir) { +pub(crate) async fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { + let state = match state::read_state(git_dir).await { Ok(state) => state, Err(error) => { return MutationScopeAdapterHealth::new( @@ -82,722 +82,3 @@ pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { "Recovery is pending with unresolved attempts; the recovery barrier's flush path only runs once attempts are empty, so future tracked PreToolUse calls deny without self-clearing.", ) } - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - use std::sync::atomic::{AtomicU64, Ordering}; - - use anyhow::anyhow; - - use super::super::{ - abandon_attempt, apply_recovery_barrier, cleanup_attempts_matching, repair_blocked, - AttemptKey, BarrierOutcome, RepairOutcome, - }; - use super::*; - use crate::services::observability::traits::Logger; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-claude-mutation-scope-health-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(tool_use_id: &str) -> AttemptKey { - AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: tool_use_id.to_string(), - } - } - - fn failing_seam( - _root: &Path, - _payload: &str, - _logger: Option<&dyn Logger>, - ) -> anyhow::Result { - Err(anyhow!("seam failure injected by test")) - } - - fn unreachable_seam( - _root: &Path, - payload: &str, - _logger: Option<&dyn Logger>, - ) -> anyhow::Result { - panic!( - "the ingress seam must not be called while the recovery barrier is armed with non-empty attempts: {payload}" - ); - } - - #[test] - fn absent_state_file_is_healthy() { - let git_dir = unique_test_git_dir("absent"); - - let health = classify_health(&git_dir); - - assert_eq!(health.adapter, ActorKind::ClaudeCode); - assert_eq!(health.status, MutationScopeHealthStatus::Healthy); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn malformed_state_file_is_invalid_with_the_read_error_surfaced() { - let git_dir = unique_test_git_dir("malformed"); - let path = state::state_path(&git_dir); - std::fs::create_dir_all(path.parent().expect("state path has a parent")) - .expect("state dir should be created"); - std::fs::write(&path, b"not json").expect("malformed file should be writable"); - - let health = classify_health(&git_dir); - - assert_eq!(health.status, MutationScopeHealthStatus::Invalid); - assert!( - health - .detail - .as_deref() - .unwrap_or_default() - .contains("malformed"), - "the read error must be surfaced in the detail: {:?}", - health.detail - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn stale_non_empty_attempts_after_a_failed_abandon_stays_blocked_across_repeated_pre_tool_use_ac3( - ) { - let git_dir = unique_test_git_dir("blocked-regression"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let allocated = state::allocate_attempt(&git_dir, &key("toolu_1"), "Write") - .expect("allocating the live attempt should succeed"); - - let error = abandon_attempt( - &git_dir, - repository_root, - &allocated.attempt, - None, - &failing_seam, - ) - .expect_err("the injected abandon seam failure must propagate"); - assert!(error.to_string().contains("seam failure")); - - let seeded_state = state::read_state(&git_dir).expect("state should be readable"); - assert!( - seeded_state.recovery_pending, - "mark_recovery_pending must have run before the seam call failed" - ); - assert_eq!( - seeded_state.attempts.len(), - 1, - "remove_attempt must never have run because the seam call failed" - ); - assert_eq!( - seeded_state.attempts[0].phase, - state::AttemptPhase::PendingAbandon, - "the attempt's abandon intent must be durably persisted before the seam call ran" - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked - ); - - for attempt_number in 1..=2 { - let outcome = - apply_recovery_barrier(&git_dir, repository_root, None, &unreachable_seam); - assert!( - matches!(outcome, BarrierOutcome::Deny), - "PreToolUse call #{attempt_number} must be denied without self-clearing" - ); - } - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "the classifier must still report Blocked after repeated denial" - ); - - assert_eq!( - assess_repairability(&git_dir), - Repairability::AutoFixable, - "a PendingAbandon attempt is an already-established abandon decision, safe to retry" - ); - - let healthy = - |_root: &Path, _payload: &str, _logger: Option<&dyn Logger>| Ok(String::new()); - let outcome = repair_blocked(&git_dir, repository_root, None, &healthy) - .expect("repair should not error"); - assert_eq!(outcome, RepairOutcome::Repaired); - - let final_status = classify_health(&git_dir).status; - assert!( - matches!( - final_status, - MutationScopeHealthStatus::Healthy | MutationScopeHealthStatus::Recovering - ), - "AC3/AC6: a reported repair must never leave the final health Blocked: {final_status:?}" - ); - assert!(state::read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_active_losing_the_race_against_an_established_pending_abandon_leaves_repairable_terminal_evidence( - ) { - let git_dir = unique_test_git_dir("mark-active-race"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let allocated = state::allocate_attempt(&git_dir, &key("toolu_1"), "Write") - .expect("allocation should succeed"); - - state::mark_recovery_pending_and_pending_abandon( - &git_dir, - std::slice::from_ref(&allocated.attempt.scope_id), - ) - .expect("atomic establishment should succeed"); - - let error = state::mark_active(&git_dir, &allocated.attempt.scope_id) - .expect_err("mark_active must fail once abandonment has been established"); - assert!(error.to_string().contains("abandon")); - - let state_after = state::read_state(&git_dir).expect("state readable"); - assert!( - state_after.recovery_pending, - "recovery_pending must remain true after losing the activation race" - ); - assert_eq!( - state_after.attempts[0].phase, - state::AttemptPhase::PendingAbandon, - "losing the activation race must leave PendingAbandon intact, not resurrect Active" - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked - ); - assert_eq!( - assess_repairability(&git_dir), - Repairability::AutoFixable, - "losing the activation race must leave repairable established terminal evidence, \ - not fall back to ManualOnly" - ); - - let healthy = - |_root: &Path, _payload: &str, _logger: Option<&dyn Logger>| Ok(String::new()); - let outcome = repair_blocked(&git_dir, repository_root, None, &healthy) - .expect("repair should not error"); - assert_eq!(outcome, RepairOutcome::Repaired); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.attempts.is_empty()); - assert!(!resolved.recovery_pending); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_removes_only_successfully_abandoned_attempts_and_keeps_recovery_pending_when_one_fails( - ) { - let git_dir = unique_test_git_dir("repair-partial-batch"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let first = state::allocate_attempt(&git_dir, &key("toolu-1"), "Write") - .expect("first allocation should succeed"); - let second = state::allocate_attempt(&git_dir, &key("toolu-2"), "Write") - .expect("second allocation should succeed"); - - state::mark_recovery_pending_and_pending_abandon( - &git_dir, - &[ - first.attempt.scope_id.clone(), - second.attempt.scope_id.clone(), - ], - ) - .expect("atomic establishment should succeed"); - - let fail_first = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> anyhow::Result { - if payload.contains(&first.attempt.scope_id) { - return Err(anyhow!( - "seam failure injected by test for the first attempt" - )); - } - Ok(String::new()) - }; - - let outcome = repair_blocked(&git_dir, repository_root, None, &fail_first) - .expect("repair should not error even though one seam call fails"); - assert_eq!(outcome, RepairOutcome::NoOp); - - let state = state::read_state(&git_dir).expect("state readable"); - assert_eq!( - state.attempts.len(), - 1, - "the successfully abandoned attempt must be removed" - ); - assert_eq!(state.attempts[0].scope_id, first.attempt.scope_id); - assert_eq!(state.attempts[0].phase, state::AttemptPhase::PendingAbandon); - assert!( - state.recovery_pending, - "the barrier must remain armed while any attempt is still unresolved" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn assess_repairability_is_manual_only_when_the_adapter_is_not_blocked() { - let git_dir = unique_test_git_dir("assess-not-blocked"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - assert_eq!(assess_repairability(&git_dir), Repairability::ManualOnly); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn assess_repairability_is_manual_only_for_a_legacy_attempt_never_marked_pending_abandon() { - let git_dir = unique_test_git_dir("assess-legacy-no-pending-abandon"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - state::allocate_attempt(&git_dir, &key("toolu_1"), "Write") - .expect("allocation should succeed"); - state::mark_recovery_pending(&git_dir).expect("marking recovery pending should succeed"); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "a legacy attempt left PendingStart under a stale recovery flag must still classify Blocked" - ); - assert_eq!( - assess_repairability(&git_dir), - Repairability::ManualOnly, - "an attempt with no established abandon intent is never treated as repairable" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn assess_repairability_is_manual_only_when_one_of_several_attempts_has_no_established_abandon_intent( - ) { - let git_dir = unique_test_git_dir("assess-mixed-phase"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let doomed = state::allocate_attempt(&git_dir, &key("toolu-doomed"), "Write") - .expect("allocating the doomed attempt should succeed"); - abandon_attempt( - &git_dir, - repository_root, - &doomed.attempt, - None, - &failing_seam, - ) - .expect_err("the injected abandon seam failure must propagate"); - - state::allocate_attempt(&git_dir, &key("toolu-live"), "Write") - .expect("allocating the untouched live attempt should succeed"); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked - ); - assert_eq!( - assess_repairability(&git_dir), - Repairability::ManualOnly, - "every attempt must be PendingAbandon before doctor may repair; a live attempt \ - with no established abandon intent must never be touched" - ); - - let outcome = repair_blocked(&git_dir, repository_root, None, &unreachable_seam) - .expect("repair should not error"); - assert_eq!( - outcome, - RepairOutcome::NoOp, - "a coexisting attempt with no established abandon intent must block the whole repair" - ); - let untouched = state::read_state(&git_dir).expect("state readable"); - assert_eq!( - untouched.attempts.len(), - 2, - "neither attempt may be touched while any one lacks PendingAbandon evidence" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_is_a_safe_no_op_when_an_attempt_is_no_longer_pending_abandon_by_the_time_the_lock_is_acquired( - ) { - let git_dir = unique_test_git_dir("repair-concurrent-race"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let allocated = state::allocate_attempt(&git_dir, &key("toolu_1"), "Write") - .expect("allocation should succeed"); - abandon_attempt( - &git_dir, - repository_root, - &allocated.attempt, - None, - &failing_seam, - ) - .expect_err("the injected abandon seam failure must propagate"); - - assert_eq!(assess_repairability(&git_dir), Repairability::AutoFixable); - - state::set_attempt_phase_for_tests( - &git_dir, - &allocated.attempt.scope_id, - state::AttemptPhase::Active, - ); - - let outcome = repair_blocked(&git_dir, repository_root, None, &unreachable_seam) - .expect("repair should not error"); - - assert_eq!( - outcome, - RepairOutcome::NoOp, - "the fresh, lock-protected re-proof must refuse to act on state assessed before it changed" - ); - let state = state::read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].phase, state::AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_interrupted_by_a_failing_seam_leaves_state_a_later_repair_completes_without_duplication( - ) { - let git_dir = unique_test_git_dir("repair-interrupted-resume"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let allocated = state::allocate_attempt(&git_dir, &key("toolu_1"), "Write") - .expect("allocation should succeed"); - abandon_attempt( - &git_dir, - repository_root, - &allocated.attempt, - None, - &failing_seam, - ) - .expect_err("the injected abandon seam failure must propagate"); - - let outcome = repair_blocked(&git_dir, repository_root, None, &failing_seam) - .expect("repair should not error even though the seam abandon call fails again"); - assert_eq!(outcome, RepairOutcome::NoOp); - - let interrupted = state::read_state(&git_dir).expect("state readable"); - assert_eq!(interrupted.attempts.len(), 1); - assert_eq!( - interrupted.attempts[0].phase, - state::AttemptPhase::PendingAbandon - ); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "an interrupted repair must remain retryable, not resurrect or duplicate the attempt" - ); - - let healthy = - |_root: &Path, _payload: &str, _logger: Option<&dyn Logger>| Ok(String::new()); - let outcome = repair_blocked(&git_dir, repository_root, None, &healthy) - .expect("the later repair should complete without error"); - assert_eq!(outcome, RepairOutcome::Repaired); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.attempts.is_empty()); - assert!(!resolved.recovery_pending); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn cleanup_attempts_matching_durably_marks_every_matched_attempt_pending_abandon_before_any_seam_call_even_when_the_first_fails( - ) { - let git_dir = unique_test_git_dir("cleanup-batch-mark"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let first = state::allocate_attempt(&git_dir, &key("toolu-1"), "Write") - .expect("first allocation should succeed"); - let second = state::allocate_attempt(&git_dir, &key("toolu-2"), "Write") - .expect("second allocation should succeed"); - - let failing_on_first = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> anyhow::Result { - if payload.contains(&first.attempt.scope_id) { - return Err(anyhow!( - "seam failure injected by test for the first attempt" - )); - } - Ok(String::new()) - }; - - let error = cleanup_attempts_matching( - &git_dir, - repository_root, - None, - &failing_on_first, - |_attempt| true, - ) - .expect_err("a failure abandoning one attempt must still surface an error"); - assert!(error.to_string().contains("seam failure")); - - let seeded = state::read_state(&git_dir).expect("state readable"); - let remaining_first = seeded - .attempts - .iter() - .find(|attempt| attempt.scope_id == first.attempt.scope_id) - .expect("the failed attempt must still be tracked, not lost"); - assert_eq!( - remaining_first.phase, - state::AttemptPhase::PendingAbandon, - "AC3/T04: every matched attempt must be durably marked before any seam call, \ - so the still-blocked first attempt keeps its retryable evidence" - ); - assert!( - seeded - .attempts - .iter() - .all(|attempt| attempt.scope_id != second.attempt.scope_id), - "the second attempt's own seam call must still have been attempted and succeeded" - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked - ); - assert_eq!(assess_repairability(&git_dir), Repairability::AutoFixable); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn apply_recovery_barrier_fails_closed_when_a_new_obligation_is_established_while_the_flush_seam_is_in_flight( - ) { - let git_dir = unique_test_git_dir("barrier-race-new-obligation"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - state::mark_recovery_pending(&git_dir).expect("arming the barrier should succeed"); - - let racing_seam = |_root: &Path, - _payload: &str, - _logger: Option<&dyn Logger>| - -> anyhow::Result { - let raced = state::allocate_attempt(&git_dir, &key("toolu-raced-in"), "Write") - .expect("the racing allocation should succeed"); - state::mark_recovery_pending_and_pending_abandon( - &git_dir, - std::slice::from_ref(&raced.attempt.scope_id), - ) - .expect("the racing establishment should succeed"); - Ok(String::new()) - }; - - let outcome = apply_recovery_barrier(&git_dir, repository_root, None, &racing_seam); - assert!( - matches!(outcome, BarrierOutcome::Deny), - "a fresh obligation established during the flush must fail the barrier closed, \ - not proceed on the stale pre-flush proof" - ); - - let after = state::read_state(&git_dir).expect("state readable"); - assert!( - after.recovery_pending, - "RecoveryNeverClearedWithUnresolvedAbandon: recovery must remain armed" - ); - assert_eq!(after.attempts.len(), 1); - assert_eq!(after.attempts[0].phase, state::AttemptPhase::PendingAbandon); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_fails_closed_when_a_new_obligation_is_established_while_abandoning_another() { - let git_dir = unique_test_git_dir("repair-race-new-obligation"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let repository_root = git_dir.as_path(); - - let original = state::allocate_attempt(&git_dir, &key("toolu-original"), "Write") - .expect("allocation should succeed"); - state::mark_recovery_pending_and_pending_abandon( - &git_dir, - std::slice::from_ref(&original.attempt.scope_id), - ) - .expect("atomic establishment should succeed"); - - let racing_seam = |_root: &Path, - _payload: &str, - _logger: Option<&dyn Logger>| - -> anyhow::Result { - let raced = state::allocate_attempt(&git_dir, &key("toolu-raced-in"), "Write") - .expect("the racing allocation should succeed"); - state::mark_recovery_pending_and_pending_abandon( - &git_dir, - std::slice::from_ref(&raced.attempt.scope_id), - ) - .expect("the racing establishment should succeed"); - Ok(String::new()) - }; - - let outcome = repair_blocked(&git_dir, repository_root, None, &racing_seam) - .expect("repair should not error"); - assert_eq!( - outcome, - RepairOutcome::NoOp, - "AC6: repair must not report Repaired when the atomic clear discovers another \ - unresolved obligation, even though its own abandon seam call succeeded" - ); - - let after = state::read_state(&git_dir).expect("state readable"); - assert!( - after.recovery_pending, - "RecoveryNeverClearedWithUnresolvedAbandon: recovery must remain armed while the \ - raced-in obligation is unresolved" - ); - assert!( - after - .attempts - .iter() - .all(|attempt| attempt.scope_id != original.attempt.scope_id), - "the original attempt's own abandon must still have completed" - ); - assert_eq!(after.attempts.len(), 1); - assert_eq!(after.attempts[0].phase, state::AttemptPhase::PendingAbandon); - - remove_test_git_dir(&git_dir); - } - - fn hand_constructed_attempt( - attempt_seq: u64, - tool_use_id: &str, - phase: state::AttemptPhase, - ) -> state::AdapterAttempt { - state::AdapterAttempt { - attempt_seq, - scope_id: format!("claude|s=session-1:a|c={attempt_seq}:{tool_use_id}"), - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: tool_use_id.to_string(), - tool_name: "Edit".to_string(), - phase, - } - } - - fn write_hand_constructed_state(git_dir: &Path, state: &state::AdapterState) { - let path = state::state_path(git_dir); - std::fs::create_dir_all(path.parent().expect("state path has a parent")) - .expect("state dir should be created"); - std::fs::write( - &path, - serde_json::to_vec(state).expect("serialize adapter state"), - ) - .expect("write hand-constructed adapter state"); - } - - #[test] - fn health_classification_table_covers_every_recovery_pending_and_attempt_phase_shape() { - use state::AttemptPhase::{Active, PendingAbandon, PendingStart}; - use MutationScopeHealthStatus::{Blocked, Healthy, Invalid, Recovering}; - - let rows: [(bool, &[state::AttemptPhase], MutationScopeHealthStatus); 12] = [ - (false, &[], Healthy), - (false, &[PendingStart], Healthy), - (false, &[Active], Healthy), - (false, &[PendingAbandon], Invalid), - (false, &[PendingAbandon, PendingStart], Invalid), - (false, &[PendingAbandon, Active], Invalid), - (true, &[], Recovering), - (true, &[PendingStart], Blocked), - (true, &[Active], Blocked), - (true, &[PendingAbandon], Blocked), - (true, &[PendingAbandon, PendingStart], Blocked), - (true, &[PendingAbandon, Active], Blocked), - ]; - - for (index, (recovery_pending, phases, expected)) in rows.into_iter().enumerate() { - let git_dir = unique_test_git_dir(&format!("table-{index}")); - let attempts: Vec = (1u64..) - .zip(phases) - .map(|(attempt_seq, phase)| { - hand_constructed_attempt(attempt_seq, &format!("toolu_{attempt_seq}"), *phase) - }) - .collect(); - let next_attempt_seq = attempts.last().map_or(1, |attempt| attempt.attempt_seq + 1); - write_hand_constructed_state( - &git_dir, - &state::AdapterState { - version: 1, - next_attempt_seq, - recovery_pending, - attempts, - }, - ); - - assert_eq!( - classify_health(&git_dir).status, - expected, - "row {index}: recovery_pending={recovery_pending} phases={phases:?}", - ); - - remove_test_git_dir(&git_dir); - } - } - - #[test] - fn assess_repairability_is_manual_only_for_an_impossible_clear_recovery_with_a_pending_abandon_attempt( - ) { - let git_dir = unique_test_git_dir("assess-invalid-clear-pending-abandon"); - write_hand_constructed_state( - &git_dir, - &state::AdapterState { - version: 1, - next_attempt_seq: 2, - recovery_pending: false, - attempts: vec![hand_constructed_attempt( - 1, - "toolu_1", - state::AttemptPhase::PendingAbandon, - )], - }, - ); - - assert_eq!( - assess_repairability(&git_dir), - Repairability::ManualOnly, - "doctor must not auto-repair an impossible Claude state" - ); - - remove_test_git_dir(&git_dir); - } -} diff --git a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs index 8b620a7e6..85f01c29b 100644 --- a/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/claude_mutation_scope/lifecycle.rs @@ -4,7 +4,6 @@ use anyhow::Result; use crate::services::hooks; use crate::services::mutation_trace::runtime::resolve_git_dir; -use crate::services::observability::traits::Logger; use super::events::BASH_TOOL_NAME; use super::state; @@ -15,10 +14,6 @@ use super::{ ClaudeToolIdentity, ToolClassification, }; -pub(super) type GitDirResolver<'a> = &'a dyn Fn(&str) -> Result; - -pub(super) type IngressSeam<'a> = &'a dyn Fn(&Path, &str, Option<&dyn Logger>) -> Result; - pub(super) const ACTOR_KIND_CLAUDE_CODE: &str = "claude_code"; pub(super) const FAIL_CLOSED_DENY_REASON: &str = @@ -31,11 +26,8 @@ pub(super) const PRE_TOOL_USE_FAIL_CLOSED_EVENT: &str = pub(super) const MODEL_STATE_UNAVAILABLE_EVENT: &str = "sce.hooks.claude_mutation_scope.model_state_unavailable"; -pub(super) type ClaudeModelStateResolver<'a> = - &'a dyn Fn(&Path, &str, &str) -> Result>; - -pub(super) fn log_pre_tool_use_fail_closed( - logger: Option<&dyn Logger>, +pub(super) fn log_pre_tool_use_fail_closed( + logger: Option<&L>, context: &str, error: &anyhow::Error, ) { @@ -49,66 +41,36 @@ pub(super) fn log_pre_tool_use_fail_closed( } } -pub(crate) fn run_claude_mutation_scope_subcommand(logger: Option<&dyn Logger>) -> Result { - let stdin_payload = hooks::read_hook_stdin()?; - run_claude_mutation_scope_from_payload(&stdin_payload, logger) -} - -pub(crate) fn run_claude_mutation_scope_from_payload( - stdin_payload: &str, - logger: Option<&dyn Logger>, +pub(crate) async fn run_claude_mutation_scope_subcommand< + L: crate::services::observability::traits::Logger, +>( + logger: Option<&L>, ) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let model_state_resolver = - |repository_root: &Path, session_id: &str, agent_id: &str| -> Result> { - let db = hooks::open_agent_trace_db_for_hook_runtime( - repository_root, - "Failed to open Agent Trace DB for Claude mutation-scope model resolution.", - )?; - Ok(db - .claude_model_state_by_session_and_agent(session_id, agent_id)? - .map(|state| state.model_id)) - }; - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { - hooks::mutation_scope::run_mutation_scope_from_payload(repository_root, payload, logger) - }; - - run_claude_mutation_scope_from_payload_with_resolver( - stdin_payload, - logger, - &resolve_git_dir_fn, - &model_state_resolver, - &seam_fn, - ) + let stdin_payload = hooks::read_hook_stdin()?; + run_claude_mutation_scope_from_payload(&stdin_payload, logger).await } -#[cfg(test)] -pub(crate) fn run_claude_mutation_scope_from_payload_at_state_root( - state_root: &Path, +pub(crate) async fn run_claude_mutation_scope_from_payload< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let model_state_root = state_root.to_path_buf(); - let seam_state_root = state_root.to_path_buf(); - let model_state_resolver = - move |repository_root: &Path, session_id: &str, agent_id: &str| -> Result> { - let db = hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - repository_root, - &model_state_root, - "Failed to open Agent Trace DB for Claude mutation-scope model resolution.", - )?; - Ok(db - .claude_model_state_by_session_and_agent(session_id, agent_id)? - .map(|state| state.model_id)) - }; - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { - hooks::mutation_scope::run_mutation_scope_from_payload_at_state_root( + let resolve_git_dir_fn = async |cwd: &str| resolve_git_dir(Path::new(cwd)).await; + let model_state_resolver = async |repository_root: &Path, session_id: &str, agent_id: &str| { + let db = hooks::open_agent_trace_db_for_hook_runtime( repository_root, - &seam_state_root, - payload, - logger, + "Failed to open Agent Trace DB for Claude mutation-scope model resolution.", ) + .await?; + Ok(db + .claude_model_state_by_session_and_agent(session_id, agent_id) + .await? + .map(|state| state.model_id)) + }; + let seam_fn = async |repository_root: &Path, payload: &str, logger: Option<&L>| { + hooks::mutation_scope::run_mutation_scope_from_payload(repository_root, payload, logger) + .await }; run_claude_mutation_scope_from_payload_with_resolver( @@ -118,45 +80,30 @@ pub(crate) fn run_claude_mutation_scope_from_payload_at_state_root( &model_state_resolver, &seam_fn, ) + .await } -#[cfg(test)] -pub(super) fn run_claude_mutation_scope_from_payload_with( - stdin_payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, -) -> Result { - let unavailable_model_state = |_repository_root: &Path, - _session_id: &str, - _agent_id: &str| - -> Result> { Ok(None) }; - run_claude_mutation_scope_from_payload_with_resolver( - stdin_payload, - logger, - resolve_git_dir, - &unavailable_model_state, - seam, - ) -} - -pub(super) fn run_claude_mutation_scope_from_payload_with_resolver( +pub(super) async fn run_claude_mutation_scope_from_payload_with_resolver< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - model_state_resolver: ClaudeModelStateResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + model_state_resolver: &impl std::ops::AsyncFn(&Path, &str, &str) -> Result>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { let event = parse_claude_hook_event(stdin_payload)?; - dispatch_claude_hook_event(event, logger, resolve_git_dir, model_state_resolver, seam) + dispatch_claude_hook_event(event, logger, resolve_git_dir, model_state_resolver, seam).await } -pub(super) fn dispatch_claude_hook_event( +pub(super) async fn dispatch_claude_hook_event< + L: crate::services::observability::traits::Logger, +>( event: ClaudeHookEvent, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - model_state_resolver: ClaudeModelStateResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + model_state_resolver: &impl std::ops::AsyncFn(&Path, &str, &str) -> Result>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { match event { ClaudeHookEvent::PreToolUse(execution) => Ok(handle_pre_tool_use( @@ -165,9 +112,10 @@ pub(super) fn dispatch_claude_hook_event( resolve_git_dir, model_state_resolver, seam, - )), + ) + .await), ClaudeHookEvent::PostToolUse(identity) | ClaudeHookEvent::PostToolUseFailure(identity) => { - let git_dir = resolve_git_dir(&identity.cwd)?; + let git_dir = resolve_git_dir(&identity.cwd).await?; let repository_root = Path::new(&identity.cwd); handle_close( &git_dir, @@ -176,9 +124,10 @@ pub(super) fn dispatch_claude_hook_event( logger, seam, ) + .await } ClaudeHookEvent::PermissionDenied(identity) => { - let git_dir = resolve_git_dir(&identity.cwd)?; + let git_dir = resolve_git_dir(&identity.cwd).await?; let repository_root = Path::new(&identity.cwd); handle_permission_denied( &git_dir, @@ -187,55 +136,61 @@ pub(super) fn dispatch_claude_hook_event( logger, seam, ) + .await } ClaudeHookEvent::Stop(session) | ClaudeHookEvent::StopFailure(session) => { - let git_dir = resolve_git_dir(&session.cwd)?; + let git_dir = resolve_git_dir(&session.cwd).await?; let repository_root = Path::new(&session.cwd); let session_id = session.session_id.clone(); cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id && attempt.agent_id.is_none() }) + .await } ClaudeHookEvent::UserPromptSubmit(session) => { - let git_dir = resolve_git_dir(&session.cwd)?; + let git_dir = resolve_git_dir(&session.cwd).await?; let repository_root = Path::new(&session.cwd); let session_id = session.session_id.clone(); cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id && attempt.agent_id.is_none() }) + .await } ClaudeHookEvent::SubagentStop(agent) => { - let git_dir = resolve_git_dir(&agent.cwd)?; + let git_dir = resolve_git_dir(&agent.cwd).await?; let repository_root = Path::new(&agent.cwd); let session_id = agent.session_id.clone(); let agent_id = agent.agent_id.clone(); cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id && attempt.agent_id.as_deref() == Some(&agent_id) }) + .await } ClaudeHookEvent::SessionEnd(session) => { - let git_dir = resolve_git_dir(&session.cwd)?; + let git_dir = resolve_git_dir(&session.cwd).await?; let repository_root = Path::new(&session.cwd); let session_id = session.session_id.clone(); cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id }) + .await } ClaudeHookEvent::WorktreeRemove(worktree_remove) => { - let git_dir = resolve_git_dir(&worktree_remove.worktree_path)?; + let git_dir = resolve_git_dir(&worktree_remove.worktree_path).await?; let repository_root = Path::new(&worktree_remove.worktree_path); cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |_attempt| true) + .await } ClaudeHookEvent::SessionStart | ClaudeHookEvent::SubagentStart => Ok(String::new()), } } -pub(super) fn handle_pre_tool_use( +pub(super) async fn handle_pre_tool_use( execution: &ClaudeToolExecution, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - model_state_resolver: ClaudeModelStateResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + model_state_resolver: &impl std::ops::AsyncFn(&Path, &str, &str) -> Result>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> String { let identity = &execution.identity; @@ -255,7 +210,7 @@ pub(super) fn handle_pre_tool_use( } let repository_root = Path::new(&identity.cwd); - let git_dir = match resolve_git_dir(&identity.cwd) { + let git_dir = match resolve_git_dir(&identity.cwd).await { Ok(git_dir) => git_dir, Err(error) => { log_pre_tool_use_fail_closed(logger, "resolve_git_dir", &error); @@ -264,7 +219,7 @@ pub(super) fn handle_pre_tool_use( }; if matches!( - apply_recovery_barrier(&git_dir, repository_root, logger, seam), + apply_recovery_barrier(&git_dir, repository_root, logger, seam).await, BarrierOutcome::Deny ) { return pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON); @@ -277,7 +232,9 @@ pub(super) fn handle_pre_tool_use( logger, model_state_resolver, seam, - ) { + ) + .await + { Ok(()) => String::new(), Err(error) => { log_pre_tool_use_fail_closed(logger, "establish_start", &error); @@ -295,13 +252,13 @@ pub(super) enum BarrierOutcome { Deny, } -pub(super) fn apply_recovery_barrier( +pub(super) async fn apply_recovery_barrier( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> BarrierOutcome { - let state = match state::read_state(git_dir) { + let state = match state::read_state(git_dir).await { Ok(state) => state, Err(error) => { log_pre_tool_use_fail_closed(logger, "recovery_barrier.read_state", &error); @@ -317,8 +274,8 @@ pub(super) fn apply_recovery_barrier( return BarrierOutcome::Deny; } - match seam(repository_root, &flush_payload(), logger) { - Ok(_) => match state::clear_recovery_pending_if_quiescent(git_dir) { + match seam(repository_root, &flush_payload(), logger).await { + Ok(_) => match state::clear_recovery_pending_if_quiescent(git_dir).await { Ok(state::ClearRecoveryOutcome::Cleared) => BarrierOutcome::Proceed, Ok(state::ClearRecoveryOutcome::StillPending) => BarrierOutcome::Deny, Err(error) => { @@ -337,33 +294,35 @@ pub(super) fn apply_recovery_barrier( } } -pub(super) fn establish_start( +pub(super) async fn establish_start( git_dir: &Path, repository_root: &Path, identity: &ClaudeToolIdentity, - logger: Option<&dyn Logger>, - model_state_resolver: ClaudeModelStateResolver, - seam: IngressSeam, + logger: Option<&L>, + model_state_resolver: &impl std::ops::AsyncFn(&Path, &str, &str) -> Result>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { - let allocated = state::allocate_attempt(git_dir, &identity.attempt_key(), &identity.tool_name)?; + let allocated = + state::allocate_attempt(git_dir, &identity.attempt_key(), &identity.tool_name).await?; let scope_id = &allocated.attempt.scope_id; let canonical_session_id = hooks::prefixed_diff_trace_session_id(hooks::CLAUDE_TOOL_NAME, &identity.session_id); let agent_id = identity.agent_id.as_deref().unwrap_or(""); - let model_id = match model_state_resolver(repository_root, &canonical_session_id, agent_id) { - Ok(model_id) => model_id.and_then(|model| hooks::normalize_claude_model_id(&model)), - Err(error) => { - if let Some(log) = logger { - log.warn( - MODEL_STATE_UNAVAILABLE_EVENT, - &error.to_string(), - &[("agent_id", agent_id)], - Some(&canonical_session_id), - ); + let model_id = + match model_state_resolver(repository_root, &canonical_session_id, agent_id).await { + Ok(model_id) => model_id.and_then(|model| hooks::normalize_claude_model_id(&model)), + Err(error) => { + if let Some(log) = logger { + log.warn( + MODEL_STATE_UNAVAILABLE_EVENT, + &error.to_string(), + &[("agent_id", agent_id)], + Some(&canonical_session_id), + ); + } + None } - None - } - }; + }; let start_payload = scope_start_payload( scope_id, &claude_scope_start_event_id(scope_id), @@ -371,19 +330,19 @@ pub(super) fn establish_start( model_id.as_deref(), ); - seam(repository_root, &start_payload, logger)?; - state::mark_active(git_dir, scope_id)?; + seam(repository_root, &start_payload, logger).await?; + state::mark_active(git_dir, scope_id).await?; Ok(()) } -pub(super) fn handle_close( +pub(super) async fn handle_close( git_dir: &Path, repository_root: &Path, key: &AttemptKey, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let Some(attempt) = current .attempts .iter() @@ -394,7 +353,7 @@ pub(super) fn handle_close( }; if attempt.phase == state::AttemptPhase::PendingStart { - abandon_attempt(git_dir, repository_root, &attempt, logger, seam)?; + abandon_attempt(git_dir, repository_root, &attempt, logger, seam).await?; return Ok(String::new()); } @@ -404,22 +363,22 @@ pub(super) fn handle_close( &claude_scope_close_event_id(&attempt.scope_id), ); - if seam(repository_root, &close_payload, logger).is_ok() { - state::remove_attempt(git_dir, &attempt.scope_id)?; + if seam(repository_root, &close_payload, logger).await.is_ok() { + state::remove_attempt(git_dir, &attempt.scope_id).await?; } else { - abandon_attempt(git_dir, repository_root, &attempt, logger, seam)?; + abandon_attempt(git_dir, repository_root, &attempt, logger, seam).await?; } Ok(String::new()) } -pub(super) fn handle_permission_denied( +pub(super) async fn handle_permission_denied( git_dir: &Path, repository_root: &Path, key: &AttemptKey, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let Some(attempt) = current .attempts .iter() @@ -429,18 +388,18 @@ pub(super) fn handle_permission_denied( return Ok(String::new()); }; - abandon_attempt(git_dir, repository_root, &attempt, logger, seam)?; + abandon_attempt(git_dir, repository_root, &attempt, logger, seam).await?; Ok(String::new()) } -pub(super) fn cleanup_attempts_matching( +pub(super) async fn cleanup_attempts_matching( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, predicate: impl Fn(&state::AdapterAttempt) -> bool, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let stale: Vec = current .attempts .into_iter() @@ -455,12 +414,12 @@ pub(super) fn cleanup_attempts_matching( .iter() .map(|attempt| attempt.scope_id.clone()) .collect(); - state::mark_recovery_pending_and_pending_abandon(git_dir, &stale_scope_ids)?; + state::mark_recovery_pending_and_pending_abandon(git_dir, &stale_scope_ids).await?; let mut first_error: Option = None; for attempt in &stale { if let Err(error) = - abandon_marked_attempt(git_dir, repository_root, &attempt.scope_id, logger, seam) + abandon_marked_attempt(git_dir, repository_root, &attempt.scope_id, logger, seam).await { if first_error.is_none() { first_error = Some(error); @@ -480,30 +439,31 @@ pub(super) fn attempt_matches_key(attempt: &state::AdapterAttempt, key: &Attempt && attempt.tool_use_id == key.tool_use_id } -pub(super) fn abandon_attempt( +pub(super) async fn abandon_attempt( git_dir: &Path, repository_root: &Path, attempt: &state::AdapterAttempt, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { state::mark_recovery_pending_and_pending_abandon( git_dir, std::slice::from_ref(&attempt.scope_id), - )?; + ) + .await?; - abandon_marked_attempt(git_dir, repository_root, &attempt.scope_id, logger, seam) + abandon_marked_attempt(git_dir, repository_root, &attempt.scope_id, logger, seam).await } -fn abandon_marked_attempt( +async fn abandon_marked_attempt( git_dir: &Path, repository_root: &Path, scope_id: &str, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { - seam(repository_root, &abandon_payload(scope_id), logger)?; - state::remove_attempt(git_dir, scope_id)?; + seam(repository_root, &abandon_payload(scope_id), logger).await?; + state::remove_attempt(git_dir, scope_id).await?; Ok(()) } @@ -513,19 +473,20 @@ pub(crate) enum RepairOutcome { NoOp, } -pub(crate) fn repair_blocked( +pub(crate) async fn repair_blocked( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let Some(attempts) = state::reprove_pending_abandon(git_dir)? else { + let Some(attempts) = state::reprove_pending_abandon(git_dir).await? else { return Ok(RepairOutcome::NoOp); }; let mut any_failed = false; for attempt in &attempts { if abandon_marked_attempt(git_dir, repository_root, &attempt.scope_id, logger, seam) + .await .is_err() { any_failed = true; @@ -533,7 +494,7 @@ pub(crate) fn repair_blocked( } let cleared = matches!( - state::clear_recovery_pending_if_quiescent(git_dir)?, + state::clear_recovery_pending_if_quiescent(git_dir).await?, state::ClearRecoveryOutcome::Cleared ); diff --git a/cli/src/services/hooks/claude_mutation_scope/mod.rs b/cli/src/services/hooks/claude_mutation_scope/mod.rs index 2295e5fb3..3a93ed92d 100644 --- a/cli/src/services/hooks/claude_mutation_scope/mod.rs +++ b/cli/src/services/hooks/claude_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; @@ -37,15 +35,11 @@ use events::{ }; #[allow(unused_imports)] pub(crate) use health::{assess_repairability, Repairability}; -#[cfg(test)] -pub(crate) use lifecycle::run_claude_mutation_scope_from_payload_at_state_root; -#[cfg(test)] -#[allow(unused_imports)] -use lifecycle::ClaudeModelStateResolver; + #[allow(unused_imports)] use lifecycle::{ abandon_attempt, apply_recovery_barrier, cleanup_attempts_matching, BarrierOutcome, - GitDirResolver, IngressSeam, EXPLICIT_BACKGROUND_SHELL_DENY_REASON, FAIL_CLOSED_DENY_REASON, + EXPLICIT_BACKGROUND_SHELL_DENY_REASON, FAIL_CLOSED_DENY_REASON, }; #[allow(unused_imports)] pub(crate) use lifecycle::{repair_blocked, RepairOutcome}; @@ -53,17 +47,9 @@ pub(crate) use lifecycle::{repair_blocked, RepairOutcome}; pub(crate) use lifecycle::{ run_claude_mutation_scope_from_payload, run_claude_mutation_scope_subcommand, }; -#[cfg(test)] -#[allow(unused_imports)] -use lifecycle::{ - run_claude_mutation_scope_from_payload_with, - run_claude_mutation_scope_from_payload_with_resolver, -}; + #[allow(unused_imports)] use payload::{ abandon_payload, flush_payload, pre_tool_use_deny_json, scope_boundary_payload, scope_start_payload, }; - -#[cfg(test)] -mod tests; diff --git a/cli/src/services/hooks/claude_mutation_scope/state.rs b/cli/src/services/hooks/claude_mutation_scope/state.rs index 08cc668c0..9cf2cfd8b 100644 --- a/cli/src/services/hooks/claude_mutation_scope/state.rs +++ b/cli/src/services/hooks/claude_mutation_scope/state.rs @@ -1,19 +1,17 @@ -use std::fs::{File, OpenOptions, TryLockError}; +use std::fs::OpenOptions; use std::io::Write as _; use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant}; use anyhow::{anyhow, Context, Result}; use serde::{Deserialize, Serialize}; use super::{format_claude_scope_id, AttemptKey}; +use crate::services::hooks::mutation_scope_lock::AdapterLockSpec; const SCE_STATE_DIR: &str = "sce"; const ADAPTER_STATE_FILE: &str = "claude-mutation-scope-state.json"; -const ADAPTER_STATE_LOCK_FILE: &str = "claude-mutation-scope-state.lock"; -const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(20); -const DEFAULT_LOCK_TIMEOUT: Duration = Duration::from_secs(10); +const STATE_LOCK: AdapterLockSpec = AdapterLockSpec::state("claude-mutation-scope-state.lock"); const ADAPTER_STATE_VERSION: u32 = 1; @@ -77,95 +75,14 @@ pub(crate) fn state_path(git_dir: &Path) -> PathBuf { state_dir(git_dir).join(ADAPTER_STATE_FILE) } -fn lock_path(git_dir: &Path) -> PathBuf { - state_dir(git_dir).join(ADAPTER_STATE_LOCK_FILE) +pub(crate) async fn read_state(git_dir: &Path) -> Result { + let git_dir = git_dir.to_owned(); + tokio::task::spawn_blocking(move || read_state_sync(&git_dir)) + .await + .context("Adapter state read worker failed")? } -struct AdapterStateLock { - file: File, -} - -#[derive(Debug)] -pub(crate) enum AdapterStateLockError { - TimedOut { path: PathBuf, timeout: Duration }, - Io(anyhow::Error), -} - -impl std::fmt::Display for AdapterStateLockError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - AdapterStateLockError::TimedOut { path, timeout } => write!( - f, - "Timed out after {timeout:?} waiting for adapter-state lock '{}'", - path.display() - ), - AdapterStateLockError::Io(source) => write!(f, "{source}"), - } - } -} - -impl std::error::Error for AdapterStateLockError {} - -impl AdapterStateLock { - fn acquire( - git_dir: &Path, - timeout: Duration, - ) -> Result { - let dir = state_dir(git_dir); - std::fs::create_dir_all(&dir) - .with_context(|| { - format!( - "Failed to create adapter state directory '{}'", - dir.display() - ) - }) - .map_err(AdapterStateLockError::Io)?; - - let path = lock_path(git_dir); - let file = OpenOptions::new() - .write(true) - .create(true) - .truncate(false) - .open(&path) - .with_context(|| { - format!( - "Failed to open adapter-state lock file '{}'", - path.display() - ) - }) - .map_err(AdapterStateLockError::Io)?; - - let deadline = Instant::now() + timeout; - loop { - match file.try_lock() { - Ok(()) => return Ok(AdapterStateLock { file }), - Err(TryLockError::WouldBlock) => { - let now = Instant::now(); - if now >= deadline { - return Err(AdapterStateLockError::TimedOut { path, timeout }); - } - std::thread::sleep(LOCK_POLL_INTERVAL.min(deadline - now)); - } - Err(TryLockError::Error(source)) => { - return Err(AdapterStateLockError::Io( - anyhow::Error::new(source).context(format!( - "Failed to acquire adapter-state lock '{}'", - path.display() - )), - )); - } - } - } - } -} - -impl Drop for AdapterStateLock { - fn drop(&mut self) { - let _ = self.file.unlock(); - } -} - -pub(crate) fn read_state(git_dir: &Path) -> Result { +fn read_state_sync(git_dir: &Path) -> Result { let path = state_path(git_dir); if !path.exists() { return Ok(AdapterState::default()); @@ -190,11 +107,12 @@ fn parse_adapter_state(content: &str, path: &Path) -> Result { Ok(state) } -fn write_state_durably(git_dir: &Path, state: &AdapterState) -> Result<()> { - write_state_durably_inner(git_dir, state, |_, _| Ok(())) +#[cfg(test)] +fn write_state_durably_sync(git_dir: &Path, state: &AdapterState) -> Result<()> { + write_state_durably_sync_inner(git_dir, state, |_, _| Ok(())) } -fn write_state_durably_inner( +fn write_state_durably_sync_inner( git_dir: &Path, state: &AdapterState, before_rename: F, @@ -261,73 +179,110 @@ where Ok(()) } -pub(crate) fn allocate_attempt( +enum StateTransaction { + Unchanged(T), + Persist(T), +} + +async fn with_locked_state(git_dir: &Path, operation: F) -> Result +where + T: Send + 'static, + F: FnOnce(&mut AdapterState) -> Result> + Send + 'static, +{ + with_locked_state_inner(git_dir, |_, _| Ok(()), operation).await +} + +async fn with_locked_state_inner( + git_dir: &Path, + before_rename: B, + operation: F, +) -> Result +where + T: Send + 'static, + B: FnOnce(&Path, &Path) -> Result<()> + Send + 'static, + F: FnOnce(&mut AdapterState) -> Result> + Send + 'static, +{ + let git_dir = git_dir.to_owned(); + STATE_LOCK + .run_locked_blocking(&state_dir(&git_dir), move || { + let mut state = read_state_sync(&git_dir)?; + match operation(&mut state)? { + StateTransaction::Unchanged(result) => Ok(result), + StateTransaction::Persist(result) => { + write_state_durably_sync_inner(&git_dir, &state, before_rename)?; + Ok(result) + } + } + }) + .await +} + +pub(crate) async fn allocate_attempt( git_dir: &Path, key: &AttemptKey, tool_name: &str, ) -> Result { - let _lock = AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT) - .map_err(|err| anyhow!("Failed to acquire adapter-state lock: {err}"))?; - - let mut state = read_state(git_dir)?; - - if let Some(existing) = state - .attempts - .iter() - .find(|attempt| attempt.matches_key(key)) - { - return Ok(AllocatedAttempt { - attempt: existing.clone(), - reused: true, - }); - } + let key = key.clone(); + let tool_name = tool_name.to_string(); + with_locked_state(git_dir, move |state| { + if let Some(existing) = state + .attempts + .iter() + .find(|attempt| attempt.matches_key(&key)) + { + return Ok(StateTransaction::Unchanged(AllocatedAttempt { + attempt: existing.clone(), + reused: true, + })); + } - let attempt_seq = state.next_attempt_seq; - let scope_id = format_claude_scope_id(attempt_seq, key); - let attempt = AdapterAttempt { - attempt_seq, - scope_id, - session_id: key.session_id.clone(), - agent_id: key.agent_id.clone(), - tool_use_id: key.tool_use_id.clone(), - tool_name: tool_name.to_string(), - phase: AttemptPhase::PendingStart, - }; + let attempt_seq = state.next_attempt_seq; + let scope_id = format_claude_scope_id(attempt_seq, &key); + let attempt = AdapterAttempt { + attempt_seq, + scope_id, + session_id: key.session_id.clone(), + agent_id: key.agent_id.clone(), + tool_use_id: key.tool_use_id.clone(), + tool_name, + phase: AttemptPhase::PendingStart, + }; - state.attempts.push(attempt.clone()); - state.next_attempt_seq += 1; - write_state_durably(git_dir, &state)?; + state.attempts.push(attempt.clone()); + state.next_attempt_seq += 1; - Ok(AllocatedAttempt { - attempt, - reused: false, + Ok(StateTransaction::Persist(AllocatedAttempt { + attempt, + reused: false, + })) }) + .await } -pub(crate) fn mark_active(git_dir: &Path, scope_id: &str) -> Result<()> { - let _lock = AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT) - .map_err(|err| anyhow!("Failed to acquire adapter-state lock: {err}"))?; - - let mut state = read_state(git_dir)?; - let attempt = state - .attempts - .iter_mut() - .find(|attempt| attempt.scope_id == scope_id) - .ok_or_else(|| anyhow!("No adapter-state attempt found for scope_id '{scope_id}'"))?; +pub(crate) async fn mark_active(git_dir: &Path, scope_id: &str) -> Result<()> { + let scope_id = scope_id.to_string(); + with_locked_state(git_dir, move |state| { + let attempt = state + .attempts + .iter_mut() + .find(|attempt| attempt.scope_id == scope_id) + .ok_or_else(|| anyhow!("No adapter-state attempt found for scope_id '{scope_id}'"))?; - match attempt.phase { - AttemptPhase::PendingStart => { - attempt.phase = AttemptPhase::Active; - } - AttemptPhase::Active => return Ok(()), - AttemptPhase::PendingAbandon => { - return Err(anyhow!( - "Cannot mark mutation-scope attempt '{scope_id}' active after abandonment was established" - )); + match attempt.phase { + AttemptPhase::PendingStart => { + attempt.phase = AttemptPhase::Active; + } + AttemptPhase::Active => return Ok(StateTransaction::Unchanged(())), + AttemptPhase::PendingAbandon => { + return Err(anyhow!( + "Cannot mark mutation-scope attempt '{scope_id}' active after abandonment was established" + )); + } } - } - write_state_durably(git_dir, &state) + Ok(StateTransaction::Persist(())) + }) + .await } fn transition_to_pending_abandon(state: &mut AdapterState, scope_ids: &[String]) { @@ -341,7 +296,7 @@ fn transition_to_pending_abandon(state: &mut AdapterState, scope_ids: &[String]) } } -pub(crate) fn mark_recovery_pending_and_pending_abandon( +pub(crate) async fn mark_recovery_pending_and_pending_abandon( git_dir: &Path, scope_ids: &[String], ) -> Result<()> { @@ -349,59 +304,48 @@ pub(crate) fn mark_recovery_pending_and_pending_abandon( return Ok(()); } - let _lock = AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT) - .map_err(|err| anyhow!("Failed to acquire adapter-state lock: {err}"))?; - - let mut state = read_state(git_dir)?; - state.recovery_pending = true; - transition_to_pending_abandon(&mut state, scope_ids); - write_state_durably(git_dir, &state) + let scope_ids = scope_ids.to_vec(); + with_locked_state(git_dir, move |state| { + state.recovery_pending = true; + transition_to_pending_abandon(state, &scope_ids); + Ok(StateTransaction::Persist(())) + }) + .await } -pub(crate) fn reprove_pending_abandon(git_dir: &Path) -> Result>> { - let _lock = AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT) - .map_err(|err| anyhow!("Failed to acquire adapter-state lock: {err}"))?; - - let state = read_state(git_dir)?; - - if !state.recovery_pending || state.attempts.is_empty() { - return Ok(None); - } - - let every_attempt_is_pending_abandon = state - .attempts - .iter() - .all(|attempt| attempt.phase == AttemptPhase::PendingAbandon); - - if !every_attempt_is_pending_abandon { - return Ok(None); - } +pub(crate) async fn reprove_pending_abandon(git_dir: &Path) -> Result>> { + with_locked_state(git_dir, |state| { + if !state.recovery_pending || state.attempts.is_empty() { + return Ok(StateTransaction::Unchanged(None)); + } - Ok(Some(state.attempts)) -} + let every_attempt_is_pending_abandon = state + .attempts + .iter() + .all(|attempt| attempt.phase == AttemptPhase::PendingAbandon); -pub(crate) fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { - let _lock = AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT) - .map_err(|err| anyhow!("Failed to acquire adapter-state lock: {err}"))?; + if !every_attempt_is_pending_abandon { + return Ok(StateTransaction::Unchanged(None)); + } - let mut state = read_state(git_dir)?; - let before = state.attempts.len(); - state - .attempts - .retain(|attempt| attempt.scope_id != scope_id); - if state.attempts.len() == before { - return Ok(()); - } - write_state_durably(git_dir, &state) + Ok(StateTransaction::Unchanged(Some(state.attempts.clone()))) + }) + .await } -pub(crate) fn mark_recovery_pending(git_dir: &Path) -> Result<()> { - let _lock = AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT) - .map_err(|err| anyhow!("Failed to acquire adapter-state lock: {err}"))?; - - let mut state = read_state(git_dir)?; - state.recovery_pending = true; - write_state_durably(git_dir, &state) +pub(crate) async fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { + let scope_id = scope_id.to_string(); + with_locked_state(git_dir, move |state| { + let before = state.attempts.len(); + state + .attempts + .retain(|attempt| attempt.scope_id != scope_id); + if state.attempts.len() == before { + return Ok(StateTransaction::Unchanged(())); + } + Ok(StateTransaction::Persist(())) + }) + .await } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -410,683 +354,131 @@ pub(crate) enum ClearRecoveryOutcome { StillPending, } -pub(crate) fn clear_recovery_pending_if_quiescent(git_dir: &Path) -> Result { - clear_recovery_pending_if_quiescent_inner(git_dir, |_, _| Ok(())) +pub(crate) async fn clear_recovery_pending_if_quiescent( + git_dir: &Path, +) -> Result { + clear_recovery_pending_if_quiescent_inner(git_dir, |_, _| Ok(())).await } -fn clear_recovery_pending_if_quiescent_inner( +async fn clear_recovery_pending_if_quiescent_inner( git_dir: &Path, before_rename: F, ) -> Result where - F: FnOnce(&Path, &Path) -> Result<()>, + F: FnOnce(&Path, &Path) -> Result<()> + Send + 'static, { - let _lock = AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT) - .map_err(|err| anyhow!("Failed to acquire adapter-state lock: {err}"))?; - - let mut state = read_state(git_dir)?; - - if !state.recovery_pending { - return Ok(ClearRecoveryOutcome::Cleared); - } - - if !state.attempts.is_empty() { - return Ok(ClearRecoveryOutcome::StillPending); - } + with_locked_state_inner(git_dir, before_rename, |state| { + if !state.recovery_pending { + return Ok(StateTransaction::Unchanged(ClearRecoveryOutcome::Cleared)); + } - state.recovery_pending = false; - write_state_durably_inner(git_dir, &state, before_rename)?; - Ok(ClearRecoveryOutcome::Cleared) -} + if !state.attempts.is_empty() { + return Ok(StateTransaction::Unchanged( + ClearRecoveryOutcome::StillPending, + )); + } -#[cfg(test)] -pub(crate) fn set_attempt_phase_for_tests( - git_dir: &Path, - scope_id: &str, - phase: AttemptPhase, -) -> AdapterAttempt { - let _lock = - AdapterStateLock::acquire(git_dir, DEFAULT_LOCK_TIMEOUT).expect("test phase-override lock"); - let mut state = read_state(git_dir).expect("test phase-override read"); - let attempt = state - .attempts - .iter_mut() - .find(|attempt| attempt.scope_id == scope_id) - .expect("attempt to override must already exist"); - attempt.phase = phase; - let updated = attempt.clone(); - write_state_durably(git_dir, &state).expect("test phase-override write"); - updated + state.recovery_pending = false; + Ok(StateTransaction::Persist(ClearRecoveryOutcome::Cleared)) + }) + .await } #[cfg(test)] mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - use std::thread; - - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-claude-mutation-scope-state-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(session_id: &str, agent_id: Option<&str>, tool_use_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - agent_id: agent_id.map(str::to_string), - tool_use_id: tool_use_id.to_string(), - } - } - - #[test] - fn read_state_returns_default_when_file_is_absent() { - let git_dir = unique_test_git_dir("read-default"); - - let state = read_state(&git_dir).expect("missing state file should read as default"); - assert_eq!(state, AdapterState::default()); + use std::sync::mpsc; + use std::time::{Duration, Instant}; - remove_test_git_dir(&git_dir); - } - - #[test] - fn attempt_seq_allocation_is_sequential_across_distinct_keys() { - let git_dir = unique_test_git_dir("sequential-allocation"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let first = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("first allocation should succeed"); - let second = allocate_attempt(&git_dir, &key("session-1", None, "toolu_2"), "Bash") - .expect("second allocation should succeed"); - let third = allocate_attempt( - &git_dir, - &key("session-1", Some("agent-1"), "toolu_3"), - "Edit", - ) - .expect("third allocation should succeed"); - - assert_eq!(first.attempt.attempt_seq, 1); - assert_eq!(second.attempt.attempt_seq, 2); - assert_eq!(third.attempt.attempt_seq, 3); - assert!(!first.reused); - assert!(!second.reused); - assert!(!third.reused); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.next_attempt_seq, 4); - assert_eq!(state.attempts.len(), 3); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn duplicate_live_attempt_reuses_the_same_attempt_seq_and_scope_id() { - let git_dir = unique_test_git_dir("duplicate-reuse"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("session-1", None, "toolu_1"); - - let first = allocate_attempt(&git_dir, &attempt_key, "Write") - .expect("first allocation should succeed"); - let second = allocate_attempt(&git_dir, &attempt_key, "Write") - .expect("duplicate delivery should still succeed"); - - assert!(!first.reused, "the first allocation is not a reuse"); - assert!( - second.reused, - "AC4: duplicate live delivery must be reported as reused" - ); - assert_eq!( - first.attempt.attempt_seq, second.attempt.attempt_seq, - "AC4: duplicate delivery must reuse the same attempt_seq" - ); - assert_eq!( - first.attempt.scope_id, second.attempt.scope_id, - "AC4: duplicate delivery must reuse the same ScopeId" - ); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!( - state.attempts.len(), - 1, - "reusing a live attempt must not create a second bookkeeping entry" - ); - assert_eq!( - state.next_attempt_seq, 2, - "reusing a live attempt must not advance the monotonic counter" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_terminal_attempt_is_followed_by_a_fresh_allocation_never_reusing_the_scope_id() { - let git_dir = unique_test_git_dir("terminal-then-fresh"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("session-1", None, "toolu_1"); - - let first = allocate_attempt(&git_dir, &attempt_key, "Write") - .expect("first allocation should succeed"); - mark_active(&git_dir, &first.attempt.scope_id).expect("attempt should become active"); - remove_attempt(&git_dir, &first.attempt.scope_id) - .expect("terminal attempt should be removable"); - - let second = allocate_attempt(&git_dir, &attempt_key, "Write") - .expect("a later execution of the same tool_use_id should allocate a fresh attempt"); - - assert!( - !second.reused, - "the attempt is a fresh allocation, not a reuse" - ); - assert_ne!( - first.attempt.attempt_seq, second.attempt.attempt_seq, - "AC5: attempt_seq must never be reused after the prior attempt became terminal" - ); - assert_ne!( - first.attempt.scope_id, second.attempt.scope_id, - "AC5: a terminal ScopeId must never be reused" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_active_transitions_phase_from_pending_start_to_active() { - let git_dir = unique_test_git_dir("mark-active"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - assert_eq!(allocated.attempt.phase, AttemptPhase::PendingStart); - - mark_active(&git_dir, &allocated.attempt.scope_id).expect("mark_active should succeed"); - - let state = read_state(&git_dir).expect("state should be readable"); - let persisted = state - .attempts - .iter() - .find(|attempt| attempt.scope_id == allocated.attempt.scope_id) - .expect("attempt should still be present"); - assert_eq!(persisted.phase, AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_active_is_idempotent_when_already_active() { - let git_dir = unique_test_git_dir("mark-active-idempotent"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - mark_active(&git_dir, &allocated.attempt.scope_id) - .expect("first activation should succeed"); - - mark_active(&git_dir, &allocated.attempt.scope_id) - .expect("re-delivery of PreToolUse after activation must be a safe idempotent no-op"); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_active_is_forbidden_once_pending_abandon_is_established() { - let git_dir = unique_test_git_dir("mark-active-forbidden-pending-abandon"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - - mark_recovery_pending_and_pending_abandon( - &git_dir, - std::slice::from_ref(&allocated.attempt.scope_id), - ) - .expect("atomic establishment should succeed"); - - let error = mark_active(&git_dir, &allocated.attempt.scope_id) - .expect_err("PendingAbandon -> Active must be forbidden"); - assert!(error.to_string().contains("abandon")); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!( - state.attempts[0].phase, - AttemptPhase::PendingAbandon, - "a rejected activation must not mutate the persisted phase" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_recovery_pending_and_pending_abandon_establishes_both_facts_in_one_durable_write() { - let git_dir = unique_test_git_dir("atomic-recovery-and-pending-abandon"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - mark_active(&git_dir, &allocated.attempt.scope_id).expect("attempt should become active"); + use super::{ + clear_recovery_pending_if_quiescent_inner, read_state, write_state_durably_sync, + AdapterState, ClearRecoveryOutcome, + }; - mark_recovery_pending_and_pending_abandon( - &git_dir, - std::slice::from_ref(&allocated.attempt.scope_id), + const INJECTED_WRITE_DELAY: Duration = Duration::from_millis(500); + const UNRELATED_TIMER: Duration = Duration::from_millis(10); + const RESPONSIVENESS_BUDGET: Duration = Duration::from_millis(250); + const HOOK_RELEASE_TIMEOUT: Duration = Duration::from_secs(5); + + fn seed_quiescent_recovery_pending(git_dir: &std::path::Path) { + write_state_durably_sync( + git_dir, + &AdapterState { + recovery_pending: true, + ..AdapterState::default() + }, ) - .expect("atomic establishment should succeed"); - - let state = read_state(&git_dir).expect("state should be readable"); - assert!( - state.recovery_pending, - "T04: the recovery barrier must be armed by the atomic operation" - ); - assert_eq!( - state.attempts[0].phase, - AttemptPhase::PendingAbandon, - "T04: abandonment evidence must be established alongside the barrier in the same write" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_recovery_pending_and_pending_abandon_failed_write_leaves_no_partial_invariant() { - let git_dir = unique_test_git_dir("atomic-injected-failure"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - mark_active(&git_dir, &allocated.attempt.scope_id).expect("attempt should become active"); - - let mut state = read_state(&git_dir).expect("state should be readable"); - state.recovery_pending = true; - transition_to_pending_abandon( - &mut state, - std::slice::from_ref(&allocated.attempt.scope_id), - ); - - let result = write_state_durably_inner(&git_dir, &state, |_, _| { - Err(anyhow!("injected interruption before rename")) - }); - assert!( - result.is_err(), - "write_state_durably_inner should surface the injected interruption" - ); - - let after = - read_state(&git_dir).expect("state should be readable after the injected failure"); - assert!( - !after.recovery_pending, - "an interrupted write must not leave the barrier half-armed" - ); - assert_eq!( - after.attempts[0].phase, - AttemptPhase::Active, - "an interrupted write must not leave the attempt half-transitioned to PendingAbandon" - ); - - remove_test_git_dir(&git_dir); + .expect("seeding adapter state should succeed"); } - #[test] - fn removing_an_already_removed_attempt_is_a_safe_no_op() { - let git_dir = unique_test_git_dir("remove-idempotent"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - - remove_attempt(&git_dir, &allocated.attempt.scope_id) - .expect("first removal should succeed"); - remove_attempt(&git_dir, &allocated.attempt.scope_id) - .expect("D9: duplicate terminal delivery after cleanup must be a safe no-op"); - - let state = read_state(&git_dir).expect("state should be readable"); - assert!(state.attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn malformed_state_file_is_rejected_without_fabricating_bookkeeping() { - let git_dir = unique_test_git_dir("malformed-json"); - let dir = state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write(state_path(&git_dir), b"not json") - .expect("malformed file should be written"); - - let error = read_state(&git_dir).expect_err("malformed state file must be rejected"); - assert!(error.to_string().contains("malformed")); + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn state_transaction_persistence_does_not_starve_the_tokio_worker() { + let git_dir = tempfile::tempdir().expect("temp dir"); + seed_quiescent_recovery_pending(git_dir.path()); - remove_test_git_dir(&git_dir); - } - - #[test] - fn unsupported_version_is_rejected_without_fabricating_bookkeeping() { - let git_dir = unique_test_git_dir("unsupported-version"); - let dir = state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write( - state_path(&git_dir), - serde_json::json!({ - "version": 99, - "next_attempt_seq": 1, - "recovery_pending": false, - "attempts": [] + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let transaction_git_dir = git_dir.path().to_owned(); + let transaction = tokio::spawn(async move { + clear_recovery_pending_if_quiescent_inner(&transaction_git_dir, move |_, _| { + let _ = entered_tx.send(Instant::now()); + std::thread::sleep(INJECTED_WRITE_DELAY); + Ok(()) }) - .to_string(), - ) - .expect("state file with unsupported version should be written"); - - let error = read_state(&git_dir).expect_err("unsupported version must be rejected"); - assert!(error.to_string().contains("unsupported version")); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn interruption_before_rename_leaves_the_canonical_path_unaffected() { - let git_dir = unique_test_git_dir("interrupted-before-rename"); - let dir = state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - - let state = AdapterState { - next_attempt_seq: 5, - ..AdapterState::default() - }; - let result = write_state_durably_inner(&git_dir, &state, |tmp_path, canonical_path| { - assert!( - tmp_path.exists(), - "temp file should exist by the time the pre-rename hook runs" - ); - assert!( - !canonical_path.exists(), - "canonical path should still be absent at the pre-rename hook" - ); - Err(anyhow!("injected interruption before rename")) + .await }); - assert!( - result.is_err(), - "write_state_durably_inner should surface the injected interruption" - ); - assert!( - !state_path(&git_dir).exists(), - "atomic replacement: canonical state path must stay absent when interrupted before rename" - ); - - let after = - read_state(&git_dir).expect("read should not error on an absent canonical file"); - assert_eq!( - after, - AdapterState::default(), - "an interrupted write must not partially apply" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_leftover_lock_file_with_no_active_os_lock_does_not_block_a_new_acquirer() { - let git_dir = unique_test_git_dir("leftover-lock-file"); - let dir = state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write(lock_path(&git_dir), b"leftover") - .expect("leftover lock file should be writable"); + let entered_at = entered_rx.await.expect("transaction should reach the hook"); + let unrelated = tokio::spawn(async { + tokio::time::sleep(UNRELATED_TIMER).await; + Instant::now() + }); + let unrelated_done_at = unrelated.await.expect("unrelated task should complete"); - let result = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write"); assert!( - result.is_ok(), - "a lock file with no active OS lock held against it must not block a new acquirer" + unrelated_done_at.duration_since(entered_at) < RESPONSIVENESS_BUDGET, + "unrelated Tokio task was starved for {:?} by the state transaction", + unrelated_done_at.duration_since(entered_at) ); - remove_test_git_dir(&git_dir); + let outcome = transaction + .await + .expect("transaction task should join") + .expect("transaction should succeed"); + assert_eq!(outcome, ClearRecoveryOutcome::Cleared); } - const PARALLEL_WRITER_COUNT: u64 = 8; - - #[test] - fn parallel_writers_for_distinct_keys_all_converge_without_lost_updates() { - let git_dir = unique_test_git_dir("parallel-writers"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn aborted_caller_does_not_interrupt_a_started_state_transaction() { + let git_dir = tempfile::tempdir().expect("temp dir"); + seed_quiescent_recovery_pending(git_dir.path()); - let handles: Vec<_> = (0..PARALLEL_WRITER_COUNT) - .map(|index| { - let git_dir = git_dir.clone(); - thread::spawn(move || { - let attempt_key = key("session-1", None, &format!("toolu_{index}")); - allocate_attempt(&git_dir, &attempt_key, "Write") - .expect("each concurrent allocation should durably succeed") - }) + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + let transaction_git_dir = git_dir.path().to_owned(); + let transaction = tokio::spawn(async move { + clear_recovery_pending_if_quiescent_inner(&transaction_git_dir, move |_, _| { + let _ = entered_tx.send(()); + let _ = release_rx.recv_timeout(HOOK_RELEASE_TIMEOUT); + Ok(()) }) - .collect(); - - let allocated: Vec = handles - .into_iter() - .map(|handle| handle.join().expect("writer thread should not panic")) - .collect(); - - let mut attempt_seqs: Vec = allocated.iter().map(|a| a.attempt.attempt_seq).collect(); - attempt_seqs.sort_unstable(); - attempt_seqs.dedup(); - assert_eq!( - attempt_seqs.len(), - usize::try_from(PARALLEL_WRITER_COUNT).unwrap(), - "concurrent writers must not lose updates or collide on attempt_seq" - ); - - let state = read_state(&git_dir).expect("state should be readable after concurrent writes"); - assert_eq!( - state.attempts.len(), - usize::try_from(PARALLEL_WRITER_COUNT).unwrap() - ); - assert_eq!(state.next_attempt_seq, PARALLEL_WRITER_COUNT + 1); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_second_acquirer_blocks_until_the_first_releases() { - use std::sync::mpsc; - - let git_dir = unique_test_git_dir("lock-contention"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let holder = AdapterStateLock::acquire(&git_dir, Duration::from_secs(5)) - .expect("first acquirer should succeed immediately"); - - let (result_tx, result_rx) = mpsc::channel(); - let git_dir_clone = git_dir.clone(); - let handle = thread::spawn(move || { - let result = AdapterStateLock::acquire(&git_dir_clone, Duration::from_secs(5)); - let _ = result_tx.send(()); - result + .await }); - assert!( - result_rx.recv_timeout(Duration::from_millis(300)).is_err(), - "second acquirer should not succeed while the first still holds the lock" - ); - - drop(holder); - - result_rx - .recv_timeout(Duration::from_secs(5)) - .expect("second acquirer should complete once the first releases the lock"); - assert!(handle - .join() - .expect("second acquirer thread should not panic") - .is_ok()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_recovery_pending_arms_the_barrier_without_touching_attempts() { - let git_dir = unique_test_git_dir("mark-recovery-pending"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - - mark_recovery_pending(&git_dir).expect("marking recovery pending should succeed"); - - let state = read_state(&git_dir).expect("state should be readable"); - assert!(state.recovery_pending, "D19: the barrier must be armed"); - assert_eq!( - state.attempts.len(), - 1, - "marking recovery pending must not remove or otherwise touch tracked attempts" - ); - assert_eq!(state.attempts[0].scope_id, allocated.attempt.scope_id); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_recovery_pending_is_idempotent() { - let git_dir = unique_test_git_dir("mark-recovery-pending-idempotent"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - mark_recovery_pending(&git_dir).expect("first marking should succeed"); - mark_recovery_pending(&git_dir).expect("second marking should succeed"); - - let state = read_state(&git_dir).expect("state should be readable"); - assert!(state.recovery_pending); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_pending_if_quiescent_resets_the_barrier_when_no_attempts_remain() { - let git_dir = unique_test_git_dir("clear-recovery-pending-quiescent"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - mark_recovery_pending(&git_dir).expect("marking recovery pending should succeed"); - - let outcome = clear_recovery_pending_if_quiescent(&git_dir) - .expect("clearing the barrier should succeed"); + entered_rx.await.expect("transaction should reach the hook"); + transaction.abort(); + assert!(transaction + .await + .expect_err("aborted caller should not join successfully") + .is_cancelled()); + release_tx.send(()).expect("hook should still be waiting"); + + let outcome = clear_recovery_pending_if_quiescent_inner(git_dir.path(), |_, _| Ok(())) + .await + .expect("state lock should be released and state should parse"); assert_eq!(outcome, ClearRecoveryOutcome::Cleared); - let state = read_state(&git_dir).expect("state should be readable"); + let state = read_state(git_dir.path()) + .await + .expect("persisted state should parse"); assert!(!state.recovery_pending); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_pending_if_quiescent_is_a_no_op_when_recovery_is_already_clear() { - let git_dir = unique_test_git_dir("clear-recovery-pending-already-clear"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let outcome = clear_recovery_pending_if_quiescent(&git_dir) - .expect("clearing an already-clear barrier should succeed"); - assert_eq!(outcome, ClearRecoveryOutcome::Cleared); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_pending_if_quiescent_leaves_recovery_armed_when_an_attempt_remains() { - let git_dir = unique_test_git_dir("clear-recovery-pending-still-pending"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let allocated = allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - mark_recovery_pending_and_pending_abandon( - &git_dir, - std::slice::from_ref(&allocated.attempt.scope_id), - ) - .expect("atomic establishment should succeed"); - - let outcome = clear_recovery_pending_if_quiescent(&git_dir) - .expect("proving quiescence should not error"); - assert_eq!(outcome, ClearRecoveryOutcome::StillPending); - - let state = read_state(&git_dir).expect("state should be readable"); - assert!( - state.recovery_pending, - "recovery must remain armed when an unresolved attempt survives the proof" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_pending_if_quiescent_interrupted_before_rename_keeps_recovery_armed() { - let git_dir = unique_test_git_dir("clear-recovery-pending-interrupted-before-rename"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - mark_recovery_pending(&git_dir).expect("marking recovery pending should succeed"); - - let result = - clear_recovery_pending_if_quiescent_inner(&git_dir, |tmp_path, canonical_path| { - assert!( - tmp_path.exists(), - "temp replacement file should exist by the time the pre-rename hook runs" - ); - assert!( - canonical_path.exists(), - "canonical state file should still exist at the pre-rename hook" - ); - Err(anyhow!("injected interruption before rename")) - }); - - assert!( - result.is_err(), - "an interrupted durable write must surface the injected error" - ); - - let after = - read_state(&git_dir).expect("state should be readable after the injected interruption"); - assert!( - after.recovery_pending, - "an interrupted clear must leave the canonical state fail-closed with recovery armed" - ); - assert!( - after.attempts.is_empty(), - "the interrupted clear must not fabricate or lose attempt bookkeeping" - ); - - let outcome = clear_recovery_pending_if_quiescent(&git_dir) - .expect("a retried clear should succeed once nothing interrupts the rename"); - assert_eq!(outcome, ClearRecoveryOutcome::Cleared); - - let final_state = - read_state(&git_dir).expect("state should be readable after the successful retry"); - assert!( - !final_state.recovery_pending, - "the retried clear must durably disarm the barrier" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn adapter_state_files_live_only_below_git_dir_sce() { - let git_dir = unique_test_git_dir("path-boundary"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - allocate_attempt(&git_dir, &key("session-1", None, "toolu_1"), "Write") - .expect("allocation should succeed"); - - let sce_dir = git_dir.join(SCE_STATE_DIR); - assert!(state_path(&git_dir).starts_with(&sce_dir)); - assert!(lock_path(&git_dir).starts_with(&sce_dir)); - - let mut found_state_file = false; - for entry in std::fs::read_dir(&sce_dir).expect("sce dir should be readable") { - let entry = entry.expect("dir entry should be readable"); - assert!( - entry.path().starts_with(&sce_dir), - "AC19: adapter state must live only under /sce/" - ); - if entry.path() == state_path(&git_dir) { - found_state_file = true; - } - } - assert!( - found_state_file, - "state file should exist under /sce/" - ); - - remove_test_git_dir(&git_dir); } } diff --git a/cli/src/services/hooks/claude_mutation_scope/tests.rs b/cli/src/services/hooks/claude_mutation_scope/tests.rs deleted file mode 100644 index 4c244d28a..000000000 --- a/cli/src/services/hooks/claude_mutation_scope/tests.rs +++ /dev/null @@ -1,3960 +0,0 @@ -use super::*; - -fn pre_tool_use_json(overrides: &[(&str, Value)]) -> String { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_PRE_TOOL_USE.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert( - CWD_FIELD.to_string(), - Value::String("/repo/checkout".to_string()), - ); - object.insert( - TOOL_NAME_FIELD.to_string(), - Value::String("Write".to_string()), - ); - object.insert( - TOOL_USE_ID_FIELD.to_string(), - Value::String("toolu_1".to_string()), - ); - for (field, value) in overrides { - object.insert((*field).to_string(), value.clone()); - } - Value::Object(object).to_string() -} - -fn identity(session_id: &str, agent_id: Option<&str>, tool_use_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - agent_id: agent_id.map(str::to_string), - tool_use_id: tool_use_id.to_string(), - } -} - -mod ingress_conformance { - use super::*; - use crate::services::hooks::mutation_scope_ingress_conformance::{ - self as conformance, mutation_scope_ingress_conformance_tests, IngressConformance, - }; - - struct ClaudeIngressConformance; - - impl IngressConformance for ClaudeIngressConformance { - const ADAPTER: &'static str = "Claude"; - const EVENT_NAME_FIELD: &'static str = HOOK_EVENT_NAME_FIELD; - const REQUIRED_START_FIELDS: &'static [&'static str] = &[ - SESSION_ID_FIELD, - CWD_FIELD, - TOOL_NAME_FIELD, - TOOL_USE_ID_FIELD, - ]; - const OPTIONAL_START_FIELDS: &'static [(&'static str, &'static str)] = &[ - (AGENT_ID_FIELD, "agent-1"), - (PROMPT_ID_FIELD, "prompt-1"), - (AGENT_TYPE_FIELD, "general-purpose"), - ]; - const UNSUPPORTED_EVENT_NAMES: &'static [&'static str] = - &["PostToolBatch", "PreCompact", "Notification"]; - - fn tracked_start() -> Map { - serde_json::from_str(&pre_tool_use_json(&[])) - .expect("the tracked PreToolUse fixture is a JSON object") - } - - fn parse(payload: &str) -> Result<()> { - parse_claude_hook_event(payload).map(|_| ()) - } - - fn run(payload: &str) -> Result { - run_claude_mutation_scope_from_payload(payload, None) - } - - fn run_with_seams( - payload: &str, - resolve_git_dir: conformance::GitDirResolver, - seam: conformance::IngressSeam, - ) -> Result { - run_claude_mutation_scope_from_payload_with(payload, None, resolve_git_dir, seam) - } - } - - mutation_scope_ingress_conformance_tests!(ClaudeIngressConformance); -} - -#[test] -fn pre_tool_use_parses_required_and_optional_fields() { - let payload = pre_tool_use_json(&[ - (AGENT_ID_FIELD, Value::String("agent-1".to_string())), - (PROMPT_ID_FIELD, Value::String("prompt-1".to_string())), - ( - AGENT_TYPE_FIELD, - Value::String("general-purpose".to_string()), - ), - ]); - - let event = parse_claude_hook_event(&payload).expect("valid PreToolUse parses"); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - - assert_eq!(execution.identity.session_id, "session-1"); - assert_eq!(execution.identity.cwd, "/repo/checkout"); - assert_eq!(execution.identity.tool_name, "Write"); - assert_eq!(execution.identity.tool_use_id, "toolu_1"); - assert_eq!(execution.identity.agent_id.as_deref(), Some("agent-1")); - assert_eq!(execution.prompt_id.as_deref(), Some("prompt-1")); - assert_eq!(execution.agent_type.as_deref(), Some("general-purpose")); - assert!(!execution.run_in_background); -} - -#[test] -fn pre_tool_use_agent_id_absent_means_main_thread() { - let payload = pre_tool_use_json(&[]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - - assert_eq!(execution.identity.agent_id, None); - assert!(!execution.identity.is_subagent()); -} - -#[test] -fn pre_tool_use_agent_id_present_means_subagent() { - let payload = pre_tool_use_json(&[(AGENT_ID_FIELD, Value::String("agent-1".to_string()))]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - - assert!(execution.identity.is_subagent()); -} - -#[test] -fn pre_tool_use_prompt_id_and_agent_type_are_optional() { - let payload = pre_tool_use_json(&[]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - - assert_eq!(execution.prompt_id, None); - assert_eq!(execution.agent_type, None); -} - -#[test] -fn null_optional_field_is_none() { - let payload = pre_tool_use_json(&[(AGENT_ID_FIELD, Value::Null)]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - assert_eq!(execution.identity.agent_id, None); -} - -#[test] -fn post_tool_use_and_failure_and_permission_denied_share_tool_identity_shape() { - for event_name in [ - HOOK_EVENT_POST_TOOL_USE, - HOOK_EVENT_POST_TOOL_USE_FAILURE, - HOOK_EVENT_PERMISSION_DENIED, - ] { - let payload = - pre_tool_use_json(&[(HOOK_EVENT_NAME_FIELD, Value::String(event_name.to_string()))]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let identity = match event { - ClaudeHookEvent::PostToolUse(identity) - | ClaudeHookEvent::PostToolUseFailure(identity) - | ClaudeHookEvent::PermissionDenied(identity) => identity, - other => panic!("expected a tool-identity event, got {other:?}"), - }; - assert_eq!(identity.session_id, "session-1"); - assert_eq!(identity.tool_use_id, "toolu_1"); - } -} - -#[test] -fn session_scoped_lifecycle_events_parse_session_identity() { - for event_name in [ - HOOK_EVENT_STOP, - HOOK_EVENT_STOP_FAILURE, - HOOK_EVENT_USER_PROMPT_SUBMIT, - HOOK_EVENT_SESSION_END, - ] { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String("/repo".to_string())); - let payload = Value::Object(object).to_string(); - - let event = parse_claude_hook_event(&payload).unwrap(); - let identity = match event { - ClaudeHookEvent::Stop(identity) - | ClaudeHookEvent::StopFailure(identity) - | ClaudeHookEvent::UserPromptSubmit(identity) - | ClaudeHookEvent::SessionEnd(identity) => identity, - other => panic!("expected a session-identity event, got {other:?}"), - }; - assert_eq!(identity.session_id, "session-1"); - assert_eq!(identity.cwd, "/repo"); - } -} - -#[test] -fn subagent_stop_requires_agent_id() { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_SUBAGENT_STOP.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String("/repo".to_string())); - let payload = Value::Object(object).to_string(); - - let error = parse_claude_hook_event(&payload).unwrap_err(); - assert!(error.to_string().contains("agent_id")); -} - -#[test] -fn subagent_stop_parses_agent_identity() { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_SUBAGENT_STOP.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String("/repo".to_string())); - object.insert( - AGENT_ID_FIELD.to_string(), - Value::String("agent-1".to_string()), - ); - let payload = Value::Object(object).to_string(); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::SubagentStop(identity) = event else { - panic!("expected SubagentStop"); - }; - assert_eq!(identity.agent_id, "agent-1"); -} - -#[test] -fn worktree_remove_requires_worktree_path_not_cwd() { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_WORKTREE_REMOVE.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert( - WORKTREE_PATH_FIELD.to_string(), - Value::String("/repo/.claude/worktrees/agent-1".to_string()), - ); - let payload = Value::Object(object).to_string(); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::WorktreeRemove(worktree_remove) = event else { - panic!("expected WorktreeRemove"); - }; - assert_eq!(worktree_remove.session_id, "session-1"); - assert_eq!( - worktree_remove.worktree_path, - "/repo/.claude/worktrees/agent-1" - ); -} - -#[test] -fn session_start_and_subagent_start_establish_no_scope_payload() { - for event_name in [HOOK_EVENT_SESSION_START, HOOK_EVENT_SUBAGENT_START] { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - let payload = Value::Object(object).to_string(); - - let event = parse_claude_hook_event(&payload).unwrap(); - assert!(matches!( - event, - ClaudeHookEvent::SessionStart | ClaudeHookEvent::SubagentStart - )); - } -} - -#[test] -fn run_in_background_true_is_parsed() { - let payload = pre_tool_use_json(&[( - TOOL_INPUT_FIELD, - serde_json::json!({ "run_in_background": true }), - )]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - assert!(execution.run_in_background); -} - -#[test] -fn run_in_background_false_is_parsed() { - let payload = pre_tool_use_json(&[( - TOOL_INPUT_FIELD, - serde_json::json!({ "run_in_background": false }), - )]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - assert!(!execution.run_in_background); -} - -#[test] -fn run_in_background_absent_defaults_to_false() { - let payload = pre_tool_use_json(&[( - TOOL_INPUT_FIELD, - serde_json::json!({ "command": "echo hi" }), - )]); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - assert!(!execution.run_in_background); -} - -#[test] -fn tool_input_absent_defaults_run_in_background_to_false() { - let mut object: serde_json::Map = - serde_json::from_str(&pre_tool_use_json(&[])).unwrap(); - object.remove(TOOL_INPUT_FIELD); - let payload = Value::Object(object).to_string(); - - let event = parse_claude_hook_event(&payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - assert!(!execution.run_in_background); -} - -#[test] -fn run_in_background_wrong_type_is_rejected() { - let payload = pre_tool_use_json(&[( - TOOL_INPUT_FIELD, - serde_json::json!({ "run_in_background": "yes" }), - )]); - - let error = parse_claude_hook_event(&payload).unwrap_err(); - assert!(error.to_string().contains("run_in_background")); -} - -#[test] -fn tool_input_wrong_type_is_rejected() { - let payload = pre_tool_use_json(&[(TOOL_INPUT_FIELD, Value::String("nope".to_string()))]); - - let error = parse_claude_hook_event(&payload).unwrap_err(); - assert!(error.to_string().contains("tool_input")); -} - -#[test] -fn known_mutation_capable_tools_are_classified_mutation_capable() { - for tool_name in [ - "Bash", - "PowerShell", - "Write", - "Edit", - "NotebookEdit", - "MultiEdit", - ] { - assert_eq!( - classify_tool(tool_name), - ToolClassification::MutationCapable, - "expected {tool_name} to be MutationCapable" - ); - } -} - -#[test] -fn known_read_only_tools_are_classified_read_only() { - for tool_name in [ - "Read", - "Glob", - "Grep", - "WebFetch", - "WebSearch", - "AskUserQuestion", - ] { - assert_eq!( - classify_tool(tool_name), - ToolClassification::ReadOnly, - "expected {tool_name} to be ReadOnly" - ); - } -} - -#[test] -fn agent_is_classified_delegation() { - assert_eq!(classify_tool("Agent"), ToolClassification::Delegation); -} - -#[test] -fn mcp_tools_are_classified_mutation_capable() { - assert_eq!( - classify_tool("mcp__claude-in-chrome__navigate"), - ToolClassification::MutationCapable - ); -} - -#[test] -fn unknown_tool_names_are_conservatively_mutation_capable() { - assert_eq!( - classify_tool("SomeBrandNewTool"), - ToolClassification::MutationCapable - ); -} - -const PROBE14_BASH_RUN_IN_BACKGROUND_TRUE: &str = - include_str!("fixtures/probe14-run-in-background-true.pre_tool_use.json"); -const PROBE15_BASH_RUN_IN_BACKGROUND_FALSE: &str = - include_str!("fixtures/probe15-run-in-background-false-hard-gate.pre_tool_use.json"); - -fn parsed_pre_tool_use(payload: &str) -> ClaudeToolExecution { - let event = parse_claude_hook_event(payload).unwrap(); - let ClaudeHookEvent::PreToolUse(execution) = event else { - panic!("expected PreToolUse"); - }; - execution -} - -#[test] -fn real_bash_run_in_background_true_fixture_is_explicit_background_shell() { - let execution = parsed_pre_tool_use(PROBE14_BASH_RUN_IN_BACKGROUND_TRUE); - - assert_eq!(execution.identity.tool_name, "Bash"); - assert!(execution.run_in_background); - assert!(is_explicit_background_shell( - &execution.identity.tool_name, - execution.run_in_background - )); -} - -#[test] -fn real_bash_run_in_background_false_fixture_is_not_explicit_background_shell() { - let execution = parsed_pre_tool_use(PROBE15_BASH_RUN_IN_BACKGROUND_FALSE); - - assert_eq!(execution.identity.tool_name, "Bash"); - assert!(!execution.run_in_background); - assert!(!is_explicit_background_shell( - &execution.identity.tool_name, - execution.run_in_background - )); -} - -#[test] -fn powershell_with_run_in_background_true_is_explicit_background_shell() { - assert!(is_explicit_background_shell("PowerShell", true)); -} - -#[test] -fn powershell_with_run_in_background_false_is_not_explicit_background_shell() { - assert!(!is_explicit_background_shell("PowerShell", false)); -} - -#[test] -fn write_with_run_in_background_true_is_not_explicit_background_shell() { - assert!(!is_explicit_background_shell("Write", true)); -} - -#[test] -fn same_attempt_seq_and_key_is_deterministic() { - let key = identity("session-1", Some("agent-1"), "toolu_1"); - - let first = format_claude_scope_id(3, &key); - let second = format_claude_scope_id(3, &key); - - assert_eq!( - first, second, - "AC4: duplicate delivery must reuse the same ScopeId" - ); - assert_eq!( - claude_scope_start_event_id(&first), - claude_scope_start_event_id(&second) - ); -} - -#[test] -fn fresh_attempt_seq_yields_a_new_scope_id() { - let key = identity("session-1", None, "toolu_1"); - - let first = format_claude_scope_id(1, &key); - let second = format_claude_scope_id(2, &key); - - assert_ne!( - first, second, - "AC5: a fresh attempt_seq for the same tool_use_id must get a new ScopeId" - ); -} - -#[test] -fn main_and_distinct_agents_produce_distinct_scope_ids() { - let main = identity("session-1", None, "toolu_1"); - let agent_a = identity("session-1", Some("A"), "toolu_1"); - let agent_b = identity("session-1", Some("B"), "toolu_1"); - - let main_scope = format_claude_scope_id(1, &main); - let scope_for_a = format_claude_scope_id(1, &agent_a); - let scope_for_b = format_claude_scope_id(1, &agent_b); - - assert_ne!( - main_scope, scope_for_a, - "AC6: main vs agent_id=A must differ" - ); - assert_ne!( - main_scope, scope_for_b, - "AC6: main vs agent_id=B must differ" - ); - assert_ne!( - scope_for_a, scope_for_b, - "AC6: agent_id=A vs agent_id=B must differ" - ); -} - -#[test] -fn event_id_derivation_is_a_pure_function_of_scope_id() { - let scope_id = format_claude_scope_id(7, &identity("session-1", None, "toolu_1")); - - assert_eq!( - claude_scope_start_event_id(&scope_id), - format!("{scope_id}|start") - ); - assert_eq!( - claude_scope_close_event_id(&scope_id), - format!("{scope_id}|close") - ); - assert_ne!( - claude_scope_start_event_id(&scope_id), - claude_scope_close_event_id(&scope_id) - ); -} - -#[test] -fn length_prefixing_disambiguates_delimiter_characters_inside_fields() { - let tricky = identity( - "sess|a=0:x|t=1:y", - Some("agent|with|pipes"), - "tool:with:colons", - ); - - let scope_id = format_claude_scope_id(1, &tricky); - - let agent_id = tricky.agent_id.as_deref().unwrap(); - let expected = format!( - "cc-tool-v1|n=1|s={}:{}|a={}:{}|t={}:{}", - tricky.session_id.len(), - tricky.session_id, - agent_id.len(), - agent_id, - tricky.tool_use_id.len(), - tricky.tool_use_id, - ); - - assert_eq!(scope_id, expected); -} - -#[test] -fn attempt_key_projects_only_the_execution_key_fields() { - let identity_a = ClaudeToolIdentity { - session_id: "session-1".to_string(), - cwd: "/repo".to_string(), - agent_id: Some("agent-1".to_string()), - tool_name: "Write".to_string(), - tool_use_id: "toolu_1".to_string(), - }; - let identity_b = ClaudeToolIdentity { - tool_name: "Bash".to_string(), - cwd: "/other".to_string(), - ..identity_a.clone() - }; - - assert_eq!( - identity_a.attempt_key(), - identity_b.attempt_key(), - "attempt_key must depend only on (session_id, agent_id, tool_use_id)" - ); -} - -mod driver { - use std::cell::{Cell, RefCell}; - use std::path::PathBuf; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::{Arc, Mutex}; - - use anyhow::anyhow; - - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-claude-mutation-scope-driver-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - #[allow(clippy::unnecessary_wraps)] - fn ok_seam(_root: &Path, _payload: &str, _logger: Option<&dyn Logger>) -> Result { - Ok(String::new()) - } - - fn unreachable_seam( - _root: &Path, - payload: &str, - _logger: Option<&dyn Logger>, - ) -> Result { - panic!("the ingress seam must not be called for this payload: {payload}"); - } - - fn seam_failing_on( - operation: &'static str, - ) -> impl Fn(&Path, &str, Option<&dyn Logger>) -> Result { - seam_failing_on_any(vec![operation]) - } - - fn seam_failing_on_any( - operations: Vec<&'static str>, - ) -> impl Fn(&Path, &str, Option<&dyn Logger>) -> Result { - move |_root, payload, _logger| { - if operations - .iter() - .any(|operation| payload.contains(&format!(r#""operation":"{operation}""#))) - { - Err(anyhow!( - "seam failure injected by test for one of {operations:?}" - )) - } else { - Ok(String::new()) - } - } - } - - fn fixed_resolver(git_dir: PathBuf) -> impl Fn(&str) -> Result { - move |_cwd| Ok(git_dir.clone()) - } - - fn start_with_model_resolver( - payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - model_state_resolver: ClaudeModelStateResolver, - seam: IngressSeam, - ) -> Result { - run_claude_mutation_scope_from_payload_with_resolver( - payload, - logger, - resolve_git_dir, - model_state_resolver, - seam, - ) - } - - #[derive(Clone, Default)] - struct RecordingLogger { - warnings: Arc>>, - } - - impl RecordingLogger { - fn warnings(&self) -> Vec<(String, String)> { - self.warnings - .lock() - .expect("recording logger mutex must not be poisoned") - .clone() - } - } - - impl Logger for RecordingLogger { - fn info(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn debug(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - - fn warn(&self, event_id: &str, message: &str, _: &[(&str, &str)], _: Option<&str>) { - self.warnings - .lock() - .expect("recording logger mutex must not be poisoned") - .push((event_id.to_string(), message.to_string())); - } - - fn error(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - - fn log_cli_error(&self, _: &crate::services::error::CliError, _: Option<&str>) {} - } - - fn session_scoped_payload(event_name: &str, session_id: &str, cwd: &str) -> String { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String(session_id.to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String(cwd.to_string())); - Value::Object(object).to_string() - } - - #[test] - fn pre_tool_use_resolves_main_and_subagent_model_state_exactly_at_admission() { - let git_dir = unique_test_git_dir("model-state-admission"); - let git_dir_resolver = fixed_resolver(git_dir.clone()); - let resolver_calls: RefCell> = RefCell::new(Vec::new()); - let model_state_resolver = |_: &Path, session_id: &str, agent_id: &str| { - resolver_calls - .borrow_mut() - .push((session_id.to_string(), agent_id.to_string())); - Ok(Some(if agent_id.is_empty() { - "claude/sonnet".to_string() - } else { - "claude/opus".to_string() - })) - }; - let starts: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - let value: Value = serde_json::from_str(payload).expect("payload is JSON"); - if value.get("operation") == Some(&Value::String("start".to_string())) { - starts.borrow_mut().push(value); - } - Ok(String::new()) - }; - - let main_payload = pre_tool_use_json(&[]); - let subagent_payload = pre_tool_use_json(&[ - ( - TOOL_USE_ID_FIELD, - Value::String("toolu_subagent".to_string()), - ), - (AGENT_ID_FIELD, Value::String("agent-1".to_string())), - ]); - start_with_model_resolver( - &main_payload, - None, - &git_dir_resolver, - &model_state_resolver, - &seam, - ) - .expect("main-agent Start should succeed"); - start_with_model_resolver( - &subagent_payload, - None, - &git_dir_resolver, - &model_state_resolver, - &seam, - ) - .expect("subagent Start should succeed"); - - assert_eq!( - resolver_calls.into_inner(), - vec![ - ("cc_session-1".to_string(), String::new()), - ("cc_session-1".to_string(), "agent-1".to_string()), - ] - ); - let starts = starts.into_inner(); - assert_eq!(starts.len(), 2); - assert_eq!( - starts[0]["provenance"], - json!({"session_id": "cc_session-1", "model_id": "claude/sonnet"}) - ); - assert_eq!( - starts[1]["provenance"], - json!({"session_id": "cc_session-1", "model_id": "claude/opus"}) - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn subagent_without_exact_model_state_does_not_inherit_main_model() { - let git_dir = unique_test_git_dir("subagent-model-state-missing"); - let git_dir_resolver = fixed_resolver(git_dir.clone()); - let starts: RefCell> = RefCell::new(Vec::new()); - let model_state_resolver = |_: &Path, _: &str, agent_id: &str| { - Ok(if agent_id.is_empty() { - Some("claude/sonnet".to_string()) - } else { - None - }) - }; - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - starts - .borrow_mut() - .push(serde_json::from_str(payload).expect("payload is JSON")); - Ok(String::new()) - }; - - for (tool_use_id, agent_id) in [("toolu_main", None), ("toolu_subagent", Some("agent-1"))] { - let overrides = agent_id - .map(|agent_id| vec![(AGENT_ID_FIELD, Value::String(agent_id.to_string()))]) - .unwrap_or_default(); - let mut overrides = overrides; - overrides.push((TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string()))); - let payload = pre_tool_use_json(&overrides); - start_with_model_resolver( - &payload, - None, - &git_dir_resolver, - &model_state_resolver, - &seam, - ) - .expect("both Starts should succeed"); - } - - let starts = starts.into_inner(); - assert_eq!(starts.len(), 2); - assert_eq!(starts[0]["provenance"]["model_id"], "claude/sonnet"); - assert_eq!(starts[1]["provenance"]["session_id"], "cc_session-1"); - assert_eq!(starts[1]["provenance"]["model_id"], Value::Null); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn missing_or_failed_model_resolution_keeps_session_provenance_and_allows_start() { - let git_dir = unique_test_git_dir("model-state-unavailable"); - let git_dir_resolver = fixed_resolver(git_dir.clone()); - let starts: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - starts - .borrow_mut() - .push(serde_json::from_str(payload).expect("payload is JSON")); - Ok(String::new()) - }; - let response_index = Cell::new(0); - let model_state_resolver = |_: &Path, _: &str, _: &str| -> Result> { - let index = response_index.get(); - response_index.set(index + 1); - if index == 0 { - Ok(None) - } else { - Err(anyhow!("local model-state DB is unavailable")) - } - }; - - for tool_use_id in ["toolu_missing", "toolu_failed"] { - let payload = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string()))]); - let output = start_with_model_resolver( - &payload, - None, - &git_dir_resolver, - &model_state_resolver, - &seam, - ) - .expect("model unavailability must not fail Start"); - assert_eq!(output, ""); - } - - let starts = starts.into_inner(); - assert_eq!(starts.len(), 2); - for start in starts { - assert_eq!( - start["provenance"], - json!({"session_id": "cc_session-1", "model_id": null}) - ); - } - - remove_test_git_dir(&git_dir); - } - - #[test] - fn read_only_tool_creates_no_scope_and_never_touches_the_seam_or_git_dir() { - let resolver = |_: &str| -> Result { - panic!("a read-only tool must never resolve a git dir") - }; - let payload = pre_tool_use_json(&[(TOOL_NAME_FIELD, Value::String("Read".to_string()))]); - - let output = run_claude_mutation_scope_from_payload_with( - &payload, - None, - &resolver, - &unreachable_seam, - ) - .expect("read-only PreToolUse should succeed"); - - assert_eq!(output, ""); - } - - #[test] - fn delegation_tool_creates_no_scope_ac3() { - let resolver = |_: &str| -> Result { - panic!("Agent delegation must never resolve a git dir") - }; - let payload = pre_tool_use_json(&[(TOOL_NAME_FIELD, Value::String("Agent".to_string()))]); - - let output = run_claude_mutation_scope_from_payload_with( - &payload, - None, - &resolver, - &unreachable_seam, - ) - .expect("Agent delegation PreToolUse should succeed"); - - assert_eq!(output, ""); - } - - #[test] - fn session_start_and_subagent_start_establish_no_scope_ac3() { - let resolver = |_: &str| -> Result { - panic!("a lifecycle-only event must never resolve a git dir") - }; - - for event_name in [HOOK_EVENT_SESSION_START, HOOK_EVENT_SUBAGENT_START] { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - let payload = Value::Object(object).to_string(); - - let output = run_claude_mutation_scope_from_payload_with( - &payload, - None, - &resolver, - &unreachable_seam, - ) - .expect("a lifecycle-only event should succeed with no scope"); - assert_eq!(output, ""); - } - } - - fn background_bash_pre_tool_use_json(tool_use_id: &str) -> String { - pre_tool_use_json(&[ - (TOOL_NAME_FIELD, Value::String("Bash".to_string())), - (TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string())), - ( - TOOL_INPUT_FIELD, - serde_json::json!({ "run_in_background": true }), - ), - ]) - } - - fn background_bash_terminal_json(event_name: &str, tool_use_id: &str) -> String { - pre_tool_use_json(&[ - (HOOK_EVENT_NAME_FIELD, Value::String(event_name.to_string())), - (TOOL_NAME_FIELD, Value::String("Bash".to_string())), - (TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string())), - ( - TOOL_INPUT_FIELD, - serde_json::json!({ "run_in_background": true }), - ), - ]) - } - - #[test] - fn explicit_background_bash_is_allowed_untracked_before_any_repository_or_seam_access() { - let resolver = |_: &str| -> Result { - panic!("untracked background Bash must never resolve a git dir") - }; - let model_state_resolver = |_: &Path, _: &str, _: &str| -> Result> { - panic!("untracked background Bash must never resolve model state") - }; - - let output = start_with_model_resolver( - &background_bash_pre_tool_use_json("toolu_background"), - None, - &resolver, - &model_state_resolver, - &unreachable_seam, - ) - .expect("background Bash should be allowed"); - - assert_eq!(output, ""); - } - - #[test] - fn explicit_background_bash_is_allowed_while_the_recovery_barrier_is_armed() { - let git_dir = unique_test_git_dir("background-bash-recovery-barrier"); - let resolver = fixed_resolver(git_dir.clone()); - state::mark_recovery_pending(&git_dir).expect("recovery should arm"); - let state_before = adapter_state_file_bytes(&git_dir); - - let output = run_claude_mutation_scope_from_payload_with( - &background_bash_pre_tool_use_json("toolu_background"), - None, - &resolver, - &unreachable_seam, - ) - .expect("background Bash should be allowed"); - - assert_eq!(output, ""); - assert_eq!( - adapter_state_file_bytes(&git_dir), - state_before, - "untracked background Bash must not read through or clear the recovery barrier" - ); - - remove_test_git_dir(&git_dir); - } - - fn adapter_state_file_bytes(git_dir: &Path) -> Vec { - std::fs::read(state::state_path(git_dir)).expect("adapter state file should be readable") - } - - #[test] - fn terminal_hooks_for_untracked_background_bash_are_no_ops_with_no_live_attempt() { - for event_name in [HOOK_EVENT_POST_TOOL_USE, HOOK_EVENT_POST_TOOL_USE_FAILURE] { - let git_dir = unique_test_git_dir("background-bash-terminal-no-attempt"); - let resolver = fixed_resolver(git_dir.clone()); - - let output = run_claude_mutation_scope_from_payload_with( - &background_bash_terminal_json(event_name, "toolu_background"), - None, - &resolver, - &unreachable_seam, - ) - .expect("a terminal hook for untracked background Bash must be a safe no-op"); - - assert_eq!(output, ""); - assert!( - !state::state_path(&git_dir).exists(), - "{event_name} for untracked background Bash must not create adapter state" - ); - - remove_test_git_dir(&git_dir); - } - } - - #[test] - fn terminal_hooks_for_untracked_background_bash_leave_an_unrelated_live_attempt_untouched() { - for event_name in [HOOK_EVENT_POST_TOOL_USE, HOOK_EVENT_POST_TOOL_USE_FAILURE] { - let git_dir = unique_test_git_dir("background-bash-terminal-unrelated-attempt"); - let resolver = fixed_resolver(git_dir.clone()); - - run_claude_mutation_scope_from_payload_with( - &pre_tool_use_json(&[]), - None, - &resolver, - &ok_seam, - ) - .expect("the unrelated foreground PreToolUse should establish an attempt"); - run_claude_mutation_scope_from_payload_with( - &background_bash_pre_tool_use_json("toolu_background"), - None, - &resolver, - &unreachable_seam, - ) - .expect("background Bash should be allowed"); - - let state_before = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!(state_before.attempts.len(), 1); - assert_eq!(state_before.attempts[0].tool_use_id, "toolu_1"); - let state_bytes_before = adapter_state_file_bytes(&git_dir); - - let output = run_claude_mutation_scope_from_payload_with( - &background_bash_terminal_json(event_name, "toolu_background"), - None, - &resolver, - &unreachable_seam, - ) - .expect("a terminal hook for untracked background Bash must be a safe no-op"); - - assert_eq!(output, ""); - assert_eq!( - state::read_state(&git_dir).expect("state should be readable"), - state_before, - "{event_name} for untracked background Bash must not transition another attempt" - ); - assert_eq!(adapter_state_file_bytes(&git_dir), state_bytes_before); - - remove_test_git_dir(&git_dir); - } - } - - #[test] - fn explicit_background_powershell_is_denied_ac21() { - let git_dir = unique_test_git_dir("explicit-background-powershell"); - let resolver = fixed_resolver(git_dir.clone()); - let payload = pre_tool_use_json(&[ - (TOOL_NAME_FIELD, Value::String("PowerShell".to_string())), - ( - TOOL_INPUT_FIELD, - serde_json::json!({ "run_in_background": true }), - ), - ]); - - let output = run_claude_mutation_scope_from_payload_with( - &payload, - None, - &resolver, - &unreachable_seam, - ) - .expect("explicit background PowerShell should be denied"); - - assert_eq!( - output, - pre_tool_use_deny_json(EXPLICIT_BACKGROUND_SHELL_DENY_REASON) - ); - assert!( - !git_dir.exists(), - "the PowerShell denial must precede any adapter-state I/O" - ); - } - - const PROBE17_SELF_DETACHING_FOREGROUND_PRE_TOOL_USE: &str = - include_str!("fixtures/probe17-detached-child-after-post-tool-use.pre_tool_use.json"); - const PROBE17_SELF_DETACHING_FOREGROUND_POST_TOOL_USE: &str = - include_str!("fixtures/probe17-detached-child-after-post-tool-use.post_tool_use.json"); - - fn recorded_operations(payloads: &RefCell>) -> Vec { - payloads - .borrow() - .iter() - .map(|payload| { - let value: Value = serde_json::from_str(payload).expect("seam payload is JSON"); - value["operation"] - .as_str() - .expect("seam payload carries an operation") - .to_string() - }) - .collect() - } - - fn tool_payload( - event_name: &str, - tool_name: &str, - tool_use_id: &str, - tool_input: Option<&Value>, - ) -> String { - let mut overrides = vec![ - (HOOK_EVENT_NAME_FIELD, Value::String(event_name.to_string())), - (TOOL_NAME_FIELD, Value::String(tool_name.to_string())), - (TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string())), - ]; - if let Some(tool_input) = tool_input { - overrides.push((TOOL_INPUT_FIELD, tool_input.clone())); - } - pre_tool_use_json(&overrides) - } - - fn assert_tracked_foreground_lifecycle( - label: &str, - tool_name: &str, - tool_input: Option<&Value>, - terminal_event: &str, - ) { - let git_dir = unique_test_git_dir(label); - let resolver = fixed_resolver(git_dir.clone()); - let payloads: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - payloads.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - let case = format!("{tool_name} with tool_input {tool_input:?} then {terminal_event}"); - - let output = run_claude_mutation_scope_from_payload_with( - &tool_payload( - HOOK_EVENT_PRE_TOOL_USE, - tool_name, - "toolu_foreground", - tool_input, - ), - None, - &resolver, - &seam, - ) - .expect("foreground PreToolUse should succeed"); - - assert_eq!(output, "", "{case}"); - assert_eq!(recorded_operations(&payloads), vec!["start"], "{case}"); - let started = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!(started.attempts.len(), 1, "{case}"); - assert_eq!(started.attempts[0].tool_use_id, "toolu_foreground"); - assert_eq!(started.attempts[0].tool_name, tool_name); - assert_eq!( - started.attempts[0].phase, - state::AttemptPhase::Active, - "{case}" - ); - assert!(!started.recovery_pending, "{case}"); - - let output = run_claude_mutation_scope_from_payload_with( - &tool_payload(terminal_event, tool_name, "toolu_foreground", tool_input), - None, - &resolver, - &seam, - ) - .expect("the foreground terminal hook should close the scope"); - - assert_eq!(output, "", "{case}"); - assert_eq!( - recorded_operations(&payloads), - vec!["start", "close"], - "{case}" - ); - let closed = state::read_state(&git_dir).expect("state should be readable"); - assert!(closed.attempts.is_empty(), "{case}"); - assert!(!closed.recovery_pending, "{case}"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn foreground_bash_with_run_in_background_false_or_absent_keeps_the_tracked_lifecycle() { - let tool_inputs = [ - None, - Some(serde_json::json!({ "command": "true" })), - Some(serde_json::json!({ "command": "true", "run_in_background": false })), - Some(serde_json::json!({ "command": "true", "run_in_background": null })), - ]; - - for tool_input in &tool_inputs { - for terminal_event in [HOOK_EVENT_POST_TOOL_USE, HOOK_EVENT_POST_TOOL_USE_FAILURE] { - assert_tracked_foreground_lifecycle( - "foreground-bash-tracked", - "Bash", - tool_input.as_ref(), - terminal_event, - ); - } - } - } - - #[test] - fn foreground_powershell_keeps_the_tracked_lifecycle() { - let tool_input = serde_json::json!({ "run_in_background": false }); - - for terminal_event in [HOOK_EVENT_POST_TOOL_USE, HOOK_EVENT_POST_TOOL_USE_FAILURE] { - assert_tracked_foreground_lifecycle( - "foreground-powershell-tracked", - "PowerShell", - Some(&tool_input), - terminal_event, - ); - } - } - - #[test] - fn non_shell_mutation_capable_tools_stay_tracked_even_with_run_in_background_true() { - let tool_input = serde_json::json!({ "run_in_background": true }); - - for tool_name in [ - "Write", - "Edit", - "NotebookEdit", - "MultiEdit", - "mcp__claude-in-chrome__navigate", - "SomeBrandNewTool", - ] { - for terminal_event in [HOOK_EVENT_POST_TOOL_USE, HOOK_EVENT_POST_TOOL_USE_FAILURE] { - assert_tracked_foreground_lifecycle( - "non-shell-run-in-background-tracked", - tool_name, - Some(&tool_input), - terminal_event, - ); - } - } - } - - #[test] - fn read_only_and_delegation_tools_with_run_in_background_true_create_no_scope() { - let resolver = |_: &str| -> Result { - panic!("a read-only or delegation tool must never resolve a git dir") - }; - let model_state_resolver = |_: &Path, _: &str, _: &str| -> Result> { - panic!("a read-only or delegation tool must never resolve model state") - }; - let tool_input = serde_json::json!({ "run_in_background": true }); - - for tool_name in [ - "Read", - "Glob", - "Grep", - "WebFetch", - "WebSearch", - "AskUserQuestion", - "Agent", - ] { - let output = start_with_model_resolver( - &tool_payload( - HOOK_EVENT_PRE_TOOL_USE, - tool_name, - "toolu_untracked", - Some(&tool_input), - ), - None, - &resolver, - &model_state_resolver, - &unreachable_seam, - ) - .expect("read-only and delegation PreToolUse should succeed"); - - assert_eq!(output, "", "{tool_name}"); - } - } - - #[test] - fn self_detaching_foreground_bash_fixture_keeps_the_ordinary_tracked_path() { - let execution = parsed_pre_tool_use(PROBE17_SELF_DETACHING_FOREGROUND_PRE_TOOL_USE); - assert_eq!(execution.identity.tool_name, "Bash"); - assert!(!execution.run_in_background); - - let git_dir = unique_test_git_dir("probe17-self-detaching-foreground"); - let resolver = fixed_resolver(git_dir.clone()); - let payloads: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - payloads.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - - let output = run_claude_mutation_scope_from_payload_with( - PROBE17_SELF_DETACHING_FOREGROUND_PRE_TOOL_USE, - None, - &resolver, - &seam, - ) - .expect("the self-detaching foreground PreToolUse should succeed"); - - assert_eq!(output, ""); - assert_eq!(recorded_operations(&payloads), vec!["start"]); - let started = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!(started.attempts.len(), 1); - assert_eq!( - started.attempts[0].tool_use_id, - execution.identity.tool_use_id - ); - assert_eq!(started.attempts[0].phase, state::AttemptPhase::Active); - - let output = run_claude_mutation_scope_from_payload_with( - PROBE17_SELF_DETACHING_FOREGROUND_POST_TOOL_USE, - None, - &resolver, - &seam, - ) - .expect("the self-detaching foreground PostToolUse should close the scope"); - - assert_eq!(output, ""); - assert_eq!(recorded_operations(&payloads), vec!["start", "close"]); - let closed = state::read_state(&git_dir).expect("state should be readable"); - assert!(closed.attempts.is_empty()); - assert!(!closed.recovery_pending); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn write_ahead_pending_start_persists_before_the_seam_start_call_ac7() { - let git_dir = unique_test_git_dir("write-ahead"); - let resolver = fixed_resolver(git_dir.clone()); - let git_dir_for_seam = git_dir.clone(); - let phase_seen_before_start: RefCell> = RefCell::new(None); - let observed_roots: RefCell> = RefCell::new(Vec::new()); - let seam = |root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - if payload.contains(r#""operation":"start""#) { - observed_roots.borrow_mut().push(root.to_path_buf()); - let observed = state::read_state(&git_dir_for_seam) - .expect("state should be readable under git_dir inside the seam call"); - *phase_seen_before_start.borrow_mut() = - observed.attempts.first().map(|attempt| attempt.phase); - } - Ok(String::new()) - }; - - let payload = pre_tool_use_json(&[]); - let output = run_claude_mutation_scope_from_payload_with(&payload, None, &resolver, &seam) - .expect("mutation-capable PreToolUse should succeed"); - - assert_eq!(output, ""); - assert_eq!( - phase_seen_before_start.into_inner(), - Some(state::AttemptPhase::PendingStart), - "AC7: the attempt must be durably pending_start (under git_dir) before the seam Start call" - ); - assert_eq!( - observed_roots.into_inner(), - vec![PathBuf::from("/repo/checkout")], - "the seam must receive the raw Claude cwd as repository_root, never the resolved git_dir" - ); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!( - final_state.attempts[0].phase, - state::AttemptPhase::Active, - "phase must become active after a successful Start" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn linked_worktree_cwd_and_git_dir_are_never_conflated_for_pre_tool_use_start() { - let git_dir = unique_test_git_dir("cwd-vs-git-dir-start"); - let raw_cwd = "/repo/.claude/worktrees/agent-123"; - let resolver = fixed_resolver(git_dir.clone()); - - let observed_roots: RefCell> = RefCell::new(Vec::new()); - let seam = |root: &Path, _payload: &str, _logger: Option<&dyn Logger>| -> Result { - observed_roots.borrow_mut().push(root.to_path_buf()); - Ok(String::new()) - }; - - let payload = pre_tool_use_json(&[(CWD_FIELD, Value::String(raw_cwd.to_string()))]); - run_claude_mutation_scope_from_payload_with(&payload, None, &resolver, &seam) - .expect("PreToolUse Start should succeed"); - - assert_ne!( - PathBuf::from(raw_cwd), - git_dir, - "test sanity: the raw checkout path and the resolved git_dir must be deliberately distinct" - ); - assert_eq!( - observed_roots.into_inner(), - vec![PathBuf::from(raw_cwd)], - "the ingress seam must receive the raw Claude cwd, never git_dir" - ); - - let state = state::read_state(&git_dir).expect("state should be readable under git_dir"); - assert_eq!( - state.attempts.len(), - 1, - "adapter bookkeeping must be written under the resolved git_dir" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn post_tool_use_close_uses_git_dir_for_state_and_raw_cwd_for_the_seam() { - let git_dir = unique_test_git_dir("cwd-vs-git-dir-close"); - let raw_cwd = "/repo/.claude/worktrees/agent-123"; - let resolver = fixed_resolver(git_dir.clone()); - - let pre_payload = pre_tool_use_json(&[(CWD_FIELD, Value::String(raw_cwd.to_string()))]); - run_claude_mutation_scope_from_payload_with(&pre_payload, None, &resolver, &ok_seam) - .expect("PreToolUse should establish an active attempt"); - - let observed_roots: RefCell> = RefCell::new(Vec::new()); - let seam = |root: &Path, _payload: &str, _logger: Option<&dyn Logger>| -> Result { - observed_roots.borrow_mut().push(root.to_path_buf()); - Ok(String::new()) - }; - let post_payload = pre_tool_use_json(&[ - (CWD_FIELD, Value::String(raw_cwd.to_string())), - ( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_POST_TOOL_USE.to_string()), - ), - ]); - run_claude_mutation_scope_from_payload_with(&post_payload, None, &resolver, &seam) - .expect("PostToolUse Close should succeed"); - - assert_eq!( - observed_roots.into_inner(), - vec![PathBuf::from(raw_cwd)], - "Close must invoke the seam with the raw Claude cwd, never git_dir" - ); - - let state = state::read_state(&git_dir).expect("state should be readable under git_dir"); - assert!( - state.attempts.is_empty(), - "the closed attempt must be removed from git_dir bookkeeping" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn abandon_via_permission_denied_uses_git_dir_for_state_and_raw_cwd_for_the_seam() { - let git_dir = unique_test_git_dir("cwd-vs-git-dir-abandon"); - let raw_cwd = "/repo/.claude/worktrees/agent-123"; - let resolver = fixed_resolver(git_dir.clone()); - - let pre_payload = pre_tool_use_json(&[(CWD_FIELD, Value::String(raw_cwd.to_string()))]); - run_claude_mutation_scope_from_payload_with(&pre_payload, None, &resolver, &ok_seam) - .expect("PreToolUse should establish an active attempt"); - - let observed_roots: RefCell> = RefCell::new(Vec::new()); - let seam = |root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - if payload.contains(r#""operation":"abandon""#) { - observed_roots.borrow_mut().push(root.to_path_buf()); - } - Ok(String::new()) - }; - let denied_payload = pre_tool_use_json(&[ - (CWD_FIELD, Value::String(raw_cwd.to_string())), - ( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_PERMISSION_DENIED.to_string()), - ), - ]); - run_claude_mutation_scope_from_payload_with(&denied_payload, None, &resolver, &seam) - .expect("PermissionDenied should succeed"); - - assert_eq!( - observed_roots.into_inner(), - vec![PathBuf::from(raw_cwd)], - "Abandon must invoke the seam with the raw Claude cwd, never git_dir" - ); - - let state = state::read_state(&git_dir).expect("state should be readable under git_dir"); - assert!(state.attempts.is_empty()); - assert!(state.recovery_pending); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_flush_uses_raw_cwd_for_the_seam_and_git_dir_for_state() { - let git_dir = unique_test_git_dir("cwd-vs-git-dir-flush"); - let raw_cwd = "/repo/.claude/worktrees/agent-123"; - let resolver = fixed_resolver(git_dir.clone()); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let seeded = state::allocate_attempt( - &git_dir, - &AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_seed".to_string(), - }, - "Write", - ) - .expect("seed allocation should succeed"); - state::mark_recovery_pending(&git_dir).expect("arming the barrier should succeed"); - state::remove_attempt(&git_dir, &seeded.attempt.scope_id) - .expect("removing the seed attempt should succeed"); - - let observed_roots: RefCell> = RefCell::new(Vec::new()); - let seam = |root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - if payload.contains(r#""operation":"flush""#) { - observed_roots.borrow_mut().push(root.to_path_buf()); - } - Ok(String::new()) - }; - - let new_pre = pre_tool_use_json(&[ - (CWD_FIELD, Value::String(raw_cwd.to_string())), - (TOOL_USE_ID_FIELD, Value::String("toolu_new".to_string())), - ]); - run_claude_mutation_scope_from_payload_with(&new_pre, None, &resolver, &seam) - .expect("quiescent recovery should flush against the raw checkout path"); - - assert_eq!( - observed_roots.into_inner(), - vec![PathBuf::from(raw_cwd)], - "flush must run against the raw Claude cwd, not git_dir" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn duplicate_live_pre_tool_use_reuses_the_same_scope_and_start_event_id_ac4() { - let git_dir = unique_test_git_dir("duplicate-delivery"); - let resolver = fixed_resolver(git_dir.clone()); - let start_event_ids: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - if payload.contains(r#""operation":"start""#) { - let value: Value = serde_json::from_str(payload).unwrap(); - start_event_ids - .borrow_mut() - .push(value["event_id"].as_str().unwrap().to_string()); - } - Ok(String::new()) - }; - - let payload = pre_tool_use_json(&[]); - run_claude_mutation_scope_from_payload_with(&payload, None, &resolver, &seam) - .expect("first delivery should succeed"); - run_claude_mutation_scope_from_payload_with(&payload, None, &resolver, &seam) - .expect("duplicate delivery should succeed"); - - let ids = start_event_ids.into_inner(); - assert_eq!(ids.len(), 2); - assert_eq!( - ids[0], ids[1], - "AC4: duplicate delivery must reuse the same Start EventId" - ); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!( - final_state.attempts.len(), - 1, - "duplicate delivery must not create a second bookkeeping entry" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn seam_start_failure_denies_and_leaves_the_attempt_pending_start_d8_d11() { - let git_dir = unique_test_git_dir("start-failure"); - let resolver = fixed_resolver(git_dir.clone()); - let seam = seam_failing_on("start"); - - let payload = pre_tool_use_json(&[]); - let output = run_claude_mutation_scope_from_payload_with(&payload, None, &resolver, &seam) - .expect("a Start failure must still return Ok with a deny payload, not propagate"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!( - final_state.attempts[0].phase, - state::AttemptPhase::PendingStart, - "D11: a failed Start must not be marked active nor removed" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_start_attempt_is_abandoned_not_late_started_on_a_terminal_signal_d11() { - let git_dir = unique_test_git_dir("pending-start-then-terminal"); - let resolver = fixed_resolver(git_dir.clone()); - - let start_failing_seam = seam_failing_on("start"); - let pre_payload = pre_tool_use_json(&[]); - run_claude_mutation_scope_from_payload_with( - &pre_payload, - None, - &resolver, - &start_failing_seam, - ) - .expect("the failed Start must still return Ok with a deny payload"); - - let seen_operations: RefCell> = RefCell::new(Vec::new()); - let recording = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - seen_operations.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - let post_payload = pre_tool_use_json(&[( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_POST_TOOL_USE.to_string()), - )]); - let output = - run_claude_mutation_scope_from_payload_with(&post_payload, None, &resolver, &recording) - .expect("PostToolUse for a pending_start attempt should succeed"); - - assert_eq!(output, ""); - let operations = seen_operations.into_inner(); - assert_eq!(operations.len(), 1); - assert!( - operations[0].contains(r#""operation":"abandon""#), - "D11: a pending_start attempt must be abandoned, not late-started, got: {operations:?}" - ); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!(final_state.attempts.is_empty()); - assert!(final_state.recovery_pending); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn close_seam_failure_is_retired_through_abandonment_not_a_replayed_close_d12() { - let git_dir = unique_test_git_dir("close-failure"); - let resolver = fixed_resolver(git_dir.clone()); - - let pre_payload = pre_tool_use_json(&[]); - run_claude_mutation_scope_from_payload_with(&pre_payload, None, &resolver, &ok_seam) - .expect("PreToolUse should establish an active attempt"); - - let close_failing_seam = seam_failing_on("close"); - let post_payload = pre_tool_use_json(&[( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_POST_TOOL_USE.to_string()), - )]); - let output = run_claude_mutation_scope_from_payload_with( - &post_payload, - None, - &resolver, - &close_failing_seam, - ) - .expect("a Close failure must still succeed via abandonment"); - - assert_eq!(output, ""); - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!( - final_state.attempts.is_empty(), - "D12: after abandonment the attempt must be retired" - ); - assert!(final_state.recovery_pending); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn post_tool_use_failure_also_closes_the_scope_d10() { - let git_dir = unique_test_git_dir("post-tool-use-failure"); - let resolver = fixed_resolver(git_dir.clone()); - - let pre_payload = pre_tool_use_json(&[]); - run_claude_mutation_scope_from_payload_with(&pre_payload, None, &resolver, &ok_seam) - .expect("PreToolUse should establish an active attempt"); - - let seen_operations: RefCell> = RefCell::new(Vec::new()); - let recording = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - seen_operations.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - let failure_payload = pre_tool_use_json(&[( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_POST_TOOL_USE_FAILURE.to_string()), - )]); - let output = run_claude_mutation_scope_from_payload_with( - &failure_payload, - None, - &resolver, - &recording, - ) - .expect("PostToolUseFailure should close the scope"); - - assert_eq!(output, ""); - let operations = seen_operations.into_inner(); - assert_eq!(operations.len(), 1); - assert!(operations[0].contains(r#""operation":"close""#)); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!(final_state.attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn post_tool_use_with_no_live_attempt_is_a_safe_no_op_d9() { - let payload = pre_tool_use_json(&[( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_POST_TOOL_USE.to_string()), - )]); - let resolver = fixed_resolver(std::env::temp_dir().join("sce-unused-nonexistent-git-dir")); - - let output = run_claude_mutation_scope_from_payload_with( - &payload, - None, - &resolver, - &unreachable_seam, - ) - .expect("a PostToolUse with no live attempt must be a safe no-op"); - - assert_eq!(output, ""); - } - - #[test] - fn permission_denied_abandons_a_live_attempt_d13() { - let git_dir = unique_test_git_dir("permission-denied"); - let resolver = fixed_resolver(git_dir.clone()); - - let pre_payload = pre_tool_use_json(&[]); - run_claude_mutation_scope_from_payload_with(&pre_payload, None, &resolver, &ok_seam) - .expect("PreToolUse should establish an active attempt"); - - let seen_operations: RefCell> = RefCell::new(Vec::new()); - let recording = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - seen_operations.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - let denied_payload = pre_tool_use_json(&[( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_PERMISSION_DENIED.to_string()), - )]); - let output = run_claude_mutation_scope_from_payload_with( - &denied_payload, - None, - &resolver, - &recording, - ) - .expect("PermissionDenied should succeed"); - - assert_eq!(output, ""); - let operations = seen_operations.into_inner(); - assert_eq!(operations.len(), 1); - assert!(operations[0].contains(r#""operation":"abandon""#)); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!(final_state.attempts.is_empty()); - assert!(final_state.recovery_pending); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn stop_abandons_only_stale_main_thread_attempts_d14() { - let git_dir = unique_test_git_dir("stop-cleanup"); - let resolver = fixed_resolver(git_dir.clone()); - - let main_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_main".to_string()))]); - let subagent_pre = pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("toolu_agent".to_string())), - (AGENT_ID_FIELD, Value::String("agent-1".to_string())), - ]); - run_claude_mutation_scope_from_payload_with(&main_pre, None, &resolver, &ok_seam) - .expect("main-thread PreToolUse should succeed"); - run_claude_mutation_scope_from_payload_with(&subagent_pre, None, &resolver, &ok_seam) - .expect("subagent PreToolUse should succeed"); - - let stop_payload = session_scoped_payload(HOOK_EVENT_STOP, "session-1", "/repo/checkout"); - let output = - run_claude_mutation_scope_from_payload_with(&stop_payload, None, &resolver, &ok_seam) - .expect("Stop cleanup should succeed"); - assert_eq!(output, ""); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!( - final_state.attempts.len(), - 1, - "only the subagent attempt should remain" - ); - assert_eq!(final_state.attempts[0].tool_use_id, "toolu_agent"); - assert!( - final_state.recovery_pending, - "abandoning the stale main attempt must arm the barrier" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn stop_failure_abandons_stale_main_thread_attempts_the_same_way_d15() { - let git_dir = unique_test_git_dir("stop-failure-cleanup"); - let resolver = fixed_resolver(git_dir.clone()); - - let main_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_main".to_string()))]); - run_claude_mutation_scope_from_payload_with(&main_pre, None, &resolver, &ok_seam) - .expect("main-thread PreToolUse should succeed"); - - let stop_failure_payload = - session_scoped_payload(HOOK_EVENT_STOP_FAILURE, "session-1", "/repo/checkout"); - run_claude_mutation_scope_from_payload_with( - &stop_failure_payload, - None, - &resolver, - &ok_seam, - ) - .expect("StopFailure cleanup should succeed"); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!(final_state.attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn user_prompt_submit_abandons_only_stale_main_thread_attempts_d16() { - let git_dir = unique_test_git_dir("user-prompt-submit-cleanup"); - let resolver = fixed_resolver(git_dir.clone()); - - let main_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_main".to_string()))]); - let subagent_pre = pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("toolu_agent".to_string())), - (AGENT_ID_FIELD, Value::String("agent-1".to_string())), - ]); - run_claude_mutation_scope_from_payload_with(&main_pre, None, &resolver, &ok_seam) - .expect("main-thread PreToolUse should succeed"); - run_claude_mutation_scope_from_payload_with(&subagent_pre, None, &resolver, &ok_seam) - .expect("subagent PreToolUse should succeed"); - - let prompt_payload = - session_scoped_payload(HOOK_EVENT_USER_PROMPT_SUBMIT, "session-1", "/repo/checkout"); - run_claude_mutation_scope_from_payload_with(&prompt_payload, None, &resolver, &ok_seam) - .expect("UserPromptSubmit cleanup should succeed"); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!( - final_state.attempts.len(), - 1, - "the subagent attempt must survive" - ); - assert_eq!(final_state.attempts[0].tool_use_id, "toolu_agent"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn subagent_stop_abandons_only_the_matching_agent_id_attempts_d17() { - let git_dir = unique_test_git_dir("subagent-stop-cleanup"); - let resolver = fixed_resolver(git_dir.clone()); - - let first_agent_payload = pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("toolu_a".to_string())), - (AGENT_ID_FIELD, Value::String("agent-a".to_string())), - ]); - let second_agent_payload = pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("toolu_b".to_string())), - (AGENT_ID_FIELD, Value::String("agent-b".to_string())), - ]); - run_claude_mutation_scope_from_payload_with( - &first_agent_payload, - None, - &resolver, - &ok_seam, - ) - .expect("agent-a PreToolUse should succeed"); - run_claude_mutation_scope_from_payload_with( - &second_agent_payload, - None, - &resolver, - &ok_seam, - ) - .expect("agent-b PreToolUse should succeed"); - - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_SUBAGENT_STOP.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert( - CWD_FIELD.to_string(), - Value::String("/repo/checkout".to_string()), - ); - object.insert( - AGENT_ID_FIELD.to_string(), - Value::String("agent-a".to_string()), - ); - let subagent_stop_payload = Value::Object(object).to_string(); - - run_claude_mutation_scope_from_payload_with( - &subagent_stop_payload, - None, - &resolver, - &ok_seam, - ) - .expect("SubagentStop cleanup should succeed"); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "toolu_b"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn session_end_abandons_every_attempt_regardless_of_agent_id_d18() { - let git_dir = unique_test_git_dir("session-end-cleanup"); - let resolver = fixed_resolver(git_dir.clone()); - - let main_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_main".to_string()))]); - let subagent_pre = pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("toolu_agent".to_string())), - (AGENT_ID_FIELD, Value::String("agent-1".to_string())), - ]); - run_claude_mutation_scope_from_payload_with(&main_pre, None, &resolver, &ok_seam) - .expect("main-thread PreToolUse should succeed"); - run_claude_mutation_scope_from_payload_with(&subagent_pre, None, &resolver, &ok_seam) - .expect("subagent PreToolUse should succeed"); - - let session_end_payload = - session_scoped_payload(HOOK_EVENT_SESSION_END, "session-1", "/repo/checkout"); - run_claude_mutation_scope_from_payload_with( - &session_end_payload, - None, - &resolver, - &ok_seam, - ) - .expect("SessionEnd cleanup should succeed"); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!(final_state.attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn worktree_remove_resolves_git_dir_from_worktree_path_not_cwd_d22() { - let main_git_dir = unique_test_git_dir("worktree-remove-main"); - let worktree_git_dir = unique_test_git_dir("worktree-remove-isolated"); - let main_git_dir_for_resolver = main_git_dir.clone(); - let worktree_git_dir_for_resolver = worktree_git_dir.clone(); - - let resolver = move |cwd: &str| -> Result { - if cwd == "/repo/.claude/worktrees/agent-1" { - Ok(worktree_git_dir_for_resolver.clone()) - } else { - Ok(main_git_dir_for_resolver.clone()) - } - }; - - let subagent_pre = pre_tool_use_json(&[ - ( - CWD_FIELD, - Value::String("/repo/.claude/worktrees/agent-1".to_string()), - ), - (AGENT_ID_FIELD, Value::String("agent-1".to_string())), - ]); - run_claude_mutation_scope_from_payload_with(&subagent_pre, None, &resolver, &ok_seam) - .expect("isolated-worktree PreToolUse should succeed"); - - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_WORKTREE_REMOVE.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert( - WORKTREE_PATH_FIELD.to_string(), - Value::String("/repo/.claude/worktrees/agent-1".to_string()), - ); - let worktree_remove_payload = Value::Object(object).to_string(); - - run_claude_mutation_scope_from_payload_with( - &worktree_remove_payload, - None, - &resolver, - &ok_seam, - ) - .expect("WorktreeRemove cleanup should succeed"); - - let worktree_state = - state::read_state(&worktree_git_dir).expect("worktree state should be readable"); - assert!( - worktree_state.attempts.is_empty(), - "D22: WorktreeRemove must retire attempts under the worktree_path's git dir" - ); - - remove_test_git_dir(&main_git_dir); - remove_test_git_dir(&worktree_git_dir); - } - - #[test] - fn recovery_barrier_denies_new_mutation_capable_pre_tool_use_while_attempts_remain_d19() { - let git_dir = unique_test_git_dir("barrier-attempts-remain"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - - let surviving = state::allocate_attempt( - &git_dir, - &AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_surviving".to_string(), - }, - "Write", - ) - .expect("seeding a surviving attempt should succeed"); - let retiring = state::allocate_attempt( - &git_dir, - &AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_retiring".to_string(), - }, - "Write", - ) - .expect("seeding a retiring attempt should succeed"); - state::mark_recovery_pending(&git_dir).expect("arming the barrier should succeed"); - state::remove_attempt(&git_dir, &retiring.attempt.scope_id) - .expect("removing the retiring attempt should succeed"); - let _ = surviving; - - let new_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_new".to_string()))]); - let output = run_claude_mutation_scope_from_payload_with( - &new_pre, - None, - &resolver, - &unreachable_seam, - ) - .expect("D19: barrier denial must still return Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_flushes_once_quiescent_and_clears_before_starting_d19() { - let git_dir = unique_test_git_dir("barrier-flush-success"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - - let seeded = state::allocate_attempt( - &git_dir, - &AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_seed".to_string(), - }, - "Write", - ) - .expect("seeding the retired attempt should succeed"); - state::mark_recovery_pending(&git_dir).expect("arming the barrier should succeed"); - state::remove_attempt(&git_dir, &seeded.attempt.scope_id) - .expect("removing the seeded attempt should succeed"); - - let seen_operations: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - seen_operations.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - - let new_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_new".to_string()))]); - let output = run_claude_mutation_scope_from_payload_with(&new_pre, None, &resolver, &seam) - .expect("D19: a quiescent recovery should flush then proceed"); - - assert_eq!(output, ""); - let operations = seen_operations.into_inner(); - assert_eq!( - operations.len(), - 2, - "expected flush then start, got: {operations:?}" - ); - assert!(operations[0].contains(r#""operation":"flush""#)); - assert!(operations[1].contains(r#""operation":"start""#)); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!( - !final_state.recovery_pending, - "a successful flush must clear the barrier" - ); - assert_eq!(final_state.attempts.len(), 1); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_stays_fail_closed_when_flush_fails_d19() { - let git_dir = unique_test_git_dir("barrier-flush-failure"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - - let seeded = state::allocate_attempt( - &git_dir, - &AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_seed".to_string(), - }, - "Write", - ) - .expect("seeding the retired attempt should succeed"); - state::mark_recovery_pending(&git_dir).expect("arming the barrier should succeed"); - state::remove_attempt(&git_dir, &seeded.attempt.scope_id) - .expect("removing the seeded attempt should succeed"); - - let seam = seam_failing_on("flush"); - let new_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_new".to_string()))]); - let output = run_claude_mutation_scope_from_payload_with(&new_pre, None, &resolver, &seam) - .expect("D19: a failed flush must still return Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert!( - final_state.recovery_pending, - "a failed flush must keep the barrier armed" - ); - assert!( - final_state.attempts.is_empty(), - "a denied PreToolUse must not allocate a new attempt" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn failed_close_and_failed_abandon_keep_recovery_armed_and_the_attempt_tracked_d12_d19() { - let git_dir = unique_test_git_dir("close-and-abandon-failure"); - let resolver = fixed_resolver(git_dir.clone()); - - let pre_payload = pre_tool_use_json(&[]); - run_claude_mutation_scope_from_payload_with(&pre_payload, None, &resolver, &ok_seam) - .expect("PreToolUse should establish an active attempt"); - - let failing_seam = seam_failing_on_any(vec!["close", "abandon"]); - let post_payload = pre_tool_use_json(&[( - HOOK_EVENT_NAME_FIELD, - Value::String(HOOK_EVENT_POST_TOOL_USE.to_string()), - )]); - let error = run_claude_mutation_scope_from_payload_with( - &post_payload, - None, - &resolver, - &failing_seam, - ) - .expect_err( - "a failed Close followed by a failed Abandon must propagate, not silently succeed", - ); - assert!(error.to_string().contains("abandon")); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!( - final_state.attempts.len(), - 1, - "D12: an attempt whose abandonment failed must remain tracked" - ); - assert!( - final_state.recovery_pending, - "D19: recovery must be armed even though abandonment itself failed" - ); - - let new_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_new".to_string()))]); - let output = run_claude_mutation_scope_from_payload_with( - &new_pre, - None, - &resolver, - &unreachable_seam, - ) - .expect("the barrier denial must still return Ok with a deny payload"); - assert_eq!( - output, - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "the next mutation-capable PreToolUse must be denied, and no new Start may occur" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn lifecycle_cleanup_with_a_failed_abandon_keeps_recovery_armed_and_the_attempt_tracked() { - let git_dir = unique_test_git_dir("lifecycle-cleanup-abandon-failure"); - let resolver = fixed_resolver(git_dir.clone()); - - let main_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_main".to_string()))]); - run_claude_mutation_scope_from_payload_with(&main_pre, None, &resolver, &ok_seam) - .expect("main-thread PreToolUse should succeed"); - - let abandon_failing_seam = seam_failing_on("abandon"); - let stop_payload = session_scoped_payload(HOOK_EVENT_STOP, "session-1", "/repo/checkout"); - let error = run_claude_mutation_scope_from_payload_with( - &stop_payload, - None, - &resolver, - &abandon_failing_seam, - ) - .expect_err("a failed abandonment during Stop cleanup must propagate"); - assert!(error.to_string().contains("abandon")); - - let final_state = state::read_state(&git_dir).expect("state should be readable"); - assert_eq!( - final_state.attempts.len(), - 1, - "the attempt whose abandonment failed must remain tracked" - ); - assert!( - final_state.recovery_pending, - "the barrier must remain armed even though cleanup abandonment failed" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn resolver_failure_logs_the_detailed_error_and_denies_with_the_stable_reason_ac8_d8() { - let logger = RecordingLogger::default(); - let resolver = - |_: &str| -> Result { Err(anyhow!("boom: git rev-parse --git-dir failed")) }; - - let payload = pre_tool_use_json(&[]); - let output = run_claude_mutation_scope_from_payload_with( - &payload, - Some(&logger), - &resolver, - &unreachable_seam, - ) - .expect("a resolver failure must still return Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - assert!( - !output.contains("boom"), - "the detailed internal error must never leak into Claude's deny reason" - ); - assert!( - !output.contains("allow"), - "a fail-closed PreToolUse must never emit an allow decision" - ); - - let warnings = logger.warnings(); - assert_eq!(warnings.len(), 1); - assert!( - warnings[0].1.contains("boom"), - "the detailed error must be logged for operators, got: {warnings:?}" - ); - } - - #[test] - fn start_seam_failure_logs_the_detailed_error_and_denies_with_the_stable_reason() { - let git_dir = unique_test_git_dir("start-failure-logged"); - let resolver = fixed_resolver(git_dir.clone()); - let logger = RecordingLogger::default(); - let seam = seam_failing_on("start"); - - let payload = pre_tool_use_json(&[]); - let output = - run_claude_mutation_scope_from_payload_with(&payload, Some(&logger), &resolver, &seam) - .expect("a Start failure must still return Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - let warnings = logger.warnings(); - assert!(!warnings.is_empty(), "the Start failure must be logged"); - assert!(warnings - .iter() - .any(|(_, message)| message.to_lowercase().contains("start"))); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_flush_failure_logs_the_detailed_error() { - let git_dir = unique_test_git_dir("barrier-flush-failure-logged"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - let logger = RecordingLogger::default(); - - let seeded = state::allocate_attempt( - &git_dir, - &AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_seed".to_string(), - }, - "Write", - ) - .expect("seed allocation should succeed"); - state::mark_recovery_pending(&git_dir).expect("arming the barrier should succeed"); - state::remove_attempt(&git_dir, &seeded.attempt.scope_id) - .expect("removing the seeded attempt should succeed"); - - let seam = seam_failing_on("flush"); - let new_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("toolu_new".to_string()))]); - let output = - run_claude_mutation_scope_from_payload_with(&new_pre, Some(&logger), &resolver, &seam) - .expect("a failed flush must still return Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - let warnings = logger.warnings(); - assert!(!warnings.is_empty(), "the flush failure must be logged"); - - remove_test_git_dir(&git_dir); - } -} - -mod production_regressions { - use std::fs; - use std::path::{Path, PathBuf}; - use std::process::Command; - - use super::*; - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use crate::services::agent_trace_db::{ClaudeModelStateObservation, ObservationKind}; - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - use crate::services::mutation_trace::runtime::{resolve_git_dir, resolve_worktree_id}; - use crate::services::mutation_trace::store::decode_revision; - - fn git(dir: &Path, args: &[&str]) -> String { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).expect("git output should be UTF-8") - } - - struct ClaudeRepo { - temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, - } - - impl ClaudeRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-claude-mutation-scope-regression-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); - - Self { - temp, - root, - state_root, - } - } - - fn drive(&self, payload: &str) -> Result { - run_claude_mutation_scope_from_payload_at_state_root(&self.state_root, payload, None) - } - - fn drive_generic(&self, payload: &str) -> Result { - crate::services::hooks::mutation_scope::run_mutation_scope_from_payload_at_state_root( - &self.root, - &self.state_root, - payload, - None, - ) - } - - fn drive_flush(&self) -> Result { - self.drive_generic(&flush_payload()) - } - - fn db(&self) -> RepositoryAgentTraceDb { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &self.root, - &self.state_root, - "claude mutation-scope regression test assertions", - ) - .expect("assertion DB should open") - } - - fn cwd(&self) -> String { - self.root.to_string_lossy().into_owned() - } - - fn cwd_at(root: &Path) -> String { - root.to_string_lossy().into_owned() - } - - fn working_tree_at(root: &Path) -> String { - git(root, &["add", "-A"]); - git(root, &["write-tree"]).trim().to_owned() - } - - fn working_tree(&self) -> String { - Self::working_tree_at(&self.root) - } - - fn git_dir_at(root: &Path) -> PathBuf { - resolve_git_dir(root).expect("git dir should resolve") - } - - fn git_dir(&self) -> PathBuf { - Self::git_dir_at(&self.root) - } - - fn adapter_state_at(root: &Path) -> state::AdapterState { - state::read_state(&Self::git_dir_at(root)).expect("adapter state should be readable") - } - - fn adapter_state(&self) -> state::AdapterState { - Self::adapter_state_at(&self.root) - } - - fn worktree_id_at(root: &Path) -> String { - resolve_worktree_id(root) - .expect("worktree id should resolve") - .0 - } - - fn worktree_id(&self) -> String { - Self::worktree_id_at(&self.root) - } - - fn add_worktree(&self, name: &str) -> PathBuf { - let worktree_path = self.temp.path().join(name); - git( - &self.root, - &[ - "worktree", - "add", - "-q", - worktree_path.to_str().expect("utf-8 worktree path"), - ], - ); - worktree_path - } - } - - fn count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("a count row should exist") - } - - fn assert_raw_agent_trace_tables_untouched(db: &RepositoryAgentTraceDb) { - assert_eq!(count(db, "diff_traces"), 0); - assert_eq!(count(db, "post_commit_patch_intersections"), 0); - assert_eq!(count(db, "agent_traces"), 0); - } - - fn worktree_row(db: &RepositoryAgentTraceDb, worktree_id: &str) -> Option<(u64, String, bool)> { - db.query_map( - "SELECT revision, cursor_tree, needs_rebaseline FROM mutation_trace_worktrees \ - WHERE worktree_id = ?1", - (worktree_id,), - |row| { - let blob: Vec = row.get(0).map_err(anyhow::Error::from)?; - let revision = decode_revision(&blob)?; - let cursor_tree = row.get::(1).map_err(anyhow::Error::from)?; - let needs_rebaseline = row.get::(2).map_err(anyhow::Error::from)? != 0; - Ok((revision, cursor_tree, needs_rebaseline)) - }, - ) - .expect("worktree-row query should succeed") - .into_iter() - .next() - } - - fn processed_events(db: &RepositoryAgentTraceDb) -> Vec<(String, String)> { - db.query_map( - "SELECT scope_id, event_id FROM mutation_trace_processed_events \ - ORDER BY scope_id, event_id", - (), - |row| { - let scope_id = row.get::(0).map_err(anyhow::Error::from)?; - let event_id = row.get::(1).map_err(anyhow::Error::from)?; - Ok((scope_id, event_id)) - }, - ) - .expect("processed-events query should succeed") - } - - fn scope_status(db: &RepositoryAgentTraceDb, scope_id: &str) -> Option<(String, String)> { - db.query_map( - "SELECT actor_kind, status FROM mutation_trace_scopes WHERE scope_id = ?1", - (scope_id,), - |row| { - let actor_kind = row.get::(0).map_err(anyhow::Error::from)?; - let status = row.get::(1).map_err(anyhow::Error::from)?; - Ok((actor_kind, status)) - }, - ) - .expect("scope query should succeed") - .into_iter() - .next() - } - - fn scope_provenance( - db: &RepositoryAgentTraceDb, - scope_id: &str, - ) -> Option<(String, Option)> { - db.query_map( - "SELECT session_id, model_id FROM mutation_trace_scope_provenance \ - WHERE scope_id = ?1", - (scope_id,), - |row| { - let session_id = row.get::(0).map_err(anyhow::Error::from)?; - let model_id = row.get::>(1).map_err(anyhow::Error::from)?; - Ok((session_id, model_id)) - }, - ) - .expect("scope-provenance query should succeed") - .into_iter() - .next() - } - - fn mutation_events_for( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - ) -> Vec<(String, Option, String)> { - db.query_map( - "SELECT attribution_kind, attribution_scope_id, boundary_kind \ - FROM mutation_trace_events WHERE worktree_id = ?1 ORDER BY revision", - (worktree_id,), - |row| { - let attribution_kind = row.get::(0).map_err(anyhow::Error::from)?; - let attribution_scope_id = - row.get::>(1).map_err(anyhow::Error::from)?; - let boundary_kind = row.get::(2).map_err(anyhow::Error::from)?; - Ok((attribution_kind, attribution_scope_id, boundary_kind)) - }, - ) - .expect("mutation-events query should succeed") - } - - fn tool_identity_json( - event_name: &str, - cwd: &str, - session_id: &str, - tool_name: &str, - tool_use_id: &str, - agent_id: Option<&str>, - ) -> String { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String(session_id.to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String(cwd.to_string())); - object.insert( - TOOL_NAME_FIELD.to_string(), - Value::String(tool_name.to_string()), - ); - object.insert( - TOOL_USE_ID_FIELD.to_string(), - Value::String(tool_use_id.to_string()), - ); - if let Some(agent_id) = agent_id { - object.insert( - AGENT_ID_FIELD.to_string(), - Value::String(agent_id.to_string()), - ); - } - Value::Object(object).to_string() - } - - fn pre_tool_use_for( - cwd: &str, - session_id: &str, - tool_name: &str, - tool_use_id: &str, - agent_id: Option<&str>, - ) -> String { - tool_identity_json( - HOOK_EVENT_PRE_TOOL_USE, - cwd, - session_id, - tool_name, - tool_use_id, - agent_id, - ) - } - - fn background_pre_tool_use_for( - cwd: &str, - session_id: &str, - tool_use_id: &str, - agent_id: Option<&str>, - ) -> String { - let mut object: serde_json::Map = serde_json::from_str(&pre_tool_use_for( - cwd, - session_id, - "Bash", - tool_use_id, - agent_id, - )) - .expect("base PreToolUse payload should parse"); - object.insert( - TOOL_INPUT_FIELD.to_string(), - json!({ "run_in_background": true }), - ); - Value::Object(object).to_string() - } - - fn post_tool_use_for( - cwd: &str, - session_id: &str, - tool_name: &str, - tool_use_id: &str, - agent_id: Option<&str>, - ) -> String { - tool_identity_json( - HOOK_EVENT_POST_TOOL_USE, - cwd, - session_id, - tool_name, - tool_use_id, - agent_id, - ) - } - - fn post_tool_use_failure_for( - cwd: &str, - session_id: &str, - tool_name: &str, - tool_use_id: &str, - agent_id: Option<&str>, - ) -> String { - tool_identity_json( - HOOK_EVENT_POST_TOOL_USE_FAILURE, - cwd, - session_id, - tool_name, - tool_use_id, - agent_id, - ) - } - - fn permission_denied_for( - cwd: &str, - session_id: &str, - tool_name: &str, - tool_use_id: &str, - agent_id: Option<&str>, - ) -> String { - tool_identity_json( - HOOK_EVENT_PERMISSION_DENIED, - cwd, - session_id, - tool_name, - tool_use_id, - agent_id, - ) - } - - fn session_json(event_name: &str, cwd: &str, session_id: &str) -> String { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String(session_id.to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String(cwd.to_string())); - Value::Object(object).to_string() - } - - fn agent_json(event_name: &str, cwd: &str, session_id: &str, agent_id: &str) -> String { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String(session_id.to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String(cwd.to_string())); - object.insert( - AGENT_ID_FIELD.to_string(), - Value::String(agent_id.to_string()), - ); - Value::Object(object).to_string() - } - - fn worktree_remove_json(session_id: &str, worktree_path: &str) -> String { - let mut object = serde_json::Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_WORKTREE_REMOVE.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String(session_id.to_string()), - ); - object.insert( - WORKTREE_PATH_FIELD.to_string(), - Value::String(worktree_path.to_string()), - ); - Value::Object(object).to_string() - } - - #[test] - fn test1_foreground_write_closes_ai_exclusive() { - let repo = ClaudeRepo::new("test1-foreground-write"); - let cwd = repo.cwd(); - - assert_eq!( - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None - )) - .expect("PreToolUse should succeed"), - "" - ); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - fs::write(repo.root.join("file.txt"), "one\ntwo\n") - .expect("the tool's own edit should write"); - - assert_eq!( - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None - )) - .expect("PostToolUse should succeed"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "the closed attempt must be removed from adapter bookkeeping" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("claude_code".to_string(), "closed".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id.clone()), - "close".to_string(), - )] - ); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(repo.working_tree()) - ); - assert_eq!( - processed_events(&db), - vec![ - (scope_id.clone(), claude_scope_close_event_id(&scope_id)), - (scope_id.clone(), claude_scope_start_event_id(&scope_id)), - ], - "rows are ordered by (scope_id, event_id), and 'close' sorts before 'start'" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test2_failed_bash_partial_write_still_closes_ai_exclusive() { - let repo = ClaudeRepo::new("test2-failed-bash"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Bash", - "toolu_1", - None, - )) - .expect("PreToolUse should succeed"); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - fs::write(repo.root.join("file.txt"), "one\npartial\n") - .expect("the failed tool's partial edit should write"); - - assert_eq!( - repo.drive(&post_tool_use_failure_for( - &cwd, - "session-1", - "Bash", - "toolu_1", - None - )) - .expect("PostToolUseFailure should succeed"), - "" - ); - - assert!(repo.adapter_state().attempts.is_empty()); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("claude_code".to_string(), "closed".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id.clone()), - "close".to_string(), - )] - ); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(repo.working_tree()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test4_duplicate_pre_and_post_replay_has_no_duplicate_transition() { - let repo = ClaudeRepo::new("test4-duplicate-replay"); - let cwd = repo.cwd(); - let pre = pre_tool_use_for(&cwd, "session-1", "Write", "toolu_1", None); - - repo.drive(&pre).expect("first PreToolUse should succeed"); - assert_eq!( - repo.drive(&pre) - .expect("duplicate PreToolUse should be idempotent"), - "" - ); - assert_eq!( - repo.adapter_state().attempts.len(), - 1, - "AC4: duplicate PreToolUse delivery must reuse the same attempt" - ); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - fs::write(repo.root.join("file.txt"), "one\ntwo\n").expect("the edit should write"); - - let post = post_tool_use_for(&cwd, "session-1", "Write", "toolu_1", None); - repo.drive(&post).expect("first PostToolUse should succeed"); - - let db = repo.db(); - let (revision_before, events_before, processed_before) = ( - worktree_row(&db, &repo.worktree_id()) - .map(|(revision, _, _)| revision) - .expect("a worktree row should exist"), - count(&db, "mutation_trace_events"), - count(&db, "mutation_trace_processed_events"), - ); - - assert_eq!( - repo.drive(&post) - .expect("duplicate PostToolUse delivery must be a safe no-op"), - "" - ); - - let db = repo.db(); - assert_eq!( - worktree_row(&db, &repo.worktree_id()).map(|(revision, _, _)| revision), - Some(revision_before) - ); - assert_eq!(count(&db, "mutation_trace_events"), events_before); - assert_eq!( - count(&db, "mutation_trace_processed_events"), - processed_before - ); - assert_eq!( - processed_events(&db) - .into_iter() - .filter(|(scope, event)| scope == &scope_id - && event == &claude_scope_close_event_id(&scope_id)) - .count(), - 1 - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test5_auto_permission_denied_abandons_and_requires_rebaseline() { - let repo = ClaudeRepo::new("test5-permission-denied"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None, - )) - .expect("PreToolUse should succeed"); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - assert_eq!( - repo.drive(&permission_denied_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None - )) - .expect("PermissionDenied should succeed"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "the denied attempt must be retired from adapter bookkeeping" - ); - assert!( - repo.adapter_state().recovery_pending, - "D19: abandonment must arm the recovery barrier" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("claude_code".to_string(), "abandoned".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert!( - worktree_row(&db, &worktree_id) - .is_some_and(|(_, _, needs_rebaseline)| needs_rebaseline), - "AC12: a denied execution must leave the worktree needing rebaseline" - ); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test3_two_parallel_subagent_tools_produce_ai_contended() { - let repo = ClaudeRepo::new("test3-parallel-subagents"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_a", - Some("agent-a"), - )) - .expect("agent-a PreToolUse should succeed"); - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_b", - Some("agent-b"), - )) - .expect("agent-b PreToolUse should succeed"); - assert_eq!(repo.adapter_state().attempts.len(), 2); - - fs::write(repo.root.join("file.txt"), "one\ncontended\n") - .expect("the racing edit should write"); - - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_a", - Some("agent-a"), - )) - .expect("agent-a PostToolUse (closing while agent-b is still active) should succeed"); - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_b", - Some("agent-b"), - )) - .expect("agent-b PostToolUse should succeed"); - - assert!(repo.adapter_state().attempts.is_empty()); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![("ai_contended".to_string(), None, "close".to_string())], - "AC11: a tree transition observed while two scopes are live must be AiContended" - ); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(repo.working_tree()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test6_other_hook_denial_is_retired_by_stop_cleanup() { - let repo = ClaudeRepo::new("test6-stop-cleanup"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Bash", - "toolu_1", - None, - )) - .expect("PreToolUse should succeed"); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - assert_eq!( - repo.drive(&session_json(HOOK_EVENT_STOP, &cwd, "session-1")) - .expect("Stop should succeed"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "AC13: Stop must retire the stale main-thread attempt" - ); - assert!(repo.adapter_state().recovery_pending); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("claude_code".to_string(), "abandoned".to_string())) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test7_interrupted_main_turn_is_retired_by_next_user_prompt_submit() { - let repo = ClaudeRepo::new("test7-user-prompt-submit-cleanup"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None, - )) - .expect("PreToolUse should succeed"); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - fs::write(repo.root.join("file.txt"), "one\ninterrupted\n") - .expect("the interrupted edit should write"); - - assert_eq!( - repo.drive(&session_json( - HOOK_EVENT_USER_PROMPT_SUBMIT, - &cwd, - "session-1" - )) - .expect("UserPromptSubmit should succeed"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "AC14: UserPromptSubmit must retire the stale main-thread attempt \ - before another mutation-capable tool can start" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("claude_code".to_string(), "abandoned".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert!(worktree_row(&db, &worktree_id) - .is_some_and(|(_, _, needs_rebaseline)| needs_rebaseline)); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test8_resumed_subagent_tool_use_id_gets_a_fresh_scope_id() { - let repo = ClaudeRepo::new("test8-resumed-subagent"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_resumed", - Some("agent-a"), - )) - .expect("first PreToolUse should succeed"); - let first_scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - fs::write(repo.root.join("file.txt"), "one\nfirst\n").expect("first edit should write"); - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_resumed", - Some("agent-a"), - )) - .expect("first PostToolUse should succeed"); - assert!(repo.adapter_state().attempts.is_empty()); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_resumed", - Some("agent-a"), - )) - .expect("resumed PreToolUse should succeed"); - let second_scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - assert_ne!( - first_scope_id, second_scope_id, - "AC15: a resumed subagent's new tool attempt must receive a fresh ScopeId" - ); - - fs::write(repo.root.join("file.txt"), "one\nfirst\nsecond\n") - .expect("second edit should write"); - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_resumed", - Some("agent-a"), - )) - .expect("second PostToolUse should succeed"); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &first_scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - assert_eq!( - scope_status(&db, &second_scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test9_main_and_subagent_concurrent_mutation_is_ai_contended() { - let repo = ClaudeRepo::new("test9-main-plus-subagent"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_main", - None, - )) - .expect("main-thread PreToolUse should succeed"); - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_sub", - Some("agent-a"), - )) - .expect("subagent PreToolUse should succeed"); - assert_eq!(repo.adapter_state().attempts.len(), 2); - - fs::write(repo.root.join("file.txt"), "one\nboth-writing\n") - .expect("the racing edit should write"); - - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_main", - None, - )) - .expect("main-thread PostToolUse should succeed"); - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_sub", - Some("agent-a"), - )) - .expect("subagent PostToolUse should succeed"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![("ai_contended".to_string(), None, "close".to_string())], - "AC11: main + subagent concurrent mutation must be AiContended" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test10_isolated_subagent_worktree_advances_only_its_own_cursor() { - let repo = ClaudeRepo::new("test10-isolated-worktree"); - let worktree_path = repo.add_worktree("subagent-worktree"); - let worktree_cwd = ClaudeRepo::cwd_at(&worktree_path); - - let main_worktree_id = repo.worktree_id(); - let sub_worktree_id = ClaudeRepo::worktree_id_at(&worktree_path); - assert_ne!( - main_worktree_id, sub_worktree_id, - "a linked worktree must resolve to a distinct WorktreeId" - ); - - repo.drive_flush() - .expect("main-checkout baseline flush should succeed"); - let main_cursor_before = worktree_row(&repo.db(), &main_worktree_id) - .map(|(_, cursor_tree, _)| cursor_tree) - .expect("main checkout should have a baseline worktree row"); - - repo.drive(&pre_tool_use_for( - &worktree_cwd, - "session-1", - "Write", - "toolu_sub", - Some("agent-a"), - )) - .expect("subagent PreToolUse in the isolated worktree should succeed"); - fs::write(worktree_path.join("file.txt"), "one\nisolated\n") - .expect("the isolated worktree's own edit should write"); - repo.drive(&post_tool_use_for( - &worktree_cwd, - "session-1", - "Write", - "toolu_sub", - Some("agent-a"), - )) - .expect("subagent PostToolUse in the isolated worktree should succeed"); - - let db = repo.db(); - assert_eq!( - worktree_row(&db, &main_worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(main_cursor_before), - "AC17: the main checkout's mutation cursor must be unchanged" - ); - let sub_row = worktree_row(&db, &sub_worktree_id).expect("subagent worktree row"); - assert_eq!( - sub_row.1, - ClaudeRepo::working_tree_at(&worktree_path), - "AC16/AC17: the isolated worktree's own cursor must advance" - ); - assert_eq!( - mutation_events_for(&db, &sub_worktree_id) - .into_iter() - .map(|(attribution, _, boundary)| (attribution, boundary)) - .collect::>(), - vec![("ai_exclusive".to_string(), "close".to_string())] - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test11_worktree_remove_cleans_only_that_worktrees_outstanding_attempt() { - let repo = ClaudeRepo::new("test11-worktree-remove"); - let worktree_path = repo.add_worktree("removed-worktree"); - let worktree_cwd = ClaudeRepo::cwd_at(&worktree_path); - let main_cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &main_cwd, - "session-1", - "Write", - "toolu_main", - None, - )) - .expect("main-thread PreToolUse should succeed"); - repo.drive(&pre_tool_use_for( - &worktree_cwd, - "session-1", - "Write", - "toolu_sub", - Some("agent-a"), - )) - .expect("subagent PreToolUse in the isolated worktree should succeed"); - - assert_eq!(ClaudeRepo::adapter_state_at(&repo.root).attempts.len(), 1); - assert_eq!( - ClaudeRepo::adapter_state_at(&worktree_path).attempts.len(), - 1 - ); - - assert_eq!( - repo.drive(&worktree_remove_json("session-1", &worktree_cwd)) - .expect("WorktreeRemove should succeed"), - "" - ); - - assert!( - ClaudeRepo::adapter_state_at(&worktree_path) - .attempts - .is_empty(), - "AC13/D22: WorktreeRemove must retire the outstanding attempt for that worktree" - ); - assert_eq!( - ClaudeRepo::adapter_state_at(&repo.root).attempts.len(), - 1, - "WorktreeRemove for one worktree must not touch the main checkout's attempts" - ); - - assert_raw_agent_trace_tables_untouched(&repo.db()); - } - - #[test] - fn test12_pending_start_crash_before_start_is_recovered_conservatively() { - let repo = ClaudeRepo::new("test12-pending-start-crash"); - let cwd = repo.cwd(); - let git_dir = repo.git_dir(); - - let key = AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_crashed".to_string(), - }; - let allocated = - state::allocate_attempt(&git_dir, &key, "Write").expect("allocation should succeed"); - assert_eq!(allocated.attempt.phase, state::AttemptPhase::PendingStart); - - assert_eq!( - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_crashed", - None - )) - .expect("D11: PostToolUse on a pending_start attempt must abandon, not late-start"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "the never-started attempt must be retired" - ); - assert!(repo.adapter_state().recovery_pending); - let db = repo.db(); - assert_eq!( - scope_status(&db, &allocated.attempt.scope_id), - None, - "a Start that never committed must never appear as a real scope" - ); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_fresh", - None, - )) - .expect("the next PreToolUse should proceed after the quiescent flush"); - assert!(!repo.adapter_state().recovery_pending); - assert_eq!(repo.adapter_state().attempts.len(), 1); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test13_start_committed_before_state_settlement_is_recovered_by_abandonment() { - let repo = ClaudeRepo::new("test13-start-committed-crash"); - let cwd = repo.cwd(); - let git_dir = repo.git_dir(); - - let key = AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: "toolu_crashed".to_string(), - }; - let allocated = - state::allocate_attempt(&git_dir, &key, "Write").expect("allocation should succeed"); - let scope_id = allocated.attempt.scope_id.clone(); - - repo.drive_generic(&scope_boundary_payload( - "start", - &scope_id, - &claude_scope_start_event_id(&scope_id), - )) - .expect("the runtime Start should commit durably"); - assert_eq!( - state::read_state(&git_dir).unwrap().attempts[0].phase, - state::AttemptPhase::PendingStart - ); - - assert_eq!( - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_crashed", - None - )) - .expect("D11: a pending_start attempt with a committed Start must be abandoned"), - "" - ); - - assert!(repo.adapter_state().attempts.is_empty()); - assert!(repo.adapter_state().recovery_pending); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("claude_code".to_string(), "abandoned".to_string())), - "the runtime's own committed Start must settle as a real abandonment" - ); - let worktree_id = repo.worktree_id(); - assert!(worktree_row(&db, &worktree_id) - .is_some_and(|(_, _, needs_rebaseline)| needs_rebaseline)); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test14_terminal_runtime_success_before_state_cleanup_is_replay_safe() { - let repo = ClaudeRepo::new("test14-close-committed-crash"); - let cwd = repo.cwd(); - let git_dir = repo.git_dir(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None, - )) - .expect("PreToolUse should succeed"); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - fs::write(repo.root.join("file.txt"), "one\ntwo\n").expect("the edit should write"); - - repo.drive_generic(&scope_boundary_payload( - "close", - &scope_id, - &claude_scope_close_event_id(&scope_id), - )) - .expect("the runtime Close should commit durably"); - assert_eq!(state::read_state(&git_dir).unwrap().attempts.len(), 1); - - let db = repo.db(); - let (revision_before, events_before) = ( - worktree_row(&db, &repo.worktree_id()) - .map(|(revision, _, _)| revision) - .expect("a worktree row should exist"), - count(&db, "mutation_trace_events"), - ); - - assert_eq!( - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None - )) - .expect("a replayed Close against an already-durable commit must be safe"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "the stale bookkeeping must finally be cleared" - ); - - let db = repo.db(); - assert_eq!( - worktree_row(&db, &repo.worktree_id()).map(|(revision, _, _)| revision), - Some(revision_before), - "a durably completed Close must never be re-applied as a second transition" - ); - assert_eq!(count(&db, "mutation_trace_events"), events_before); - assert_eq!( - scope_status(&db, &scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test15_explicit_background_bash_is_allowed_untracked_with_no_scope_or_attempt() { - let repo = ClaudeRepo::new("test15-explicit-background-bash"); - let cwd = repo.cwd(); - let state_path = state::state_path(&repo.git_dir()); - - let output = repo - .drive(&background_pre_tool_use_for( - &cwd, - "session-1", - "toolu_background", - None, - )) - .expect("background Bash should be allowed"); - - assert_eq!(output, ""); - assert!( - !state_path.exists(), - "untracked background Bash must create no adapter state" - ); - - for (terminal_event, payload) in [ - ( - HOOK_EVENT_POST_TOOL_USE, - post_tool_use_for(&cwd, "session-1", "Bash", "toolu_background", None), - ), - ( - HOOK_EVENT_POST_TOOL_USE_FAILURE, - post_tool_use_failure_for(&cwd, "session-1", "Bash", "toolu_background", None), - ), - ] { - assert_eq!( - repo.drive(&payload) - .expect("a terminal hook for untracked background Bash must be a no-op"), - "" - ); - assert!( - !state_path.exists(), - "{terminal_event} for untracked background Bash must create no adapter state" - ); - } - - let db = repo.db(); - assert_eq!(count(&db, "mutation_trace_scopes"), 0); - assert_eq!(count(&db, "mutation_trace_events"), 0); - assert!(processed_events(&db).is_empty()); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test19_terminal_hooks_for_untracked_background_bash_do_not_transition_a_live_foreground_attempt( - ) { - let repo = ClaudeRepo::new("test19-background-terminal-unrelated"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_foreground", - None, - )) - .expect("the foreground PreToolUse should succeed"); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - assert_eq!( - repo.drive(&background_pre_tool_use_for( - &cwd, - "session-1", - "toolu_background", - None, - )) - .expect("background Bash should be allowed"), - "" - ); - - let state_before = repo.adapter_state(); - let scope_before = scope_status(&repo.db(), &scope_id); - let processed_before = processed_events(&repo.db()); - - for payload in [ - post_tool_use_for(&cwd, "session-1", "Bash", "toolu_background", None), - post_tool_use_failure_for(&cwd, "session-1", "Bash", "toolu_background", None), - ] { - assert_eq!( - repo.drive(&payload) - .expect("a terminal hook for untracked background Bash must be a no-op"), - "" - ); - assert_eq!(repo.adapter_state(), state_before); - } - - let db = repo.db(); - assert_eq!(count(&db, "mutation_trace_scopes"), 1); - assert_eq!(scope_status(&db, &scope_id), scope_before); - assert_eq!(processed_events(&db), processed_before); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test20_untracked_background_bash_mutation_is_observed_at_foreground_close_with_no_attribution_guarantee( - ) { - let repo = ClaudeRepo::new("test20-background-a-foreground-b"); - let cwd = repo.cwd(); - - assert_eq!( - repo.drive(&background_pre_tool_use_for( - &cwd, - "session-1", - "toolu_background_a", - None, - )) - .expect("background A should be allowed"), - "" - ); - assert!(!state::state_path(&repo.git_dir()).exists()); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_foreground_b", - None, - )) - .expect("foreground B PreToolUse should succeed"); - let state_with_b = repo.adapter_state(); - assert_eq!(state_with_b.attempts.len(), 1); - assert_eq!(state_with_b.attempts[0].tool_use_id, "toolu_foreground_b"); - let scope_b = state_with_b.attempts[0].scope_id.clone(); - - fs::write(repo.root.join("background-a.txt"), "written by A\n") - .expect("untracked background A's write should write"); - - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_foreground_b", - None, - )) - .expect("foreground B PostToolUse should succeed"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!(count(&db, "mutation_trace_scopes"), 1); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_b), - "close".to_string(), - )], - "no attribution guarantee: untracked background A's mutation is observed at \ - tracked foreground B's close boundary and attributed to B" - ); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(repo.working_tree()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test18_model_switch_does_not_rewrite_scope_provenance() { - let repo = ClaudeRepo::new("model-switch-provenance"); - let session_id = "session-model-switch"; - let db = repo.db(); - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: "cc_session-model-switch".to_string(), - agent_id: String::new(), - model_id: "claude/sonnet".to_string(), - observation_kind: ObservationKind::SessionStart, - source: "test".to_string(), - observed_at_ms: 1, - }) - .expect("initial Claude model state should persist"); - - let pre_payload = - pre_tool_use_for(&repo.cwd(), session_id, "Bash", "toolu_model_switch", None); - repo.drive(&pre_payload) - .expect("initial PreToolUse should establish a scope"); - let scope_id = repo - .adapter_state() - .attempts - .first() - .expect("the scope should remain live") - .scope_id - .clone(); - let before = scope_provenance(&repo.db(), &scope_id); - assert_eq!( - before, - Some(( - "cc_session-model-switch".to_string(), - Some("claude/sonnet".to_string()), - )) - ); - - repo.db() - .upsert_claude_model_state(ClaudeModelStateObservation { - session_id: "cc_session-model-switch".to_string(), - agent_id: String::new(), - model_id: "claude/opus".to_string(), - observation_kind: ObservationKind::PostModelSwitch, - source: "test".to_string(), - observed_at_ms: 2, - }) - .expect("model switch should persist"); - repo.drive(&pre_payload) - .expect("replayed PreToolUse should remain idempotent"); - - assert_eq!(scope_provenance(&repo.db(), &scope_id), before); - } - - #[test] - fn test16_regression_matrix_leaves_raw_agent_trace_tables_untouched() { - let repo = ClaudeRepo::new("test16-raw-tables-untouched"); - let cwd = repo.cwd(); - - let before = { - let db = repo.db(); - ( - count(&db, "diff_traces"), - count(&db, "post_commit_patch_intersections"), - count(&db, "agent_traces"), - ) - }; - assert_eq!(before, (0, 0, 0)); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None, - )) - .expect("PreToolUse should succeed"); - fs::write(repo.root.join("file.txt"), "one\ntwo\n").expect("the edit should write"); - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_1", - None, - )) - .expect("PostToolUse should succeed"); - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Write", - "toolu_2", - None, - )) - .expect("second PreToolUse should succeed"); - repo.drive(&permission_denied_for( - &cwd, - "session-1", - "Write", - "toolu_2", - None, - )) - .expect("PermissionDenied should succeed"); - - let db = repo.db(); - let after = ( - count(&db, "diff_traces"), - count(&db, "post_commit_patch_intersections"), - count(&db, "agent_traces"), - ); - assert_eq!( - after, - (0, 0, 0), - "AC20: Claude mutation-scope-only regressions must leave the raw \ - Agent Trace tables unchanged" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test17_detached_descendant_write_after_post_tool_use_is_not_folded_into_the_closed_scope() { - let repo = ClaudeRepo::new("test17-detached-descendant"); - let cwd = repo.cwd(); - - repo.drive(&pre_tool_use_for( - &cwd, - "session-1", - "Bash", - "toolu_1", - None, - )) - .expect("PreToolUse should succeed"); - let scope_id = repo.adapter_state().attempts[0].scope_id.clone(); - - fs::write(repo.root.join("file.txt"), "one\nforeground-output\n") - .expect("the tool's own foreground write should write"); - let tree_at_close = repo.working_tree(); - - repo.drive(&post_tool_use_for( - &cwd, - "session-1", - "Bash", - "toolu_1", - None, - )) - .expect("PostToolUse should succeed"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!( - scope_status(&db, &scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(tree_at_close.clone()), - "the scope must close at the tool's own observed tree" - ); - - fs::write( - repo.root.join("file.txt"), - "one\nforeground-output\ndetached-descendant\n", - ) - .expect("the detached descendant's later write should write"); - let tree_after_descendant = repo.working_tree(); - assert_ne!(tree_after_descendant, tree_at_close); - - let events_before_flush = mutation_events_for(&db, &worktree_id); - - repo.drive_flush() - .expect("a later recovery/diagnostic flush should succeed"); - - let db = repo.db(); - let events_after_flush = mutation_events_for(&db, &worktree_id); - assert_eq!( - events_after_flush.len(), - events_before_flush.len() + 1, - "the detached descendant's mutation must surface as its own event" - ); - let (attribution_kind, attribution_scope_id, _) = events_after_flush - .last() - .expect("a flush event should exist"); - assert_ne!( - attribution_scope_id.as_deref(), - Some(scope_id.as_str()), - "the detached descendant's mutation must never be attributed to the \ - already-closed tool scope" - ); - assert_eq!(attribution_kind, "ineligible_unscoped"); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(tree_after_descendant) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } -} diff --git a/cli/src/services/hooks/claude_transcript.rs b/cli/src/services/hooks/claude_transcript.rs index b2aa26b1a..f2dfdc42b 100644 --- a/cli/src/services/hooks/claude_transcript.rs +++ b/cli/src/services/hooks/claude_transcript.rs @@ -81,69 +81,3 @@ fn extract_claude_transcript_model_from_reader( None } - -#[cfg(test)] -mod tests { - use std::io::{Cursor, Error, ErrorKind}; - - use super::*; - - fn transcript_reader(content: &str) -> Cursor<&[u8]> { - Cursor::new(content.as_bytes()) - } - - #[test] - fn claude_transcript_reads_real_assistant_envelope_and_skips_malformed_records() { - let transcript = concat!( - "{malformed unrelated record}\n", - r#"{"type":"user","message":{"role":"user","content":[{"type":"tool_use","id":"tool-123"}]}}"#, - "\n", - r#"{"type":"assistant","message":{"role":"assistant","model":"claude-opus-4-1","content":[{"type":"text","text":"working"},{"type":"tool_use","id":"tool-123","name":"Write"}]}}"#, - "\n" - ); - - let model = extract_claude_transcript_model_from_reader( - Ok(transcript_reader(transcript)), - "tool-123", - ); - - assert_eq!(model.as_deref(), Some("claude-opus-4-1")); - } - - #[test] - fn claude_transcript_returns_none_when_transcript_cannot_be_read() { - let unavailable = Err(Error::new(ErrorKind::NotFound, "transcript unavailable")); - - assert_eq!( - extract_claude_transcript_model_from_reader::>(unavailable, "tool-123"), - None - ); - } - - #[test] - fn claude_transcript_returns_none_for_unmatched_tool_use_or_missing_model() { - let unmatched = concat!( - r#"{"type":"assistant","message":{"role":"assistant","model":"claude-opus-4-1","content":[{"type":"tool_use","id":"other-tool"}]}}"#, - "\n" - ); - let missing_model = concat!( - r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"tool-123"}]}}"#, - "\n" - ); - - assert_eq!( - extract_claude_transcript_model_from_reader( - Ok(transcript_reader(unmatched)), - "tool-123" - ), - None - ); - assert_eq!( - extract_claude_transcript_model_from_reader( - Ok(transcript_reader(missing_model)), - "tool-123" - ), - None - ); - } -} diff --git a/cli/src/services/hooks/codex/apply_patch/mod.rs b/cli/src/services/hooks/codex/apply_patch/mod.rs index aab79885f..f8f03fbe1 100644 --- a/cli/src/services/hooks/codex/apply_patch/mod.rs +++ b/cli/src/services/hooks/codex/apply_patch/mod.rs @@ -11,7 +11,6 @@ use crate::services::agent_trace_db::{DiffTraceInsert, PAYLOAD_TYPE_PATCH}; use crate::services::agent_trace_storage::{ resolve_agent_trace_storage_for_hook_runtime_at_state_root, AgentTraceStorageContext, }; -use crate::services::observability::traits::Logger; use normalize::normalize_codex_patch; #[allow(unused_imports)] @@ -27,19 +26,19 @@ use super::super::{ }; use super::CodexHookEvent; -pub(super) fn handle( +pub(super) async fn handle( repository_root: &Path, event: &CodexHookEvent, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { - handle_with_state_root(repository_root, event, None, logger) + handle_with_state_root(repository_root, event, None, logger).await } -pub(super) fn handle_with_state_root( +pub(super) async fn handle_with_state_root( repository_root: &Path, event: &CodexHookEvent, state_root: Option<&Path>, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { required_session_id(event.session_id.as_deref())?; @@ -134,15 +133,19 @@ pub(super) fn handle_with_state_root( }, state_root, ) + .await .map(|storage| storage.db) .context("Failed to open Agent Trace DB for Codex apply_patch persistence."), - None => open_agent_trace_db_for_hook_runtime( - repository_root, - "Failed to open Agent Trace DB for Codex apply_patch persistence.", - ), + None => { + open_agent_trace_db_for_hook_runtime( + repository_root, + "Failed to open Agent Trace DB for Codex apply_patch persistence.", + ) + .await + } }?; - persist_with(&db, event, &normalized_patch, time_ms) + persist_with(&db, event, &normalized_patch, time_ms).await } fn apply_patch_command_from_event(event: &CodexHookEvent) -> Option<&str> { @@ -162,7 +165,7 @@ fn required_session_id(value: Option<&str>) -> Result<&str> { } } -fn persist_with( +async fn persist_with( db: &RepositoryAgentTraceDb, event: &CodexHookEvent, normalized_patch: &str, @@ -183,454 +186,8 @@ fn persist_with( tool_version: None, payload_type: PAYLOAD_TYPE_PATCH, }) + .await .context("Failed to persist Codex apply_patch diff-trace row.")?; Ok(String::new()) } - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::{Path, PathBuf}, - time::{SystemTime, UNIX_EPOCH}, - }; - - use serde_json::json; - - use crate::services::agent_trace::{build_agent_trace, AgentTraceMetadataInput}; - use crate::services::patch::{ - FileChangeKind, ParsedPatch, PatchFileChange, PatchHunk, TouchedLine, TouchedLineKind, - }; - - use super::*; - - const ADD_AND_UPDATE_PATCH: &str = "*** Begin Patch\n\ -*** Add File: new_file.txt\n\ -+hello world\n\ -*** Update File: src/lib.rs\n\ -@@\n\ --old_line\n\ -+new_line\n\ -*** End Patch"; - - const MOVE_WITH_EDITS_PATCH: &str = "*** Begin Patch\n\ -*** Update File: old_name.txt\n\ -*** Move to: new_name.txt\n\ -@@\n\ --old\n\ -+new\n\ -*** End Patch"; - - const PURE_RENAME_PATCH: &str = "*** Begin Patch\n\ -*** Update File: old_name.txt\n\ -*** Move to: new_name.txt\n\ -*** End Patch"; - - const DELETE_ONLY_PATCH: &str = "*** Begin Patch\n\ -*** Delete File: obsolete.txt\n\ -*** End Patch"; - - const MIXED_PATCH: &str = "*** Begin Patch\n\ -*** Add File: a.txt\n\ -+hello\n\ -*** Delete File: b.txt\n\ -*** Update File: c.txt\n\ -@@\n\ --old\n\ -+new\n\ -*** End Patch"; - - const MALFORMED_PATCH: &str = "not a real apply_patch payload"; - - const UPDATE_ONLY_PATCH: &str = "*** Begin Patch\n\ -*** Update File: src/lib.rs\n\ -@@\n\ --old_line\n\ -+new_line\n\ -*** End Patch"; - - fn event(session_id: &str, model: Option<&str>, command: &str) -> CodexHookEvent { - event_with_tool_input(session_id, model, Some(json!({ "command": command }))) - } - - fn event_with_tool_input( - session_id: &str, - model: Option<&str>, - tool_input: Option, - ) -> CodexHookEvent { - CodexHookEvent { - hook_event_name: "PostToolUse".to_string(), - session_id: Some(session_id.to_string()), - turn_id: Some("turn-1".to_string()), - cwd: None, - model: model.map(str::to_string), - tool_name: Some("apply_patch".to_string()), - tool_use_id: Some("tool-1".to_string()), - tool_input, - tool_response: None, - prompt: None, - last_assistant_message: super::super::NullableField::Missing, - } - } - - fn normalized(raw: &str) -> String { - normalize_codex_patch( - &parse_codex_apply_patch(raw).expect("fixture patch should parse"), - "tool-1", - ) - .expect("fixture tool identity should normalize") - } - - fn unique_test_db_path(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-codex-apply-patch-{label}-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") - } - - fn remove_test_db(db_path: &Path) { - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); - } - } - - #[test] - fn handle_fails_open_silently_when_tool_input_missing() { - let output = handle( - Path::new("/nonexistent"), - &event_with_tool_input("session-1", None, None), - None, - ) - .expect("apply_patch handling should succeed"); - assert_eq!(output, ""); - } - - #[test] - fn handle_fails_open_silently_when_command_missing() { - let output = handle( - Path::new("/nonexistent"), - &event_with_tool_input("session-1", None, Some(json!({}))), - None, - ) - .expect("apply_patch handling should succeed"); - assert_eq!(output, ""); - } - - #[test] - fn handle_fails_open_silently_when_command_non_string() { - let output = handle( - Path::new("/nonexistent"), - &event_with_tool_input("session-1", None, Some(json!({"command": 42}))), - None, - ) - .expect("apply_patch handling should succeed"); - assert_eq!(output, ""); - } - - #[test] - fn handle_fails_open_silently_on_malformed_patch_text() { - let output = handle( - Path::new("/nonexistent"), - &event("session-1", None, MALFORMED_PATCH), - None, - ) - .expect("apply_patch handling should succeed"); - assert_eq!(output, ""); - } - - #[test] - fn handle_is_a_successful_no_op_for_a_delete_only_patch() { - let output = handle( - Path::new("/nonexistent"), - &event("session-1", None, DELETE_ONLY_PATCH), - None, - ) - .expect("apply_patch handling should succeed"); - assert_eq!(output, ""); - } - - #[test] - fn persist_with_rejects_missing_empty_and_whitespace_session_ids_without_rows() { - let db_path = unique_test_db_path("invalid-session"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let normalized_patch = normalized(UPDATE_ONLY_PATCH); - - for session_id in [None, Some(""), Some(" ")] { - let mut invalid_event = event("session-1", None, UPDATE_ONLY_PATCH); - invalid_event.session_id = session_id.map(str::to_string); - assert!(persist_with(&db, &invalid_event, &normalized_patch, 1_000).is_err()); - } - - let recent = db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 0); - - remove_test_db(&db_path); - } - - #[test] - fn persist_with_trims_valid_session_ids_before_prefixing() { - let db_path = unique_test_db_path("trimmed-session"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let normalized_patch = normalized(UPDATE_ONLY_PATCH); - let mut trimmed_event = event(" session-1 ", None, UPDATE_ONLY_PATCH); - - persist_with(&db, &trimmed_event, &normalized_patch, 1_000) - .expect("trimmed session should persist"); - trimmed_event.session_id = Some("cx_session-2".to_string()); - persist_with(&db, &trimmed_event, &normalized_patch, 2_000) - .expect("already prefixed session should persist"); - - let rows = db - .query_map( - "SELECT session_id FROM diff_traces ORDER BY id ASC", - (), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("diff trace query should succeed"); - assert_eq!(rows, vec!["cx_session-1", "cx_session-2"]); - - remove_test_db(&db_path); - } - - #[test] - fn handle_is_a_successful_no_op_for_a_pure_rename_with_no_changed_lines() { - let output = handle( - Path::new("/nonexistent"), - &event("session-1", None, PURE_RENAME_PATCH), - None, - ) - .expect("apply_patch handling should succeed"); - assert_eq!(output, ""); - } - - #[test] - fn apply_patch_persists_one_row_with_expected_field_values_for_add_and_update() { - let db_path = unique_test_db_path("add-update"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let normalized_patch = normalized(ADD_AND_UPDATE_PATCH); - let output = persist_with( - &db, - &event("session-1", Some("gpt-5-codex"), ADD_AND_UPDATE_PATCH), - &normalized_patch, - 1_000, - ) - .expect("persist_with should succeed"); - assert_eq!(output, ""); - - let recent = db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 1); - assert_eq!(recent.skipped_count(), 0); - - let row = &recent.patches[0]; - assert_eq!(row.session_id, "cx_session-1"); - assert_eq!(row.tool_name.as_deref(), Some("codex")); - assert_eq!(row.tool_version, None); - assert_eq!(row.payload_type, "patch"); - assert_eq!( - row.patch.files.len(), - 2, - "Add File and Update File both persist evidence" - ); - assert!(row - .patch - .files - .iter() - .flat_map(|file| &file.hunks) - .all(|hunk| hunk.model_id.as_deref() == Some("gpt-5-codex"))); - - remove_test_db(&db_path); - } - - #[test] - fn apply_patch_persists_truthful_model_ids_without_fabricating_openai() { - let db_path = unique_test_db_path("model-provenance"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let normalized_patch = normalized(UPDATE_ONLY_PATCH); - let cases = [ - (Some("openai/gpt-x"), Some("openai/gpt-x")), - ( - Some("qualified/custom-provider/model"), - Some("qualified/custom-provider/model"), - ), - (Some("custom-model"), Some("custom-model")), - (Some(" "), None), - (None, None), - ]; - - for (index, (model, _expected)) in cases.iter().enumerate() { - persist_with( - &db, - &event(&format!("session-{index}"), *model, UPDATE_ONLY_PATCH), - &normalized_patch, - i64::try_from(index).expect("test index should fit") + 1_000, - ) - .expect("model provenance should persist"); - } - - let recent = db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - let model_ids: Vec> = recent - .patches - .iter() - .map(|row| row.patch.files[0].hunks[0].model_id.clone()) - .collect(); - let expected: Vec> = cases - .iter() - .map(|(_, expected)| expected.map(str::to_string)) - .collect(); - assert_eq!(model_ids, expected); - - remove_test_db(&db_path); - } - - #[test] - fn apply_patch_move_with_edits_persists_row_with_expected_paths() { - let db_path = unique_test_db_path("move-edits"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let normalized_patch = normalized(MOVE_WITH_EDITS_PATCH); - persist_with( - &db, - &event("session-1", None, MOVE_WITH_EDITS_PATCH), - &normalized_patch, - 1_000, - ) - .expect("persist_with should succeed"); - - let recent = db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 1); - let file = &recent.patches[0].patch.files[0]; - assert_eq!(file.old_path, "old_name.txt"); - assert_eq!(file.new_path, "new_name.txt"); - assert_eq!(file.kind, FileChangeKind::Renamed); - - remove_test_db(&db_path); - } - - #[test] - fn apply_patch_mixed_operations_persists_only_add_and_update_evidence() { - let db_path = unique_test_db_path("mixed"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let normalized_patch = normalized(MIXED_PATCH); - persist_with( - &db, - &event("session-1", None, MIXED_PATCH), - &normalized_patch, - 1_000, - ) - .expect("persist_with should succeed"); - - let recent = db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 1); - let file_paths: Vec<&str> = recent.patches[0] - .patch - .files - .iter() - .map(|file| file.new_path.as_str()) - .collect(); - assert_eq!( - file_paths, - vec!["a.txt", "c.txt"], - "Delete File must not appear" - ); - - remove_test_db(&db_path); - } - - #[test] - fn apply_patch_diff_trace_attributes_through_agent_trace_pipeline_at_different_real_lines() { - let db_path = unique_test_db_path("agent-trace-pipeline"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let normalized_patch = normalized(UPDATE_ONLY_PATCH); - persist_with( - &db, - &event("session-1", Some("gpt-5-codex"), UPDATE_ONLY_PATCH), - &normalized_patch, - 1_000, - ) - .expect("persist_with should succeed"); - - let recent = db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 1); - let constructed = &recent.patches[0]; - - let post_commit_patch = ParsedPatch { - files: vec![PatchFileChange { - old_path: "src/lib.rs".to_string(), - new_path: "src/lib.rs".to_string(), - kind: FileChangeKind::Modified, - hunks: vec![PatchHunk { - old_start: 42, - old_count: 1, - new_start: 42, - new_count: 2, - model_id: None, - lines: vec![ - TouchedLine { - kind: TouchedLineKind::Removed, - line_number: 42, - content: "old_line".to_string(), - session_id: None, - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 42, - content: "new_line".to_string(), - session_id: None, - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 43, - content: "unrelated_line".to_string(), - session_id: None, - }, - ], - }], - }], - }; - - let agent_trace = build_agent_trace( - &constructed.patch, - &post_commit_patch, - AgentTraceMetadataInput { - commit_timestamp: "2026-04-23T10:20:30Z", - commit_revision: "abc123def456", - vcs_type: None, - tool_name: constructed.tool_name.as_deref(), - tool_version: constructed.tool_version.as_deref(), - }, - ) - .expect("Agent Trace should build from the post-commit intersection"); - let agent_trace_json = - serde_json::to_value(&agent_trace).expect("Agent Trace should serialize"); - - assert_eq!(agent_trace_json["tool"]["name"], "codex"); - assert_eq!( - agent_trace_json["files"][0]["conversations"][0]["contributor"]["model_id"], - "gpt-5-codex" - ); - - remove_test_db(&db_path); - } -} diff --git a/cli/src/services/hooks/codex/apply_patch/normalize.rs b/cli/src/services/hooks/codex/apply_patch/normalize.rs index 4ffd55fa4..3a91288e8 100644 --- a/cli/src/services/hooks/codex/apply_patch/normalize.rs +++ b/cli/src/services/hooks/codex/apply_patch/normalize.rs @@ -29,7 +29,6 @@ fn normalize_error(message: impl Into) -> CodexPatchNormalizeError { } } -#[allow(dead_code)] pub(crate) fn normalize_codex_patch( patch: &CodexPatch, tool_use_id: &str, @@ -210,556 +209,3 @@ fn line_count(count: usize) -> Result { fn render_file_section(old_path: &str, new_path: &str, body: &str) -> String { format!("Index: {new_path}\n{PATCH_INDEX_SEPARATOR}\n--- {old_path}\n+++ {new_path}\n{body}") } - -#[cfg(test)] -mod tests { - use super::super::parser::parse_codex_apply_patch; - use super::*; - use crate::services::patch::{ - combine_patches, intersect_patches, parse_patch, FileChangeKind, ParsedPatch, - PatchFileChange, PatchHunk, TouchedLine, TouchedLineKind, - }; - - fn parse(raw: &str) -> CodexPatch { - parse_codex_apply_patch(raw).expect("fixture patch should parse") - } - - fn normalized(raw: &str) -> String { - normalize_codex_patch(&parse(raw), "tool-1").expect("fixture should normalize") - } - - fn test_base() -> u64 { - synthetic_base("tool-1").expect("test identity should hash") - } - - #[test] - fn normalizes_add_file_into_a_parseable_added_hunk() { - let patch = parse( - "*** Begin Patch\n\ - *** Add File: foo.txt\n\ - +line one\n\ - +line two\n\ - *** End Patch", - ); - - let normalized = normalize_codex_patch(&patch, "tool-1").expect("should normalize"); - let base = test_base(); - - assert_eq!( - normalized, - format!( - "Index: foo.txt\n\ - ===================================================================\n\ - --- foo.txt\n\ - +++ foo.txt\n\ - @@ -0,0 +{base},2 @@\n\ - +line one\n\ - +line two\n" - ) - ); - - let parsed = parse_patch(&normalized, Some("cx_test")).expect("should parse"); - assert_eq!(parsed.files.len(), 1); - let file = &parsed.files[0]; - assert_eq!(file.kind, FileChangeKind::Added); - assert_eq!(file.hunks.len(), 1); - assert_eq!( - file.hunks[0].lines, - vec![ - TouchedLine { - kind: TouchedLineKind::Added, - line_number: base, - content: "line one".to_string(), - session_id: Some("cx_test".to_string()), - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: base + 1, - content: "line two".to_string(), - session_id: Some("cx_test".to_string()), - }, - ] - ); - } - - #[test] - fn normalizes_update_file_dropping_context_lines() { - let patch = parse( - "*** Begin Patch\n\ - *** Update File: src/lib.rs\n\ - @@ fn main() {\n\ - \x20 unchanged\n\ - - old_line\n\ - + new_line\n\ - *** End Patch", - ); - - let normalized = normalize_codex_patch(&patch, "tool-1").expect("should normalize"); - let base = test_base(); - - assert_eq!( - normalized, - format!( - "Index: src/lib.rs\n\ - ===================================================================\n\ - --- src/lib.rs\n\ - +++ src/lib.rs\n\ - @@ -{base},1 +{base},1 @@\n\ - - old_line\n\ - + new_line\n" - ) - ); - - let parsed = parse_patch(&normalized, Some("cx_test")).expect("should parse"); - assert_eq!(parsed.files.len(), 1); - let file = &parsed.files[0]; - assert_eq!(file.kind, FileChangeKind::Modified); - assert_eq!( - file.hunks[0].lines, - vec![ - TouchedLine { - kind: TouchedLineKind::Removed, - line_number: base, - content: " old_line".to_string(), - session_id: Some("cx_test".to_string()), - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: base, - content: " new_line".to_string(), - session_id: Some("cx_test".to_string()), - }, - ] - ); - } - - #[test] - fn normalizes_update_with_move_and_changed_lines() { - let patch = parse( - "*** Begin Patch\n\ - *** Update File: old_name.txt\n\ - *** Move to: new_name.txt\n\ - @@\n\ - -old\n\ - +new\n\ - *** End Patch", - ); - - let normalized = normalize_codex_patch(&patch, "tool-1").expect("should normalize"); - let base = test_base(); - - assert_eq!( - normalized, - format!( - "Index: new_name.txt\n\ - ===================================================================\n\ - --- old_name.txt\n\ - +++ new_name.txt\n\ - @@ -{base},1 +{base},1 @@\n\ - -old\n\ - +new\n" - ) - ); - - let parsed = parse_patch(&normalized, Some("cx_test")).expect("should parse"); - assert_eq!(parsed.files.len(), 1); - let file = &parsed.files[0]; - assert_eq!(file.old_path, "old_name.txt"); - assert_eq!(file.new_path, "new_name.txt"); - assert_eq!(file.kind, FileChangeKind::Renamed); - } - - #[test] - fn drops_pure_rename_with_no_changed_lines() { - let patch = parse( - "*** Begin Patch\n\ - *** Update File: old_name.txt\n\ - *** Move to: new_name.txt\n\ - *** End Patch", - ); - - assert_eq!( - normalize_codex_patch(&patch, "tool-1").expect("should normalize"), - "" - ); - } - - #[test] - fn normalizes_delete_only_patch_to_empty_string() { - let patch = parse( - "*** Begin Patch\n\ - *** Delete File: obsolete.txt\n\ - *** End Patch", - ); - - assert_eq!( - normalize_codex_patch(&patch, "tool-1").expect("should normalize"), - "" - ); - } - - #[test] - fn mixed_patch_keeps_only_add_and_update_evidence() { - let normalized = normalized( - "*** Begin Patch\n\ - *** Add File: a.txt\n\ - +hello\n\ - *** Delete File: b.txt\n\ - *** Update File: c.txt\n\ - @@\n\ - -old\n\ - +new\n\ - *** End Patch", - ); - let parsed = parse_patch(&normalized, Some("cx_test")).expect("should parse"); - - assert_eq!(parsed.files.len(), 2); - assert_eq!(parsed.files[0].new_path, "a.txt"); - assert_eq!(parsed.files[0].kind, FileChangeKind::Added); - assert_eq!(parsed.files[1].new_path, "c.txt"); - assert_eq!(parsed.files[1].kind, FileChangeKind::Modified); - } - - #[test] - fn multiple_hunks_advance_positions_cumulatively() { - let patch = parse( - "*** Begin Patch\n\ - *** Update File: d.txt\n\ - @@ fn one() {\n\ - -a\n\ - +b\n\ - @@ fn two() {\n\ - -c\n\ - +d\n\ - *** End Patch", - ); - - let normalized = normalize_codex_patch(&patch, "tool-1").expect("should normalize"); - let base = test_base(); - - assert_eq!( - normalized, - format!( - "Index: d.txt\n\ - ===================================================================\n\ - --- d.txt\n\ - +++ d.txt\n\ - @@ -{base},1 +{base},1 @@\n\ - -a\n\ - +b\n\ - @@ -{next},1 +{next},1 @@\n\ - -c\n\ - +d\n", - next = base + 1 - ) - ); - - parse_patch(&normalized, Some("cx_test")).expect("should parse"); - } - - #[test] - fn event_scoped_normalization_is_deterministic_and_allocates_across_files() { - let patch = parse( - "*** Begin Patch\n\ - *** Add File: first.txt\n\ - +first\n\ - +second\n\ - *** Update File: second.txt\n\ - @@\n\ - -old\n\ - +new\n\ - *** Add File: third.txt\n\ - +third\n\ - *** End Patch", - ); - - let first = normalize_codex_patch(&patch, "event-1").expect("should normalize"); - let repeated = normalize_codex_patch(&patch, "event-1").expect("should normalize"); - assert_eq!(first, repeated); - - let parsed = parse_patch(&first, None).expect("normalized patch should parse"); - let line_numbers: Vec = parsed - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .flat_map(|hunk| hunk.lines.iter()) - .map(|line| line.line_number) - .collect(); - assert_eq!(line_numbers.len(), 5); - assert!(line_numbers.iter().all(|line| *line > 1)); - assert_eq!( - line_numbers, - vec![ - test_base_for("event-1"), - test_base_for("event-1") + 1, - test_base_for("event-1") + 2, - test_base_for("event-1") + 2, - test_base_for("event-1") + 3, - ] - ); - } - - #[test] - fn different_event_ids_use_separate_synthetic_ranges() { - let patch = parse( - "*** Begin Patch\n\ - *** Add File: same.txt\n\ - +same content\n\ - *** End Patch", - ); - - let first = normalize_codex_patch(&patch, "event-1").expect("should normalize"); - let second = normalize_codex_patch(&patch, "event-2").expect("should normalize"); - assert_ne!(synthetic_base("event-1"), synthetic_base("event-2")); - assert_ne!(first, second); - - let first_line = parse_patch(&first, None) - .expect("first patch should parse") - .files[0] - .hunks[0] - .lines[0] - .line_number; - let second_line = parse_patch(&second, None) - .expect("second patch should parse") - .files[0] - .hunks[0] - .lines[0] - .line_number; - assert_ne!(first_line, second_line); - } - - #[test] - fn rejects_faulted_identity_inputs_and_checked_range_overflow() { - let patch = parse( - "*** Begin Patch\n\ - *** Add File: file.txt\n\ - +one\n\ - +two\n\ - *** End Patch", - ); - - assert!(normalize_codex_patch(&patch, "").is_err()); - assert!(normalize_codex_patch(&patch, " ").is_err()); - assert!(normalize_codex_patch(&patch, " event-1").is_err()); - assert!(normalize_codex_patch_with_base(&patch, u64::MAX).is_err()); - } - - #[test] - fn same_content_events_survive_combination_and_match_two_commit_additions() { - let patch = parse( - "*** Begin Patch\n\ - *** Add File: same.txt\n\ - +same content\n\ - *** End Patch", - ); - let first = parse_patch( - &normalize_codex_patch(&patch, "event-1").expect("first event should normalize"), - Some("cx_event-1"), - ) - .expect("first normalized patch should parse"); - let second = parse_patch( - &normalize_codex_patch(&patch, "event-2").expect("second event should normalize"), - Some("cx_event-2"), - ) - .expect("second normalized patch should parse"); - - let combined = combine_patches(&[first, second]); - let combined_lines: Vec<&TouchedLine> = combined.files[0] - .hunks - .iter() - .flat_map(|hunk| hunk.lines.iter()) - .collect(); - assert_eq!(combined_lines.len(), 2); - assert_ne!(combined_lines[0].line_number, combined_lines[1].line_number); - - let post_commit = ParsedPatch { - files: vec![PatchFileChange { - old_path: String::new(), - new_path: "same.txt".to_string(), - kind: FileChangeKind::Added, - hunks: vec![PatchHunk { - old_start: 0, - old_count: 0, - new_start: 40, - new_count: 2, - model_id: None, - lines: vec![ - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 40, - content: "same content".to_string(), - session_id: None, - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 41, - content: "same content".to_string(), - session_id: None, - }, - ], - }], - }], - }; - - let overlap = intersect_patches(&combined, &post_commit); - let overlap_lines: Vec<&TouchedLine> = overlap.files[0] - .hunks - .iter() - .flat_map(|hunk| hunk.lines.iter()) - .collect(); - assert_eq!(overlap_lines.len(), 2); - assert_eq!(overlap_lines[0].line_number, 40); - assert_eq!(overlap_lines[1].line_number, 41); - } - - #[test] - fn repeated_identical_content_remains_physically_ambiguous_without_line_ranges() { - let patch = parse( - "*** Begin Patch\n\ - *** Update File: repeated.txt\n\ - @@\n\ - -before\n\ - +same\n\ - @@\n\ - -before\n\ - +same\n\ - *** End Patch", - ); - let first = parse_patch( - &normalize_codex_patch(&patch, "event-1").expect("first event should normalize"), - Some("cx_event-1"), - ) - .expect("first normalized patch should parse"); - let second = parse_patch( - &normalize_codex_patch(&patch, "event-2").expect("second event should normalize"), - Some("cx_event-2"), - ) - .expect("second normalized patch should parse"); - - let combined = combine_patches(&[first, second]); - let post_commit = ParsedPatch { - files: vec![PatchFileChange { - old_path: "repeated.txt".to_string(), - new_path: "repeated.txt".to_string(), - kind: FileChangeKind::Modified, - hunks: vec![PatchHunk { - old_start: 20, - old_count: 2, - new_start: 20, - new_count: 2, - model_id: None, - lines: vec![ - TouchedLine { - kind: TouchedLineKind::Removed, - line_number: 20, - content: "before".to_string(), - session_id: None, - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 20, - content: "same".to_string(), - session_id: None, - }, - ], - }], - }], - }; - - let overlap = intersect_patches(&combined, &post_commit); - let lines: Vec<&TouchedLine> = overlap.files[0] - .hunks - .iter() - .flat_map(|hunk| hunk.lines.iter()) - .collect(); - assert_eq!(lines.len(), 2); - assert_eq!(lines[0].session_id.as_deref(), Some("cx_event-1")); - assert_eq!( - lines[1].session_id.as_deref(), - Some("cx_event-1"), - "without true line ranges, both matching physical lines are attributed to the first available event" - ); - } - - fn test_base_for(tool_use_id: &str) -> u64 { - synthetic_base(tool_use_id).expect("test identity should hash") - } - - #[test] - fn intersect_patches_matches_synthetic_lines_via_historical_fallback() { - let codex_patch = parse( - "*** Begin Patch\n\ - *** Update File: src/lib.rs\n\ - @@\n\ - -old_line\n\ - +new_line\n\ - *** End Patch", - ); - let normalized = normalize_codex_patch(&codex_patch, "tool-1").expect("should normalize"); - let constructed_patch = - parse_patch(&normalized, Some("cx_test")).expect("constructed patch should parse"); - - let post_commit_patch = real_commit_patch(); - - let overlap = intersect_patches(&constructed_patch, &post_commit_patch); - - assert_eq!(overlap.files.len(), 1); - let file = &overlap.files[0]; - assert_eq!(file.new_path, "src/lib.rs"); - assert_eq!(file.hunks.len(), 1); - assert_eq!( - file.hunks[0].lines, - vec![ - TouchedLine { - kind: TouchedLineKind::Removed, - line_number: 42, - content: "old_line".to_string(), - session_id: Some("cx_test".to_string()), - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 42, - content: "new_line".to_string(), - session_id: Some("cx_test".to_string()), - }, - ] - ); - } - - fn real_commit_patch() -> ParsedPatch { - ParsedPatch { - files: vec![PatchFileChange { - old_path: "src/lib.rs".to_string(), - new_path: "src/lib.rs".to_string(), - kind: FileChangeKind::Modified, - hunks: vec![PatchHunk { - old_start: 42, - old_count: 1, - new_start: 42, - new_count: 2, - model_id: None, - lines: vec![ - TouchedLine { - kind: TouchedLineKind::Removed, - line_number: 42, - content: "old_line".to_string(), - session_id: None, - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 42, - content: "new_line".to_string(), - session_id: None, - }, - TouchedLine { - kind: TouchedLineKind::Added, - line_number: 43, - content: "unrelated_line".to_string(), - session_id: None, - }, - ], - }], - }], - } - } -} diff --git a/cli/src/services/hooks/codex/apply_patch/parser.rs b/cli/src/services/hooks/codex/apply_patch/parser.rs index e66f01eba..46a924032 100644 --- a/cli/src/services/hooks/codex/apply_patch/parser.rs +++ b/cli/src/services/hooks/codex/apply_patch/parser.rs @@ -9,13 +9,11 @@ const END_OF_FILE_MARKER: &str = "*** End of File"; const CHANGE_CONTEXT_MARKER: &str = "@@"; const CHANGE_CONTEXT_MARKER_WITH_TEXT: &str = "@@ "; -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CodexPatch { pub(crate) operations: Vec, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum CodexFileOperation { Add { @@ -32,7 +30,6 @@ pub(crate) enum CodexFileOperation { }, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CodexHunk { pub(crate) context: Option, @@ -40,7 +37,6 @@ pub(crate) struct CodexHunk { pub(crate) is_end_of_file: bool, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum CodexHunkLine { Context(String), @@ -48,7 +44,6 @@ pub(crate) enum CodexHunkLine { Removed(String), } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct CodexPatchParseError { pub(crate) message: String, @@ -68,7 +63,6 @@ fn error(message: impl Into) -> CodexPatchParseError { } } -#[allow(dead_code)] pub(crate) fn normalize_outer_apply_patch_input(raw: &str) -> Result { let trimmed = raw.trim(); let lines: Vec<&str> = trimmed.lines().collect(); @@ -106,7 +100,6 @@ fn has_canonical_boundaries(lines: &[&str]) -> bool { && lines.last().map(|line| line.trim()) == Some(END_PATCH_MARKER) } -#[allow(dead_code)] pub(crate) fn parse_codex_apply_patch(raw: &str) -> Result { let trimmed = raw.trim(); let lines: Vec<&str> = trimmed.lines().collect(); @@ -295,434 +288,3 @@ fn validate_path(path: &str) -> Result { Ok(path.to_string()) } - -#[cfg(test)] -mod tests { - use super::*; - - const SIMPLE_CANONICAL_PATCH: &str = - "*** Begin Patch\n*** Add File: hello.txt\n+hello\n*** End Patch"; - - #[test] - fn normal_raw_patch_is_returned_unchanged() { - let raw = format!("\n{SIMPLE_CANONICAL_PATCH}\n"); - - assert_eq!( - normalize_outer_apply_patch_input(&raw).expect("raw patch should be accepted"), - raw - ); - } - - #[test] - fn normalizes_each_upstream_supported_heredoc_wrapper_exactly() { - for wrapper in ["< Result { - let git_root = resolve_git_root(repository_root)?; - let event_cwd = resolve_event_cwd(&git_root, event_cwd)?; - resolve_path_from_cwd(&git_root, &event_cwd, codex_path) -} - fn resolve_git_root(repository_root: &Path) -> Result { let output = Command::new("git") .args(["rev-parse", "--show-toplevel"]) @@ -271,271 +260,3 @@ fn path_to_utf8_slash_path(path: &Path) -> Result { } Ok(components.join("/")) } - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::{Path, PathBuf}, - process::Command, - time::{SystemTime, UNIX_EPOCH}, - }; - - use super::*; - - fn temp_repo(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - let root = std::env::temp_dir().join(format!( - "sce-codex-path-{label}-{}-{nonce}", - std::process::id() - )); - fs::create_dir_all(&root).expect("temporary repository should be created"); - let output = Command::new("git") - .args(["init", "-q"]) - .current_dir(&root) - .output() - .expect("git init should run"); - assert!(output.status.success(), "git init failed: {output:?}"); - root - } - - fn remove_repo(root: &Path) { - let _ = fs::remove_dir_all(root); - } - - #[test] - fn resolves_a_root_cwd_path_to_repository_relative_form() { - let root = temp_repo("root"); - let src = root.join("src"); - fs::create_dir(&src).expect("src directory should be created"); - let result = resolve_codex_patch_path(&root, &root.to_string_lossy(), "src/lib.rs") - .expect("root cwd path should resolve"); - assert_eq!(result, "src/lib.rs"); - remove_repo(&root); - } - - #[test] - fn resolves_nested_cwd_parent_traversal_and_dot_components() { - let root = temp_repo("nested"); - let cwd = root.join("src").join("lib"); - fs::create_dir_all(&cwd).expect("nested cwd should be created"); - - let result = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "./../lib.rs") - .expect("valid parent traversal should resolve"); - assert_eq!(result, "src/lib.rs"); - - let result = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "../../root.rs") - .expect("multiple valid parent traversals should resolve"); - assert_eq!(result, "root.rs"); - - let result = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "./nested/file.rs") - .expect("nested relative path should resolve"); - assert_eq!(result, "src/lib/nested/file.rs"); - remove_repo(&root); - } - - #[test] - fn accepts_absolute_inside_worktree_paths() { - let root = temp_repo("absolute-inside"); - let cwd = root.join("src"); - fs::create_dir(&cwd).expect("src directory should be created"); - let target = cwd.join("../lib.rs"); - - let result = - resolve_codex_patch_path(&root, &cwd.to_string_lossy(), &target.to_string_lossy()) - .expect("absolute path inside the worktree should resolve"); - assert_eq!(result, "lib.rs"); - remove_repo(&root); - } - - #[test] - fn accepts_missing_add_targets_and_normalizes_their_parent_components() { - let root = temp_repo("missing-target"); - let cwd = root.join("src").join("lib"); - fs::create_dir_all(&cwd).expect("nested cwd should be created"); - - let result = - resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "../generated/new file.rs") - .expect("missing add target should resolve"); - assert_eq!(result, "src/generated/new file.rs"); - - let result = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "../../new.rs") - .expect("missing target after parent traversal should resolve"); - assert_eq!(result, "new.rs"); - remove_repo(&root); - } - - #[test] - fn resolves_move_source_and_destination_independently() { - let root = temp_repo("move"); - let cwd = root.join("src"); - fs::create_dir(&cwd).expect("src directory should be created"); - let source = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "old.rs") - .expect("move source should resolve"); - let destination = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "new.rs") - .expect("move destination should resolve"); - assert_eq!(source, "src/old.rs"); - assert_eq!(destination, "src/new.rs"); - remove_repo(&root); - } - - #[test] - fn rejects_repository_escape_absolute_and_malformed_paths() { - let root = temp_repo("invalid-path"); - let outside = root - .parent() - .expect("temporary root should have a parent") - .to_path_buf(); - - for path in ["../outside.txt", "/etc/passwd", "./..", "", "bad\0path"] { - let error = resolve_codex_patch_path(&root, &root.to_string_lossy(), path) - .expect_err("unsafe path should be rejected"); - assert!(!error.to_string().is_empty()); - } - - let error = resolve_codex_patch_path(&root, &outside.to_string_lossy(), "file.rs") - .expect_err("outside cwd should be rejected"); - assert!(error.to_string().contains("outside Git repository")); - - let error = resolve_codex_patch_path(&root, "", "file.rs") - .expect_err("missing cwd should be rejected"); - assert!(error.to_string().contains("missing or malformed")); - - let error = resolve_codex_patch_path(&root, "relative/cwd", "file.rs") - .expect_err("relative cwd should be rejected"); - assert!(error.to_string().contains("absolute")); - remove_repo(&root); - } - - #[cfg(unix)] - #[test] - fn rejects_existing_and_missing_paths_that_escape_through_symlinks() { - use std::os::unix::fs::symlink; - - let root = temp_repo("symlink-escape"); - let outside = root - .parent() - .expect("temporary root should have a parent") - .join(format!("sce-codex-path-outside-{}", std::process::id())); - fs::create_dir_all(&outside).expect("outside directory should be created"); - fs::write(outside.join("existing.rs"), "outside").expect("outside file should be created"); - - let alias = root.join("alias"); - let existing_link = root.join("existing-link"); - let missing_link = root.join("missing-link"); - symlink(&outside, &alias).expect("alias escape symlink should be created"); - symlink(&outside, &existing_link).expect("existing escape symlink should be created"); - symlink(&outside, &missing_link).expect("missing escape symlink should be created"); - - let eliminated = - resolve_codex_patch_path(&root, &root.to_string_lossy(), "alias/../foo.rs") - .expect("a symlink removed by lexical parent traversal must not be inspected"); - assert_eq!(eliminated, "foo.rs"); - assert!( - resolve_codex_patch_path(&root, &root.to_string_lossy(), "alias/foo.rs").is_err(), - "an actually traversed escape symlink must be rejected" - ); - assert!(resolve_codex_patch_path( - &root, - &root.to_string_lossy(), - "existing-link/existing.rs" - ) - .is_err()); - assert!( - resolve_codex_patch_path(&root, &root.to_string_lossy(), "missing-link/new.rs") - .is_err() - ); - - remove_repo(&root); - let _ = fs::remove_dir_all(outside); - } - - #[test] - fn clamps_excessive_parent_traversal_at_filesystem_root() { - let root = temp_repo("root-clamp"); - let cwd = root.join("src"); - fs::create_dir(&cwd).expect("src directory should be created"); - let parent_path = root - .parent() - .expect("temporary repository should have a parent") - .strip_prefix(Path::new("/")) - .expect("temporary repository parent should be absolute") - .to_string_lossy(); - let root_name = root - .file_name() - .expect("temporary repository should have a name") - .to_str() - .expect("temporary repository name should be UTF-8"); - - let cwd_depth = cwd.components().count(); - let excess_traversal = "../".repeat(cwd_depth + 8); - let path = format!("{excess_traversal}{parent_path}/{root_name}/clamped.rs"); - - let result = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), &path) - .expect("excessive parent traversal should clamp at filesystem root"); - assert_eq!(result, "clamped.rs"); - remove_repo(&root); - } - - #[test] - fn preserves_spaces_in_repository_relative_paths() { - let root = temp_repo("spaces"); - let cwd = root.join("folder with spaces"); - fs::create_dir(&cwd).expect("spaced cwd should be created"); - let result = resolve_codex_patch_path(&root, &cwd.to_string_lossy(), "file with spaces.rs") - .expect("spaced paths should resolve"); - assert_eq!(result, "folder with spaces/file with spaces.rs"); - remove_repo(&root); - } - - #[test] - fn resolves_all_operation_paths_in_one_event() { - let root = temp_repo("operations"); - let cwd = root.join("src"); - fs::create_dir(&cwd).expect("src directory should be created"); - let mut patch = CodexPatch { - operations: vec![ - CodexFileOperation::Add { - path: "new.rs".to_string(), - lines: vec!["new".to_string()], - }, - CodexFileOperation::Update { - old_path: "old.rs".to_string(), - new_path: Some("moved.rs".to_string()), - hunks: Vec::new(), - }, - CodexFileOperation::Delete { - path: "gone.rs".to_string(), - }, - ], - }; - - resolve_codex_patch_paths(&root, &cwd.to_string_lossy(), &mut patch) - .expect("all operation paths should resolve"); - assert_eq!( - patch.operations[0], - CodexFileOperation::Add { - path: "src/new.rs".to_string(), - lines: vec!["new".to_string()], - } - ); - assert_eq!( - patch.operations[1], - CodexFileOperation::Update { - old_path: "src/old.rs".to_string(), - new_path: Some("src/moved.rs".to_string()), - hunks: Vec::new(), - } - ); - assert_eq!( - patch.operations[2], - CodexFileOperation::Delete { - path: "src/gone.rs".to_string(), - } - ); - remove_repo(&root); - } -} diff --git a/cli/src/services/hooks/codex/bash_policy.rs b/cli/src/services/hooks/codex/bash_policy.rs index f668fd218..13be22179 100644 --- a/cli/src/services/hooks/codex/bash_policy.rs +++ b/cli/src/services/hooks/codex/bash_policy.rs @@ -7,8 +7,6 @@ use crate::services::bash_policy::{ evaluate_bash_command_policy, format_policy_block_message, PolicyEvaluation, }; use crate::services::config; -#[cfg(test)] -use crate::services::config::policy::BashPolicyConfig; use super::CodexHookEvent; @@ -73,234 +71,6 @@ pub(super) fn handle(repository_root: &Path, event: &CodexHookEvent) -> Result, -) -> Result { - Ok( - match decision_from_evaluation(evaluate_bash_command_policy(command, policy_config))? { - CodexBashPolicyDecision::Allowed => String::new(), - CodexBashPolicyDecision::Blocked(response) => response, - }, - ) -} - fn bash_command_from_event(event: &CodexHookEvent) -> Result<&str> { bash_command_from_tool_input(event.tool_input.as_ref()) } - -#[cfg(test)] -mod tests { - use std::{ - path::{Path, PathBuf}, - process::Command, - time::{SystemTime, UNIX_EPOCH}, - }; - - use serde_json::json; - - use super::super::NullableField; - use super::*; - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - use crate::services::config::policy::CustomBashPolicyEntry; - - fn event_with_tool_input(tool_input: Option) -> CodexHookEvent { - CodexHookEvent { - hook_event_name: "PreToolUse".to_string(), - session_id: Some("session-1".to_string()), - turn_id: Some("turn-1".to_string()), - cwd: None, - model: None, - tool_name: Some("Bash".to_string()), - tool_use_id: Some("tool-1".to_string()), - tool_input, - tool_response: None, - prompt: None, - last_assistant_message: NullableField::Missing, - } - } - - fn blocking_policy_config() -> BashPolicyConfig { - BashPolicyConfig { - presets: Vec::new(), - custom: vec![CustomBashPolicyEntry { - id: "block-rm".to_string(), - argv_prefix: vec!["rm".to_string()], - satisfied_by: Vec::new(), - message: "This repository does not allow `rm` via the bash tool.".to_string(), - }], - } - } - - #[test] - fn bash_command_from_event_reads_tool_input_command() { - let event = event_with_tool_input(Some(json!({"command": "echo hi"}))); - assert_eq!(bash_command_from_event(&event).unwrap(), "echo hi"); - } - - #[test] - fn bash_command_from_event_rejects_missing_tool_input() { - let event = event_with_tool_input(None); - let error = bash_command_from_event(&event).expect_err("missing tool_input should error"); - assert!(error.to_string().contains("tool_input.command")); - } - - #[test] - fn bash_command_from_event_rejects_blank_command() { - let event = event_with_tool_input(Some(json!({"command": " "}))); - assert!(bash_command_from_event(&event).is_err()); - } - - #[test] - fn shared_policy_evaluation_is_identical_for_the_handler_and_the_mutation_scope_preflight() { - let repo = unique_temp_dir("shared-policy-parity"); - std::fs::create_dir_all(repo.join(".sce")).expect("create .sce dir"); - std::fs::write( - repo.join(".sce").join("config.json"), - concat!( - r#"{"policies":{"bash":{"custom":[{"id":"no-rm","#, - r#""match":{"argv_prefix":["rm"]},"#, - r#""message":"rm is blocked in this repository"}]}}}"#, - ), - ) - .expect("write repo bash policy config"); - - let blocked_event = event_with_tool_input(Some(json!({"command": "rm -rf build"}))); - let handler_output = handle(&repo, &blocked_event).expect("handler evaluates policy"); - match evaluate_codex_bash_policy(&repo, "rm -rf build").expect("shared evaluator") { - CodexBashPolicyDecision::Blocked(response) => assert_eq!( - handler_output, response, - "the handler's rendered deny must match the shared evaluator's Blocked response", - ), - CodexBashPolicyDecision::Allowed => panic!("expected a policy block for `rm`"), - } - - let allowed_event = event_with_tool_input(Some(json!({"command": "echo ok > ok.txt"}))); - assert_eq!( - handle(&repo, &allowed_event).expect("handler evaluates policy"), - "", - "an allowed command is silent through the handler", - ); - assert!( - matches!( - evaluate_codex_bash_policy(&repo, "echo ok > ok.txt").expect("shared evaluator"), - CodexBashPolicyDecision::Allowed - ), - "the shared evaluator agrees the command is allowed", - ); - - std::fs::remove_dir_all(&repo).ok(); - } - - #[test] - fn render_bash_policy_response_is_silent_for_an_allowed_command() { - let output = render_bash_policy_response("echo generated > generated.txt", None) - .expect("evaluation should succeed"); - assert_eq!(output, ""); - } - - #[test] - fn render_bash_policy_response_denies_with_codex_native_shape_for_a_blocked_command() { - let config = blocking_policy_config(); - let output = render_bash_policy_response("rm -rf /tmp/x", Some(&config)) - .expect("evaluation should succeed"); - - let parsed: serde_json::Value = - serde_json::from_str(&output).expect("deny output should be valid JSON"); - assert_eq!( - parsed["hookSpecificOutput"]["hookEventName"], - json!("PreToolUse") - ); - assert_eq!( - parsed["hookSpecificOutput"]["permissionDecision"], - json!("deny") - ); - let reason = parsed["hookSpecificOutput"]["permissionDecisionReason"] - .as_str() - .expect("reason should be a string"); - assert!(reason.contains("block-rm")); - assert!(reason.contains("does not allow `rm`")); - } - - fn unique_temp_dir(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - let dir = std::env::temp_dir().join(format!( - "sce-codex-bash-policy-{label}-{}-{nonce}", - std::process::id() - )); - std::fs::create_dir_all(&dir).expect("create temp dir"); - dir - } - - fn git(repo_root: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(repo_root) - .output() - .unwrap_or_else(|error| panic!("git {args:?} failed to spawn: {error}")); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - #[test] - fn codex_bash_pre_tool_use_path_creates_no_diff_trace_for_a_filesystem_mutation_command() { - let repo_root = unique_temp_dir("repo"); - git(&repo_root, &["init", "-q"]); - git( - &repo_root, - &[ - "remote", - "add", - "origin", - "https://example.invalid/codex-bash-policy-test.git", - ], - ); - let state_root = unique_temp_dir("state"); - - let payload = json!({ - "hook_event_name": "PreToolUse", - "session_id": "session-1", - "turn_id": "turn-1", - "tool_name": "Bash", - "tool_use_id": "tool-1", - "tool_input": {"command": "echo generated > generated.txt"} - }) - .to_string(); - - let output = super::super::run_codex_subcommand_from_payload(&repo_root, &payload, None) - .expect("Codex Bash PreToolUse dispatch should succeed"); - assert_eq!(output, "", "an allowed command must be silent"); - - let storage = resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &repo_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("Agent Trace storage should resolve for the scratch repo"); - - let recent = storage - .db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!( - recent.loaded_count(), - 0, - "the Codex Bash hook path must create no diff_traces rows" - ); - - std::fs::remove_dir_all(&repo_root).ok(); - std::fs::remove_dir_all(&state_root).ok(); - } -} diff --git a/cli/src/services/hooks/codex/mod.rs b/cli/src/services/hooks/codex/mod.rs index c6f4744b0..3f9d5b9a3 100644 --- a/cli/src/services/hooks/codex/mod.rs +++ b/cli/src/services/hooks/codex/mod.rs @@ -4,8 +4,6 @@ use anyhow::{Context, Result}; use serde::{Deserialize, Deserializer}; use serde_json::Value; -use crate::services::observability::traits::Logger; - use super::read_hook_stdin; mod apply_patch; @@ -28,25 +26,7 @@ pub(crate) enum NullableField { Value(T), } -impl NullableField { - #[cfg(test)] - pub(crate) fn is_missing(&self) -> bool { - matches!(self, NullableField::Missing) - } - - #[cfg(test)] - pub(crate) fn is_null(&self) -> bool { - matches!(self, NullableField::Null) - } - - #[cfg(test)] - pub(crate) fn as_value(&self) -> Option<&T> { - match self { - NullableField::Value(value) => Some(value), - NullableField::Missing | NullableField::Null => None, - } - } -} +impl NullableField {} fn deserialize_nullable_field<'de, T, D>( deserializer: D, @@ -62,7 +42,6 @@ where } #[derive(Debug, Deserialize)] -#[allow(dead_code)] pub(crate) struct CodexHookEvent { pub(crate) hook_event_name: String, #[serde(default)] @@ -80,8 +59,6 @@ pub(crate) struct CodexHookEvent { #[serde(default)] pub(crate) tool_input: Option, #[serde(default)] - pub(crate) tool_response: Option, - #[serde(default)] pub(crate) prompt: Option, #[serde(default, deserialize_with = "deserialize_nullable_field")] pub(crate) last_assistant_message: NullableField, @@ -110,874 +87,70 @@ pub(crate) fn classify_codex_event(event: &CodexHookEvent) -> CodexDispatchArm { } } -pub(super) fn run_codex_subcommand(repository_root: &Path, logger: Option<&dyn Logger>) -> String { +pub(super) async fn run_codex_subcommand( + repository_root: &Path, + logger: Option<&L>, +) -> String { let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, Err(error) => return log_codex_fail_open(&error, logger), }; - match run_codex_subcommand_from_payload(repository_root, &stdin_payload, logger) { + match run_codex_subcommand_from_payload(repository_root, &stdin_payload, logger).await { Ok(output) => output, Err(error) => log_codex_fail_open(&error, logger), } } -fn run_codex_subcommand_from_payload( +async fn run_codex_subcommand_from_payload( repository_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { run_codex_subcommand_from_payload_with_state_root(repository_root, stdin_payload, logger, None) + .await } -#[cfg(test)] -fn run_codex_subcommand_from_payload_at_state_root( +async fn run_codex_subcommand_from_payload_with_state_root< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, - state_root: &Path, -) -> Result { - run_codex_subcommand_from_payload_with_state_root( - repository_root, - stdin_payload, - logger, - Some(state_root), - ) -} - -fn run_codex_subcommand_from_payload_with_state_root( - repository_root: &Path, - stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, state_root: Option<&Path>, ) -> Result { let event: CodexHookEvent = serde_json::from_str(stdin_payload) .context("Invalid Codex hook payload from STDIN: expected valid JSON.")?; Ok(match classify_codex_event(&event) { - CodexDispatchArm::UserPromptSubmit => user_prompt_submit::handle(repository_root, &event)?, - CodexDispatchArm::Stop => stop::handle(repository_root, &event)?, + CodexDispatchArm::UserPromptSubmit => { + user_prompt_submit::handle(repository_root, &event).await? + } + CodexDispatchArm::Stop => stop::handle(repository_root, &event).await?, CodexDispatchArm::PreToolUseBash => bash_policy::handle(repository_root, &event)?, CodexDispatchArm::PostToolUseApplyPatch => match state_root { - Some(state_root) => apply_patch::handle_with_state_root( - repository_root, - &event, - Some(state_root), - logger, - )?, - None => apply_patch::handle(repository_root, &event, logger)?, + Some(state_root) => { + apply_patch::handle_with_state_root( + repository_root, + &event, + Some(state_root), + logger, + ) + .await? + } + None => apply_patch::handle(repository_root, &event, logger).await?, }, CodexDispatchArm::NoOp => String::new(), }) } -fn log_codex_fail_open(error: &anyhow::Error, logger: Option<&dyn Logger>) -> String { +fn log_codex_fail_open( + error: &anyhow::Error, + logger: Option<&L>, +) -> String { if let Some(log) = logger { log.error("sce.hooks.codex.error", &error.to_string(), &[], None); } String::new() } - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::{Path, PathBuf}, - process::Command, - time::{SystemTime, UNIX_EPOCH}, - }; - - use serde_json::json; - - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, - resolve_agent_trace_storage_for_hook_runtime_at_state_root, AgentTraceStorageContext, - }; - use crate::services::patch::FileChangeKind; - - use super::*; - - fn event(hook_event_name: &str, tool_name: Option<&str>) -> CodexHookEvent { - CodexHookEvent { - hook_event_name: hook_event_name.to_string(), - session_id: Some("abc123".to_string()), - turn_id: Some("turn-1".to_string()), - cwd: None, - model: None, - tool_name: tool_name.map(str::to_string), - tool_use_id: None, - tool_input: None, - tool_response: None, - prompt: None, - last_assistant_message: NullableField::Missing, - } - } - - #[test] - fn classify_codex_event_routes_user_prompt_submit() { - assert_eq!( - classify_codex_event(&event("UserPromptSubmit", None)), - CodexDispatchArm::UserPromptSubmit - ); - } - - #[test] - fn classify_codex_event_routes_stop() { - assert_eq!( - classify_codex_event(&event("Stop", None)), - CodexDispatchArm::Stop - ); - } - - #[test] - fn classify_codex_event_routes_pre_tool_use_bash() { - assert_eq!( - classify_codex_event(&event("PreToolUse", Some("Bash"))), - CodexDispatchArm::PreToolUseBash - ); - } - - #[test] - fn classify_codex_event_routes_pre_tool_use_apply_patch_to_no_op() { - assert_eq!( - classify_codex_event(&event("PreToolUse", Some("apply_patch"))), - CodexDispatchArm::NoOp - ); - } - - #[test] - fn classify_codex_event_routes_post_tool_use_apply_patch() { - assert_eq!( - classify_codex_event(&event("PostToolUse", Some("apply_patch"))), - CodexDispatchArm::PostToolUseApplyPatch - ); - } - - #[test] - fn classify_codex_event_routes_unknown_pre_tool_use_tool_name_to_no_op() { - assert_eq!( - classify_codex_event(&event("PreToolUse", Some("Edit"))), - CodexDispatchArm::NoOp - ); - } - - #[test] - fn classify_codex_event_routes_pre_tool_use_with_no_tool_name_to_no_op() { - assert_eq!( - classify_codex_event(&event("PreToolUse", None)), - CodexDispatchArm::NoOp - ); - } - - #[test] - fn classify_codex_event_routes_post_tool_use_bash_to_no_op() { - assert_eq!( - classify_codex_event(&event("PostToolUse", Some("Bash"))), - CodexDispatchArm::NoOp - ); - } - - #[test] - fn classify_codex_event_routes_unrecognized_hook_event_name_to_no_op() { - assert_eq!( - classify_codex_event(&event("SessionStart", None)), - CodexDispatchArm::NoOp - ); - } - - #[test] - fn run_codex_subcommand_from_payload_no_ops_unsupported_combination_without_error() { - let payload = r#"{"hook_event_name":"PreToolUse","session_id":"s1","tool_name":"Read"}"#; - - let output = run_codex_subcommand_from_payload(Path::new("/tmp"), payload, None) - .expect("no-op dispatch should succeed"); - - assert_eq!(output, ""); - } - - #[test] - fn run_codex_subcommand_from_payload_no_ops_unrecognized_hook_event_name_without_error() { - let payload = r#"{"hook_event_name":"SessionStart","session_id":"s1"}"#; - - let output = run_codex_subcommand_from_payload(Path::new("/tmp"), payload, None) - .expect("no-op dispatch should succeed"); - - assert_eq!(output, ""); - } - - #[test] - fn run_codex_subcommand_from_payload_rejects_non_json_stdin() { - let error = run_codex_subcommand_from_payload(Path::new("/tmp"), "not json", None) - .expect_err("malformed payload should fail parsing"); - - assert!(error.to_string().contains("Invalid Codex hook payload")); - } - - #[test] - fn run_codex_subcommand_fails_open_on_malformed_stdin_payload() { - let error = anyhow::anyhow!("Invalid Codex hook payload from STDIN: expected valid JSON."); - - let output = log_codex_fail_open(&error, None); - - assert_eq!(output, ""); - } - - #[derive(Clone, Default)] - struct RecordingLogger { - errors: std::sync::Arc>>, - } - - impl Logger for RecordingLogger { - fn info(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn debug(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn warn(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - - fn error(&self, _event_id: &str, message: &str, _: &[(&str, &str)], _: Option<&str>) { - self.errors - .lock() - .expect("recording logger mutex must not be poisoned") - .push(message.to_string()); - } - - fn log_cli_error(&self, _error: &crate::services::error::CliError, _: Option<&str>) {} - } - - #[test] - fn log_codex_fail_open_logs_a_propagated_timestamp_failure_and_returns_empty_stdout() { - let logger = RecordingLogger::default(); - let error = anyhow::anyhow!("clock failed"); - - let output = log_codex_fail_open(&error, Some(&logger)); - - assert_eq!(output, ""); - let errors = logger.errors.lock().expect("mutex must not be poisoned"); - assert_eq!(errors.as_slice(), ["clock failed"]); - } - - #[derive(Debug, Clone, Copy)] - struct StopRowCounts { - messages: i64, - parts: i64, - } - - fn stop_row_counts( - storage: &crate::services::agent_trace_storage::ResolvedAgentTraceStorage, - ) -> StopRowCounts { - let messages = storage - .db - .query_map("SELECT COUNT(*) FROM messages", (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("messages count query should succeed")[0]; - let parts = storage - .db - .query_map("SELECT COUNT(*) FROM parts", (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("parts count query should succeed")[0]; - StopRowCounts { messages, parts } - } - - fn reopen_storage_for_counts( - repository_root: &Path, - state_root: &Path, - ) -> crate::services::agent_trace_storage::ResolvedAgentTraceStorage { - resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &AgentTraceStorageContext { - repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - state_root, - ) - .expect("repository Agent Trace DB should reopen") - } - - #[test] - fn stop_dispatch_propagates_a_missing_last_assistant_message_field_for_the_outer_fail_open_boundary( - ) { - let (repository_root, state_root) = initialize_repository("stop-dispatch-missing-field"); - let payload = json!({ - "hook_event_name": "Stop", - "session_id": "s1", - "turn_id": "t1" - }) - .to_string(); - - let error = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect_err( - "a Stop payload missing last_assistant_message must error so the outer boundary can fail open", - ); - assert!(error.to_string().contains("last_assistant_message")); - assert_eq!(log_codex_fail_open(&error, None), ""); - - let storage = reopen_storage_for_counts(&repository_root, &state_root); - let counts = stop_row_counts(&storage); - assert_eq!(counts.messages, 0); - assert_eq!(counts.parts, 0); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - #[test] - fn stop_dispatch_is_a_silent_no_op_for_an_explicit_null_last_assistant_message() { - let (repository_root, state_root) = initialize_repository("stop-dispatch-null"); - let payload = json!({ - "hook_event_name": "Stop", - "session_id": "s1", - "turn_id": "t1", - "last_assistant_message": null - }) - .to_string(); - - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("explicit null last_assistant_message should be a successful no-op"); - assert_eq!(output, ""); - - let storage = reopen_storage_for_counts(&repository_root, &state_root); - let counts = stop_row_counts(&storage); - assert_eq!(counts.messages, 0); - assert_eq!(counts.parts, 0); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - #[test] - fn codex_hook_event_deserializes_missing_last_assistant_message_as_missing() { - let event: CodexHookEvent = - serde_json::from_str(r#"{"hook_event_name":"Stop","session_id":"s1","turn_id":"t1"}"#) - .expect("payload without last_assistant_message should still deserialize"); - - assert!(event.last_assistant_message.is_missing()); - } - - #[test] - fn codex_hook_event_deserializes_explicit_null_last_assistant_message_as_null() { - let event: CodexHookEvent = serde_json::from_str( - r#"{"hook_event_name":"Stop","session_id":"s1","turn_id":"t1","last_assistant_message":null}"#, - ) - .expect("payload with explicit null last_assistant_message should deserialize"); - - assert!(event.last_assistant_message.is_null()); - } - - #[test] - fn codex_hook_event_deserializes_empty_string_last_assistant_message_as_value() { - let event: CodexHookEvent = serde_json::from_str( - r#"{"hook_event_name":"Stop","session_id":"s1","turn_id":"t1","last_assistant_message":""}"#, - ) - .expect("payload with empty string last_assistant_message should deserialize"); - - assert_eq!( - event.last_assistant_message.as_value().map(String::as_str), - Some("") - ); - } - - #[test] - fn codex_hook_event_deserializes_present_text_last_assistant_message_as_value() { - let event: CodexHookEvent = serde_json::from_str( - r#"{"hook_event_name":"Stop","session_id":"s1","turn_id":"t1","last_assistant_message":"hello"}"#, - ) - .expect("payload with text last_assistant_message should deserialize"); - - assert_eq!( - event.last_assistant_message.as_value().map(String::as_str), - Some("hello") - ); - } - - fn unique_temp_dir(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - let path = std::env::temp_dir().join(format!( - "sce-codex-pipeline-{label}-{}-{nonce}", - std::process::id() - )); - fs::create_dir_all(&path).expect("temporary directory should be created"); - path - } - - fn git(repository_root: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(repository_root) - .output() - .unwrap_or_else(|error| panic!("git {args:?} failed to spawn: {error}")); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - fn initialize_repository(label: &str) -> (PathBuf, PathBuf) { - let repository_root = unique_temp_dir(&format!("{label}-repo")); - git(&repository_root, &["init", "-q"]); - git( - &repository_root, - &[ - "remote", - "add", - "origin", - "https://example.invalid/codex-t19.git", - ], - ); - let state_root = unique_temp_dir(&format!("{label}-state")); - let context = AgentTraceStorageContext { - repository_root: &repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }; - let storage = resolve_agent_trace_storage_at_state_root(&context, &state_root) - .expect("repository Agent Trace DB should initialize"); - drop(storage); - (repository_root, state_root) - } - - fn codex_apply_patch_payload( - cwd: &Path, - session_id: &str, - model: &str, - tool_use_id: &str, - command: &str, - ) -> String { - json!({ - "hook_event_name": "PostToolUse", - "session_id": session_id, - "turn_id": "turn-realistic", - "cwd": cwd, - "model": model, - "tool_name": "apply_patch", - "tool_use_id": tool_use_id, - "tool_input": {"command": command}, - "tool_response": {"success": true} - }) - .to_string() - } - - #[test] - #[allow(clippy::too_many_lines)] - fn realistic_post_tool_use_patch_flows_through_repository_db_and_post_commit_attribution() { - let (repository_root, state_root) = initialize_repository("end-to-end"); - let source_dir = repository_root.join("src"); - fs::create_dir_all(&source_dir).expect("source directory should be created"); - fs::write(source_dir.join("lib.rs"), "prefix\nold_line\nsuffix\n") - .expect("initial source should be written"); - git(&repository_root, &["add", "."]); - git( - &repository_root, - &[ - "-c", - "user.name=SCE Test", - "-c", - "user.email=sce@example.invalid", - "commit", - "-qm", - "initial", - ], - ); - - let command = "<<\"EOF\"\n*** Begin Patch\n*** Update File: lib.rs\n@@\n-old_line\n+new_line\n*** End Patch\nEOF"; - let payload = codex_apply_patch_payload( - &source_dir, - " session-realistic ", - "custom/codex-model", - "tool-realistic-1", - command, - ); - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("realistic Codex PostToolUse dispatch should succeed"); - assert_eq!(output, "", "successful apply_patch hooks are silent"); - - fs::write(source_dir.join("lib.rs"), "prefix\nnew_line\nsuffix\n") - .expect("updated source should be written"); - git(&repository_root, &["add", "."]); - git( - &repository_root, - &[ - "-c", - "user.name=SCE Test", - "-c", - "user.email=sce@example.invalid", - "commit", - "-qm", - "apply patch", - ], - ); - - let context = AgentTraceStorageContext { - repository_root: &repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }; - let storage = - resolve_agent_trace_storage_for_hook_runtime_at_state_root(&context, &state_root) - .expect("repository Agent Trace DB should reopen"); - let recent = storage - .db - .recent_diff_trace_patches(0, i64::MAX) - .expect("stored Codex patch should be queryable"); - assert_eq!(recent.loaded_count(), 1); - let stored_file = &recent.patches[0].patch.files[0]; - assert_eq!(recent.patches[0].session_id, "cx_session-realistic"); - assert_eq!(recent.patches[0].tool_name.as_deref(), Some("codex")); - assert_eq!(recent.patches[0].tool_version, None); - assert_eq!(recent.patches[0].payload_type, "patch"); - assert_eq!(recent.patches[0].patch.files.len(), 1); - assert_eq!(stored_file.old_path, "src/lib.rs"); - assert_eq!(stored_file.new_path, "src/lib.rs"); - assert_eq!( - stored_file.hunks[0].model_id.as_deref(), - Some("custom/codex-model") - ); - - let flow = super::super::run_post_commit_intersection_flow_with( - &repository_root, - super::super::capture_post_commit_patch_from_git, - super::super::current_unix_time_ms, - |cutoff_ms, end_ms| storage.db.recent_diff_trace_patches(cutoff_ms, end_ms), - |insert| { - storage - .db - .insert_post_commit_patch_intersection(insert) - .map(|_| ()) - }, - ) - .expect("post-commit intersection should use the stored Codex evidence"); - let trace = super::super::run_post_commit_agent_trace_flow_with( - &flow, - None, - "https://example.invalid/codex-t19.git", - &crate::services::patch::ParsedPatch { files: Vec::new() }, - |value| { - crate::services::agent_trace::validate_agent_trace_value(value) - .map_err(|error| anyhow::anyhow!(error.to_string())) - }, - |insert| storage.db.insert_agent_trace(insert).map(|_| ()), - ) - .expect("post-commit Agent Trace should persist"); - assert_eq!( - trace.tool.as_ref().and_then(|tool| tool.name.as_deref()), - Some("codex") - ); - - let intersections = storage - .db - .query_map( - "SELECT intersection_patch FROM post_commit_patch_intersections ORDER BY id", - (), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("intersection row should be queryable"); - assert_eq!(intersections.len(), 1); - let intersection: serde_json::Value = - serde_json::from_str(&intersections[0]).expect("intersection JSON should parse"); - assert_eq!( - intersection["files"][0]["hunks"][0]["lines"][1]["session_id"], - "cx_session-realistic" - ); - - let traces = storage - .db - .query_map( - "SELECT trace_json FROM agent_traces ORDER BY id", - (), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("Agent Trace row should be queryable"); - assert_eq!(traces.len(), 1); - let trace_json: serde_json::Value = - serde_json::from_str(&traces[0]).expect("stored Agent Trace JSON should parse"); - assert_eq!(trace_json["tool"]["name"], "codex"); - assert_eq!( - trace_json["files"][0]["conversations"][0]["contributor"]["model_id"], - "custom/codex-model" - ); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - #[test] - fn delete_only_and_pure_rename_apply_patch_events_persist_no_rows() { - let (repository_root, state_root) = initialize_repository("no-row-boundaries"); - let delete_payload = codex_apply_patch_payload( - &repository_root, - "session-delete", - "custom/model", - "tool-delete", - "*** Begin Patch\n*** Delete File: obsolete.txt\n*** End Patch", - ); - let rename_payload = codex_apply_patch_payload( - &repository_root, - "session-rename", - "custom/model", - "tool-rename", - "*** Begin Patch\n*** Update File: old.txt\n*** Move to: new.txt\n*** End Patch", - ); - - for payload in [delete_payload, rename_payload] { - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("delete and pure-rename hooks should fail open successfully"); - assert_eq!(output, ""); - } - - let storage = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &AgentTraceStorageContext { - repository_root: &repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("repository Agent Trace DB should reopen"); - let recent = storage - .db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 0); - assert_eq!(recent.skipped_count(), 0); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - fn diff_trace_count(repository_root: &Path, state_root: &Path) -> usize { - let storage = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &AgentTraceStorageContext { - repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - state_root, - ) - .expect("repository Agent Trace DB should reopen"); - storage - .db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed") - .loaded_count() - } - - #[test] - fn nested_cwd_parent_traversal_path_is_accepted_and_persisted_repo_relative() { - let (repository_root, state_root) = initialize_repository("nested-cwd-traversal"); - let cwd = repository_root.join("src").join("lib"); - fs::create_dir_all(&cwd).expect("nested cwd should be created"); - - let payload = codex_apply_patch_payload( - &cwd, - "session-nested-traversal", - "custom/model", - "tool-nested-traversal", - "*** Begin Patch\n*** Add File: ../inside.rs\n+content\n*** End Patch", - ); - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("a `..` path that stays inside the repo should be accepted"); - assert_eq!(output, ""); - - let storage = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &AgentTraceStorageContext { - repository_root: &repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("repository Agent Trace DB should reopen"); - let recent = storage - .db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 1); - let file = &recent.patches[0].patch.files[0]; - assert_eq!(file.kind, FileChangeKind::Added); - assert_eq!(file.new_path, "src/inside.rs"); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - #[test] - fn absolute_path_inside_worktree_is_accepted_and_persisted_repo_relative() { - let (repository_root, state_root) = initialize_repository("absolute-inside"); - let absolute_target = repository_root.join("lib.rs"); - - let payload = codex_apply_patch_payload( - &repository_root, - "session-absolute-inside", - "custom/model", - "tool-absolute-inside", - &format!( - "*** Begin Patch\n*** Add File: {}\n+content\n*** End Patch", - absolute_target.display() - ), - ); - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("an absolute path inside the worktree should be accepted"); - assert_eq!(output, ""); - - let storage = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &AgentTraceStorageContext { - repository_root: &repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("repository Agent Trace DB should reopen"); - let recent = storage - .db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 1); - let file = &recent.patches[0].patch.files[0]; - assert_eq!(file.kind, FileChangeKind::Added); - assert_eq!(file.new_path, "lib.rs"); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - #[test] - fn parent_traversal_path_escaping_repository_is_rejected_with_no_diff_trace() { - let (repository_root, state_root) = initialize_repository("traversal-escape"); - - let payload = codex_apply_patch_payload( - &repository_root, - "session-traversal-escape", - "custom/model", - "tool-traversal-escape", - "*** Begin Patch\n*** Add File: ../outside.rs\n+content\n*** End Patch", - ); - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("a `..` path escaping the repo should fail open, not error"); - assert_eq!(output, ""); - assert_eq!(diff_trace_count(&repository_root, &state_root), 0); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - #[test] - fn absolute_path_outside_repository_is_rejected_with_no_diff_trace() { - let (repository_root, state_root) = initialize_repository("absolute-outside"); - let outside_target = std::env::temp_dir().join(format!( - "sce-codex-outside-target-{}-{}.rs", - std::process::id(), - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos() - )); - - let payload = codex_apply_patch_payload( - &repository_root, - "session-absolute-outside", - "custom/model", - "tool-absolute-outside", - &format!( - "*** Begin Patch\n*** Add File: {}\n+content\n*** End Patch", - outside_target.display() - ), - ); - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("an absolute path outside the repo should fail open, not error"); - assert_eq!(output, ""); - assert_eq!(diff_trace_count(&repository_root, &state_root), 0); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } - - #[test] - fn move_to_destination_with_valid_parent_traversal_resolves_source_and_destination_independently( - ) { - let (repository_root, state_root) = initialize_repository("move-traversal"); - let cwd = repository_root.join("src"); - fs::create_dir_all(&cwd).expect("src directory should be created"); - - let payload = codex_apply_patch_payload( - &cwd, - "session-move-traversal", - "custom/model", - "tool-move-traversal", - "*** Begin Patch\n*** Update File: old.rs\n*** Move to: ../moved.rs\n@@\n-old\n+new\n*** End Patch", - ); - let output = run_codex_subcommand_from_payload_at_state_root( - &repository_root, - &payload, - None, - &state_root, - ) - .expect("a move whose destination traverses `..` inside the repo should be accepted"); - assert_eq!(output, ""); - - let storage = resolve_agent_trace_storage_for_hook_runtime_at_state_root( - &AgentTraceStorageContext { - repository_root: &repository_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("repository Agent Trace DB should reopen"); - let recent = storage - .db - .recent_diff_trace_patches(0, i64::MAX) - .expect("diff trace query should succeed"); - assert_eq!(recent.loaded_count(), 1); - let file = &recent.patches[0].patch.files[0]; - assert_eq!(file.old_path, "src/old.rs"); - assert_eq!(file.new_path, "moved.rs"); - - fs::remove_dir_all(&repository_root).ok(); - fs::remove_dir_all(&state_root).ok(); - } -} diff --git a/cli/src/services/hooks/codex/stop.rs b/cli/src/services/hooks/codex/stop.rs index 6d0fb5545..9093584ec 100644 --- a/cli/src/services/hooks/codex/stop.rs +++ b/cli/src/services/hooks/codex/stop.rs @@ -13,11 +13,15 @@ use super::super::{ }; use super::{CodexHookEvent, NullableField}; -pub(super) fn handle(repository_root: &Path, event: &CodexHookEvent) -> Result { - handle_with_clock(repository_root, event, current_unix_time_ms) +pub(super) async fn handle(repository_root: &Path, event: &CodexHookEvent) -> Result { + handle_with_clock(repository_root, event, current_unix_time_ms).await } -fn handle_with_clock(repository_root: &Path, event: &CodexHookEvent, now: F) -> Result +async fn handle_with_clock( + repository_root: &Path, + event: &CodexHookEvent, + now: F, +) -> Result where F: FnOnce() -> Result, { @@ -29,17 +33,19 @@ where let generated_at_unix_ms = now()?; - let db = open_agent_trace_db_for_hook_runtime( + let mut db = open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for Codex Stop persistence.", - )?; + ) + .await?; persist_with( - &db, + &mut db, &validated, last_assistant_message, generated_at_unix_ms, ) + .await } #[derive(Debug)] @@ -69,8 +75,8 @@ fn validate_stop_event(event: &CodexHookEvent) -> Result> { }) } -fn persist_with( - db: &RepositoryAgentTraceDb, +async fn persist_with( + db: &mut RepositoryAgentTraceDb, validated: &ValidatedStop<'_>, last_assistant_message: &str, generated_at_unix_ms: i64, @@ -94,6 +100,7 @@ fn persist_with( generated_at_unix_ms, }, ) + .await .context("Failed to insert Codex Stop message/text-part event.")?; Ok(String::new()) @@ -107,475 +114,3 @@ fn required_trimmed_field<'a>(value: Option<&'a str>, field_name: &str) -> Resul )), } } - -#[cfg(test)] -fn capture_with( - db: &RepositoryAgentTraceDb, - event: &CodexHookEvent, - generated_at_unix_ms: i64, -) -> Result { - let validated = validate_stop_event(event)?; - match validated.last_assistant_message { - Some(last_assistant_message) => { - persist_with(db, &validated, last_assistant_message, generated_at_unix_ms) - } - None => Ok(String::new()), - } -} - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::PathBuf, - time::{SystemTime, UNIX_EPOCH}, - }; - - use super::*; - - fn unique_test_db_path(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-codex-stop-{label}-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") - } - - fn remove_test_db(db_path: &Path) { - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); - } - } - - fn event(session_id: &str, turn_id: &str, last_assistant_message: &str) -> CodexHookEvent { - CodexHookEvent { - hook_event_name: "Stop".to_string(), - session_id: Some(session_id.to_string()), - turn_id: Some(turn_id.to_string()), - cwd: None, - model: None, - tool_name: None, - tool_use_id: None, - tool_input: None, - tool_response: None, - prompt: None, - last_assistant_message: NullableField::Value(last_assistant_message.to_string()), - } - } - - fn message_rows(db: &RepositoryAgentTraceDb) -> Vec<(String, String, String)> { - db.query_map( - "SELECT session_id, message_id, role FROM messages ORDER BY id ASC", - (), - |row| { - Ok(( - row.get::(0).map_err(anyhow::Error::from)?, - row.get::(1).map_err(anyhow::Error::from)?, - row.get::(2).map_err(anyhow::Error::from)?, - )) - }, - ) - .expect("messages query should succeed") - } - - fn part_rows(db: &RepositoryAgentTraceDb) -> Vec<(String, String, String, String)> { - db.query_map( - "SELECT session_id, message_id, type, text FROM parts ORDER BY id ASC", - (), - |row| { - Ok(( - row.get::(0).map_err(anyhow::Error::from)?, - row.get::(1).map_err(anyhow::Error::from)?, - row.get::(2).map_err(anyhow::Error::from)?, - row.get::(3).map_err(anyhow::Error::from)?, - )) - }, - ) - .expect("parts query should succeed") - } - - #[test] - fn capture_with_produces_one_message_and_one_part_under_the_prefixed_session() { - let db_path = unique_test_db_path("basic"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let output = capture_with(&db, &event("session-1", "turn-1", "hello back"), 1_000) - .expect("capture should succeed"); - assert_eq!(output, ""); - - assert_eq!( - message_rows(&db), - vec![( - "cx_session-1".to_string(), - "cx:turn-1:assistant".to_string(), - "assistant".to_string() - )] - ); - assert_eq!( - part_rows(&db), - vec![( - "cx_session-1".to_string(), - "cx:turn-1:assistant".to_string(), - "text".to_string(), - "hello back".to_string() - )] - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_keeps_an_already_prefixed_session_id_unchanged() { - let db_path = unique_test_db_path("prefixed"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - capture_with(&db, &event("cx_session-1", "turn-1", "hi"), 1_000) - .expect("capture should succeed"); - - assert_eq!(message_rows(&db)[0].0, "cx_session-1"); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_does_not_duplicate_the_parent_message_on_reprocess() { - let db_path = unique_test_db_path("dedupe"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let payload = event("session-1", "turn-1", "hello back"); - - capture_with(&db, &payload, 1_000).expect("first capture should succeed"); - capture_with(&db, &payload, 2_000).expect("reprocessed capture should succeed"); - - assert_eq!( - message_rows(&db).len(), - 1, - "reprocessing the same turn's Stop must not duplicate the parent message row" - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_rejects_a_missing_last_assistant_message() { - let db_path = unique_test_db_path("missing-last-assistant-message"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello back"); - payload.last_assistant_message = NullableField::Missing; - - let error = capture_with(&db, &payload, 1_000) - .expect_err("missing last_assistant_message should error"); - assert!(error.to_string().contains("'last_assistant_message'")); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_is_a_no_op_for_a_null_last_assistant_message() { - let db_path = unique_test_db_path("null-last-assistant-message"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello back"); - payload.last_assistant_message = NullableField::Null; - - let output = capture_with(&db, &payload, 1_000) - .expect("null last_assistant_message is a valid no-op, not an error"); - assert_eq!(output, ""); - assert_eq!(message_rows(&db).len(), 0); - assert_eq!(part_rows(&db).len(), 0); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_rejects_a_missing_turn_id() { - let db_path = unique_test_db_path("missing-turn-id"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello back"); - payload.turn_id = None; - - let error = capture_with(&db, &payload, 1_000).expect_err("missing turn_id should error"); - assert!(error.to_string().contains("'turn_id'")); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_trims_padded_session_and_turn_ids_before_persisting() { - let db_path = unique_test_db_path("trimmed-ids"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event(" session-1 ", " turn-1 ", "hello back"); - payload.session_id = Some(" session-1 ".to_string()); - payload.turn_id = Some(" turn-1 ".to_string()); - - capture_with(&db, &payload, 1_000).expect("padded ids should persist trimmed"); - - assert_eq!( - message_rows(&db), - vec![( - "cx_session-1".to_string(), - "cx:turn-1:assistant".to_string(), - "assistant".to_string() - )] - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_rejects_a_whitespace_only_session_id() { - let db_path = unique_test_db_path("blank-session-id"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello back"); - payload.session_id = Some(" ".to_string()); - - let error = capture_with(&db, &payload, 1_000).expect_err("blank session_id should error"); - assert!(error.to_string().contains("'session_id'")); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_persists_an_explicit_empty_last_assistant_message() { - let db_path = unique_test_db_path("explicit-empty"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let payload = event("session-1", "turn-1", ""); - - let output = - capture_with(&db, &payload, 1_000).expect("explicit empty text should persist"); - assert_eq!(output, ""); - - assert_eq!( - part_rows(&db), - vec![( - "cx_session-1".to_string(), - "cx:turn-1:assistant".to_string(), - "text".to_string(), - String::new() - )], - "an explicit empty string is a present value, unlike null, and persists like any other text" - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_persists_deserialized_raw_json_with_an_explicit_empty_string() { - let db_path = unique_test_db_path("raw-json-empty-string"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let payload: CodexHookEvent = serde_json::from_str( - r#"{"hook_event_name":"Stop","session_id":"s1","turn_id":"t1","last_assistant_message":""}"#, - ) - .expect("raw JSON with an explicit empty string should deserialize"); - - let output = capture_with(&db, &payload, 1_000) - .expect("deserialized explicit empty string should persist"); - assert_eq!(output, ""); - assert_eq!(message_rows(&db).len(), 1); - assert_eq!( - part_rows(&db), - vec![( - "cx_s1".to_string(), - "cx:t1:assistant".to_string(), - "text".to_string(), - String::new() - )] - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_persists_deserialized_raw_json_with_normal_text() { - let db_path = unique_test_db_path("raw-json-normal-text"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let payload: CodexHookEvent = serde_json::from_str( - r#"{"hook_event_name":"Stop","session_id":"s1","turn_id":"t1","last_assistant_message":"hello"}"#, - ) - .expect("raw JSON with normal text should deserialize"); - - let output = - capture_with(&db, &payload, 1_000).expect("deserialized normal text should persist"); - assert_eq!(output, ""); - assert_eq!(message_rows(&db).len(), 1); - assert_eq!( - part_rows(&db), - vec![( - "cx_s1".to_string(), - "cx:t1:assistant".to_string(), - "text".to_string(), - "hello".to_string() - )] - ); - - remove_test_db(&db_path); - } - - #[test] - fn handle_is_a_silent_no_op_for_a_null_last_assistant_message_without_opening_the_db() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.last_assistant_message = NullableField::Null; - - let output = handle(Path::new("/nonexistent-repository-root"), &payload) - .expect("null last_assistant_message should be a silent successful no-op"); - assert_eq!(output, ""); - } - - #[test] - fn handle_with_clock_errors_for_a_missing_last_assistant_message_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.last_assistant_message = NullableField::Missing; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a missing last_assistant_message") - }) - .expect_err("missing last_assistant_message should error"); - assert!(error.to_string().contains("'last_assistant_message'")); - } - - #[test] - fn handle_with_clock_is_a_silent_no_op_for_null_without_calling_the_clock_or_opening_the_db() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.last_assistant_message = NullableField::Null; - - let output = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for an explicit null last_assistant_message") - }) - .expect("null last_assistant_message should be a silent successful no-op"); - assert_eq!(output, ""); - } - - #[test] - fn handle_with_clock_propagates_a_timestamp_failure_as_an_error_with_no_persistence() { - let payload = event("session-1", "turn-1", "hello back"); - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - Err(anyhow::anyhow!("clock failed")) - }) - .expect_err("a failed clock must propagate as an error for the outer fail-open boundary"); - assert!(error.to_string().contains("clock failed")); - } - - #[test] - fn handle_with_clock_rejects_a_null_stop_with_a_missing_session_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.session_id = None; - payload.last_assistant_message = NullableField::Null; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed Stop payload") - }) - .expect_err("a null Stop with a missing session_id must still be rejected as malformed"); - assert!(error.to_string().contains("'session_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_null_stop_with_an_empty_session_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.session_id = Some(String::new()); - payload.last_assistant_message = NullableField::Null; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed Stop payload") - }) - .expect_err("a null Stop with an empty session_id must still be rejected as malformed"); - assert!(error.to_string().contains("'session_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_null_stop_with_a_whitespace_only_session_id_without_calling_the_clock( - ) { - let mut payload = event("session-1", "turn-1", "unused"); - payload.session_id = Some(" ".to_string()); - payload.last_assistant_message = NullableField::Null; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed Stop payload") - }) - .expect_err( - "a null Stop with a whitespace-only session_id must still be rejected as malformed", - ); - assert!(error.to_string().contains("'session_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_null_stop_with_a_missing_turn_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.turn_id = None; - payload.last_assistant_message = NullableField::Null; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed Stop payload") - }) - .expect_err("a null Stop with a missing turn_id must still be rejected as malformed"); - assert!(error.to_string().contains("'turn_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_null_stop_with_an_empty_turn_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.turn_id = Some(String::new()); - payload.last_assistant_message = NullableField::Null; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed Stop payload") - }) - .expect_err("a null Stop with an empty turn_id must still be rejected as malformed"); - assert!(error.to_string().contains("'turn_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_null_stop_with_a_whitespace_only_turn_id_without_calling_the_clock( - ) { - let mut payload = event("session-1", "turn-1", "unused"); - payload.turn_id = Some(" ".to_string()); - payload.last_assistant_message = NullableField::Null; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed Stop payload") - }) - .expect_err( - "a null Stop with a whitespace-only turn_id must still be rejected as malformed", - ); - assert!(error.to_string().contains("'turn_id'")); - } - - #[test] - fn handle_with_clock_is_a_silent_no_op_for_null_with_padded_ids_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.session_id = Some(" session-1 ".to_string()); - payload.turn_id = Some(" turn-1 ".to_string()); - payload.last_assistant_message = NullableField::Null; - - let output = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for an explicit null last_assistant_message") - }) - .expect("a null Stop with padded-but-valid identifiers should still be a successful no-op"); - assert_eq!(output, ""); - } - - #[test] - fn validate_stop_event_rejects_a_missing_last_assistant_message_with_valid_ids() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.last_assistant_message = NullableField::Missing; - - let error = validate_stop_event(&payload) - .expect_err("missing last_assistant_message should be rejected"); - assert!(error.to_string().contains("'last_assistant_message'")); - } - - #[test] - fn validate_stop_event_returns_none_for_an_explicit_null_with_valid_ids() { - let mut payload = event("session-1", "turn-1", "unused"); - payload.last_assistant_message = NullableField::Null; - - let validated = - validate_stop_event(&payload).expect("valid ids with a null message should validate"); - assert_eq!(validated.session_id, "session-1"); - assert_eq!(validated.turn_id, "turn-1"); - assert_eq!(validated.last_assistant_message, None); - } -} diff --git a/cli/src/services/hooks/codex/user_prompt_submit.rs b/cli/src/services/hooks/codex/user_prompt_submit.rs index 7d99d4add..34a2dca10 100644 --- a/cli/src/services/hooks/codex/user_prompt_submit.rs +++ b/cli/src/services/hooks/codex/user_prompt_submit.rs @@ -13,11 +13,15 @@ use super::super::{ }; use super::CodexHookEvent; -pub(super) fn handle(repository_root: &Path, event: &CodexHookEvent) -> Result { - handle_with_clock(repository_root, event, current_unix_time_ms) +pub(super) async fn handle(repository_root: &Path, event: &CodexHookEvent) -> Result { + handle_with_clock(repository_root, event, current_unix_time_ms).await } -fn handle_with_clock(repository_root: &Path, event: &CodexHookEvent, now: F) -> Result +async fn handle_with_clock( + repository_root: &Path, + event: &CodexHookEvent, + now: F, +) -> Result where F: FnOnce() -> Result, { @@ -25,12 +29,13 @@ where let generated_at_unix_ms = now()?; - let db = open_agent_trace_db_for_hook_runtime( + let mut db = open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for Codex UserPromptSubmit persistence.", - )?; + ) + .await?; - persist_with(&db, &validated, generated_at_unix_ms) + persist_with(&mut db, &validated, generated_at_unix_ms).await } struct ValidatedUserPromptSubmit<'a> { @@ -53,8 +58,8 @@ fn validate_user_prompt_submit_event( }) } -fn persist_with( - db: &RepositoryAgentTraceDb, +async fn persist_with( + db: &mut RepositoryAgentTraceDb, validated: &ValidatedUserPromptSubmit<'_>, generated_at_unix_ms: i64, ) -> Result { @@ -77,6 +82,7 @@ fn persist_with( generated_at_unix_ms, }, ) + .await .context("Failed to insert Codex UserPromptSubmit message/text-part event.")?; Ok(String::new()) @@ -99,296 +105,3 @@ fn required_trimmed_field<'a>(value: Option<&'a str>, field_name: &str) -> Resul )), } } - -#[cfg(test)] -fn capture_with( - db: &RepositoryAgentTraceDb, - event: &CodexHookEvent, - generated_at_unix_ms: i64, -) -> Result { - let validated = validate_user_prompt_submit_event(event)?; - persist_with(db, &validated, generated_at_unix_ms) -} - -#[cfg(test)] -mod tests { - use std::{ - fs, - path::PathBuf, - time::{SystemTime, UNIX_EPOCH}, - }; - - use super::super::NullableField; - use super::*; - - fn unique_test_db_path(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-codex-user-prompt-submit-{label}-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") - } - - fn remove_test_db(db_path: &Path) { - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); - } - } - - fn event(session_id: &str, turn_id: &str, prompt: &str) -> CodexHookEvent { - CodexHookEvent { - hook_event_name: "UserPromptSubmit".to_string(), - session_id: Some(session_id.to_string()), - turn_id: Some(turn_id.to_string()), - cwd: None, - model: None, - tool_name: None, - tool_use_id: None, - tool_input: None, - tool_response: None, - prompt: Some(prompt.to_string()), - last_assistant_message: NullableField::Missing, - } - } - - fn message_rows(db: &RepositoryAgentTraceDb) -> Vec<(String, String, String)> { - db.query_map( - "SELECT session_id, message_id, role FROM messages ORDER BY id ASC", - (), - |row| { - Ok(( - row.get::(0).map_err(anyhow::Error::from)?, - row.get::(1).map_err(anyhow::Error::from)?, - row.get::(2).map_err(anyhow::Error::from)?, - )) - }, - ) - .expect("messages query should succeed") - } - - fn part_rows(db: &RepositoryAgentTraceDb) -> Vec<(String, String, String, String)> { - db.query_map( - "SELECT session_id, message_id, type, text FROM parts ORDER BY id ASC", - (), - |row| { - Ok(( - row.get::(0).map_err(anyhow::Error::from)?, - row.get::(1).map_err(anyhow::Error::from)?, - row.get::(2).map_err(anyhow::Error::from)?, - row.get::(3).map_err(anyhow::Error::from)?, - )) - }, - ) - .expect("parts query should succeed") - } - - #[test] - fn capture_with_produces_one_message_and_one_part_under_the_prefixed_session() { - let db_path = unique_test_db_path("basic"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - let output = capture_with(&db, &event("session-1", "turn-1", "hello world"), 1_000) - .expect("capture should succeed"); - assert_eq!(output, ""); - - assert_eq!( - message_rows(&db), - vec![( - "cx_session-1".to_string(), - "cx:turn-1:user".to_string(), - "user".to_string() - )] - ); - assert_eq!( - part_rows(&db), - vec![( - "cx_session-1".to_string(), - "cx:turn-1:user".to_string(), - "text".to_string(), - "hello world".to_string() - )] - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_keeps_an_already_prefixed_session_id_unchanged() { - let db_path = unique_test_db_path("prefixed"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - - capture_with(&db, &event("cx_session-1", "turn-1", "hi"), 1_000) - .expect("capture should succeed"); - - assert_eq!(message_rows(&db)[0].0, "cx_session-1"); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_does_not_duplicate_the_parent_message_on_reprocess() { - let db_path = unique_test_db_path("dedupe"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let payload = event("session-1", "turn-1", "hello world"); - - capture_with(&db, &payload, 1_000).expect("first capture should succeed"); - capture_with(&db, &payload, 2_000).expect("reprocessed capture should succeed"); - - assert_eq!( - message_rows(&db).len(), - 1, - "reprocessing the same turn's UserPromptSubmit must not duplicate the parent message row" - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_rejects_a_missing_prompt() { - let db_path = unique_test_db_path("missing-prompt"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello world"); - payload.prompt = None; - - let error = capture_with(&db, &payload, 1_000).expect_err("missing prompt should error"); - assert!(error.to_string().contains("'prompt'")); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_rejects_a_missing_turn_id() { - let db_path = unique_test_db_path("missing-turn-id"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello world"); - payload.turn_id = None; - - let error = capture_with(&db, &payload, 1_000).expect_err("missing turn_id should error"); - assert!(error.to_string().contains("'turn_id'")); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_trims_padded_session_and_turn_ids_before_persisting() { - let db_path = unique_test_db_path("trimmed-ids"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello world"); - payload.session_id = Some(" session-1 ".to_string()); - payload.turn_id = Some(" turn-1 ".to_string()); - - capture_with(&db, &payload, 1_000).expect("padded ids should persist trimmed"); - - assert_eq!( - message_rows(&db), - vec![( - "cx_session-1".to_string(), - "cx:turn-1:user".to_string(), - "user".to_string() - )] - ); - - remove_test_db(&db_path); - } - - #[test] - fn capture_with_rejects_a_whitespace_only_turn_id() { - let db_path = unique_test_db_path("blank-turn-id"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("repository DB should open"); - let mut payload = event("session-1", "turn-1", "hello world"); - payload.turn_id = Some(" ".to_string()); - - let error = capture_with(&db, &payload, 1_000).expect_err("blank turn_id should error"); - assert!(error.to_string().contains("'turn_id'")); - - remove_test_db(&db_path); - } - - #[test] - fn handle_with_clock_propagates_a_timestamp_failure_as_an_error_with_no_persistence() { - let payload = event("session-1", "turn-1", "hello world"); - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - Err(anyhow::anyhow!("clock failed")) - }) - .expect_err("a failed clock must propagate as an error for the outer fail-open boundary"); - assert!(error.to_string().contains("clock failed")); - } - - #[test] - fn handle_with_clock_rejects_a_missing_session_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "hello world"); - payload.session_id = None; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed UserPromptSubmit payload") - }) - .expect_err("missing session_id should be rejected before the clock is consulted"); - assert!(error.to_string().contains("'session_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_whitespace_only_session_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "hello world"); - payload.session_id = Some(" ".to_string()); - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed UserPromptSubmit payload") - }) - .expect_err("whitespace-only session_id should be rejected before the clock is consulted"); - assert!(error.to_string().contains("'session_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_missing_turn_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "hello world"); - payload.turn_id = None; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed UserPromptSubmit payload") - }) - .expect_err("missing turn_id should be rejected before the clock is consulted"); - assert!(error.to_string().contains("'turn_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_whitespace_only_turn_id_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "hello world"); - payload.turn_id = Some(" ".to_string()); - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed UserPromptSubmit payload") - }) - .expect_err("whitespace-only turn_id should be rejected before the clock is consulted"); - assert!(error.to_string().contains("'turn_id'")); - } - - #[test] - fn handle_with_clock_rejects_a_missing_prompt_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "hello world"); - payload.prompt = None; - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed UserPromptSubmit payload") - }) - .expect_err("missing prompt should be rejected before the clock is consulted"); - assert!(error.to_string().contains("'prompt'")); - } - - #[test] - fn handle_with_clock_rejects_a_whitespace_only_prompt_without_calling_the_clock() { - let mut payload = event("session-1", "turn-1", "hello world"); - payload.prompt = Some(" ".to_string()); - - let error = handle_with_clock(Path::new("/nonexistent-repository-root"), &payload, || { - panic!("clock must not be called for a malformed UserPromptSubmit payload") - }) - .expect_err("whitespace-only prompt should be rejected before the clock is consulted"); - assert!(error.to_string().contains("'prompt'")); - } -} diff --git a/cli/src/services/hooks/codex_mutation_scope/events.rs b/cli/src/services/hooks/codex_mutation_scope/events.rs index c83864032..80dccd113 100644 --- a/cli/src/services/hooks/codex_mutation_scope/events.rs +++ b/cli/src/services/hooks/codex_mutation_scope/events.rs @@ -50,10 +50,6 @@ impl CodexToolIdentity { tool_use_id: self.tool_use_id.clone(), } } - - pub(crate) fn is_subagent(&self) -> bool { - self.agent_id.is_some() - } } #[derive(Clone, Debug, Eq, PartialEq)] @@ -104,12 +100,6 @@ pub(crate) enum ToolClassification { pub(super) const TRACKED_MUTATION_TOOL_NAMES: &[&str] = &["Bash", "apply_patch"]; pub(super) const DELEGATION_TOOL_NAMES: &[&str] = &["collaborationspawn_agent", "collaborationwait_agent"]; -pub(super) const MCP_TOOL_NAME_PREFIX: &str = "mcp__"; - -pub(crate) fn is_mcp_tool_name(tool_name: &str) -> bool { - tool_name.starts_with(MCP_TOOL_NAME_PREFIX) -} - pub(crate) fn classify_tool(tool_name: &str) -> ToolClassification { if TRACKED_MUTATION_TOOL_NAMES.contains(&tool_name) { ToolClassification::TrackedMutation diff --git a/cli/src/services/hooks/codex_mutation_scope/health.rs b/cli/src/services/hooks/codex_mutation_scope/health.rs index 52968cb83..32c8ebbe2 100644 --- a/cli/src/services/hooks/codex_mutation_scope/health.rs +++ b/cli/src/services/hooks/codex_mutation_scope/health.rs @@ -7,8 +7,8 @@ use crate::services::mutation_trace::types::ActorKind; use super::state::{self, RecoveryState}; -pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { - let state = match state::read_state(git_dir) { +pub(crate) async fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { + let state = match state::read_state(git_dir).await { Ok(state) => state, Err(error) => { return MutationScopeAdapterHealth::new( @@ -52,213 +52,3 @@ pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { ), } } - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - use std::sync::atomic::{AtomicU64, Ordering}; - - use super::super::AttemptKey; - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-codex-mutation-scope-health-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(tool_use_id: &str) -> AttemptKey { - AttemptKey { - session_id: "session-1".to_string(), - agent_id: None, - tool_use_id: tool_use_id.to_string(), - } - } - - #[test] - fn absent_state_file_is_healthy() { - let git_dir = unique_test_git_dir("absent"); - - let health = classify_health(&git_dir); - - assert_eq!(health.adapter, ActorKind::Codex); - assert_eq!(health.status, MutationScopeHealthStatus::Healthy); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_is_healthy_even_with_live_attempts() { - let git_dir = unique_test_git_dir("clear-with-attempts"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - state::seed_attempt_for_tests( - &git_dir, - &key("exec-1"), - "turn-1", - "Bash", - state::AttemptPhase::Active, - ); - - let health = classify_health(&git_dir); - - assert_eq!(health.status, MutationScopeHealthStatus::Healthy); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_recovery_with_empty_attempts_is_recovering_and_the_next_admission_claims_the_flush() - { - let git_dir = unique_test_git_dir("pending-empty-recovering"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let generation = state::arm_recovery(&git_dir).expect("arming recovery should succeed"); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering - ); - - assert_eq!( - state::admit_tracked_attempt(&git_dir, &key("exec-new"), "turn-1", "Bash") - .expect("admit should not error"), - state::AdmitDecision::FlushClaimed { generation }, - "Recovering must be proven by the next admission actually claiming the flush" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_non_empty_recovery_denies_unrelated_admission_without_losing_the_recovery_path() { - let git_dir = unique_test_git_dir("pending-non-empty-recovering"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - state::seed_attempt_for_tests( - &git_dir, - &key("exec-stuck"), - "turn-1", - "Bash", - state::AttemptPhase::Active, - ); - state::arm_recovery(&git_dir).expect("arming recovery should succeed"); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "a proven same-lane self-healing path exists even though this state denies unrelated admission" - ); - - for attempt_number in 1..=2 { - assert_eq!( - state::admit_tracked_attempt(&git_dir, &key("exec-other"), "turn-2", "Bash") - .expect("admit should not error"), - state::AdmitDecision::RecoveryBlocked, - "admission #{attempt_number} in an unrelated lane must be denied without self-clearing" - ); - } - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "the classifier must still report Recovering after repeated unrelated denial; \ - Recovering does not mean every future call succeeds, only that a proven normal \ - self-healing route exists" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn orphaned_flushing_with_no_attempts_is_recovering_and_reclaimed_by_the_next_admission() { - let git_dir = unique_test_git_dir("orphaned-flushing-recovering"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let generation = state::arm_recovery(&git_dir).expect("arming recovery should succeed"); - assert_eq!( - state::admit_tracked_attempt(&git_dir, &key("seed"), "seed-turn", "Bash") - .expect("seeding the flush claim should not error"), - state::AdmitDecision::FlushClaimed { generation }, - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering - ); - - state::normalize_recovery_after_boundary_lock_acquired(&git_dir) - .expect("normalize should succeed"); - assert_eq!( - state::admit_tracked_attempt(&git_dir, &key("exec-new"), "turn-1", "Bash") - .expect("admit should not error"), - state::AdmitDecision::FlushClaimed { generation }, - "Recovering must be proven by the reclaimed flush being retried" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn flushing_with_unresolved_attempts_is_a_structurally_impossible_state_classified_invalid() { - let git_dir = unique_test_git_dir("flushing-non-empty-invalid"); - let dir = state::adapter_state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write( - state::state_path(&git_dir), - serde_json::json!({ - "version": 3, - "next_attempt_seq": 2, - "next_recovery_generation": 2, - "recovery": { "phase": "flushing", "generation": 1 }, - "attempts": [{ - "attempt_seq": 1, - "scope_id": "cx-tool-v1|n=1|s=9:session-1|a=0:|t=6:exec-1", - "session_id": "session-1", - "turn_id": "turn-1", - "agent_id": null, - "tool_use_id": "exec-1", - "tool_name": "Bash", - "phase": "active", - }], - }) - .to_string(), - ) - .expect("hand-seeded state file should be writable"); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Invalid - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn malformed_state_file_is_invalid_with_the_read_error_surfaced() { - let git_dir = unique_test_git_dir("malformed"); - let dir = state::adapter_state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write(state::state_path(&git_dir), b"not json") - .expect("malformed file should be writable"); - - let health = classify_health(&git_dir); - - assert_eq!(health.status, MutationScopeHealthStatus::Invalid); - assert!( - health - .detail - .as_deref() - .unwrap_or_default() - .contains("malformed"), - "the read error must be surfaced in the detail: {:?}", - health.detail - ); - - remove_test_git_dir(&git_dir); - } -} diff --git a/cli/src/services/hooks/codex_mutation_scope/lifecycle.rs b/cli/src/services/hooks/codex_mutation_scope/lifecycle.rs index abef09050..49508938a 100644 --- a/cli/src/services/hooks/codex_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/codex_mutation_scope/lifecycle.rs @@ -10,7 +10,6 @@ use crate::services::hooks::{ normalize_codex_model_id, prefixed_diff_trace_session_id, CODEX_TOOL_NAME, }; use crate::services::mutation_trace::runtime::resolve_git_dir; -use crate::services::observability::traits::Logger; use super::events::CODEX_TRACKED_TOOL_BASH; use super::state; @@ -20,13 +19,6 @@ use super::{ scope_start_payload, AttemptKey, CodexHookEvent, CodexToolExecution, ToolClassification, }; -pub(super) type GitDirResolver<'a> = &'a dyn Fn(&str) -> Result; - -pub(super) type IngressSeam<'a> = &'a dyn Fn(&Path, &str, Option<&dyn Logger>) -> Result; - -pub(super) type BashPolicyEvaluator<'a> = - &'a dyn Fn(&Path, &str) -> Result; - pub(super) const ACTOR_KIND_CODEX: &str = "codex"; #[derive(Clone, Debug, Eq, PartialEq)] @@ -51,8 +43,8 @@ pub(super) const FAIL_CLOSED_DENY_REASON: &str = pub(super) const PRE_TOOL_USE_FAIL_CLOSED_EVENT: &str = "sce.hooks.codex_mutation_scope.pre_tool_use_fail_closed"; -pub(super) fn log_pre_tool_use_fail_closed( - logger: Option<&dyn Logger>, +pub(super) fn log_pre_tool_use_fail_closed( + logger: Option<&L>, context: &str, error: &anyhow::Error, ) { @@ -66,18 +58,25 @@ pub(super) fn log_pre_tool_use_fail_closed( } } -pub(crate) fn run_codex_mutation_scope_subcommand(logger: Option<&dyn Logger>) -> Result { +pub(crate) async fn run_codex_mutation_scope_subcommand< + L: crate::services::observability::traits::Logger, +>( + logger: Option<&L>, +) -> Result { let stdin_payload = hooks::read_hook_stdin()?; - run_codex_mutation_scope_from_payload(&stdin_payload, logger) + run_codex_mutation_scope_from_payload(&stdin_payload, logger).await } -pub(crate) fn run_codex_mutation_scope_from_payload( +pub(crate) async fn run_codex_mutation_scope_from_payload< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { + let resolve_git_dir_fn = async |cwd: &str| resolve_git_dir(Path::new(cwd)).await; + let seam_fn = async |repository_root: &Path, payload: &str, logger: Option<&L>| { hooks::mutation_scope::run_mutation_scope_from_payload(repository_root, payload, logger) + .await }; let bash_policy_fn = |repository_root: &Path, command: &str| { evaluate_codex_bash_policy(repository_root, command) @@ -90,87 +89,28 @@ pub(crate) fn run_codex_mutation_scope_from_payload( &seam_fn, &bash_policy_fn, ) + .await } -#[cfg(test)] -pub(crate) fn run_codex_mutation_scope_from_payload_at_state_root( - state_root: &Path, - stdin_payload: &str, - logger: Option<&dyn Logger>, -) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { - hooks::mutation_scope::run_mutation_scope_from_payload_at_state_root( - repository_root, - state_root, - payload, - logger, - ) - }; - let bash_policy_fn = |repository_root: &Path, command: &str| { - evaluate_codex_bash_policy(repository_root, command) - }; - - run_codex_mutation_scope_from_payload_with_seams( - stdin_payload, - logger, - &resolve_git_dir_fn, - &seam_fn, - &bash_policy_fn, - ) -} - -#[cfg(test)] -pub(super) fn run_codex_mutation_scope_from_payload_with( +pub(super) async fn run_codex_mutation_scope_from_payload_with_seams< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, -) -> Result { - let allow_all = |_repository_root: &Path, _command: &str| Ok(CodexBashPolicyDecision::Allowed); - run_codex_mutation_scope_from_payload_with_seams( - stdin_payload, - logger, - resolve_git_dir, - seam, - &allow_all, - ) -} - -#[cfg(test)] -pub(super) fn run_codex_mutation_scope_from_payload_with_bash_policy( - stdin_payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, - evaluate_bash_policy: BashPolicyEvaluator, -) -> Result { - run_codex_mutation_scope_from_payload_with_seams( - stdin_payload, - logger, - resolve_git_dir, - seam, - evaluate_bash_policy, - ) -} - -pub(super) fn run_codex_mutation_scope_from_payload_with_seams( - stdin_payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, - evaluate_bash_policy: BashPolicyEvaluator, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, + evaluate_bash_policy: &impl Fn(&Path, &str) -> Result, ) -> Result { let event = parse_codex_hook_event(stdin_payload)?; - dispatch_codex_hook_event(event, logger, resolve_git_dir, seam, evaluate_bash_policy) + dispatch_codex_hook_event(event, logger, resolve_git_dir, seam, evaluate_bash_policy).await } -pub(super) fn dispatch_codex_hook_event( +pub(super) async fn dispatch_codex_hook_event( event: CodexHookEvent, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, - evaluate_bash_policy: BashPolicyEvaluator, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, + evaluate_bash_policy: &impl Fn(&Path, &str) -> Result, ) -> Result { match event { CodexHookEvent::PreToolUse(execution) => Ok(handle_pre_tool_use( @@ -179,7 +119,8 @@ pub(super) fn dispatch_codex_hook_event( resolve_git_dir, seam, evaluate_bash_policy, - )), + ) + .await), CodexHookEvent::PostToolUse(identity) => { if !matches!( classify_tool(&identity.tool_name), @@ -188,9 +129,9 @@ pub(super) fn dispatch_codex_hook_event( return Ok(String::new()); } - let git_dir = resolve_git_dir(&identity.cwd)?; + let git_dir = resolve_git_dir(&identity.cwd).await?; let repository_root = Path::new(&identity.cwd); - with_boundary_lock(&git_dir, || { + with_boundary_lock(&git_dir, async || { handle_close( &git_dir, repository_root, @@ -198,67 +139,78 @@ pub(super) fn dispatch_codex_hook_event( logger, seam, ) + .await }) + .await } CodexHookEvent::Stop(turn) => { - let git_dir = resolve_git_dir(&turn.cwd)?; + let git_dir = resolve_git_dir(&turn.cwd).await?; let repository_root = Path::new(&turn.cwd); let session_id = turn.session_id.clone(); - with_boundary_lock(&git_dir, || { + with_boundary_lock(&git_dir, async || { cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id && attempt.agent_id.is_none() }) + .await }) + .await } CodexHookEvent::Interrupt(turn) => { - let git_dir = resolve_git_dir(&turn.cwd)?; + let git_dir = resolve_git_dir(&turn.cwd).await?; let repository_root = Path::new(&turn.cwd); let session_id = turn.session_id.clone(); - with_boundary_lock(&git_dir, || { + with_boundary_lock(&git_dir, async || { cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id }) + .await }) + .await } CodexHookEvent::SubagentStop(agent) => { - let git_dir = resolve_git_dir(&agent.cwd)?; + let git_dir = resolve_git_dir(&agent.cwd).await?; let repository_root = Path::new(&agent.cwd); let session_id = agent.session_id.clone(); let agent_id = agent.agent_id.clone(); - with_boundary_lock(&git_dir, || { + with_boundary_lock(&git_dir, async || { cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id && attempt.agent_id.as_deref() == Some(&agent_id) }) + .await }) + .await } CodexHookEvent::SessionEnd(session) => { - let git_dir = resolve_git_dir(&session.cwd)?; + let git_dir = resolve_git_dir(&session.cwd).await?; let repository_root = Path::new(&session.cwd); let session_id = session.session_id.clone(); - with_boundary_lock(&git_dir, || { + with_boundary_lock(&git_dir, async || { cleanup_attempts_matching(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == session_id }) + .await }) + .await } } } -pub(super) fn with_boundary_lock( +pub(super) async fn with_boundary_lock( git_dir: &Path, - operation: impl FnOnce() -> Result, + operation: impl std::ops::AsyncFnOnce() -> Result, ) -> Result { - let _boundary = state::BOUNDARY_LOCK.acquire(&state::adapter_state_dir(git_dir))?; - operation() + state::BOUNDARY_LOCK + .run_under_boundary(&state::adapter_state_dir(git_dir), operation) + .await } -pub(super) fn handle_pre_tool_use( +pub(super) async fn handle_pre_tool_use( execution: &CodexToolExecution, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, - evaluate_bash_policy: BashPolicyEvaluator, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, + evaluate_bash_policy: &impl Fn(&Path, &str) -> Result, ) -> String { let identity = &execution.identity; @@ -282,7 +234,7 @@ pub(super) fn handle_pre_tool_use( } } - let git_dir = match resolve_git_dir(&identity.cwd) { + let git_dir = match resolve_git_dir(&identity.cwd).await { Ok(git_dir) => git_dir, Err(error) => { log_pre_tool_use_fail_closed(logger, "resolve_git_dir", &error); @@ -293,10 +245,11 @@ pub(super) fn handle_pre_tool_use( let key = identity.attempt_key(); let turn_id = identity.turn_id.as_str(); let provenance = codex_scope_provenance(execution); - let outcome = with_boundary_lock(&git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(&git_dir)?; + let outcome = with_boundary_lock(&git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(&git_dir).await?; - sweep_stale_lane_predecessors(&git_dir, repository_root, &key, turn_id, logger, seam)?; + sweep_stale_lane_predecessors(&git_dir, repository_root, &key, turn_id, logger, seam) + .await?; match admit_or_recover( &git_dir, @@ -306,7 +259,9 @@ pub(super) fn handle_pre_tool_use( &identity.tool_name, logger, seam, - )? { + ) + .await? + { Admission::Admitted(allocated) => { establish_start( &git_dir, @@ -315,12 +270,14 @@ pub(super) fn handle_pre_tool_use( &provenance, logger, seam, - )?; + ) + .await?; Ok(PreToolUseOutcome::Continue) } Admission::Denied => Ok(PreToolUseOutcome::Deny), } - }); + }) + .await; match outcome { Ok(PreToolUseOutcome::Continue) => String::new(), @@ -346,7 +303,7 @@ pub(super) enum BashPolicyPreflight { pub(super) fn codex_bash_policy_preflight( repository_root: &Path, execution: &CodexToolExecution, - evaluate_bash_policy: BashPolicyEvaluator, + evaluate_bash_policy: &impl Fn(&Path, &str) -> Result, ) -> BashPolicyPreflight { let command = match bash_command_from_tool_input(execution.tool_input.as_ref()) { Ok(command) => command, @@ -365,16 +322,18 @@ pub(super) enum Admission { Denied, } -pub(super) fn sweep_stale_lane_predecessors( +pub(super) async fn sweep_stale_lane_predecessors< + L: crate::services::observability::traits::Logger, +>( git_dir: &Path, repository_root: &Path, key: &AttemptKey, turn_id: &str, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { loop { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let Some(stale) = current .attempts .iter() @@ -386,35 +345,35 @@ pub(super) fn sweep_stale_lane_predecessors( else { return Ok(()); }; - abandon_attempt(git_dir, repository_root, &stale, logger, seam)?; + abandon_attempt(git_dir, repository_root, &stale, logger, seam).await?; } } -pub(super) fn admit_or_recover( +pub(super) async fn admit_or_recover( git_dir: &Path, repository_root: &Path, key: &AttemptKey, turn_id: &str, tool_name: &str, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - match state::admit_tracked_attempt(git_dir, key, turn_id, tool_name)? { + match state::admit_tracked_attempt(git_dir, key, turn_id, tool_name).await? { state::AdmitDecision::Admitted(allocated) => Ok(Admission::Admitted(allocated)), state::AdmitDecision::RecoveryBlocked | state::AdmitDecision::UncertainAttemptBlocked | state::AdmitDecision::StalePredecessorBlocked => Ok(Admission::Denied), state::AdmitDecision::FlushClaimed { generation } => { - match seam(repository_root, &flush_payload(), logger) { - Ok(_) => match state::complete_recovery_flush(git_dir, generation)? { + match seam(repository_root, &flush_payload(), logger).await { + Ok(_) => match state::complete_recovery_flush(git_dir, generation).await? { state::RecoveryFlushCompletion::Cleared => { - readmit_after_flush(git_dir, key, turn_id, tool_name) + readmit_after_flush(git_dir, key, turn_id, tool_name).await } state::RecoveryFlushCompletion::Superseded => Ok(Admission::Denied), }, Err(error) => { log_pre_tool_use_fail_closed(logger, "recovery_flush", &error); - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; Ok(Admission::Denied) } } @@ -422,16 +381,16 @@ pub(super) fn admit_or_recover( } } -pub(super) fn readmit_after_flush( +pub(super) async fn readmit_after_flush( git_dir: &Path, key: &AttemptKey, turn_id: &str, tool_name: &str, ) -> Result { - match state::admit_tracked_attempt(git_dir, key, turn_id, tool_name)? { + match state::admit_tracked_attempt(git_dir, key, turn_id, tool_name).await? { state::AdmitDecision::Admitted(allocated) => Ok(Admission::Admitted(allocated)), state::AdmitDecision::FlushClaimed { generation } => { - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; Ok(Admission::Denied) } state::AdmitDecision::RecoveryBlocked @@ -440,13 +399,13 @@ pub(super) fn readmit_after_flush( } } -pub(super) fn establish_start( +pub(super) async fn establish_start( git_dir: &Path, repository_root: &Path, allocated: &state::AllocatedAttempt, provenance: &CodexScopeProvenance, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { let scope_id = &allocated.attempt.scope_id; @@ -457,19 +416,19 @@ pub(super) fn establish_start( let start_payload = scope_start_payload(scope_id, &codex_scope_start_event_id(scope_id), provenance); - seam(repository_root, &start_payload, logger)?; - state::mark_active(git_dir, scope_id)?; + seam(repository_root, &start_payload, logger).await?; + state::mark_active(git_dir, scope_id).await?; Ok(()) } -pub(super) fn handle_close( +pub(super) async fn handle_close( git_dir: &Path, repository_root: &Path, key: &AttemptKey, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let Some(attempt) = current .attempts .iter() @@ -480,7 +439,7 @@ pub(super) fn handle_close( }; if attempt.phase == state::AttemptPhase::PendingStart { - abandon_attempt(git_dir, repository_root, &attempt, logger, seam)?; + abandon_attempt(git_dir, repository_root, &attempt, logger, seam).await?; return Ok(String::new()); } @@ -490,22 +449,22 @@ pub(super) fn handle_close( &codex_scope_close_event_id(&attempt.scope_id), ); - if seam(repository_root, &close_payload, logger).is_ok() { - state::remove_attempt(git_dir, &attempt.scope_id)?; + if seam(repository_root, &close_payload, logger).await.is_ok() { + state::remove_attempt(git_dir, &attempt.scope_id).await?; } else { - abandon_attempt(git_dir, repository_root, &attempt, logger, seam)?; + abandon_attempt(git_dir, repository_root, &attempt, logger, seam).await?; } Ok(String::new()) } -pub(super) fn cleanup_attempts_matching( +pub(super) async fn cleanup_attempts_matching( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, predicate: impl Fn(&state::AdapterAttempt) -> bool, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let stale: Vec = current .attempts .into_iter() @@ -513,7 +472,7 @@ pub(super) fn cleanup_attempts_matching( .collect(); for attempt in &stale { - abandon_attempt(git_dir, repository_root, attempt, logger, seam)?; + abandon_attempt(git_dir, repository_root, attempt, logger, seam).await?; } Ok(String::new()) @@ -525,16 +484,16 @@ pub(super) fn attempt_matches_key(attempt: &state::AdapterAttempt, key: &Attempt && attempt.tool_use_id == key.tool_use_id } -pub(super) fn abandon_attempt( +pub(super) async fn abandon_attempt( git_dir: &Path, repository_root: &Path, attempt: &state::AdapterAttempt, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { - state::arm_recovery(git_dir)?; + state::arm_recovery(git_dir).await?; - seam(repository_root, &abandon_payload(&attempt.scope_id), logger)?; - state::remove_attempt(git_dir, &attempt.scope_id)?; + seam(repository_root, &abandon_payload(&attempt.scope_id), logger).await?; + state::remove_attempt(git_dir, &attempt.scope_id).await?; Ok(()) } diff --git a/cli/src/services/hooks/codex_mutation_scope/mod.rs b/cli/src/services/hooks/codex_mutation_scope/mod.rs index 365535907..e29e38078 100644 --- a/cli/src/services/hooks/codex_mutation_scope/mod.rs +++ b/cli/src/services/hooks/codex_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; @@ -25,9 +23,8 @@ use serde_json::{json, Map, Value}; #[allow(unused_imports)] pub(crate) use events::{ classify_tool, codex_scope_close_event_id, codex_scope_start_event_id, format_codex_scope_id, - is_mcp_tool_name, parse_codex_hook_event, AttemptKey, CodexAgentIdentity, CodexHookEvent, - CodexSessionIdentity, CodexToolExecution, CodexToolIdentity, CodexTurnIdentity, - ToolClassification, + parse_codex_hook_event, AttemptKey, CodexAgentIdentity, CodexHookEvent, CodexSessionIdentity, + CodexToolExecution, CodexToolIdentity, CodexTurnIdentity, ToolClassification, }; #[allow(unused_imports)] use events::{ @@ -37,19 +34,13 @@ use events::{ PROVENANCE_FIELD, SESSION_ID_FIELD, TOOL_INPUT_FIELD, TOOL_NAME_FIELD, TOOL_USE_ID_FIELD, TRACKED_MUTATION_TOOL_NAMES, TURN_ID_FIELD, }; -#[cfg(test)] -pub(crate) use lifecycle::run_codex_mutation_scope_from_payload_at_state_root; + #[allow(unused_imports)] use lifecycle::{ abandon_attempt, run_codex_mutation_scope_from_payload_with_seams, FAIL_CLOSED_DENY_REASON, PRE_TOOL_USE_FAIL_CLOSED_EVENT, }; -#[cfg(test)] -#[allow(unused_imports)] -use lifecycle::{ - codex_scope_provenance, run_codex_mutation_scope_from_payload_with, - run_codex_mutation_scope_from_payload_with_bash_policy, GitDirResolver, IngressSeam, -}; + #[allow(unused_imports)] pub(crate) use lifecycle::{ run_codex_mutation_scope_from_payload, run_codex_mutation_scope_subcommand, @@ -59,6 +50,3 @@ use payload::{ abandon_payload, flush_payload, pre_tool_use_deny_json, scope_boundary_payload, scope_start_payload, }; - -#[cfg(test)] -mod tests; diff --git a/cli/src/services/hooks/codex_mutation_scope/state.rs b/cli/src/services/hooks/codex_mutation_scope/state.rs index 72016ef49..bbd73b521 100644 --- a/cli/src/services/hooks/codex_mutation_scope/state.rs +++ b/cli/src/services/hooks/codex_mutation_scope/state.rs @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, Context, Result}; use serde::{Deserialize, Serialize}; -use crate::services::hooks::mutation_scope_lock::{AdapterLockSpec, OsAdvisoryLock}; +use crate::services::hooks::mutation_scope_lock::AdapterLockSpec; use super::{format_codex_scope_id, AttemptKey}; @@ -39,12 +39,6 @@ pub(crate) enum RecoveryState { }, } -impl RecoveryState { - pub(crate) fn is_clear(&self) -> bool { - matches!(self, RecoveryState::Clear) - } -} - #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct AdapterAttempt { pub attempt_seq: u64, @@ -125,7 +119,14 @@ pub(crate) fn state_path(git_dir: &Path) -> PathBuf { adapter_state_dir(git_dir).join(ADAPTER_STATE_FILE) } -pub(crate) fn read_state(git_dir: &Path) -> Result { +pub(crate) async fn read_state(git_dir: &Path) -> Result { + let git_dir = git_dir.to_owned(); + tokio::task::spawn_blocking(move || read_state_sync(&git_dir)) + .await + .context("Adapter state read worker failed")? +} + +fn read_state_sync(git_dir: &Path) -> Result { let path = state_path(git_dir); if !path.exists() { return Ok(AdapterState::default()); @@ -150,11 +151,11 @@ fn parse_adapter_state(content: &str, path: &Path) -> Result { Ok(state) } -fn write_state_durably(git_dir: &Path, state: &AdapterState) -> Result<()> { - write_state_durably_inner(git_dir, state, |_, _| Ok(())) +fn write_state_durably_sync(git_dir: &Path, state: &AdapterState) -> Result<()> { + write_state_durably_sync_inner(git_dir, state, |_, _| Ok(())) } -fn write_state_durably_inner( +fn write_state_durably_sync_inner( git_dir: &Path, state: &AdapterState, before_rename: F, @@ -221,8 +222,29 @@ where Ok(()) } -fn acquire_lock(git_dir: &Path) -> Result { - STATE_LOCK.acquire(&adapter_state_dir(git_dir)) +enum StateTransaction { + Unchanged(T), + Persist(T), +} + +async fn with_locked_state(git_dir: &Path, operation: F) -> Result +where + T: Send + 'static, + F: FnOnce(&mut AdapterState) -> Result> + Send + 'static, +{ + let git_dir = git_dir.to_owned(); + STATE_LOCK + .run_locked_blocking(&adapter_state_dir(&git_dir), move || { + let mut state = read_state_sync(&git_dir)?; + match operation(&mut state)? { + StateTransaction::Unchanged(result) => Ok(result), + StateTransaction::Persist(result) => { + write_state_durably_sync(&git_dir, &state)?; + Ok(result) + } + } + }) + .await } fn allocate_pending_start( @@ -247,977 +269,155 @@ fn allocate_pending_start( attempt } -pub(crate) fn admit_tracked_attempt( +pub(crate) async fn admit_tracked_attempt( git_dir: &Path, key: &AttemptKey, turn_id: &str, tool_name: &str, ) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - match state.recovery { - RecoveryState::Flushing { .. } => return Ok(AdmitDecision::RecoveryBlocked), - RecoveryState::Pending { generation } => { - if !state.attempts.is_empty() { - return Ok(AdmitDecision::RecoveryBlocked); + let key = key.clone(); + let turn_id = turn_id.to_string(); + let tool_name = tool_name.to_string(); + with_locked_state(git_dir, move |state| { + match state.recovery { + RecoveryState::Flushing { .. } => { + return Ok(StateTransaction::Unchanged(AdmitDecision::RecoveryBlocked)) } - state.recovery = RecoveryState::Flushing { generation }; - write_state_durably(git_dir, &state)?; - return Ok(AdmitDecision::FlushClaimed { generation }); + RecoveryState::Pending { generation } => { + if !state.attempts.is_empty() { + return Ok(StateTransaction::Unchanged(AdmitDecision::RecoveryBlocked)); + } + state.recovery = RecoveryState::Flushing { generation }; + return Ok(StateTransaction::Persist(AdmitDecision::FlushClaimed { + generation, + })); + } + RecoveryState::Clear => {} } - RecoveryState::Clear => {} - } - if let Some(existing) = state - .attempts - .iter() - .find(|attempt| attempt.matches_key(key)) - { - return Ok(AdmitDecision::Admitted(AllocatedAttempt { - attempt: existing.clone(), - reused: true, - })); - } + if let Some(existing) = state + .attempts + .iter() + .find(|attempt| attempt.matches_key(&key)) + { + return Ok(StateTransaction::Unchanged(AdmitDecision::Admitted( + AllocatedAttempt { + attempt: existing.clone(), + reused: true, + }, + ))); + } - if state.attempts.iter().any(|attempt| { - attempt.in_builtin_lane(&key.session_id, turn_id) && !attempt.matches_key(key) - }) { - return Ok(AdmitDecision::StalePredecessorBlocked); - } + if state.attempts.iter().any(|attempt| { + attempt.in_builtin_lane(&key.session_id, &turn_id) && !attempt.matches_key(&key) + }) { + return Ok(StateTransaction::Unchanged( + AdmitDecision::StalePredecessorBlocked, + )); + } - if state - .attempts - .iter() - .any(|attempt| attempt.phase == AttemptPhase::PendingStart) - { - return Ok(AdmitDecision::UncertainAttemptBlocked); - } + if state + .attempts + .iter() + .any(|attempt| attempt.phase == AttemptPhase::PendingStart) + { + return Ok(StateTransaction::Unchanged( + AdmitDecision::UncertainAttemptBlocked, + )); + } - let attempt = allocate_pending_start(&mut state, key, turn_id, tool_name); - write_state_durably(git_dir, &state)?; - Ok(AdmitDecision::Admitted(AllocatedAttempt { - attempt, - reused: false, - })) + let attempt = allocate_pending_start(state, &key, &turn_id, &tool_name); + Ok(StateTransaction::Persist(AdmitDecision::Admitted( + AllocatedAttempt { + attempt, + reused: false, + }, + ))) + }) + .await } -pub(crate) fn mark_active(git_dir: &Path, scope_id: &str) -> Result<()> { - #[cfg(test)] - if fault::take_mark_active_failure() { - return Err(anyhow!("injected mark_active failure for tests")); - } - - let _lock = acquire_lock(git_dir)?; - - let mut state = read_state(git_dir)?; - let attempt = state - .attempts - .iter_mut() - .find(|attempt| attempt.scope_id == scope_id) - .ok_or_else(|| anyhow!("No adapter-state attempt found for scope_id '{scope_id}'"))?; - attempt.phase = AttemptPhase::Active; - write_state_durably(git_dir, &state) +pub(crate) async fn mark_active(git_dir: &Path, scope_id: &str) -> Result<()> { + let scope_id = scope_id.to_string(); + with_locked_state(git_dir, move |state| { + let attempt = state + .attempts + .iter_mut() + .find(|attempt| attempt.scope_id == scope_id) + .ok_or_else(|| anyhow!("No adapter-state attempt found for scope_id '{scope_id}'"))?; + attempt.phase = AttemptPhase::Active; + Ok(StateTransaction::Persist(())) + }) + .await } -pub(crate) fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - - let mut state = read_state(git_dir)?; - let before = state.attempts.len(); - state - .attempts - .retain(|attempt| attempt.scope_id != scope_id); - if state.attempts.len() == before { - return Ok(()); - } - write_state_durably(git_dir, &state) +pub(crate) async fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { + let scope_id = scope_id.to_string(); + with_locked_state(git_dir, move |state| { + let before = state.attempts.len(); + state + .attempts + .retain(|attempt| attempt.scope_id != scope_id); + if state.attempts.len() == before { + return Ok(StateTransaction::Unchanged(())); + } + Ok(StateTransaction::Persist(())) + }) + .await } -pub(crate) fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let RecoveryState::Flushing { generation } = state.recovery { - state.recovery = RecoveryState::Pending { generation }; - write_state_durably(git_dir, &state)?; - } - Ok(()) +pub(crate) async fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { + with_locked_state(git_dir, |state| { + if let RecoveryState::Flushing { generation } = state.recovery { + state.recovery = RecoveryState::Pending { generation }; + return Ok(StateTransaction::Persist(())); + } + Ok(StateTransaction::Unchanged(())) + }) + .await } -pub(crate) fn arm_recovery(git_dir: &Path) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - let generation = match state.recovery { - RecoveryState::Pending { generation } => generation, - RecoveryState::Clear | RecoveryState::Flushing { .. } => { - let generation = state.next_recovery_generation; - state.next_recovery_generation += 1; - generation - } - }; - state.recovery = RecoveryState::Pending { generation }; - write_state_durably(git_dir, &state)?; - Ok(generation) +pub(crate) async fn arm_recovery(git_dir: &Path) -> Result { + with_locked_state(git_dir, |state| { + let generation = match state.recovery { + RecoveryState::Pending { generation } => generation, + RecoveryState::Clear | RecoveryState::Flushing { .. } => { + let generation = state.next_recovery_generation; + state.next_recovery_generation += 1; + generation + } + }; + state.recovery = RecoveryState::Pending { generation }; + Ok(StateTransaction::Persist(generation)) + }) + .await } -pub(crate) fn complete_recovery_flush( +pub(crate) async fn complete_recovery_flush( git_dir: &Path, generation: u64, ) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - match state.recovery { + with_locked_state(git_dir, move |state| match state.recovery { RecoveryState::Flushing { generation: owned } if owned == generation => { state.recovery = RecoveryState::Clear; - write_state_durably(git_dir, &state)?; - Ok(RecoveryFlushCompletion::Cleared) + Ok(StateTransaction::Persist(RecoveryFlushCompletion::Cleared)) } - _ => Ok(RecoveryFlushCompletion::Superseded), - } -} - -pub(crate) fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let RecoveryState::Flushing { generation: owned } = state.recovery { - if owned == generation { - state.recovery = RecoveryState::Pending { generation: owned }; - write_state_durably(git_dir, &state)?; - } - } - Ok(()) -} - -#[cfg(test)] -pub(crate) fn seed_attempt_for_tests( - git_dir: &Path, - key: &AttemptKey, - turn_id: &str, - tool_name: &str, - phase: AttemptPhase, -) -> AdapterAttempt { - let _lock = acquire_lock(git_dir).expect("test seed lock"); - let mut state = read_state(git_dir).expect("test seed read"); - allocate_pending_start(&mut state, key, turn_id, tool_name); - let seeded = state.attempts.last_mut().expect("attempt was just pushed"); - seeded.phase = phase; - let attempt = seeded.clone(); - write_state_durably(git_dir, &state).expect("test seed write"); - attempt -} - -#[cfg(test)] -mod fault { - use std::cell::Cell; - - thread_local! { - static FAIL_NEXT_MARK_ACTIVE: Cell = const { Cell::new(false) }; - } - - pub(super) fn arm_mark_active_failure() { - FAIL_NEXT_MARK_ACTIVE.with(|cell| cell.set(true)); - } - - pub(super) fn take_mark_active_failure() -> bool { - FAIL_NEXT_MARK_ACTIVE.with(Cell::take) - } -} - -#[cfg(test)] -pub(crate) fn arm_mark_active_failure_for_tests() { - fault::arm_mark_active_failure(); -} - -#[cfg(test)] -mod state_conformance { - use super::*; - use crate::services::hooks::mutation_scope_state_conformance::{ - mutation_scope_state_conformance_tests, CompletionView, FlushClaimView, RecoveryView, - StateConformance, - }; - - const FIXTURE_SESSION: &str = "session-conformance"; - const FIXTURE_PHASES: [AttemptPhase; 3] = [ - AttemptPhase::Active, - AttemptPhase::Active, - AttemptPhase::PendingStart, - ]; - - struct CodexStateConformance; - - fn fixture_key(index: usize) -> AttemptKey { - AttemptKey { - session_id: FIXTURE_SESSION.to_string(), - agent_id: (index % 2 == 1).then(|| format!("agent-{index}")), - tool_use_id: format!("exec-{index}"), - } - } - - impl StateConformance for CodexStateConformance { - type State = AdapterState; - - const ADAPTER: &'static str = "codex"; - const SUPPORTED_VERSION: u32 = ADAPTER_STATE_VERSION; - - fn state_path(git_dir: &Path) -> PathBuf { - state_path(git_dir) - } - - fn fixture_state( - recovery: RecoveryView, - next_recovery_generation: u64, - attempt_count: usize, - ) -> AdapterState { - let mut state = AdapterState { - next_recovery_generation, - recovery: match recovery { - RecoveryView::Clear => RecoveryState::Clear, - RecoveryView::Pending(generation) => RecoveryState::Pending { generation }, - RecoveryView::Flushing(generation) => RecoveryState::Flushing { generation }, - }, - ..AdapterState::default() - }; - for index in 0..attempt_count { - allocate_pending_start( - &mut state, - &fixture_key(index), - &format!("turn-{index}"), - "apply_patch", - ); - state.attempts[index].phase = FIXTURE_PHASES[index % FIXTURE_PHASES.len()]; - } - state - } - - fn read_state(git_dir: &Path) -> Result { - read_state(git_dir) - } - - fn persist(git_dir: &Path, state: &AdapterState) -> Result<()> { - write_state_durably(git_dir, state) - } - - fn persist_with_before_rename_hook( - git_dir: &Path, - state: &AdapterState, - before_rename: F, - ) -> Result<()> - where - F: FnOnce(&Path, &Path) -> Result<()>, - { - write_state_durably_inner(git_dir, state, before_rename) - } - - fn recovery_view(state: &AdapterState) -> RecoveryView { - match state.recovery { - RecoveryState::Clear => RecoveryView::Clear, - RecoveryState::Pending { generation } => RecoveryView::Pending(generation), - RecoveryState::Flushing { generation } => RecoveryView::Flushing(generation), - } - } - - fn next_recovery_generation(state: &AdapterState) -> u64 { - state.next_recovery_generation - } - - fn scope_ids(state: &AdapterState) -> Vec { - state - .attempts - .iter() - .map(|attempt| attempt.scope_id.clone()) - .collect() - } - - fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { - remove_attempt(git_dir, scope_id) - } - - fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { - normalize_recovery_after_boundary_lock_acquired(git_dir) - } - - fn complete_recovery_flush(git_dir: &Path, generation: u64) -> Result { - Ok(match complete_recovery_flush(git_dir, generation)? { - RecoveryFlushCompletion::Cleared => CompletionView::Cleared, - RecoveryFlushCompletion::Superseded => CompletionView::Superseded, - }) - } - - fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { - relinquish_recovery_flush(git_dir, generation) - } - - fn claim_flush_by_admitting_a_new_attempt( - git_dir: &Path, - contender: usize, - ) -> Result { - let decision = admit_tracked_attempt( - git_dir, - &fixture_key(contender), - &format!("turn-{contender}"), - "Bash", - )?; - Ok(match decision { - AdmitDecision::FlushClaimed { generation } => FlushClaimView::Claimed(generation), - AdmitDecision::RecoveryBlocked => FlushClaimView::Blocked, - other => FlushClaimView::Other(format!("{other:?}")), - }) - } - } - - mutation_scope_state_conformance_tests!(CodexStateConformance); + _ => Ok(StateTransaction::Unchanged( + RecoveryFlushCompletion::Superseded, + )), + }) + .await } -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - use std::thread; - use std::time::Duration; - - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-codex-mutation-scope-state-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(session_id: &str, agent_id: Option<&str>, tool_use_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - agent_id: agent_id.map(str::to_string), - tool_use_id: tool_use_id.to_string(), - } - } - - const TEST_TURN: &str = "turn-1"; - - fn admit(git_dir: &Path, key: &AttemptKey, tool_name: &str) -> AdmitDecision { - admit_in_turn(git_dir, key, TEST_TURN, tool_name) - } - - fn admit_in_turn( - git_dir: &Path, - key: &AttemptKey, - turn_id: &str, - tool_name: &str, - ) -> AdmitDecision { - admit_tracked_attempt(git_dir, key, turn_id, tool_name).expect("admit should not error") - } - - fn admit_and_activate(git_dir: &Path, key: &AttemptKey, tool_name: &str) -> AdapterAttempt { - admit_and_activate_in_turn(git_dir, key, TEST_TURN, tool_name) - } - - fn admit_and_activate_in_turn( - git_dir: &Path, - key: &AttemptKey, - turn_id: &str, - tool_name: &str, - ) -> AdapterAttempt { - match admit_in_turn(git_dir, key, turn_id, tool_name) { - AdmitDecision::Admitted(allocated) => { - mark_active(git_dir, &allocated.attempt.scope_id) - .expect("mark_active should succeed"); - allocated.attempt +pub(crate) async fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { + with_locked_state(git_dir, move |state| { + if let RecoveryState::Flushing { generation: owned } = state.recovery { + if owned == generation { + state.recovery = RecoveryState::Pending { generation: owned }; + return Ok(StateTransaction::Persist(())); } - other => panic!("expected Admitted, got {other:?}"), } - } - - #[test] - fn admit_allocates_sequential_attempt_seqs_across_distinct_keys() { - let git_dir = unique_test_git_dir("sequential-allocation"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let first = admit_and_activate_in_turn( - &git_dir, - &key("session-1", None, "exec-1"), - "turn-1", - "Bash", - ); - let second = admit_and_activate_in_turn( - &git_dir, - &key("session-1", None, "exec-2"), - "turn-2", - "apply_patch", - ); - let third = admit_and_activate_in_turn( - &git_dir, - &key("session-1", Some("agent-1"), "exec-3"), - "turn-3", - "Bash", - ); - - assert_eq!(first.attempt_seq, 1); - assert_eq!(second.attempt_seq, 2); - assert_eq!(third.attempt_seq, 3); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.next_attempt_seq, 4); - assert_eq!(state.attempts.len(), 3); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_state_is_checkout_local_to_its_git_dir() { - let git_dir_a = unique_test_git_dir("checkout-local-a"); - let git_dir_b = unique_test_git_dir("checkout-local-b"); - std::fs::create_dir_all(&git_dir_a).expect("git dir A should be created"); - std::fs::create_dir_all(&git_dir_b).expect("git dir B should be created"); - - admit_and_activate_in_turn( - &git_dir_a, - &key("session-1", None, "exec-1"), - "turn-1", - "Bash", - ); - admit_and_activate_in_turn( - &git_dir_a, - &key("session-1", None, "exec-2"), - "turn-2", - "Bash", - ); - - let first_b = admit_and_activate_in_turn( - &git_dir_b, - &key("session-1", None, "exec-1"), - "turn-1", - "Bash", - ); - assert_eq!( - first_b.attempt_seq, 1, - "checkout B's monotonic counter must be independent of checkout A's" - ); - - assert_eq!( - read_state(&git_dir_a) - .expect("state A readable") - .attempts - .len(), - 2 - ); - assert_eq!( - read_state(&git_dir_b) - .expect("state B readable") - .attempts - .len(), - 1 - ); - - remove_test_git_dir(&git_dir_a); - remove_test_git_dir(&git_dir_b); - } - - #[test] - fn duplicate_live_delivery_reuses_the_same_attempt_seq_and_scope_id() { - let git_dir = unique_test_git_dir("duplicate-reuse"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("session-1", None, "exec-1"); - - let AdmitDecision::Admitted(first) = admit(&git_dir, &attempt_key, "Bash") else { - panic!("first admission should be Admitted"); - }; - assert!(!first.reused); - - let AdmitDecision::Admitted(second) = admit(&git_dir, &attempt_key, "Bash") else { - panic!("duplicate delivery should still be Admitted"); - }; - assert!( - second.reused, - "duplicate live delivery must be reported as reused" - ); - assert_eq!(first.attempt.attempt_seq, second.attempt.attempt_seq); - assert_eq!(first.attempt.scope_id, second.attempt.scope_id); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.next_attempt_seq, 2); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_terminal_attempt_is_followed_by_a_fresh_allocation_never_reusing_the_scope_id() { - let git_dir = unique_test_git_dir("terminal-then-fresh"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("session-1", None, "exec-1"); - - let first = admit_and_activate(&git_dir, &attempt_key, "Bash"); - remove_attempt(&git_dir, &first.scope_id).expect("terminal attempt should be removable"); - - let AdmitDecision::Admitted(second) = admit(&git_dir, &attempt_key, "Bash") else { - panic!("a later execution should be Admitted with a fresh attempt"); - }; - assert!(!second.reused); - assert_ne!(first.attempt_seq, second.attempt.attempt_seq); - assert_ne!(first.scope_id, second.attempt.scope_id); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_persists_the_pending_start_attempt_before_returning() { - let git_dir = unique_test_git_dir("admit-persists-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let AdmitDecision::Admitted(allocated) = admit( - &git_dir, - &key("session-7", Some("agent-2"), "exec-9"), - "apply_patch", - ) else { - panic!("expected Admitted"); - }; - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!( - state.attempts.len(), - 1, - "I2: PendingStart must be durable before admit returns" - ); - assert_eq!(state.attempts[0], allocated.attempt); - assert_eq!(state.attempts[0].phase, AttemptPhase::PendingStart); - assert_eq!(state.attempts[0].session_id, "session-7"); - assert_eq!(state.attempts[0].turn_id, TEST_TURN); - assert_eq!(state.attempts[0].agent_id.as_deref(), Some("agent-2")); - assert_eq!(state.attempts[0].tool_use_id, "exec-9"); - assert_eq!(state.attempts[0].tool_name, "apply_patch"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_blocks_a_successor_while_an_unrelated_pending_start_in_another_lane_is_unresolved() { - let git_dir = unique_test_git_dir("admit-blocks-on-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let AdmitDecision::Admitted(_) = admit_in_turn( - &git_dir, - &key("session-1", None, "exec-a"), - "turn-1", - "Bash", - ) else { - panic!("first admission should be Admitted"); - }; - - assert_eq!( - admit_in_turn( - &git_dir, - &key("session-1", None, "exec-b"), - "turn-2", - "Bash" - ), - AdmitDecision::UncertainAttemptBlocked, - "I5: an unrelated unresolved PendingStart must block a successor Start" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_blocks_a_new_key_in_the_same_builtin_lane_as_an_outstanding_attempt() { - let git_dir = unique_test_git_dir("admit-blocks-same-lane"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - admit_and_activate_in_turn( - &git_dir, - &key("session-1", None, "exec-a"), - "turn-1", - "Bash", - ); - - assert_eq!( - admit_in_turn( - &git_dir, - &key("session-1", None, "exec-b"), - "turn-1", - "Bash" - ), - AdmitDecision::StalePredecessorBlocked, - "a new tracked attempt must not be admitted while an older different \ - AttemptKey in the same (session_id, turn_id) built-in lane is outstanding", - ); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts.len(), - 1, - "the backstop admits nothing while the predecessor remains", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_blocks_a_new_key_when_a_same_lane_predecessor_is_only_pending_start() { - let git_dir = unique_test_git_dir("admit-blocks-same-lane-pending"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - seed_attempt_for_tests( - &git_dir, - &key("session-1", None, "exec-a"), - "turn-1", - "Bash", - AttemptPhase::PendingStart, - ); - - assert_eq!( - admit_in_turn(&git_dir, &key("session-1", None, "exec-b"), "turn-1", "Bash"), - AdmitDecision::StalePredecessorBlocked, - "a same-lane PendingStart predecessor blocks admission before the uncertain-attempt path", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_never_treats_duplicate_delivery_as_a_same_lane_predecessor() { - let git_dir = unique_test_git_dir("admit-duplicate-not-predecessor"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("session-1", None, "exec-1"); - - admit_and_activate_in_turn(&git_dir, &attempt_key, "turn-1", "Bash"); - - let AdmitDecision::Admitted(again) = - admit_in_turn(&git_dir, &attempt_key, "turn-1", "Bash") - else { - panic!("duplicate delivery in the same lane must still reuse, never StalePredecessorBlocked"); - }; - assert!(again.reused); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_allows_a_new_key_alongside_an_active_attempt_in_a_different_lane() { - let git_dir = unique_test_git_dir("admit-alongside-active"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - admit_and_activate_in_turn( - &git_dir, - &key("session-1", None, "exec-a"), - "turn-1", - "Bash", - ); - - let AdmitDecision::Admitted(second) = admit_in_turn( - &git_dir, - &key("session-1", None, "exec-b"), - "turn-2", - "Bash", - ) else { - panic!("a distinct-lane execution may run alongside an Active attempt (D14)"); - }; - assert!(!second.reused); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts.len(), - 2 - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_refuses_while_recovery_is_pending_with_outstanding_attempts() { - let git_dir = unique_test_git_dir("admit-recovery-blocked"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - seed_attempt_for_tests( - &git_dir, - &key("session-1", None, "exec-live"), - "turn-1", - "Bash", - AttemptPhase::Active, - ); - arm_recovery(&git_dir).expect("arming recovery should succeed"); - - assert_eq!( - admit(&git_dir, &key("session-1", None, "exec-new"), "Bash"), - AdmitDecision::RecoveryBlocked, - "I1: no new tracked Start while recovery is pending with outstanding attempts" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_claims_the_flush_when_recovery_is_quiescent() { - let git_dir = unique_test_git_dir("admit-claims-flush"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let generation = arm_recovery(&git_dir).expect("arming recovery should succeed"); - - assert_eq!( - admit(&git_dir, &key("session-1", None, "exec-new"), "Bash"), - AdmitDecision::FlushClaimed { generation }, - ); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Flushing { generation }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn arm_recovery_moves_clear_to_pending_with_a_fresh_generation() { - let git_dir = unique_test_git_dir("arm-clear-to-pending"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let generation = arm_recovery(&git_dir).expect("arming should succeed"); - assert_eq!(generation, 1); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.recovery, RecoveryState::Pending { generation: 1 }); - assert_eq!(state.next_recovery_generation, 2); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn arm_recovery_keeps_the_same_generation_when_already_pending() { - let git_dir = unique_test_git_dir("arm-pending-idempotent"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - assert_eq!(arm_recovery(&git_dir).expect("first arm"), 1); - assert_eq!(arm_recovery(&git_dir).expect("second arm"), 1); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.recovery, RecoveryState::Pending { generation: 1 }); - assert_eq!(state.next_recovery_generation, 2); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn arm_recovery_supersedes_a_flushing_generation_with_a_newer_one() { - let git_dir = unique_test_git_dir("arm-supersedes-flushing"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - arm_recovery(&git_dir).expect("arm g1"); - admit(&git_dir, &key("session-1", None, "exec-new"), "Bash"); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Flushing { generation: 1 }, - ); - - let generation = arm_recovery(&git_dir).expect("re-arm during flush"); - assert_eq!(generation, 2); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 2 }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn complete_recovery_flush_is_a_no_op_when_a_newer_recovery_was_armed() { - let git_dir = unique_test_git_dir("complete-superseded-by-newer"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - arm_recovery(&git_dir).expect("arm g1"); - admit(&git_dir, &key("session-1", None, "exec-new"), "Bash"); - arm_recovery(&git_dir).expect("re-arm to g2 while flushing g1"); - - assert_eq!( - complete_recovery_flush(&git_dir, 1).expect("stale completion"), - RecoveryFlushCompletion::Superseded, - ); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 2 }, - "I4: a stale Flush(g1) completion must not clear recovery armed for g2" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn stale_generation_completion_against_clear_state_is_a_safe_no_op() { - let git_dir = unique_test_git_dir("complete-stale-against-clear"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - assert_eq!( - complete_recovery_flush(&git_dir, 7).expect("stale completion against clear"), - RecoveryFlushCompletion::Superseded, - ); - assert!(read_state(&git_dir) - .expect("state readable") - .recovery - .is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn normalize_after_boundary_lock_is_a_no_op_for_clear_or_pending() { - let git_dir = unique_test_git_dir("normalize-noop"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - normalize_recovery_after_boundary_lock_acquired(&git_dir).expect("normalize on clear"); - assert!(read_state(&git_dir) - .expect("state readable") - .recovery - .is_clear()); - - arm_recovery(&git_dir).expect("arm"); - normalize_recovery_after_boundary_lock_acquired(&git_dir).expect("normalize on pending"); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 1 }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn relinquish_recovery_flush_is_a_no_op_for_a_superseded_generation() { - let git_dir = unique_test_git_dir("relinquish-superseded"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - arm_recovery(&git_dir).expect("arm g1"); - admit(&git_dir, &key("session-1", None, "exec-new"), "Bash"); - arm_recovery(&git_dir).expect("re-arm to g2"); - - relinquish_recovery_flush(&git_dir, 1).expect("stale relinquish"); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 2 }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_active_transitions_phase_from_pending_start_to_active() { - let git_dir = unique_test_git_dir("mark-active"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let AdmitDecision::Admitted(allocated) = - admit(&git_dir, &key("session-1", None, "exec-1"), "Bash") - else { - panic!("expected Admitted"); - }; - assert_eq!(allocated.attempt.phase, AttemptPhase::PendingStart); - - mark_active(&git_dir, &allocated.attempt.scope_id).expect("mark_active should succeed"); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_active_for_an_unknown_scope_id_is_rejected_without_fabricating_an_attempt() { - let git_dir = unique_test_git_dir("mark-active-unknown"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let error = mark_active(&git_dir, "cx-tool-v1|n=99|s=1:x|a=0:|t=1:y") - .expect_err("marking an unknown scope active must be rejected"); - assert!(error.to_string().contains("No adapter-state attempt found")); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - remove_test_git_dir(&git_dir); - } - - const PARALLEL_ADMISSION_COUNT: u64 = 6; - - #[test] - fn parallel_admissions_serialize_and_converge_without_lost_updates() { - let git_dir = unique_test_git_dir("parallel-admissions"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let handles: Vec<_> = (0..PARALLEL_ADMISSION_COUNT) - .map(|index| { - let git_dir = git_dir.clone(); - thread::spawn(move || { - let attempt_key = key("session-1", None, &format!("exec-{index}")); - let turn_id = format!("turn-{index}"); - loop { - match admit_tracked_attempt(&git_dir, &attempt_key, &turn_id, "Bash") - .expect("admit should not error") - { - AdmitDecision::Admitted(allocated) => { - mark_active(&git_dir, &allocated.attempt.scope_id) - .expect("mark_active should succeed"); - break allocated.attempt.attempt_seq; - } - AdmitDecision::UncertainAttemptBlocked => { - thread::sleep(Duration::from_millis(5)); - } - other => panic!("unexpected admission decision: {other:?}"), - } - } - }) - }) - .collect(); - - let mut attempt_seqs: Vec = handles - .into_iter() - .map(|handle| handle.join().expect("thread should not panic")) - .collect(); - attempt_seqs.sort_unstable(); - attempt_seqs.dedup(); - assert_eq!( - attempt_seqs.len(), - usize::try_from(PARALLEL_ADMISSION_COUNT).unwrap(), - "concurrent admissions must serialize and never collide on attempt_seq" - ); - - let state = read_state(&git_dir).expect("state should be readable after concurrent admits"); - assert_eq!( - state.attempts.len(), - usize::try_from(PARALLEL_ADMISSION_COUNT).unwrap() - ); - assert_eq!(state.next_attempt_seq, PARALLEL_ADMISSION_COUNT + 1); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::Active)); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn every_locked_helper_releases_the_state_lock_before_returning() { - let git_dir = unique_test_git_dir("lock-released-between-helpers"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("session-1", None, "exec-1"); - - let AdmitDecision::Admitted(allocated) = admit(&git_dir, &attempt_key, "Bash") else { - panic!("expected Admitted"); - }; - mark_active(&git_dir, &allocated.attempt.scope_id) - .expect("mark_active must acquire the lock admit released"); - arm_recovery(&git_dir).expect("arm_recovery must acquire the lock mark_active released"); - remove_attempt(&git_dir, &allocated.attempt.scope_id) - .expect("remove_attempt must acquire the lock arm_recovery released"); - complete_recovery_flush(&git_dir, 999) - .expect("complete_recovery_flush must acquire the lock remove_attempt released"); - relinquish_recovery_flush(&git_dir, 999) - .expect("relinquish_recovery_flush must acquire the lock"); - - drop( - STATE_LOCK - .acquire_with_timeout(&adapter_state_dir(&git_dir), Duration::from_millis(200)) - .expect("the state lock must be free once every helper has returned"), - ); - - remove_test_git_dir(&git_dir); - } + Ok(StateTransaction::Unchanged(())) + }) + .await } diff --git a/cli/src/services/hooks/codex_mutation_scope/tests.rs b/cli/src/services/hooks/codex_mutation_scope/tests.rs deleted file mode 100644 index d0ab67bb3..000000000 --- a/cli/src/services/hooks/codex_mutation_scope/tests.rs +++ /dev/null @@ -1,4698 +0,0 @@ -use super::*; - -const PROBE01_SHELL_PRE: &str = - include_str!("fixtures/probe01-apply-patch-and-shell-success.shell.pre_tool_use.json"); -const PROBE01_SHELL_POST: &str = - include_str!("fixtures/probe01-apply-patch-and-shell-success.shell.post_tool_use.json"); -const PROBE01_APPLY_PATCH_PRE: &str = - include_str!("fixtures/probe01-apply-patch-and-shell-success.apply_patch.pre_tool_use.json"); -const PROBE01_STOP: &str = include_str!("fixtures/probe01-apply-patch-and-shell-success.stop.json"); -const PROBE01_SESSION_END: &str = - include_str!("fixtures/probe01-apply-patch-and-shell-success.session_end.json"); -const PROBE02_FAILED_SHELL_POST: &str = - include_str!("fixtures/probe02-shell-partial-write-then-nonzero-exit.post_tool_use.json"); -const PROBE04_BLOCKED_PRE: &str = - include_str!("fixtures/probe04-pre-tool-use-hook-hookspecificoutput-deny.pre_tool_use.json"); -const PROBE05_SHELL_PRE: &str = - include_str!("fixtures/probe05-tool-vocabulary.shell-read-list-search.pre_tool_use.json"); -const PROBE08_SPAWN_AGENT_PRE: &str = - include_str!("fixtures/probe08-subagent-delegation.spawn_agent.pre_tool_use.json"); -const PROBE08_WAIT_AGENT_PRE: &str = - include_str!("fixtures/probe08-subagent-delegation.wait_agent.pre_tool_use.json"); -const PROBE08_AGENT_APPLY_PATCH_PRE: &str = - include_str!("fixtures/probe08-subagent-delegation.agent-apply-patch.pre_tool_use.json"); -const PROBE08_AGENT_APPLY_PATCH_POST: &str = - include_str!("fixtures/probe08-subagent-delegation.agent-apply-patch.post_tool_use.json"); -const PROBE08_SUBAGENT_STOP: &str = - include_str!("fixtures/probe08-subagent-delegation.subagent_stop.json"); -const PROBE10_WORKTREE_PRE: &str = - include_str!("fixtures/probe10-linked-worktree-cwd.pre_tool_use.json"); -const PROBE11_INTERRUPT: &str = - include_str!("fixtures/probe11-interrupt-event-on-sigint.interrupt.json"); -const PROBE12_MCP_PRE: &str = include_str!("fixtures/probe12-mcp-mutate-success.pre_tool_use.json"); -const PROBE12_MCP_POST: &str = - include_str!("fixtures/probe12-mcp-mutate-success.post_tool_use.json"); -const PROBE13_MCP_MUTATE_THEN_ERROR_PRE: &str = - include_str!("fixtures/probe13-mcp-mutate-then-error.pre_tool_use.json"); -const PROBE13_MCP_SESSION_END: &str = - include_str!("fixtures/probe13-mcp-mutate-then-error.session_end.json"); - -fn pre_tool_use_json(overrides: &[(&str, Value)]) -> String { - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_PRE_TOOL_USE.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("session-1".to_string()), - ); - object.insert( - TURN_ID_FIELD.to_string(), - Value::String("turn-1".to_string()), - ); - object.insert( - CWD_FIELD.to_string(), - Value::String("/repo/checkout".to_string()), - ); - object.insert( - TOOL_NAME_FIELD.to_string(), - Value::String("Bash".to_string()), - ); - object.insert( - TOOL_USE_ID_FIELD.to_string(), - Value::String("exec-1".to_string()), - ); - object.insert(TOOL_INPUT_FIELD.to_string(), json!({"command": "true"})); - for (field, value) in overrides { - object.insert((*field).to_string(), value.clone()); - } - Value::Object(object).to_string() -} - -fn key(session_id: &str, agent_id: Option<&str>, tool_use_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - agent_id: agent_id.map(str::to_string), - tool_use_id: tool_use_id.to_string(), - } -} - -fn pre_tool_use(payload: &str) -> CodexToolExecution { - match parse_codex_hook_event(payload).expect("valid PreToolUse parses") { - CodexHookEvent::PreToolUse(execution) => execution, - other => panic!("expected PreToolUse, got {other:?}"), - } -} - -mod ingress_conformance { - use super::*; - use crate::services::hooks::mutation_scope_ingress_conformance::{ - self as conformance, mutation_scope_ingress_conformance_tests, IngressConformance, - }; - - struct CodexIngressConformance; - - fn unreachable_bash_policy(_root: &Path, command: &str) -> Result { - panic!("the Bash policy preflight must not run for rejected ingress (command: {command})"); - } - - impl IngressConformance for CodexIngressConformance { - const ADAPTER: &'static str = "Codex"; - const EVENT_NAME_FIELD: &'static str = HOOK_EVENT_NAME_FIELD; - const REQUIRED_START_FIELDS: &'static [&'static str] = &[ - SESSION_ID_FIELD, - TURN_ID_FIELD, - CWD_FIELD, - TOOL_NAME_FIELD, - TOOL_USE_ID_FIELD, - ]; - const OPTIONAL_START_FIELDS: &'static [(&'static str, &'static str)] = &[ - (AGENT_ID_FIELD, "01a07c24-bb59-7ca0-80f7-99cf940a486e"), - (AGENT_TYPE_FIELD, "default"), - ]; - const UNSUPPORTED_EVENT_NAMES: &'static [&'static str] = &[ - "SessionStart", - "SubagentStart", - "UserPromptSubmit", - "PreCompact", - ]; - - fn tracked_start() -> Map { - serde_json::from_str(&pre_tool_use_json(&[])) - .expect("the tracked PreToolUse fixture is a JSON object") - } - - fn parse(payload: &str) -> Result<()> { - parse_codex_hook_event(payload).map(|_| ()) - } - - fn run(payload: &str) -> Result { - run_codex_mutation_scope_from_payload(payload, None) - } - - fn run_with_seams( - payload: &str, - resolve_git_dir: conformance::GitDirResolver, - seam: conformance::IngressSeam, - ) -> Result { - run_codex_mutation_scope_from_payload_with_bash_policy( - payload, - None, - resolve_git_dir, - seam, - &unreachable_bash_policy, - ) - } - } - - mutation_scope_ingress_conformance_tests!(CodexIngressConformance); -} - -#[test] -fn ac2_pre_tool_use_fixtures_parse_to_expected_identity() { - let shell = pre_tool_use(PROBE01_SHELL_PRE); - assert_eq!(shell.identity.tool_name, "Bash"); - assert_eq!( - shell.identity.tool_use_id, - "exec-414820f5-555e-457a-92e7-60ddd27d4eec" - ); - assert_eq!( - shell.identity.session_id, - "01a07c1e-e08e-7172-8032-cb9d62af21d9" - ); - assert_eq!( - shell.identity.turn_id, - "01a07c1e-e0cc-75f1-a566-e790c06cb033" - ); - assert_eq!(shell.identity.agent_id, None); - assert!(!shell.identity.is_subagent()); - assert!(shell.identity.cwd.ends_with("/probe-repo")); - - let apply_patch = pre_tool_use(PROBE01_APPLY_PATCH_PRE); - assert_eq!(apply_patch.identity.tool_name, "apply_patch"); - - let vocab = pre_tool_use(PROBE05_SHELL_PRE); - assert_eq!(vocab.identity.tool_name, "Bash"); - - let worktree = pre_tool_use(PROBE10_WORKTREE_PRE); - assert!(worktree.identity.cwd.ends_with("/probe-worktree")); - - let mcp = pre_tool_use(PROBE12_MCP_PRE); - assert_eq!(mcp.identity.tool_name, "mcp__probe__mutate_success"); - - let mcp_err = pre_tool_use(PROBE13_MCP_MUTATE_THEN_ERROR_PRE); - assert!(is_mcp_tool_name(&mcp_err.identity.tool_name)); -} - -#[test] -fn ac2_subagent_pre_tool_use_fixture_carries_agent_identity() { - let execution = pre_tool_use(PROBE08_AGENT_APPLY_PATCH_PRE); - assert_eq!( - execution.identity.agent_id.as_deref(), - Some("01a07c24-bb59-7ca0-80f7-99cf940a486e") - ); - assert!(execution.identity.is_subagent()); - assert_eq!(execution.agent_type.as_deref(), Some("default")); -} - -#[test] -fn ac3_pre_tool_use_fixtures_retain_the_codex_model() { - for fixture in [ - PROBE01_SHELL_PRE, - PROBE01_APPLY_PATCH_PRE, - PROBE05_SHELL_PRE, - PROBE08_AGENT_APPLY_PATCH_PRE, - ] { - assert_eq!(pre_tool_use(fixture).model.as_deref(), Some("gpt-5.6-sol")); - } -} - -#[test] -fn ac3_scope_provenance_canonicalizes_the_session_and_normalizes_the_model() { - let provenance = codex_scope_provenance(&pre_tool_use(PROBE01_SHELL_PRE)); - assert_eq!( - provenance.session_id, - "cx_01a07c1e-e08e-7172-8032-cb9d62af21d9" - ); - assert_eq!(provenance.model_id.as_deref(), Some("gpt-5.6-sol")); - - let apply_patch = codex_scope_provenance(&pre_tool_use(PROBE01_APPLY_PATCH_PRE)); - assert_eq!(apply_patch, provenance); -} - -#[test] -fn ac3_scope_provenance_keeps_an_already_prefixed_session_id() { - let execution = pre_tool_use(&pre_tool_use_json(&[( - SESSION_ID_FIELD, - Value::String("cx_session-1".to_string()), - )])); - assert_eq!( - codex_scope_provenance(&execution).session_id, - "cx_session-1" - ); -} - -#[test] -fn ac3_an_unusable_model_yields_no_model_id_without_rejecting_the_event() { - for model in [ - Value::Null, - Value::String(String::new()), - Value::String(" ".to_string()), - Value::Bool(true), - json!(7), - json!({ "id": "gpt-5.6-sol" }), - ] { - let payload = pre_tool_use_json(&[(MODEL_FIELD, model.clone())]); - let execution = pre_tool_use(&payload); - let provenance = codex_scope_provenance(&execution); - assert_eq!(provenance.model_id, None, "model {model:?}"); - assert_eq!(provenance.session_id, "cx_session-1", "model {model:?}"); - } - - let absent = pre_tool_use(&pre_tool_use_json(&[])); - assert_eq!(absent.model, None); - assert_eq!(codex_scope_provenance(&absent).model_id, None); -} - -#[test] -fn ac2_post_tool_use_fixtures_parse() { - for (payload, tool_name, tool_use_id) in [ - ( - PROBE01_SHELL_POST, - "Bash", - "exec-414820f5-555e-457a-92e7-60ddd27d4eec", - ), - ( - PROBE02_FAILED_SHELL_POST, - "Bash", - "exec-52155265-e98d-423f-87f8-76ee56ff33b1", - ), - ( - PROBE12_MCP_POST, - "mcp__probe__mutate_success", - "exec-00988fad-6707-48ed-81b6-07bb11933886", - ), - ] { - match parse_codex_hook_event(payload).expect("PostToolUse fixture parses") { - CodexHookEvent::PostToolUse(identity) => { - assert_eq!(identity.tool_name, tool_name); - assert_eq!(identity.tool_use_id, tool_use_id); - } - other => panic!("expected PostToolUse, got {other:?}"), - } - } -} - -#[test] -fn ac2_subagent_post_tool_use_ties_to_its_pre_tool_use() { - let CodexHookEvent::PostToolUse(post) = - parse_codex_hook_event(PROBE08_AGENT_APPLY_PATCH_POST).unwrap() - else { - panic!("expected PostToolUse"); - }; - let pre = pre_tool_use(PROBE08_AGENT_APPLY_PATCH_PRE); - assert_eq!(post.attempt_key(), pre.identity.attempt_key()); - assert!(post.attempt_key().agent_id.is_some()); -} - -#[test] -fn ac2_terminal_lifecycle_fixtures_parse() { - assert!(matches!( - parse_codex_hook_event(PROBE01_STOP).unwrap(), - CodexHookEvent::Stop(id) if id.turn_id == "01a07c1e-e0cc-75f1-a566-e790c06cb033" - )); - assert!(matches!( - parse_codex_hook_event(PROBE11_INTERRUPT).unwrap(), - CodexHookEvent::Interrupt(id) if id.session_id == "01a07c2f-ccbf-79f0-afb9-2d2ce919eea7" - )); - assert!(matches!( - parse_codex_hook_event(PROBE08_SUBAGENT_STOP).unwrap(), - CodexHookEvent::SubagentStop(id) - if id.agent_id == "01a07c24-bb59-7ca0-80f7-99cf940a486e" - )); - for session_end in [PROBE01_SESSION_END, PROBE13_MCP_SESSION_END] { - assert!(matches!( - parse_codex_hook_event(session_end).unwrap(), - CodexHookEvent::SessionEnd(_) - )); - } -} - -#[test] -fn ac2_session_end_needs_no_turn_id() { - let CodexHookEvent::SessionEnd(identity) = parse_codex_hook_event(PROBE01_SESSION_END).unwrap() - else { - panic!("expected SessionEnd"); - }; - assert_eq!(identity.session_id, "01a07c1e-e08e-7172-8032-cb9d62af21d9"); -} - -#[test] -fn ac3_classification_table() { - let cases: &[(&str, ToolClassification)] = &[ - ("Bash", ToolClassification::TrackedMutation), - ("apply_patch", ToolClassification::TrackedMutation), - ("collaborationspawn_agent", ToolClassification::Delegation), - ("collaborationwait_agent", ToolClassification::Delegation), - ("mcp__probe__mutate_success", ToolClassification::Untracked), - ("mcp__probe_par__slow_mutate", ToolClassification::Untracked), - ("mcp__", ToolClassification::Untracked), - ("Read", ToolClassification::Untracked), - ("PowerShell", ToolClassification::Untracked), - ("some_future_codex_tool", ToolClassification::Untracked), - ("", ToolClassification::Untracked), - ]; - for (tool_name, expected) in cases { - assert_eq!( - classify_tool(tool_name), - *expected, - "classify_tool({tool_name:?})" - ); - } -} - -#[test] -fn ac3_classification_is_total_and_single_valued() { - for tool_name in [ - "Bash", - "apply_patch", - "collaborationspawn_agent", - "collaborationwait_agent", - "mcp__x__y", - "unknown", - ] { - let _: ToolClassification = classify_tool(tool_name); - } -} - -#[test] -fn ac3_delegation_and_untracked_tool_fixtures_do_not_yield_a_tracked_scope() { - for payload in [ - PROBE08_SPAWN_AGENT_PRE, - PROBE08_WAIT_AGENT_PRE, - PROBE12_MCP_PRE, - PROBE13_MCP_MUTATE_THEN_ERROR_PRE, - ] { - let execution = pre_tool_use(payload); - let classification = classify_tool(&execution.identity.tool_name); - assert_ne!( - classification, - ToolClassification::TrackedMutation, - "tool {:?} must not be TrackedMutation", - execution.identity.tool_name - ); - } - - assert_eq!( - classify_tool(&pre_tool_use(PROBE04_BLOCKED_PRE).identity.tool_name), - ToolClassification::TrackedMutation - ); -} - -#[test] -fn ac3_is_mcp_tool_name() { - assert!(is_mcp_tool_name("mcp__probe__mutate_success")); - assert!(is_mcp_tool_name("mcp__")); - assert!(!is_mcp_tool_name("Bash")); - assert!(!is_mcp_tool_name("apply_patch")); - assert!(!is_mcp_tool_name("collaborationspawn_agent")); -} - -#[test] -fn ac4_scope_id_is_deterministic_for_the_same_attempt_seq_and_key() { - let k = key("session-1", None, "exec-1"); - assert_eq!(format_codex_scope_id(7, &k), format_codex_scope_id(7, &k)); - - let scope_id = format_codex_scope_id(7, &k); - assert_eq!(scope_id, "cx-tool-v1|n=7|s=9:session-1|a=0:|t=6:exec-1"); - assert_eq!( - codex_scope_start_event_id(&scope_id), - format!("{scope_id}|start") - ); - assert_eq!( - codex_scope_close_event_id(&scope_id), - format!("{scope_id}|close") - ); -} - -#[test] -fn ac4_length_prefix_disambiguates_delimiter_collisions() { - let a = key("a:b", None, "c"); - let b = key("a", None, "b:c"); - assert_ne!(format_codex_scope_id(1, &a), format_codex_scope_id(1, &b)); -} - -#[test] -fn ac4_subagent_key_encodes_the_agent_id() { - let main = key("session-1", None, "exec-1"); - let sub = key("session-1", Some("agent-1"), "exec-1"); - assert_ne!( - format_codex_scope_id(1, &main), - format_codex_scope_id(1, &sub) - ); - assert_eq!( - format_codex_scope_id(1, &sub), - "cx-tool-v1|n=1|s=9:session-1|a=7:agent-1|t=6:exec-1" - ); -} - -#[test] -fn ac5_a_fresh_attempt_seq_yields_a_new_scope_id() { - let k = key("session-1", None, "exec-1"); - assert_ne!(format_codex_scope_id(1, &k), format_codex_scope_id(2, &k)); - assert!(format_codex_scope_id(2, &k).contains("|n=2|")); -} - -#[test] -fn ac5_attempt_key_excludes_turn_id() { - let base = pre_tool_use(&pre_tool_use_json(&[( - TOOL_USE_ID_FIELD, - Value::String("exec-9".to_string()), - )])); - let other_turn = pre_tool_use(&pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("exec-9".to_string())), - (TURN_ID_FIELD, Value::String("turn-99".to_string())), - ])); - assert_eq!( - base.identity.attempt_key(), - other_turn.identity.attempt_key() - ); -} - -mod driver { - use std::cell::RefCell; - use std::path::{Path, PathBuf}; - use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; - use std::sync::mpsc; - use std::sync::{Arc, Mutex}; - use std::thread; - use std::time::Duration; - - use anyhow::{anyhow, Result}; - - use super::*; - use crate::services::observability::traits::Logger; - - const CWD: &str = "/repo/checkout"; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-codex-mutation-scope-driver-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - #[allow(clippy::unnecessary_wraps)] - fn ok_seam(_root: &Path, _payload: &str, _logger: Option<&dyn Logger>) -> Result { - Ok(String::new()) - } - - fn unreachable_seam( - _root: &Path, - payload: &str, - _logger: Option<&dyn Logger>, - ) -> Result { - panic!("the ingress seam must not be called for this payload: {payload}"); - } - - const BLOCKED_BASH_POLICY_RESPONSE: &str = concat!( - r#"{"hookSpecificOutput":{"hookEventName":"PreToolUse","#, - r#""permissionDecision":"deny","#, - r#""permissionDecisionReason":"Blocked by SCE bash-tool policy 'no-danger': danger is not allowed"}}"#, - ); - - #[allow(clippy::unnecessary_wraps)] - fn blocking_bash_policy(_root: &Path, _command: &str) -> Result { - Ok(CodexBashPolicyDecision::Blocked( - BLOCKED_BASH_POLICY_RESPONSE.to_string(), - )) - } - - #[allow(clippy::unnecessary_wraps)] - fn allow_bash_policy(_root: &Path, _command: &str) -> Result { - Ok(CodexBashPolicyDecision::Allowed) - } - - fn failing_bash_policy(_root: &Path, _command: &str) -> Result { - Err(anyhow!( - "repository Bash policy configuration is invalid and could not be evaluated" - )) - } - - fn unreachable_bash_policy(_root: &Path, command: &str) -> Result { - panic!("the Bash policy preflight must not run for this event (command: {command})"); - } - - fn seam_failing_on( - operation: &'static str, - ) -> impl Fn(&Path, &str, Option<&dyn Logger>) -> Result { - seam_failing_on_any(vec![operation]) - } - - fn seam_failing_on_any( - operations: Vec<&'static str>, - ) -> impl Fn(&Path, &str, Option<&dyn Logger>) -> Result { - move |_root, payload, _logger| { - if operations - .iter() - .any(|operation| payload.contains(&format!(r#""operation":"{operation}""#))) - { - Err(anyhow!( - "seam failure injected by test for one of {operations:?}" - )) - } else { - Ok(String::new()) - } - } - } - - fn recording_seam( - log: Arc>>, - ) -> impl Fn(&Path, &str, Option<&dyn Logger>) -> Result { - move |_root, payload, _logger| { - log.lock() - .expect("recording seam mutex") - .push(payload.to_string()); - Ok(String::new()) - } - } - - struct SeamGate { - entered: mpsc::Receiver<()>, - release: mpsc::Sender<()>, - } - - impl SeamGate { - fn wait_until_entered(&self) { - self.entered - .recv_timeout(Duration::from_secs(5)) - .expect("gated seam should be entered"); - } - - fn release(&self) { - let _ = self.release.send(()); - } - } - - #[allow(clippy::type_complexity)] - fn gated_seam( - operation: &'static str, - calls: Arc>>, - ) -> ( - impl Fn(&Path, &str, Option<&dyn Logger>) -> Result + Send, - SeamGate, - ) { - let (entered_tx, entered_rx) = mpsc::channel(); - let (release_tx, release_rx) = mpsc::channel(); - let release_rx = Mutex::new(release_rx); - let seam = move |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| { - calls - .lock() - .expect("gated seam mutex") - .push(payload.to_string()); - if payload.contains(&format!(r#""operation":"{operation}""#)) { - entered_tx.send(()).expect("gate entry signal"); - release_rx - .lock() - .expect("gate release mutex") - .recv() - .expect("gate release signal"); - } - Ok(String::new()) - }; - ( - seam, - SeamGate { - entered: entered_rx, - release: release_tx, - }, - ) - } - - fn fixed_resolver(git_dir: PathBuf) -> impl Fn(&str) -> Result + Send + Clone { - move |_cwd| Ok(git_dir.clone()) - } - - fn panicking_resolver(_cwd: &str) -> Result { - panic!("resolve_git_dir must not be called for a non-tracked tool") - } - - #[derive(Clone, Default)] - struct RecordingLogger { - warnings: Arc>>, - } - - impl RecordingLogger { - fn warnings(&self) -> Vec<(String, String)> { - self.warnings - .lock() - .expect("recording logger mutex must not be poisoned") - .clone() - } - } - - impl Logger for RecordingLogger { - fn info(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - fn debug(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - - fn warn(&self, event_id: &str, message: &str, _: &[(&str, &str)], _: Option<&str>) { - self.warnings - .lock() - .expect("recording logger mutex must not be poisoned") - .push((event_id.to_string(), message.to_string())); - } - - fn error(&self, _: &str, _: &str, _: &[(&str, &str)], _: Option<&str>) {} - - fn log_cli_error(&self, _: &crate::services::error::CliError, _: Option<&str>) {} - } - - fn tool_event_json(event_name: &str, overrides: &[(&str, Value)]) -> String { - let mut merged: Vec<(&str, Value)> = - vec![(HOOK_EVENT_NAME_FIELD, Value::String(event_name.to_string()))]; - merged.extend( - overrides - .iter() - .map(|(field, value)| (*field, value.clone())), - ); - pre_tool_use_json(&merged) - } - - fn post_tool_use_json(overrides: &[(&str, Value)]) -> String { - tool_event_json(HOOK_EVENT_POST_TOOL_USE, overrides) - } - - fn turn_scoped_payload(event_name: &str, session_id: &str, turn_id: &str) -> String { - json!({ - HOOK_EVENT_NAME_FIELD: event_name, - SESSION_ID_FIELD: session_id, - TURN_ID_FIELD: turn_id, - CWD_FIELD: CWD, - }) - .to_string() - } - - fn subagent_stop_payload(session_id: &str, turn_id: &str, agent_id: &str) -> String { - json!({ - HOOK_EVENT_NAME_FIELD: HOOK_EVENT_SUBAGENT_STOP, - SESSION_ID_FIELD: session_id, - TURN_ID_FIELD: turn_id, - CWD_FIELD: CWD, - AGENT_ID_FIELD: agent_id, - }) - .to_string() - } - - fn session_end_payload(session_id: &str) -> String { - json!({ - HOOK_EVENT_NAME_FIELD: HOOK_EVENT_SESSION_END, - SESSION_ID_FIELD: session_id, - CWD_FIELD: CWD, - }) - .to_string() - } - - fn read_state(git_dir: &Path) -> state::AdapterState { - state::read_state(git_dir).expect("adapter state should be readable") - } - - const DRIVER_TURN: &str = "turn-1"; - - fn seed_attempt( - git_dir: &Path, - session_id: &str, - agent_id: Option<&str>, - tool_use_id: &str, - phase: state::AttemptPhase, - ) -> state::AdapterAttempt { - seed_attempt_in_turn( - git_dir, - session_id, - DRIVER_TURN, - agent_id, - tool_use_id, - phase, - ) - } - - fn seed_attempt_in_turn( - git_dir: &Path, - session_id: &str, - turn_id: &str, - agent_id: Option<&str>, - tool_use_id: &str, - phase: state::AttemptPhase, - ) -> state::AdapterAttempt { - state::seed_attempt_for_tests( - git_dir, - &AttemptKey { - session_id: session_id.to_string(), - agent_id: agent_id.map(str::to_string), - tool_use_id: tool_use_id.to_string(), - }, - turn_id, - "Bash", - phase, - ) - } - - fn drive( - payload: &str, - resolver: &(impl Fn(&str) -> Result + ?Sized), - seam: &(impl Fn(&Path, &str, Option<&dyn Logger>) -> Result + ?Sized), - ) -> String { - run_codex_mutation_scope_from_payload_with(payload, None, &resolver, &seam) - .expect("driver should return Ok") - } - - #[test] - fn untracked_mcp_pre_tool_use_creates_no_scope_and_never_touches_seam_or_git_dir() { - let payload = pre_tool_use_json(&[( - TOOL_NAME_FIELD, - Value::String("mcp__probe__mutate_success".to_string()), - )]); - let output = drive(&payload, &panicking_resolver, &unreachable_seam); - assert_eq!(output, ""); - } - - #[test] - fn unknown_and_delegation_pre_tool_use_create_no_scope_ac3() { - for tool in [ - "some_future_codex_tool", - "collaborationspawn_agent", - "collaborationwait_agent", - ] { - let payload = pre_tool_use_json(&[(TOOL_NAME_FIELD, Value::String(tool.to_string()))]); - assert_eq!(drive(&payload, &panicking_resolver, &unreachable_seam), ""); - } - } - - #[test] - fn untracked_pre_tool_use_leaves_the_state_store_untouched_ac9b() { - let git_dir = unique_test_git_dir("untracked-state-untouched"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - - for tool in ["mcp__probe__mutate_success", "some_future_codex_tool"] { - let payload = pre_tool_use_json(&[(TOOL_NAME_FIELD, Value::String(tool.to_string()))]); - drive(&payload, &resolver, &ok_seam); - } - - assert!(read_state(&git_dir).attempts.is_empty()); - assert!(read_state(&git_dir).recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn tracked_pre_tool_use_writes_ahead_start_then_returns_continue_ac6() { - let git_dir = unique_test_git_dir("tracked-write-ahead"); - let resolver = fixed_resolver(git_dir.clone()); - - let seen: RefCell> = RefCell::new(Vec::new()); - let seam = |root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - let phase_is_pending = state::read_state(&git_dir) - .expect("state readable inside seam") - .attempts - .first() - .is_some_and(|attempt| attempt.phase == state::AttemptPhase::PendingStart); - seen.borrow_mut() - .push((payload.to_string(), root == Path::new(CWD))); - assert!( - phase_is_pending, - "AC6: Start driven while attempt is PendingStart" - ); - Ok(String::new()) - }; - - let output = drive(&pre_tool_use_json(&[]), &resolver, &seam); - assert_eq!(output, ""); - - let calls = seen.into_inner(); - assert_eq!(calls.len(), 1); - assert!(calls[0].0.contains(r#""operation":"start""#)); - assert!(calls[0].0.contains(r#""actor_kind":"codex""#)); - assert!( - calls[0].1, - "AC6: the seam receives the raw hook cwd as repository_root" - ); - - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].phase, state::AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - fn boundary_payload_field(payload: &str, field: &str) -> Option { - let object: Map = - serde_json::from_str(payload).expect("a boundary payload is a JSON object"); - object.get(field).cloned() - } - - fn start_provenance(payload: &str) -> Value { - assert_eq!( - boundary_payload_field(payload, "operation"), - Some(Value::String("start".to_string())) - ); - boundary_payload_field(payload, PROVENANCE_FIELD) - .expect("a Codex start payload carries provenance") - } - - fn drive_recording_start(label: &str, payload: &str) -> Vec { - let git_dir = unique_test_git_dir(label); - let resolver = fixed_resolver(git_dir.clone()); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - drive(payload, &resolver, &recording_seam(Arc::clone(&recorded))); - - let calls = recorded.lock().expect("recording seam mutex").clone(); - remove_test_git_dir(&git_dir); - calls - } - - #[test] - fn ac3_tracked_start_carries_scope_provenance_for_both_tracked_tools() { - for tool in TRACKED_MUTATION_TOOL_NAMES { - let payload = pre_tool_use_json(&[ - (TOOL_NAME_FIELD, Value::String((*tool).to_string())), - (MODEL_FIELD, Value::String("gpt-5.6-sol".to_string())), - ]); - let calls = drive_recording_start(&format!("provenance-{tool}"), &payload); - - assert_eq!(calls.len(), 1, "{tool} should drive exactly one boundary"); - assert_eq!( - start_provenance(&calls[0]), - json!({ "session_id": "cx_session-1", "model_id": "gpt-5.6-sol" }), - "AC3: {tool} must carry its canonical session and normalized model" - ); - } - } - - #[test] - fn ac3_a_start_without_a_usable_model_still_carries_its_session() { - let cases: [(&str, Option); 4] = [ - ("absent", None), - ("null", Some(Value::Null)), - ("blank", Some(Value::String(" ".to_string()))), - ("non-string", Some(Value::Bool(true))), - ]; - - for (label, model) in cases { - let overrides = model.map_or_else(Vec::new, |value| vec![(MODEL_FIELD, value)]); - let payload = pre_tool_use_json(&overrides); - let calls = drive_recording_start(&format!("provenance-model-{label}"), &payload); - - assert_eq!(calls.len(), 1, "{label} should drive exactly one boundary"); - assert_eq!( - start_provenance(&calls[0]), - json!({ "session_id": "cx_session-1", "model_id": Value::Null }), - "AC3: a {label} model records no model without losing the session" - ); - } - } - - #[test] - fn ac3_only_the_start_boundary_carries_provenance() { - let git_dir = unique_test_git_dir("provenance-start-only"); - let resolver = fixed_resolver(git_dir.clone()); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - let seam = recording_seam(Arc::clone(&recorded)); - - drive(&pre_tool_use_json(&[]), &resolver, &seam); - drive(&post_tool_use_json(&[]), &resolver, &seam); - - let calls = recorded.lock().expect("recording seam mutex").clone(); - assert_eq!(calls.len(), 2); - assert!(boundary_payload_field(&calls[0], PROVENANCE_FIELD).is_some()); - assert_eq!( - boundary_payload_field(&calls[1], "operation"), - Some(Value::String("close".to_string())) - ); - assert_eq!(boundary_payload_field(&calls[1], PROVENANCE_FIELD), None); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn duplicate_pre_tool_use_reuses_the_same_scope_id_ac4_test_e() { - let git_dir = unique_test_git_dir("duplicate-pre"); - let resolver = fixed_resolver(git_dir.clone()); - - drive(&pre_tool_use_json(&[]), &resolver, &ok_seam); - let scope_id = read_state(&git_dir).attempts[0].scope_id.clone(); - - drive(&pre_tool_use_json(&[]), &resolver, &ok_seam); - - let attempts = read_state(&git_dir).attempts; - assert_eq!( - attempts.len(), - 1, - "AC4/Test E: a replay must not fork a new attempt" - ); - assert_eq!(attempts[0].scope_id, scope_id); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn resolver_failure_denies_with_stable_reason_and_logs_the_detail_ac7() { - let logger = RecordingLogger::default(); - let resolver = - |_: &str| -> Result { Err(anyhow!("boom: git rev-parse --git-dir failed")) }; - - let output = run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[]), - Some(&logger), - &resolver, - &unreachable_seam, - ) - .expect("a resolver failure must still return Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - assert!(!output.contains("boom")); - assert!(!output.contains("allow")); - - let warnings = logger.warnings(); - assert_eq!(warnings.len(), 1); - assert_eq!(warnings[0].0, PRE_TOOL_USE_FAIL_CLOSED_EVENT); - assert!(warnings[0].1.contains("boom")); - } - - #[test] - fn start_seam_failure_denies_and_leaves_the_pending_start_attempt_as_a_barrier_ac7() { - let git_dir = unique_test_git_dir("start-seam-failure"); - let resolver = fixed_resolver(git_dir.clone()); - let logger = RecordingLogger::default(); - let seam = seam_failing_on("start"); - - let output = run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[]), - Some(&logger), - &resolver, - &seam, - ) - .expect("a Start failure must still return Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - assert!(!logger.warnings().is_empty()); - - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!( - final_state.attempts[0].phase, - state::AttemptPhase::PendingStart - ); - - let successor = pre_tool_use_json(&[ - ( - TOOL_USE_ID_FIELD, - Value::String("exec-successor".to_string()), - ), - (TURN_ID_FIELD, Value::String("turn-2".to_string())), - ]); - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &successor, - None, - &resolver, - &unreachable_seam, - ) - .expect("successor must return Ok with a deny payload"), - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "I5: an unresolved PendingStart in another lane must block a successor Start", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn delegation_and_untracked_pre_tool_use_are_never_fail_closed_ac7() { - let resolver = |_: &str| -> Result { Err(anyhow!("must not be called")) }; - for tool in [ - "mcp__probe__mutate_success", - "some_future_codex_tool", - "collaborationspawn_agent", - ] { - let payload = pre_tool_use_json(&[(TOOL_NAME_FIELD, Value::String(tool.to_string()))]); - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &payload, - None, - &resolver, - &unreachable_seam, - ) - .expect("non-tracked PreToolUse should succeed"), - "", - ); - } - } - - #[test] - fn successful_close_removes_the_attempt_ac8() { - let git_dir = unique_test_git_dir("close-success"); - let resolver = fixed_resolver(git_dir.clone()); - - drive(&pre_tool_use_json(&[]), &resolver, &ok_seam); - - let seen: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - seen.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - assert_eq!(drive(&post_tool_use_json(&[]), &resolver, &seam), ""); - - let calls = seen.into_inner(); - assert_eq!(calls.len(), 1); - assert!(calls[0].contains(r#""operation":"close""#)); - assert!(read_state(&git_dir).attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_start_close_abandons_rather_than_late_starting_d11() { - let git_dir = unique_test_git_dir("pending-start-close"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-1", - state::AttemptPhase::PendingStart, - ); - - let seen: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - seen.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - drive(&post_tool_use_json(&[]), &resolver, &seam); - - let calls = seen.into_inner(); - assert_eq!(calls.len(), 1); - assert!(calls[0].contains(r#""operation":"abandon""#)); - - let final_state = read_state(&git_dir); - assert!(final_state.attempts.is_empty()); - assert!(!final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn failed_close_abandons_and_arms_recovery_ac13() { - let git_dir = unique_test_git_dir("failed-close"); - let resolver = fixed_resolver(git_dir.clone()); - - drive(&pre_tool_use_json(&[]), &resolver, &ok_seam); - - let seam = seam_failing_on("close"); - drive(&post_tool_use_json(&[]), &resolver, &seam); - - let final_state = read_state(&git_dir); - assert!(final_state.attempts.is_empty()); - assert!( - !final_state.recovery.is_clear(), - "D11: a failed Close arms recovery" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn failed_close_and_failed_abandon_keep_the_attempt_tracked_and_recovery_armed_d11() { - let git_dir = unique_test_git_dir("failed-close-and-abandon"); - let resolver = fixed_resolver(git_dir.clone()); - - drive(&pre_tool_use_json(&[]), &resolver, &ok_seam); - - let seam = seam_failing_on_any(vec!["close", "abandon"]); - let error = run_codex_mutation_scope_from_payload_with( - &post_tool_use_json(&[]), - None, - &resolver, - &seam, - ) - .expect_err("a failed Close then failed Abandon must propagate"); - assert!(error.to_string().contains("abandon")); - - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert!(!final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn post_tool_use_with_no_matching_attempt_is_a_noop() { - let git_dir = unique_test_git_dir("close-no-attempt"); - let resolver = fixed_resolver(git_dir.clone()); - - assert_eq!( - drive( - &post_tool_use_json(&[(TOOL_NAME_FIELD, Value::String("Bash".to_string()),)]), - &resolver, - &unreachable_seam, - ), - "", - ); - assert!(read_state(&git_dir).attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn stop_sweeps_only_main_thread_attempts_d12() { - let git_dir = unique_test_git_dir("stop-sweep"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-main", - state::AttemptPhase::Active, - ); - seed_attempt( - &git_dir, - "session-1", - Some("agent-1"), - "exec-agent", - state::AttemptPhase::Active, - ); - - drive( - &turn_scoped_payload(HOOK_EVENT_STOP, "session-1", "turn-1"), - &resolver, - &ok_seam, - ); - - let attempts = read_state(&git_dir).attempts; - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].tool_use_id, "exec-agent"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn interrupt_sweeps_every_attempt_for_the_session_d12() { - let git_dir = unique_test_git_dir("interrupt-sweep"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-main", - state::AttemptPhase::Active, - ); - seed_attempt( - &git_dir, - "session-1", - Some("agent-1"), - "exec-agent", - state::AttemptPhase::Active, - ); - seed_attempt( - &git_dir, - "session-2", - None, - "exec-other", - state::AttemptPhase::Active, - ); - - drive( - &turn_scoped_payload(HOOK_EVENT_INTERRUPT, "session-1", "turn-1"), - &resolver, - &ok_seam, - ); - - let attempts = read_state(&git_dir).attempts; - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].session_id, "session-2"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn subagent_stop_sweeps_only_the_matching_agent_d12() { - let git_dir = unique_test_git_dir("subagent-stop-sweep"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - Some("agent-a"), - "exec-a", - state::AttemptPhase::Active, - ); - seed_attempt( - &git_dir, - "session-1", - Some("agent-b"), - "exec-b", - state::AttemptPhase::Active, - ); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-main", - state::AttemptPhase::Active, - ); - - drive( - &subagent_stop_payload("session-1", "turn-1", "agent-a"), - &resolver, - &ok_seam, - ); - - let mut remaining: Vec = read_state(&git_dir) - .attempts - .into_iter() - .map(|attempt| attempt.tool_use_id) - .collect(); - remaining.sort(); - assert_eq!( - remaining, - vec!["exec-b".to_string(), "exec-main".to_string()] - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn session_end_sweeps_every_attempt_for_the_session_d12() { - let git_dir = unique_test_git_dir("session-end-sweep"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-main", - state::AttemptPhase::Active, - ); - seed_attempt( - &git_dir, - "session-1", - Some("agent-1"), - "exec-agent", - state::AttemptPhase::Active, - ); - - drive(&session_end_payload("session-1"), &resolver, &ok_seam); - - assert!(read_state(&git_dir).attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn lifecycle_cleanup_with_a_failed_abandon_keeps_the_attempt_tracked_d12() { - let git_dir = unique_test_git_dir("sweep-failed-abandon"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-main", - state::AttemptPhase::Active, - ); - - let seam = seam_failing_on("abandon"); - let error = run_codex_mutation_scope_from_payload_with( - &session_end_payload("session-1"), - None, - &resolver, - &seam, - ) - .expect_err("a failed abandonment during cleanup must propagate"); - assert!(error.to_string().contains("abandon")); - - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert!(!final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_non_empty_recovery_denies_unrelated_admission_without_losing_the_recovery_path() { - use crate::services::hooks::mutation_scope_health::MutationScopeHealthStatus; - - let git_dir = unique_test_git_dir("health-recovering-unrelated-denied"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-main", - state::AttemptPhase::Active, - ); - - let seam = seam_failing_on("abandon"); - let error = run_codex_mutation_scope_from_payload_with( - &session_end_payload("session-1"), - None, - &resolver, - &seam, - ) - .expect_err("a failed abandonment during cleanup must propagate"); - assert!(error.to_string().contains("abandon")); - - assert_eq!( - health::classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "a stuck non-empty attempt with recovery armed is Recovering: unrelated \ - admission stays fail-closed, but a same-lane successor can still retry \ - the stale predecessor's abandonment" - ); - - for attempt_number in 1..=2 { - let output = drive( - &pre_tool_use_json(&[ - (SESSION_ID_FIELD, Value::String("session-2".to_string())), - (TURN_ID_FIELD, Value::String("turn-2".to_string())), - ( - TOOL_USE_ID_FIELD, - Value::String("exec-unrelated".to_string()), - ), - ]), - &resolver, - &unreachable_seam, - ); - assert_eq!( - output, - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "PreToolUse call #{attempt_number} for an unrelated session must still be denied" - ); - } - - assert_eq!( - health::classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "the classifier must still report Recovering after repeated denial from an \ - unrelated session; Recovering does not mean every future call succeeds, only \ - that a proven normal self-healing route exists" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn same_lane_successor_retries_abandon_and_reaches_healthy_after_a_failed_lifecycle_abandon_ac4( - ) { - use crate::services::hooks::mutation_scope_health::MutationScopeHealthStatus; - - let git_dir = unique_test_git_dir("health-same-lane-self-heal"); - let resolver = fixed_resolver(git_dir.clone()); - - seed_attempt_in_turn( - &git_dir, - "session-1", - "turn-1", - None, - "exec-a", - state::AttemptPhase::Active, - ); - - let failing_abandon_seam = seam_failing_on("abandon"); - let output_b = drive( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-b".to_string()))]), - &resolver, - &failing_abandon_seam, - ); - assert_eq!( - output_b, - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "B fails closed when the same-lane sweep's retried abandon of A fails" - ); - - let after_b = read_state(&git_dir); - assert_eq!( - after_b.attempts.len(), - 1, - "A remains persisted after the failed same-lane sweep" - ); - assert_eq!(after_b.attempts[0].tool_use_id, "exec-a"); - assert!(!after_b.recovery.is_clear()); - - assert_eq!( - health::classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering - ); - - let seam_calls = Arc::new(Mutex::new(Vec::new())); - let recording = recording_seam(Arc::clone(&seam_calls)); - let output_c = drive( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-c".to_string()))]), - &resolver, - &recording, - ); - assert_eq!( - output_c, "", - "C's tracked Start proceeds once the same-lane sweep clears A and the \ - quiescent flush completes" - ); - - let operations: Vec = seam_calls - .lock() - .expect("recording seam mutex") - .iter() - .filter_map(|payload| { - serde_json::from_str::(payload) - .ok() - .and_then(|value| { - value - .get("operation") - .and_then(Value::as_str) - .map(str::to_string) - }) - }) - .collect(); - let abandon_index = operations - .iter() - .position(|operation| operation == "abandon"); - let flush_index = operations.iter().position(|operation| operation == "flush"); - let start_index = operations.iter().position(|operation| operation == "start"); - assert!( - abandon_index.is_some() && flush_index.is_some() && start_index.is_some(), - "expected abandon(A), flush, and start(C) seam calls, got {operations:?}" - ); - assert!( - abandon_index < flush_index && flush_index < start_index, - "expected abandon(A) -> flush -> start(C) ordering, got {operations:?}" - ); - - let final_state = read_state(&git_dir); - assert!(final_state.recovery.is_clear()); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-c"); - - assert_eq!( - health::classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn health_classifies_recovering_then_healthy_once_the_next_pre_tool_use_flushes_ac4() { - use crate::services::hooks::mutation_scope_health::MutationScopeHealthStatus; - - let git_dir = unique_test_git_dir("health-recovering-then-healthy"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - state::arm_recovery(&git_dir).expect("arming the barrier should succeed"); - - assert_eq!( - health::classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering - ); - - let output = drive( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-new".to_string()))]), - &resolver, - &ok_seam, - ); - assert_eq!(output, ""); - - assert_eq!( - health::classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy, - "a successful flush clears recovery and returns the adapter to Healthy" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_denies_new_tracked_pre_tool_use_while_attempts_remain_ac12() { - let git_dir = unique_test_git_dir("barrier-attempts-remain"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt_in_turn( - &git_dir, - "session-1", - "turn-other", - None, - "exec-live", - state::AttemptPhase::Active, - ); - state::arm_recovery(&git_dir).expect("arming the barrier should succeed"); - - let output = run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-new".to_string()))]), - None, - &resolver, - &unreachable_seam, - ) - .expect("the barrier denial still returns Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_does_not_affect_untracked_pre_tool_use_ac12_test_f() { - let git_dir = unique_test_git_dir("barrier-untracked-unaffected"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-live", - state::AttemptPhase::Active, - ); - state::arm_recovery(&git_dir).expect("arming the barrier should succeed"); - - for tool in [ - "mcp__probe__mutate_success", - "some_future_codex_tool", - "collaborationspawn_agent", - ] { - let payload = pre_tool_use_json(&[(TOOL_NAME_FIELD, Value::String(tool.to_string()))]); - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &payload, - None, - &resolver, - &unreachable_seam, - ) - .expect("an untracked PreToolUse ignores the barrier"), - "", - "Test F: recovery must never deny an untracked tool", - ); - } - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_flushes_once_quiescent_then_starts_ac12() { - let git_dir = unique_test_git_dir("barrier-flush-success"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - state::arm_recovery(&git_dir).expect("arming the barrier should succeed"); - - let seen: RefCell> = RefCell::new(Vec::new()); - let seam = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| -> Result { - seen.borrow_mut().push(payload.to_string()); - Ok(String::new()) - }; - - let output = drive( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-new".to_string()))]), - &resolver, - &seam, - ); - assert_eq!(output, ""); - - let operations = seen.into_inner(); - assert_eq!( - operations.len(), - 2, - "expected flush then start, got {operations:?}" - ); - assert!(operations[0].contains(r#""operation":"flush""#)); - assert!(operations[1].contains(r#""operation":"start""#)); - - let final_state = read_state(&git_dir); - assert!( - final_state.recovery.is_clear(), - "a successful flush clears the barrier" - ); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].phase, state::AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn recovery_barrier_stays_closed_when_flush_fails_ac12() { - let git_dir = unique_test_git_dir("barrier-flush-failure"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - let generation = state::arm_recovery(&git_dir).expect("arming the barrier should succeed"); - - let seam = seam_failing_on("flush"); - let output = run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-new".to_string()))]), - None, - &resolver, - &seam, - ) - .expect("a failed flush still returns Ok with a deny payload"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - let final_state = read_state(&git_dir); - assert_eq!( - final_state.recovery, - state::RecoveryState::Pending { generation }, - "a failed flush hands the generation back as Pending so a later PreToolUse retries", - ); - assert!(final_state.attempts.is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mcp_mutate_then_error_leaves_no_stale_state_ac9c() { - let git_dir = unique_test_git_dir("mcp-mutate-then-error"); - let resolver = fixed_resolver(git_dir.clone()); - - let mcp_pre = pre_tool_use_json(&[ - ( - TOOL_NAME_FIELD, - Value::String("mcp__probe__mutate_then_error".to_string()), - ), - (TOOL_USE_ID_FIELD, Value::String("exec-mcp".to_string())), - ]); - drive(&mcp_pre, &resolver, &unreachable_seam); - assert!(read_state(&git_dir).attempts.is_empty()); - - drive( - &turn_scoped_payload(HOOK_EVENT_STOP, "session-1", "turn-1"), - &resolver, - &ok_seam, - ); - drive(&session_end_payload("session-1"), &resolver, &ok_seam); - - let final_state = read_state(&git_dir); - assert!(final_state.attempts.is_empty()); - assert!( - final_state.recovery.is_clear(), - "AC9c: no Start => no abandon => recovery stays clear" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn failed_mcp_then_tracked_successor_starts_clean_ac9d() { - let git_dir = unique_test_git_dir("mcp-then-tracked"); - let resolver = fixed_resolver(git_dir.clone()); - - let mcp_a = pre_tool_use_json(&[ - ( - TOOL_NAME_FIELD, - Value::String("mcp__probe__mutate_then_error".to_string()), - ), - (TOOL_USE_ID_FIELD, Value::String("exec-a".to_string())), - ]); - let bash_b = pre_tool_use_json(&[ - (TOOL_NAME_FIELD, Value::String("Bash".to_string())), - (TOOL_USE_ID_FIELD, Value::String("exec-b".to_string())), - ]); - - drive(&mcp_a, &resolver, &unreachable_seam); - drive(&bash_b, &resolver, &ok_seam); - - let attempts = read_state(&git_dir).attempts; - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].tool_use_id, "exec-b"); - assert_eq!(attempts[0].phase, state::AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn parallel_mcp_executions_create_no_scopes_ac9e() { - let git_dir = unique_test_git_dir("parallel-mcp"); - let resolver = fixed_resolver(git_dir.clone()); - - for tool_use_id in ["exec-par-a", "exec-par-b"] { - let payload = pre_tool_use_json(&[ - ( - TOOL_NAME_FIELD, - Value::String("mcp__probe_par__slow_mutate".to_string()), - ), - (TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string())), - ]); - drive(&payload, &resolver, &unreachable_seam); - assert!(read_state(&git_dir).attempts.is_empty()); - } - - let final_state = read_state(&git_dir); - assert!(final_state.attempts.is_empty()); - assert!(final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn builtin_failed_a_then_b_never_leaves_a_zombie_scope_ac9a() { - let git_dir = unique_test_git_dir("builtin-failed-a-then-b"); - let resolver = fixed_resolver(git_dir.clone()); - - let predecessor_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-a".to_string()))]); - let predecessor_post = - post_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-a".to_string()))]); - let successor_pre = - pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-b".to_string()))]); - - drive(&predecessor_pre, &resolver, &ok_seam); - drive(&predecessor_post, &resolver, &ok_seam); - drive(&successor_pre, &resolver, &ok_seam); - - let attempts = read_state(&git_dir).attempts; - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].tool_use_id, "exec-b"); - - remove_test_git_dir(&git_dir); - } - - fn spawn_pre_tool_use( - git_dir: &Path, - tool_use_id: &'static str, - seam: impl Fn(&Path, &str, Option<&dyn Logger>) -> Result + Send + 'static, - ) -> (thread::JoinHandle, mpsc::Receiver<()>) { - spawn_pre_tool_use_in_turn(git_dir, tool_use_id, DRIVER_TURN, seam) - } - - fn spawn_pre_tool_use_in_turn( - git_dir: &Path, - tool_use_id: &'static str, - turn_id: &'static str, - seam: impl Fn(&Path, &str, Option<&dyn Logger>) -> Result + Send + 'static, - ) -> (thread::JoinHandle, mpsc::Receiver<()>) { - let (done_tx, done_rx) = mpsc::channel(); - let resolver = fixed_resolver(git_dir.to_path_buf()); - let handle = thread::spawn(move || { - let output = run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string())), - (TURN_ID_FIELD, Value::String(turn_id.to_string())), - ]), - None, - &resolver, - &seam, - ) - .expect("PreToolUse should return Ok"); - let _ = done_tx.send(()); - output - }); - (handle, done_rx) - } - - fn assert_still_blocked(done_rx: &mpsc::Receiver<()>, context: &str) { - assert!( - done_rx.recv_timeout(Duration::from_millis(250)).is_err(), - "{context}: the operation must still be blocked on the boundary lock", - ); - } - - fn first_index_of(recorded: &[String], operation: &str) -> Option { - recorded - .iter() - .position(|payload| payload.contains(&format!(r#""operation":"{operation}""#))) - } - - #[test] - fn test_h_cleanup_owning_the_boundary_lock_blocks_admission_until_recovery_is_processed() { - let git_dir = unique_test_git_dir("test-h-cleanup-owns-boundary"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_attempt( - &git_dir, - "session-1", - None, - "exec-a", - state::AttemptPhase::Active, - ); - - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - let (abandon_seam, gate) = gated_seam("abandon", Arc::clone(&recorded)); - - let sweeper = { - let resolver = fixed_resolver(git_dir.clone()); - thread::spawn(move || { - run_codex_mutation_scope_from_payload_with( - &session_end_payload("session-1"), - None, - &resolver, - &abandon_seam, - ) - .expect("SessionEnd cleanup should succeed") - }) - }; - - gate.wait_until_entered(); - assert_eq!( - read_state(&git_dir).recovery, - state::RecoveryState::Pending { generation: 1 }, - "cleanup arms recovery while it owns the boundary lock", - ); - - let (b_handle, b_done) = - spawn_pre_tool_use(&git_dir, "exec-b", recording_seam(Arc::clone(&recorded))); - assert_still_blocked(&b_done, "Test H"); - assert!( - first_index_of(&recorded.lock().unwrap(), "start").is_none(), - "Test H: B must not reach Start while cleanup owns the boundary lock", - ); - - gate.release(); - sweeper.join().expect("sweeper thread should not panic"); - - let b_output = b_handle.join().expect("B thread should not panic"); - assert_eq!( - b_output, "", - "Test H: once recovery is processed B proceeds" - ); - - let recorded = recorded.lock().unwrap().clone(); - let abandon_at = first_index_of(&recorded, "abandon").expect("cleanup abandoned exec-a"); - let flush_at = first_index_of(&recorded, "flush").expect("B drove the quiescent flush"); - let start_at = first_index_of(&recorded, "start").expect("B reached Start"); - assert!( - abandon_at < flush_at && flush_at < start_at, - "Test H: the serialized order must be abandon -> flush -> start, got {recorded:?}", - ); - - let final_state = read_state(&git_dir); - assert!(final_state.recovery.is_clear()); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-b"); - assert_eq!(final_state.attempts[0].phase, state::AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn test_g_admission_completed_recovery_cannot_arm_before_start() { - let git_dir = unique_test_git_dir("test-g-admit-before-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - let (start_seam, gate) = gated_seam("start", Arc::clone(&recorded)); - - let p1 = { - let resolver = fixed_resolver(git_dir.clone()); - thread::spawn(move || { - run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-b".to_string()))]), - None, - &resolver, - &start_seam, - ) - .expect("P1 PreToolUse should return Ok") - }) - }; - - gate.wait_until_entered(); - let mid = read_state(&git_dir); - assert_eq!(mid.attempts.len(), 1); - assert_eq!(mid.attempts[0].phase, state::AttemptPhase::PendingStart); - assert!( - mid.recovery.is_clear(), - "recovery must still be Clear while P1 holds the boundary lock pre-Start", - ); - - let (p2_handle, p2_done) = spawn_pre_tool_use( - &git_dir, - "exec-cleanup-trigger", - recording_seam(Arc::clone(&recorded)), - ); - - let sweeper = { - let resolver = fixed_resolver(git_dir.clone()); - let recorded = Arc::clone(&recorded); - thread::spawn(move || { - let seam = recording_seam(recorded); - run_codex_mutation_scope_from_payload_with( - &session_end_payload("session-1"), - None, - &resolver, - &seam, - ) - .expect("SessionEnd cleanup should return Ok") - }) - }; - - assert_still_blocked(&p2_done, "Test G"); - assert!( - read_state(&git_dir).recovery.is_clear(), - "Test G: no concurrent process may arm recovery between admit(B) and Start(B)", - ); - - gate.release(); - assert_eq!(p1.join().expect("P1 should not panic"), ""); - sweeper.join().expect("sweeper should not panic"); - p2_handle.join().expect("P2 should not panic"); - - let recorded = recorded.lock().unwrap().clone(); - let start_at = first_index_of(&recorded, "start").expect("P1 drove Start(B)"); - if let Some(abandon_at) = first_index_of(&recorded, "abandon") { - assert!( - start_at < abandon_at, - "Test G: Start(B) must be serialized before any later abandon, got {recorded:?}", - ); - } - - remove_test_git_dir(&git_dir); - } - - #[test] - fn test_j_a_live_flush_owner_is_never_reclaimed_by_a_blocked_process() { - let git_dir = unique_test_git_dir("test-j-live-flush-owner"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - state::arm_recovery(&git_dir).expect("arm recovery"); - - let flush_count = Arc::new(AtomicUsize::new(0)); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - let (owner_gated, gate) = gated_seam("flush", Arc::clone(&recorded)); - - let owner = { - let resolver = fixed_resolver(git_dir.clone()); - let flush_count = Arc::clone(&flush_count); - thread::spawn(move || { - let seam = move |root: &Path, - payload: &str, - logger: Option<&dyn Logger>| - -> Result { - if payload.contains(r#""operation":"flush""#) { - flush_count.fetch_add(1, Ordering::SeqCst); - } - owner_gated(root, payload, logger) - }; - run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[( - TOOL_USE_ID_FIELD, - Value::String("exec-owner".to_string()), - )]), - None, - &resolver, - &seam, - ) - .expect("owner PreToolUse should return Ok") - }) - }; - - gate.wait_until_entered(); - assert_eq!( - read_state(&git_dir).recovery, - state::RecoveryState::Flushing { generation: 1 }, - ); - - let flush_count_p2 = Arc::clone(&flush_count); - let (p2_handle, p2_done) = - spawn_pre_tool_use_in_turn(&git_dir, "exec-2", "turn-2", move |_r, payload, _l| { - if payload.contains(r#""operation":"flush""#) { - flush_count_p2.fetch_add(1, Ordering::SeqCst); - } - Ok(String::new()) - }); - - assert_still_blocked(&p2_done, "Test J"); - assert_eq!( - read_state(&git_dir).recovery, - state::RecoveryState::Flushing { generation: 1 }, - "Test J: a blocked process must not reclaim the live owner's Flushing(g)", - ); - - gate.release(); - assert_eq!(owner.join().expect("owner should not panic"), ""); - assert_eq!(p2_handle.join().expect("P2 should not panic"), ""); - - assert_eq!( - flush_count.load(Ordering::SeqCst), - 1, - "Test J: exactly one Flush ran — the live owner's, never a reclaim", - ); - let final_state = read_state(&git_dir); - assert!(final_state.recovery.is_clear()); - assert_eq!(final_state.attempts.len(), 2); - assert!(final_state - .attempts - .iter() - .all(|a| a.phase == state::AttemptPhase::Active)); - - remove_test_git_dir(&git_dir); - } - - fn seed_orphaned_flushing(git_dir: &Path) -> u64 { - let generation = state::arm_recovery(git_dir).expect("arm recovery to seed"); - match state::admit_tracked_attempt(git_dir, &key("seed", None, "seed"), "seed-turn", "Bash") - .expect("seeding admit should not error") - { - state::AdmitDecision::FlushClaimed { - generation: claimed, - } => { - assert_eq!(claimed, generation); - } - other => panic!("expected FlushClaimed while seeding, got {other:?}"), - } - assert_eq!( - read_state(git_dir).recovery, - state::RecoveryState::Flushing { generation }, - "seed left durable Flushing(g) with no live boundary-lock owner", - ); - generation - } - - #[test] - fn test_i_orphaned_flushing_is_reclaimed_and_flush_is_retried_once() { - let git_dir = unique_test_git_dir("test-i-orphaned-flushing"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let generation = seed_orphaned_flushing(&git_dir); - let next_generation_before = read_state(&git_dir).next_recovery_generation; - - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - let resolver = fixed_resolver(git_dir.clone()); - let output = drive( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-x".to_string()))]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(output, ""); - - let ops = recorded.lock().unwrap().clone(); - assert_eq!( - ops.iter() - .filter(|p| p.contains(r#""operation":"flush""#)) - .count(), - 1, - "Test I: exactly one retry Flush for the reclaimed generation, got {ops:?}", - ); - assert!(first_index_of(&ops, "flush").unwrap() < first_index_of(&ops, "start").unwrap()); - - let final_state = read_state(&git_dir); - assert!( - final_state.recovery.is_clear(), - "Test I: no permanent RecoveryBlocked" - ); - assert_eq!( - final_state.next_recovery_generation, next_generation_before, - "Test I: reclaiming Flushing(g) preserves the generation, never bumps it", - ); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-x"); - assert_eq!(final_state.attempts[0].phase, state::AttemptPhase::Active); - let _ = generation; - - remove_test_git_dir(&git_dir); - } - - #[test] - fn test_k_crash_after_durable_flush_before_completion_write_converges() { - let git_dir = unique_test_git_dir("test-k-crash-after-flush"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_orphaned_flushing(&git_dir); - - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - let resolver = fixed_resolver(git_dir.clone()); - - let first = drive( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-1".to_string()))]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(first, ""); - assert!(read_state(&git_dir).recovery.is_clear()); - - let second = drive( - &pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("exec-2".to_string())), - (TURN_ID_FIELD, Value::String("turn-2".to_string())), - ]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(second, ""); - - let ops = recorded.lock().unwrap().clone(); - assert_eq!( - ops.iter() - .filter(|p| p.contains(r#""operation":"flush""#)) - .count(), - 1, - "Test K: the recovery retry Flush runs exactly once across convergence, got {ops:?}", - ); - let final_state = read_state(&git_dir); - assert!(final_state.recovery.is_clear()); - assert_eq!(final_state.attempts.len(), 2); - assert!(final_state - .attempts - .iter() - .all(|a| a.phase == state::AttemptPhase::Active)); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn test_l_duplicate_active_delivery_drives_no_second_start() { - let git_dir = unique_test_git_dir("test-l-duplicate-active"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let resolver = fixed_resolver(git_dir.clone()); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - let first = drive( - &pre_tool_use_json(&[]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(first, ""); - let scope_id = read_state(&git_dir).attempts[0].scope_id.clone(); - assert_eq!( - read_state(&git_dir).attempts[0].phase, - state::AttemptPhase::Active - ); - - let duplicate = drive( - &pre_tool_use_json(&[]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(duplicate, ""); - - let ops = recorded.lock().unwrap().clone(); - assert_eq!( - ops.iter() - .filter(|p| p.contains(r#""operation":"start""#)) - .count(), - 1, - "Test L: duplicate delivery of an Active execution drives no second Start, got {ops:?}", - ); - let attempts = read_state(&git_dir).attempts; - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].scope_id, scope_id); - assert_eq!(read_state(&git_dir).next_attempt_seq, 2); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn test_m_untracked_tools_never_touch_the_boundary_lock() { - let git_dir = unique_test_git_dir("test-m-untracked-no-boundary"); - let resolver = fixed_resolver(git_dir.clone()); - - for tool in [ - "mcp__probe__mutate_success", - "some_future_codex_tool", - "collaborationspawn_agent", - "collaborationwait_agent", - ] { - let payload = pre_tool_use_json(&[(TOOL_NAME_FIELD, Value::String(tool.to_string()))]); - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &payload, - None, - &panicking_resolver, - &unreachable_seam, - ) - .expect("an untracked PreToolUse is neutral"), - "", - ); - assert_eq!(drive(&payload, &resolver, &unreachable_seam), ""); - } - - assert!( - !state::BOUNDARY_LOCK - .path(&state::adapter_state_dir(&git_dir)) - .exists(), - "Test M: no untracked tool may create the adapter boundary lock", - ); - assert!( - !state::adapter_state_dir(&git_dir).exists(), - "Test M: an untracked tool resolves no git dir and touches no adapter state", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn untracked_post_tool_use_never_touches_mutation_scope_machinery() { - let git_dir = unique_test_git_dir("untracked-post-no-footprint"); - let resolver = fixed_resolver(git_dir.clone()); - - for tool in [ - "mcp__probe__mutate_success", - "some_future_codex_tool", - "collaborationspawn_agent", - "collaborationwait_agent", - ] { - let payload = post_tool_use_json(&[(TOOL_NAME_FIELD, Value::String(tool.to_string()))]); - - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &payload, - None, - &panicking_resolver, - &unreachable_seam, - ) - .expect("an untracked PostToolUse is neutral"), - "", - "untracked PostToolUse for {tool:?} must return neutral", - ); - assert_eq!( - drive(&payload, &resolver, &unreachable_seam), - "", - "untracked PostToolUse for {tool:?} must not call the ingress seam", - ); - } - - assert!( - !state::adapter_state_dir(&git_dir).exists(), - "an untracked PostToolUse resolves no git dir and creates no adapter state directory", - ); - assert!( - !state::BOUNDARY_LOCK - .path(&state::adapter_state_dir(&git_dir)) - .exists(), - "an untracked PostToolUse must not create the adapter boundary lock", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_complete_successful_mcp_lifecycle_leaves_zero_adapter_footprint() { - let git_dir = unique_test_git_dir("mcp-lifecycle-no-footprint"); - let resolver = fixed_resolver(git_dir.clone()); - - let mcp = &[( - TOOL_NAME_FIELD, - Value::String("mcp__probe__mutate_success".to_string()), - )]; - - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(mcp), - None, - &panicking_resolver, - &unreachable_seam, - ) - .expect("MCP PreToolUse is neutral"), - "", - ); - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &post_tool_use_json(mcp), - None, - &panicking_resolver, - &unreachable_seam, - ) - .expect("MCP PostToolUse is neutral"), - "", - ); - - assert_eq!( - drive(&pre_tool_use_json(mcp), &resolver, &unreachable_seam), - "" - ); - assert_eq!( - drive(&post_tool_use_json(mcp), &resolver, &unreachable_seam), - "" - ); - - let state = read_state(&git_dir); - assert!(state.attempts.is_empty(), "no attempts recorded"); - assert!(state.recovery.is_clear(), "recovery stays Clear"); - - assert!( - !state::adapter_state_dir(&git_dir).exists(), - "a complete successful MCP lifecycle creates no adapter state directory, \ - state lock, or boundary lock", - ); - assert!( - !state::BOUNDARY_LOCK - .path(&state::adapter_state_dir(&git_dir)) - .exists(), - "a complete successful MCP lifecycle creates no boundary lock", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn test_c_recovery_rearmed_while_flush_in_flight_survives_the_stale_completion() { - let git_dir = unique_test_git_dir("race-rearm-during-flush"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - state::arm_recovery(&git_dir).expect("arm g1"); - - let calls: Arc>> = Arc::new(Mutex::new(Vec::new())); - let (flush_seam, gate) = gated_seam("flush", Arc::clone(&calls)); - - let flusher = { - let git_dir = git_dir.clone(); - let resolver = fixed_resolver(git_dir.clone()); - thread::spawn(move || { - run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[( - TOOL_USE_ID_FIELD, - Value::String("exec-flusher".to_string()), - )]), - None, - &resolver, - &flush_seam, - ) - .expect("flusher PreToolUse should return Ok") - }) - }; - - gate.wait_until_entered(); - assert_eq!( - read_state(&git_dir).recovery, - state::RecoveryState::Flushing { generation: 1 }, - ); - - let second_generation = state::arm_recovery(&git_dir).expect("re-arm to g2"); - assert_eq!(second_generation, 2); - - gate.release(); - let flusher_output = flusher.join().expect("flusher thread should not panic"); - assert_eq!( - flusher_output, - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "Test C: the flusher denies because recovery was re-armed under it", - ); - - assert_eq!( - read_state(&git_dir).recovery, - state::RecoveryState::Pending { generation: 2 }, - "Test C: the stale Flush(g1) completion must not clear Pending(g2)", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn test_d_start_succeeds_but_mark_active_fails_blocks_a_successor_until_recovery() { - let git_dir = unique_test_git_dir("start-then-mark-active-fails"); - let resolver = fixed_resolver(git_dir.clone()); - - state::arm_mark_active_failure_for_tests(); - let logger = RecordingLogger::default(); - let output = run_codex_mutation_scope_from_payload_with( - &pre_tool_use_json(&[(TOOL_USE_ID_FIELD, Value::String("exec-a".to_string()))]), - Some(&logger), - &resolver, - &ok_seam, - ) - .expect("a mark_active failure still returns Ok with a deny payload"); - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - let after_start = read_state(&git_dir); - assert_eq!(after_start.attempts.len(), 1); - assert_eq!( - after_start.attempts[0].phase, - state::AttemptPhase::PendingStart - ); - assert!(after_start.recovery.is_clear()); - - let successor = pre_tool_use_json(&[ - (TOOL_USE_ID_FIELD, Value::String("exec-b".to_string())), - (TURN_ID_FIELD, Value::String("turn-2".to_string())), - ]); - assert_eq!( - run_codex_mutation_scope_from_payload_with( - &successor, - None, - &resolver, - &unreachable_seam, - ) - .expect("successor returns a deny payload"), - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "Test D: an uncertain PendingStart in another lane blocks a successor Start", - ); - - drive(&session_end_payload("session-1"), &resolver, &ok_seam); - assert!(read_state(&git_dir).attempts.is_empty()); - assert!(!read_state(&git_dir).recovery.is_clear()); - - let recording: Arc>> = Arc::new(Mutex::new(Vec::new())); - let seam = recording_seam(Arc::clone(&recording)); - let recovered = drive(&successor, &resolver, &seam); - assert_eq!(recovered, ""); - - let ops = recording.lock().expect("recording mutex").clone(); - assert_eq!(ops.len(), 2, "expected flush then start, got {ops:?}"); - assert!(ops[0].contains(r#""operation":"flush""#)); - assert!(ops[1].contains(r#""operation":"start""#)); - - let final_state = read_state(&git_dir); - assert!(final_state.recovery.is_clear()); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].phase, state::AttemptPhase::Active); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-b"); - - remove_test_git_dir(&git_dir); - } - - fn boundary_lock_exists(git_dir: &Path) -> bool { - state::BOUNDARY_LOCK - .path(&state::adapter_state_dir(git_dir)) - .exists() - } - - #[test] - fn policy_blocked_bash_pre_tool_use_creates_no_mutation_scope_state() { - let git_dir = unique_test_git_dir("policy-blocked-no-scope"); - - let output = run_codex_mutation_scope_from_payload_with_bash_policy( - &pre_tool_use_json(&[(TOOL_INPUT_FIELD, json!({"command": "danger --now"}))]), - None, - &panicking_resolver, - &unreachable_seam, - &blocking_bash_policy, - ) - .expect("a policy-blocked Bash PreToolUse still returns Ok"); - - assert_eq!( - output, BLOCKED_BASH_POLICY_RESPONSE, - "a policy block returns the Codex-native policy denial verbatim, \ - never the generic mutation-scope deny", - ); - assert!(!output.contains(FAIL_CLOSED_DENY_REASON)); - assert!( - !state::adapter_state_dir(&git_dir).exists(), - "a policy block must leave no adapter state: no PendingStart, Active, \ - Start, Abandon, Flush, or recovery", - ); - assert!( - !boundary_lock_exists(&git_dir), - "a policy block must not even acquire the boundary lock", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn policy_allowed_bash_pre_tool_use_follows_the_normal_write_ahead_start_path() { - let git_dir = unique_test_git_dir("policy-allowed-start"); - let resolver = fixed_resolver(git_dir.clone()); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - let output = run_codex_mutation_scope_from_payload_with_bash_policy( - &pre_tool_use_json(&[]), - None, - &resolver, - &recording_seam(Arc::clone(&recorded)), - &allow_bash_policy, - ) - .expect("an allowed Bash PreToolUse returns Ok"); - assert_eq!(output, ""); - - let ops = recorded.lock().unwrap().clone(); - assert_eq!( - ops.iter() - .filter(|payload| payload.contains(r#""operation":"start""#)) - .count(), - 1, - "an allowed Bash still drives exactly one write-ahead Start, got {ops:?}", - ); - let attempts = read_state(&git_dir).attempts; - assert_eq!(attempts.len(), 1); - assert_eq!(attempts[0].phase, state::AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn policy_evaluation_failure_is_fail_closed_with_no_mutation_scope_state() { - let git_dir = unique_test_git_dir("policy-eval-failure"); - let logger = RecordingLogger::default(); - - let output = run_codex_mutation_scope_from_payload_with_bash_policy( - &pre_tool_use_json(&[]), - Some(&logger), - &panicking_resolver, - &unreachable_seam, - &failing_bash_policy, - ) - .expect("a Bash policy evaluation failure still returns Ok"); - - assert_eq!( - output, - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "a policy evaluation failure fails closed with the generic mutation-scope deny", - ); - let warnings = logger.warnings(); - assert_eq!(warnings.len(), 1); - assert_eq!(warnings[0].0, PRE_TOOL_USE_FAIL_CLOSED_EVENT); - assert!(warnings[0].1.contains("could not be evaluated")); - assert!( - !state::adapter_state_dir(&git_dir).exists(), - "a fail-closed policy evaluation must leave no adapter state", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn apply_patch_pre_tool_use_never_evaluates_bash_policy() { - let git_dir = unique_test_git_dir("apply-patch-no-policy"); - let resolver = fixed_resolver(git_dir.clone()); - - let output = run_codex_mutation_scope_from_payload_with_bash_policy( - &pre_tool_use_json(&[ - (TOOL_NAME_FIELD, Value::String("apply_patch".to_string())), - (TOOL_INPUT_FIELD, Value::Null), - ]), - None, - &resolver, - &ok_seam, - &unreachable_bash_policy, - ) - .expect("an apply_patch PreToolUse returns Ok"); - assert_eq!(output, ""); - - let attempts = read_state(&git_dir).attempts; - assert_eq!(attempts.len(), 1, "apply_patch still establishes a scope"); - assert_eq!(attempts[0].phase, state::AttemptPhase::Active); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn malformed_bash_tool_input_is_fail_closed_before_the_policy_evaluator_runs() { - let git_dir = unique_test_git_dir("malformed-bash-tool-input"); - let logger = RecordingLogger::default(); - - let output = run_codex_mutation_scope_from_payload_with_bash_policy( - &pre_tool_use_json(&[(TOOL_INPUT_FIELD, json!({"not_command": "x"}))]), - Some(&logger), - &panicking_resolver, - &unreachable_seam, - &unreachable_bash_policy, - ) - .expect("a malformed Bash tool_input still returns Ok"); - - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - let warnings = logger.warnings(); - assert_eq!(warnings.len(), 1); - assert_eq!(warnings[0].0, PRE_TOOL_USE_FAIL_CLOSED_EVENT); - assert!( - warnings[0].1.contains("tool_input.command"), - "extraction reuses the shared bash_command_from_tool_input semantics", - ); - assert!(!state::adapter_state_dir(&git_dir).exists()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn production_bash_policy_evaluator_blocks_a_repo_denied_command_before_any_start() { - let repo = unique_test_git_dir("prod-policy-regression"); - std::fs::create_dir_all(repo.join(".sce")).expect("create .sce dir"); - std::fs::write( - repo.join(".sce").join("config.json"), - concat!( - r#"{"policies":{"bash":{"custom":[{"id":"no-rm","#, - r#""match":{"argv_prefix":["rm"]},"#, - r#""message":"rm is blocked in this repository"}]}}}"#, - ), - ) - .expect("write repo bash policy config"); - - let real_evaluator = |root: &Path, command: &str| evaluate_codex_bash_policy(root, command); - - let blocked = run_codex_mutation_scope_from_payload_with_bash_policy( - &pre_tool_use_json(&[ - ( - CWD_FIELD, - Value::String(repo.to_string_lossy().into_owned()), - ), - (TOOL_INPUT_FIELD, json!({"command": "rm -rf build"})), - ]), - None, - &panicking_resolver, - &unreachable_seam, - &real_evaluator, - ) - .expect("a repo-denied Bash command still returns Ok"); - - assert!(blocked.contains(r#""permissionDecision":"deny""#)); - assert!(blocked.contains("no-rm")); - assert!(blocked.contains("rm is blocked in this repository")); - assert!( - !blocked.contains(FAIL_CLOSED_DENY_REASON), - "a real policy block keeps the policy-specific UX, not the generic deny", - ); - - let allowed = run_codex_mutation_scope_from_payload_with_bash_policy( - &pre_tool_use_json(&[ - ( - CWD_FIELD, - Value::String(repo.to_string_lossy().into_owned()), - ), - (TOOL_INPUT_FIELD, json!({"command": "echo ok > ok.txt"})), - (TOOL_USE_ID_FIELD, Value::String("exec-allowed".to_string())), - ]), - None, - &fixed_resolver(repo.join(".git")), - &ok_seam, - &real_evaluator, - ) - .expect("an allowed Bash command still returns Ok"); - assert_eq!( - allowed, "", - "the same repo config lets a non-denied command through to the normal path", - ); - - remove_test_git_dir(&repo); - } - - fn operations(recorded: &[String]) -> Vec { - recorded - .iter() - .filter_map(|payload| { - for op in ["start", "close", "abandon", "flush"] { - if payload.contains(&format!(r#""operation":"{op}""#)) { - return Some(op.to_string()); - } - } - None - }) - .collect() - } - - fn pre(tool_use_id: &str, tool_name: &str, overrides: &[(&str, Value)]) -> String { - let mut merged: Vec<(&str, Value)> = vec![ - (TOOL_USE_ID_FIELD, Value::String(tool_use_id.to_string())), - (TOOL_NAME_FIELD, Value::String(tool_name.to_string())), - ]; - if tool_name != CODEX_TRACKED_TOOL_BASH { - merged.push((TOOL_INPUT_FIELD, Value::Null)); - } - merged.extend( - overrides - .iter() - .map(|(field, value)| (*field, value.clone())), - ); - pre_tool_use_json(&merged) - } - - fn assert_zombie_then_successor_sweep( - a_tool: &str, - b_tool: &str, - b_overrides: &[(&str, Value)], - ) { - let git_dir = unique_test_git_dir(&format!("zombie-successor-{a_tool}-{b_tool}")); - let resolver = fixed_resolver(git_dir.clone()); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - let started = drive( - &pre("exec-a", a_tool, &[]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(started, ""); - assert_eq!( - read_state(&git_dir).attempts[0].phase, - state::AttemptPhase::Active, - ); - - let successor = drive( - &pre("exec-b", b_tool, b_overrides), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(successor, ""); - - let ops = operations(&recorded.lock().unwrap()); - assert_eq!( - ops, - vec![ - "start".to_string(), - "abandon".to_string(), - "flush".to_string(), - "start".to_string(), - ], - "successor sequence must be Start(A) -> Abandon(A) -> Flush -> Start(B), never Start(A) -> Start(B) -> Abandon(A)", - ); - - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-b"); - assert_eq!(final_state.attempts[0].phase, state::AttemptPhase::Active); - assert!(final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn regression1_arbitrary_blocker_zombie_then_tracked_successor() { - assert_zombie_then_successor_sweep("Bash", "Bash", &[]); - } - - #[test] - fn regression2_apply_patch_successor_variants() { - assert_zombie_then_successor_sweep("Bash", "apply_patch", &[]); - assert_zombie_then_successor_sweep("apply_patch", "Bash", &[]); - assert_zombie_then_successor_sweep("apply_patch", "apply_patch", &[]); - } - - #[test] - fn regression3_parent_then_subagent_same_lane_is_swept() { - assert_zombie_then_successor_sweep( - "Bash", - "Bash", - &[(AGENT_ID_FIELD, Value::String("agent-1".to_string()))], - ); - } - - #[test] - fn regression3_subagent_then_parent_same_lane_is_swept() { - let git_dir = unique_test_git_dir("subagent-then-parent"); - let resolver = fixed_resolver(git_dir.clone()); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - drive( - &pre( - "exec-a", - "Bash", - &[(AGENT_ID_FIELD, Value::String("agent-1".to_string()))], - ), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - drive( - &pre("exec-b", "Bash", &[]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - - assert_eq!( - operations(&recorded.lock().unwrap()), - vec![ - "start".to_string(), - "abandon".to_string(), - "flush".to_string(), - "start".to_string(), - ], - ); - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-b"); - assert!(final_state.attempts[0].agent_id.is_none()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn regression4_different_session_is_not_swept() { - let git_dir = unique_test_git_dir("different-session-not-swept"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt_in_turn( - &git_dir, - "session-1", - "turn-1", - None, - "exec-a", - state::AttemptPhase::Active, - ); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - let output = drive( - &pre( - "exec-b", - "Bash", - &[(SESSION_ID_FIELD, Value::String("session-2".to_string()))], - ), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(output, ""); - - assert_eq!( - operations(&recorded.lock().unwrap()), - vec!["start".to_string()] - ); - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 2); - assert!(final_state - .attempts - .iter() - .any(|attempt| attempt.tool_use_id == "exec-a")); - assert!(final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn regression5_different_turn_is_not_swept_by_case_b_inference() { - let git_dir = unique_test_git_dir("different-turn-not-swept"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt_in_turn( - &git_dir, - "session-1", - "turn-1", - None, - "exec-a", - state::AttemptPhase::Active, - ); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - let output = drive( - &pre( - "exec-b", - "Bash", - &[(TURN_ID_FIELD, Value::String("turn-2".to_string()))], - ), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(output, ""); - - assert_eq!( - operations(&recorded.lock().unwrap()), - vec!["start".to_string()] - ); - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 2); - assert!(final_state - .attempts - .iter() - .any(|attempt| attempt.tool_use_id == "exec-a")); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn regression6_duplicate_same_attempt_key_is_not_swept() { - let git_dir = unique_test_git_dir("duplicate-not-swept"); - let resolver = fixed_resolver(git_dir.clone()); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - drive( - &pre("exec-a", "Bash", &[]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - let scope_id = read_state(&git_dir).attempts[0].scope_id.clone(); - let next_seq = read_state(&git_dir).next_attempt_seq; - - drive( - &pre("exec-a", "Bash", &[]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - - assert_eq!( - operations(&recorded.lock().unwrap()), - vec!["start".to_string()] - ); - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].scope_id, scope_id); - assert_eq!(final_state.next_attempt_seq, next_seq); - assert!(final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn regression7_pending_start_predecessor_is_swept() { - let git_dir = unique_test_git_dir("pending-start-predecessor-swept"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt_in_turn( - &git_dir, - "session-1", - "turn-1", - None, - "exec-a", - state::AttemptPhase::PendingStart, - ); - let recorded: Arc>> = Arc::new(Mutex::new(Vec::new())); - - let output = drive( - &pre("exec-b", "Bash", &[]), - &resolver, - &recording_seam(Arc::clone(&recorded)), - ); - assert_eq!(output, ""); - - assert_eq!( - operations(&recorded.lock().unwrap()), - vec![ - "abandon".to_string(), - "flush".to_string(), - "start".to_string(), - ], - ); - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-b"); - assert!(final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn regression8_abandon_failure_during_sweep_is_fail_closed() { - let git_dir = unique_test_git_dir("sweep-abandon-failure"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt_in_turn( - &git_dir, - "session-1", - "turn-1", - None, - "exec-a", - state::AttemptPhase::Active, - ); - - let output = run_codex_mutation_scope_from_payload_with( - &pre("exec-b", "Bash", &[]), - None, - &resolver, - &seam_failing_on("abandon"), - ) - .expect("a failed sweep abandon still returns Ok with a deny payload"); - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - let final_state = read_state(&git_dir); - assert_eq!(final_state.attempts.len(), 1); - assert_eq!(final_state.attempts[0].tool_use_id, "exec-a"); - assert!(!final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn regression9_flush_failure_after_sweep_is_fail_closed() { - let git_dir = unique_test_git_dir("sweep-flush-failure"); - let resolver = fixed_resolver(git_dir.clone()); - seed_attempt_in_turn( - &git_dir, - "session-1", - "turn-1", - None, - "exec-a", - state::AttemptPhase::Active, - ); - - let output = run_codex_mutation_scope_from_payload_with( - &pre("exec-b", "Bash", &[]), - None, - &resolver, - &seam_failing_on("flush"), - ) - .expect("a failed post-sweep flush still returns Ok with a deny payload"); - assert_eq!(output, pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON)); - - let final_state = read_state(&git_dir); - assert!(final_state - .attempts - .iter() - .all(|attempt| attempt.tool_use_id != "exec-b")); - assert!(!final_state.recovery.is_clear()); - - remove_test_git_dir(&git_dir); - } -} - -mod production_regressions { - use std::fs; - use std::path::{Path, PathBuf}; - use std::process::Command; - - use super::*; - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - use crate::services::mutation_trace::runtime::{resolve_git_dir, resolve_worktree_id}; - use crate::services::mutation_trace::store::decode_revision; - - const PROBE01_APPLY_PATCH_POST: &str = include_str!( - "fixtures/probe01-apply-patch-and-shell-success.apply_patch.post_tool_use.json" - ); - const PROBE02_FAILED_SHELL_PRE: &str = - include_str!("fixtures/probe02-shell-partial-write-then-nonzero-exit.pre_tool_use.json"); - const PROBE06_APPLY_PATCH_FAILURE_PRE: &str = - include_str!("fixtures/probe06-apply-patch-verification-failure-no-post.pre_tool_use.json"); - const PROBE09_DETACHED_PRE: &str = - include_str!("fixtures/probe09-self-detaching-descendant.pre_tool_use.json"); - const PROBE09_DETACHED_POST: &str = - include_str!("fixtures/probe09-self-detaching-descendant.post_tool_use.json"); - const PROBE11_INTERRUPT_PRE: &str = - include_str!("fixtures/probe11-interrupt-event-on-sigint.pre_tool_use.json"); - const PROBE13_MCP_STOP: &str = include_str!("fixtures/probe13-mcp-mutate-then-error.stop.json"); - const PROBE14_MCP_FAILED_PRE: &str = - include_str!("fixtures/probe14-mcp-failed-then-successor.failed.pre_tool_use.json"); - const PROBE14_MCP_SUCCESSOR_PRE: &str = - include_str!("fixtures/probe14-mcp-failed-then-successor.successor.pre_tool_use.json"); - const PROBE14_MCP_SUCCESSOR_POST: &str = - include_str!("fixtures/probe14-mcp-failed-then-successor.successor.post_tool_use.json"); - const PROBE16_MCP_PARALLEL_A_PRE: &str = - include_str!("fixtures/probe16-mcp-parallel-server-optin.a.pre_tool_use.json"); - const PROBE16_MCP_PARALLEL_A_POST: &str = - include_str!("fixtures/probe16-mcp-parallel-server-optin.a.post_tool_use.json"); - const PROBE16_MCP_PARALLEL_B_PRE: &str = - include_str!("fixtures/probe16-mcp-parallel-server-optin.b.pre_tool_use.json"); - const PROBE16_MCP_PARALLEL_B_POST: &str = - include_str!("fixtures/probe16-mcp-parallel-server-optin.b.post_tool_use.json"); - - const OTHER_HARNESS_ACTOR_KIND: &str = "claude_code"; - - fn git(dir: &Path, args: &[&str]) -> String { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).expect("git output should be UTF-8") - } - - struct CodexRepo { - temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, - } - - impl CodexRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-codex-mutation-scope-regression-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); - - Self { - temp, - root, - state_root, - } - } - - fn drive(&self, payload: &str) -> Result { - run_codex_mutation_scope_from_payload_at_state_root(&self.state_root, payload, None) - } - - fn drive_generic(&self, payload: &str) -> Result { - self.drive_generic_at(&self.root, payload) - } - - fn drive_generic_at(&self, repository_root: &Path, payload: &str) -> Result { - crate::services::hooks::mutation_scope::run_mutation_scope_from_payload_at_state_root( - repository_root, - &self.state_root, - payload, - None, - ) - } - - fn drive_flush(&self) -> Result { - self.drive_generic(&flush_payload()) - } - - fn db(&self) -> RepositoryAgentTraceDb { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &self.root, - &self.state_root, - "codex mutation-scope regression test assertions", - ) - .expect("assertion DB should open") - } - - fn cwd(&self) -> String { - Self::cwd_at(&self.root) - } - - fn cwd_at(root: &Path) -> String { - root.to_string_lossy().into_owned() - } - - fn working_tree_at(root: &Path) -> String { - git(root, &["add", "-A"]); - git(root, &["write-tree"]).trim().to_owned() - } - - fn working_tree(&self) -> String { - Self::working_tree_at(&self.root) - } - - fn git_dir_at(root: &Path) -> PathBuf { - resolve_git_dir(root).expect("git dir should resolve") - } - - fn git_dir(&self) -> PathBuf { - Self::git_dir_at(&self.root) - } - - fn adapter_state_at(root: &Path) -> state::AdapterState { - state::read_state(&Self::git_dir_at(root)).expect("adapter state should be readable") - } - - fn adapter_state(&self) -> state::AdapterState { - Self::adapter_state_at(&self.root) - } - - fn adapter_state_file_exists(&self) -> bool { - state::adapter_state_dir(&self.git_dir()) - .join("codex-mutation-scope-state.json") - .exists() - } - - fn worktree_id_at(root: &Path) -> String { - resolve_worktree_id(root) - .expect("worktree id should resolve") - .0 - } - - fn worktree_id(&self) -> String { - Self::worktree_id_at(&self.root) - } - - fn add_worktree(&self, name: &str) -> PathBuf { - let worktree_path = self.temp.path().join(name); - git( - &self.root, - &[ - "worktree", - "add", - "-q", - worktree_path.to_str().expect("utf-8 worktree path"), - ], - ); - worktree_path - } - - fn write(&self, name: &str, contents: &str) { - fs::write(self.root.join(name), contents).expect("regression write should succeed"); - } - - fn live_scope_id(&self) -> String { - let state = self.adapter_state(); - assert_eq!(state.attempts.len(), 1, "exactly one live attempt expected"); - state.attempts[0].scope_id.clone() - } - } - - fn count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("a count row should exist") - } - - fn raw_agent_trace_row_counts(db: &RepositoryAgentTraceDb) -> [i64; 5] { - [ - count(db, "diff_traces"), - count(db, "post_commit_patch_intersections"), - count(db, "agent_traces"), - count(db, "messages"), - count(db, "parts"), - ] - } - - fn assert_raw_agent_trace_tables_untouched(db: &RepositoryAgentTraceDb) { - assert_eq!( - raw_agent_trace_row_counts(db), - [0, 0, 0, 0, 0], - "AC20: the mutation-scope adapter must never write the raw Agent Trace tables" - ); - } - - fn worktree_row(db: &RepositoryAgentTraceDb, worktree_id: &str) -> Option<(u64, String, bool)> { - db.query_map( - "SELECT revision, cursor_tree, needs_rebaseline FROM mutation_trace_worktrees \ - WHERE worktree_id = ?1", - (worktree_id,), - |row| { - let blob: Vec = row.get(0).map_err(anyhow::Error::from)?; - let revision = decode_revision(&blob)?; - let cursor_tree = row.get::(1).map_err(anyhow::Error::from)?; - let needs_rebaseline = row.get::(2).map_err(anyhow::Error::from)? != 0; - Ok((revision, cursor_tree, needs_rebaseline)) - }, - ) - .expect("worktree-row query should succeed") - .into_iter() - .next() - } - - fn processed_events(db: &RepositoryAgentTraceDb) -> Vec<(String, String)> { - db.query_map( - "SELECT scope_id, event_id FROM mutation_trace_processed_events \ - ORDER BY scope_id, event_id", - (), - |row| { - let scope_id = row.get::(0).map_err(anyhow::Error::from)?; - let event_id = row.get::(1).map_err(anyhow::Error::from)?; - Ok((scope_id, event_id)) - }, - ) - .expect("processed-events query should succeed") - } - - fn scope_status(db: &RepositoryAgentTraceDb, scope_id: &str) -> Option<(String, String)> { - db.query_map( - "SELECT actor_kind, status FROM mutation_trace_scopes WHERE scope_id = ?1", - (scope_id,), - |row| { - let actor_kind = row.get::(0).map_err(anyhow::Error::from)?; - let status = row.get::(1).map_err(anyhow::Error::from)?; - Ok((actor_kind, status)) - }, - ) - .expect("scope query should succeed") - .into_iter() - .next() - } - - fn mutation_events_for( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - ) -> Vec<(String, Option, String)> { - db.query_map( - "SELECT attribution_kind, attribution_scope_id, boundary_kind \ - FROM mutation_trace_events WHERE worktree_id = ?1 ORDER BY revision", - (worktree_id,), - |row| { - let attribution_kind = row.get::(0).map_err(anyhow::Error::from)?; - let attribution_scope_id = - row.get::>(1).map_err(anyhow::Error::from)?; - let boundary_kind = row.get::(2).map_err(anyhow::Error::from)?; - Ok((attribution_kind, attribution_scope_id, boundary_kind)) - }, - ) - .expect("mutation-events query should succeed") - } - - fn scope_provenance( - db: &RepositoryAgentTraceDb, - scope_id: &str, - ) -> Option<(String, Option)> { - db.query_map( - "SELECT session_id, model_id FROM mutation_trace_scope_provenance \ - WHERE scope_id = ?1", - (scope_id,), - |row| { - let session_id = row.get::(0).map_err(anyhow::Error::from)?; - let model_id = row.get::>(1).map_err(anyhow::Error::from)?; - Ok((session_id, model_id)) - }, - ) - .expect("scope-provenance query should succeed") - .into_iter() - .next() - } - - fn active_scopes_for(db: &RepositoryAgentTraceDb, worktree_id: &str) -> Vec { - db.query_map( - "SELECT scope_id FROM mutation_trace_event_active_scopes \ - WHERE worktree_id = ?1 ORDER BY revision, scope_id", - (worktree_id,), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("active-scopes query should succeed") - } - - fn fixture_at(fixture: &str, cwd: &str) -> String { - let mut object: Map = - serde_json::from_str(fixture).expect("a fixture payload is a JSON object"); - object.insert(CWD_FIELD.to_string(), Value::String(cwd.to_string())); - Value::Object(object).to_string() - } - - struct ToolEvent<'a> { - event_name: &'a str, - cwd: &'a str, - session_id: &'a str, - turn_id: &'a str, - tool_name: &'a str, - tool_use_id: &'a str, - agent_id: Option<&'a str>, - } - - fn tool_event_json(event: &ToolEvent, tool_input: Option) -> String { - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(event.event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String(event.session_id.to_string()), - ); - object.insert( - TURN_ID_FIELD.to_string(), - Value::String(event.turn_id.to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String(event.cwd.to_string())); - object.insert( - TOOL_NAME_FIELD.to_string(), - Value::String(event.tool_name.to_string()), - ); - object.insert( - TOOL_USE_ID_FIELD.to_string(), - Value::String(event.tool_use_id.to_string()), - ); - if let Some(agent_id) = event.agent_id { - object.insert( - AGENT_ID_FIELD.to_string(), - Value::String(agent_id.to_string()), - ); - } - if let Some(tool_input) = tool_input { - object.insert(TOOL_INPUT_FIELD.to_string(), tool_input); - } - Value::Object(object).to_string() - } - - struct TrackedCall<'a> { - cwd: &'a str, - session_id: &'a str, - turn_id: &'a str, - tool_name: &'a str, - tool_use_id: &'a str, - agent_id: Option<&'a str>, - } - - impl TrackedCall<'_> { - fn pre(&self) -> String { - tool_event_json( - &ToolEvent { - event_name: HOOK_EVENT_PRE_TOOL_USE, - cwd: self.cwd, - session_id: self.session_id, - turn_id: self.turn_id, - tool_name: self.tool_name, - tool_use_id: self.tool_use_id, - agent_id: self.agent_id, - }, - Some(json!({ "command": "echo regression >> file.txt" })), - ) - } - - fn post(&self) -> String { - tool_event_json( - &ToolEvent { - event_name: HOOK_EVENT_POST_TOOL_USE, - cwd: self.cwd, - session_id: self.session_id, - turn_id: self.turn_id, - tool_name: self.tool_name, - tool_use_id: self.tool_use_id, - agent_id: self.agent_id, - }, - None, - ) - } - } - - fn bash_call<'a>(cwd: &'a str, session_id: &'a str, tool_use_id: &'a str) -> TrackedCall<'a> { - TrackedCall { - cwd, - session_id, - turn_id: "turn-1", - tool_name: CODEX_TRACKED_TOOL_BASH, - tool_use_id, - agent_id: None, - } - } - - fn turn_event_json(event_name: &str, cwd: &str, session_id: &str, turn_id: &str) -> String { - json!({ - HOOK_EVENT_NAME_FIELD: event_name, - SESSION_ID_FIELD: session_id, - TURN_ID_FIELD: turn_id, - CWD_FIELD: cwd, - }) - .to_string() - } - - fn session_end_json(cwd: &str, session_id: &str) -> String { - json!({ - HOOK_EVENT_NAME_FIELD: HOOK_EVENT_SESSION_END, - SESSION_ID_FIELD: session_id, - CWD_FIELD: cwd, - }) - .to_string() - } - - fn other_harness_payload(operation: &str, scope_id: &str) -> String { - json!({ - "operation": operation, - "scope_id": scope_id, - "event_id": format!("{scope_id}|{operation}"), - "actor_kind": OTHER_HARNESS_ACTOR_KIND, - }) - .to_string() - } - - #[test] - fn test1_tracked_bash_success_closes_ai_exclusive_ac8() { - let repo = CodexRepo::new("test1-bash-success"); - let cwd = repo.cwd(); - let pre = fixture_at(PROBE01_SHELL_PRE, &cwd); - let post = fixture_at(PROBE01_SHELL_POST, &cwd); - - assert_eq!( - repo.drive(&pre).expect("PreToolUse should succeed"), - "", - "a tracked PreToolUse that established Start returns the neutral response" - ); - let scope_id = repo.live_scope_id(); - - repo.write("file.txt", "one\ntwo\n"); - - assert_eq!(repo.drive(&post).expect("PostToolUse should succeed"), ""); - assert!( - repo.adapter_state().attempts.is_empty(), - "the closed attempt must be removed from adapter bookkeeping" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "closed".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id.clone()), - "close".to_string(), - )] - ); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(repo.working_tree()) - ); - assert_eq!( - processed_events(&db), - vec![ - (scope_id.clone(), codex_scope_close_event_id(&scope_id)), - (scope_id.clone(), codex_scope_start_event_id(&scope_id)), - ], - "rows are ordered by (scope_id, event_id), and 'close' sorts before 'start'" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test2_failed_bash_partial_write_still_closes_ai_exclusive_ac9() { - let repo = CodexRepo::new("test2-failed-bash"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE02_FAILED_SHELL_PRE, &cwd)) - .expect("PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - - repo.write("file.txt", "one\npartial\n"); - - assert_eq!( - repo.drive(&fixture_at(PROBE02_FAILED_SHELL_POST, &cwd)) - .expect("a non-zero-exit shell still fires PostToolUse"), - "" - ); - assert!(repo.adapter_state().attempts.is_empty()); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "closed".to_string())), - "D10: a Bash tool that partially mutated then exited non-zero closes its scope" - ); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id), - "close".to_string(), - )], - "the partial mutation is attributed to the failed tool's own scope" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test3_apply_patch_success_closes_ai_exclusive_ac8() { - let repo = CodexRepo::new("test3-apply-patch-success"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE01_APPLY_PATCH_PRE, &cwd)) - .expect("PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - - repo.write("alpha.txt", "alpha one\n"); - - repo.drive(&fixture_at(PROBE01_APPLY_PATCH_POST, &cwd)) - .expect("PostToolUse should succeed"); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "closed".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id), - "close".to_string(), - )] - ); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(repo.working_tree()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test4_apply_patch_verification_failure_mutates_nothing_and_is_swept_ac9() { - let repo = CodexRepo::new("test4-apply-patch-failure"); - let cwd = repo.cwd(); - let pre = fixture_at(PROBE06_APPLY_PATCH_FAILURE_PRE, &cwd); - let tree_before = repo.working_tree(); - - repo.drive(&pre).expect("PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - - let stop = { - let execution = pre_tool_use(&pre); - turn_event_json( - HOOK_EVENT_STOP, - &cwd, - &execution.identity.session_id, - &execution.identity.turn_id, - ) - }; - repo.drive(&stop) - .expect("Stop should retire the attempt that never received PostToolUse"); - - assert!(repo.adapter_state().attempts.is_empty()); - assert!(!repo.adapter_state().recovery.is_clear()); - assert_eq!( - repo.working_tree(), - tree_before, - "D10: apply_patch verification failure never touches the working tree" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "abandoned".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![], - "no mutation happened, so nothing is attributed" - ); - assert!(worktree_row(&db, &worktree_id) - .is_some_and(|(_, _, needs_rebaseline)| needs_rebaseline)); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test5_duplicate_tracked_lifecycle_is_idempotent_ac4() { - let repo = CodexRepo::new("test5-duplicate-lifecycle"); - let cwd = repo.cwd(); - let pre = fixture_at(PROBE01_SHELL_PRE, &cwd); - let post = fixture_at(PROBE01_SHELL_POST, &cwd); - - repo.drive(&pre).expect("first PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - assert_eq!( - repo.drive(&pre) - .expect("duplicate PreToolUse should be idempotent"), - "" - ); - assert_eq!( - repo.live_scope_id(), - scope_id, - "AC4: duplicate delivery of a live PreToolUse reuses the same ScopeId" - ); - - repo.write("file.txt", "one\ntwo\n"); - repo.drive(&post).expect("first PostToolUse should succeed"); - - let db = repo.db(); - let (revision_before, events_before, processed_before) = ( - worktree_row(&db, &repo.worktree_id()) - .map(|(revision, _, _)| revision) - .expect("a worktree row should exist"), - count(&db, "mutation_trace_events"), - count(&db, "mutation_trace_processed_events"), - ); - - assert_eq!( - repo.drive(&post) - .expect("duplicate PostToolUse delivery must be a safe no-op"), - "" - ); - - let db = repo.db(); - assert_eq!( - worktree_row(&db, &repo.worktree_id()).map(|(revision, _, _)| revision), - Some(revision_before) - ); - assert_eq!(count(&db, "mutation_trace_events"), events_before); - assert_eq!( - count(&db, "mutation_trace_processed_events"), - processed_before - ); - assert_eq!( - processed_events(&db) - .into_iter() - .filter(|(scope, event)| scope == &scope_id - && event == &codex_scope_close_event_id(&scope_id)) - .count(), - 1 - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test6_interrupted_tracked_execution_is_retired_by_interrupt_ac11() { - let repo = CodexRepo::new("test6-interrupt"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE11_INTERRUPT_PRE, &cwd)) - .expect("PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - - repo.write("file.txt", "one\ninterrupted\n"); - - assert_eq!( - repo.drive(&fixture_at(PROBE11_INTERRUPT, &cwd)) - .expect("Interrupt should succeed"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "AC11: Interrupt is a proven cleanup signal for the interrupted turn" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "abandoned".to_string())) - ); - let worktree_id = repo.worktree_id(); - assert!(worktree_row(&db, &worktree_id) - .is_some_and(|(_, _, needs_rebaseline)| needs_rebaseline)); - assert_eq!(mutation_events_for(&db, &worktree_id), vec![]); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test6b_session_end_is_the_load_bearing_backstop_ac11() { - let repo = CodexRepo::new("test6b-session-end"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE01_SHELL_PRE, &cwd)) - .expect("PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - - repo.write("file.txt", "one\nstranded\n"); - - assert_eq!( - repo.drive(&fixture_at(PROBE01_SESSION_END, &cwd)) - .expect("SessionEnd should succeed"), - "" - ); - - assert!( - repo.adapter_state().attempts.is_empty(), - "D12: SessionEnd is the load-bearing whole-session backstop" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "abandoned".to_string())) - ); - assert!(worktree_row(&db, &repo.worktree_id()) - .is_some_and(|(_, _, needs_rebaseline)| needs_rebaseline)); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test7_subagent_tracked_tool_gets_its_own_scope_identity() { - let repo = CodexRepo::new("test7-subagent"); - let cwd = repo.cwd(); - let subagent_pre = fixture_at(PROBE08_AGENT_APPLY_PATCH_PRE, &cwd); - let subagent_identity = pre_tool_use(&subagent_pre).identity; - let agent_id = subagent_identity - .agent_id - .clone() - .expect("probe08 carries a delegated-agent identity"); - let main_thread = TrackedCall { - cwd: &cwd, - session_id: &subagent_identity.session_id, - turn_id: "main-turn", - tool_name: CODEX_TRACKED_TOOL_BASH, - tool_use_id: "exec-main-thread", - agent_id: None, - }; - - repo.drive(&main_thread.pre()) - .expect("main-thread PreToolUse should succeed"); - repo.drive(&subagent_pre) - .expect("subagent PreToolUse should succeed"); - - let state = repo.adapter_state(); - assert_eq!(state.attempts.len(), 2); - let subagent_scope_id = state - .attempts - .iter() - .find(|attempt| attempt.agent_id.as_deref() == Some(agent_id.as_str())) - .map(|attempt| attempt.scope_id.clone()) - .expect("the subagent attempt carries its agent_id"); - let main_scope_id = state - .attempts - .iter() - .find(|attempt| attempt.agent_id.is_none()) - .map(|attempt| attempt.scope_id.clone()) - .expect("the main-thread attempt has no agent_id"); - assert_ne!(subagent_scope_id, main_scope_id); - assert!(subagent_scope_id.contains(&agent_id)); - - repo.drive(&fixture_at(PROBE08_SUBAGENT_STOP, &cwd)) - .expect("SubagentStop should succeed"); - - let remaining = repo.adapter_state(); - assert_eq!( - remaining - .attempts - .iter() - .map(|attempt| attempt.scope_id.clone()) - .collect::>(), - vec![main_scope_id.clone()], - "D12: SubagentStop sweeps only the ending agent's attempts" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &subagent_scope_id).map(|(_, status)| status), - Some("abandoned".to_string()) - ); - assert_eq!( - scope_status(&db, &main_scope_id).map(|(_, status)| status), - Some("active".to_string()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test8_linked_worktree_advances_only_its_own_cursor_ac14() { - let repo = CodexRepo::new("test8-linked-worktree"); - let worktree_path = repo.add_worktree("codex-worktree"); - let worktree_cwd = CodexRepo::cwd_at(&worktree_path); - - let main_worktree_id = repo.worktree_id(); - let linked_worktree_id = CodexRepo::worktree_id_at(&worktree_path); - assert_ne!(main_worktree_id, linked_worktree_id); - - repo.drive_flush() - .expect("main-checkout baseline flush should succeed"); - let main_cursor_before = worktree_row(&repo.db(), &main_worktree_id) - .map(|(_, cursor_tree, _)| cursor_tree) - .expect("main checkout should have a baseline worktree row"); - - let pre = fixture_at(PROBE10_WORKTREE_PRE, &worktree_cwd); - let post = { - let identity = pre_tool_use(&pre).identity; - tool_event_json( - &ToolEvent { - event_name: HOOK_EVENT_POST_TOOL_USE, - cwd: &worktree_cwd, - session_id: &identity.session_id, - turn_id: &identity.turn_id, - tool_name: &identity.tool_name, - tool_use_id: &identity.tool_use_id, - agent_id: None, - }, - None, - ) - }; - - repo.drive(&pre) - .expect("linked-worktree PreToolUse should succeed"); - let scope_id = CodexRepo::adapter_state_at(&worktree_path).attempts[0] - .scope_id - .clone(); - fs::write(worktree_path.join("wt.txt"), "worktree-write\n") - .expect("the linked worktree's own write should succeed"); - repo.drive(&post) - .expect("linked-worktree PostToolUse should succeed"); - - let db = repo.db(); - assert_eq!( - worktree_row(&db, &main_worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(main_cursor_before), - "AC14: the main checkout's cursor must not move" - ); - let linked_row = - worktree_row(&db, &linked_worktree_id).expect("linked worktree row should exist"); - assert_eq!( - linked_row.1, - CodexRepo::working_tree_at(&worktree_path), - "AC14: the linked worktree's own cursor advances" - ); - assert_eq!( - mutation_events_for(&db, &linked_worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id), - "close".to_string(), - )] - ); - assert_eq!(mutation_events_for(&db, &main_worktree_id), vec![]); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test9_successful_mcp_lifecycle_creates_no_mutation_scope_ac9b() { - let repo = CodexRepo::new("test9-mcp-success"); - let cwd = repo.cwd(); - - assert_eq!( - repo.drive(&fixture_at(PROBE12_MCP_PRE, &cwd)) - .expect("an MCP PreToolUse is allowed"), - "", - "AC9b: an Untracked tool gets the Codex-neutral continue response" - ); - repo.write("mcp_a.txt", "written by the MCP server\n"); - assert_eq!( - repo.drive(&fixture_at(PROBE12_MCP_POST, &cwd)) - .expect("an MCP PostToolUse is ignored"), - "" - ); - - assert!( - !repo.adapter_state_file_exists(), - "AC9b: an Untracked lifecycle writes no adapter bookkeeping at all" - ); - - let db = repo.db(); - assert_eq!(count(&db, "mutation_trace_scopes"), 0); - assert_eq!(count(&db, "mutation_trace_events"), 0); - assert_eq!(count(&db, "mutation_trace_processed_events"), 0); - assert_eq!(count(&db, "mutation_trace_worktrees"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test10_mcp_mutate_then_error_leaves_no_zombie_state_ac9c() { - let repo = CodexRepo::new("test10-mcp-mutate-then-error"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE13_MCP_MUTATE_THEN_ERROR_PRE, &cwd)) - .expect("an MCP PreToolUse is allowed"); - repo.write("mcp_b.txt", "mutated before the MCP error\n"); - - repo.drive(&fixture_at(PROBE13_MCP_STOP, &cwd)) - .expect("Stop should find nothing to retire"); - repo.drive(&fixture_at(PROBE13_MCP_SESSION_END, &cwd)) - .expect("SessionEnd should find nothing to retire"); - - assert!( - !repo.adapter_state_file_exists(), - "AC9c: no Start occurred, so there is no stale attempt and no recovery to arm" - ); - - let db = repo.db(); - assert_eq!(count(&db, "mutation_trace_scopes"), 0); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test11_failed_mcp_then_tracked_successor_starts_clean_ac9d() { - let repo = CodexRepo::new("test11-mcp-then-tracked"); - let cwd = repo.cwd(); - let failed_mcp = fixture_at(PROBE14_MCP_FAILED_PRE, &cwd); - let failed_identity = pre_tool_use(&failed_mcp).identity; - - repo.drive(&failed_mcp) - .expect("the failing MCP PreToolUse is allowed"); - repo.write("mcp_c1.txt", "mutated by the failing MCP tool\n"); - repo.drive(&fixture_at(PROBE14_MCP_SUCCESSOR_PRE, &cwd)) - .expect("the MCP successor is also Untracked"); - repo.drive(&fixture_at(PROBE14_MCP_SUCCESSOR_POST, &cwd)) - .expect("the MCP successor's PostToolUse is ignored"); - - let tracked_successor = TrackedCall { - cwd: &cwd, - session_id: &failed_identity.session_id, - turn_id: &failed_identity.turn_id, - tool_name: CODEX_TRACKED_TOOL_BASH, - tool_use_id: "exec-tracked-successor", - agent_id: None, - }; - repo.drive(&tracked_successor.pre()) - .expect("the tracked successor should Start normally"); - let scope_id = repo.live_scope_id(); - assert!( - repo.adapter_state().recovery.is_clear(), - "AC9d: no MCP attempt existed, so no successor barrier runs" - ); - - repo.write("file.txt", "one\ntracked-successor\n"); - repo.drive(&tracked_successor.post()) - .expect("the tracked successor's PostToolUse should close its scope"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!(count(&db, "mutation_trace_scopes"), 1); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id), - "close".to_string(), - )], - "AC9d: the tracked successor is the only live scope — no false AiContended" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test12_parallel_mcp_executions_create_no_scopes_ac9e() { - let repo = CodexRepo::new("test12-parallel-mcp"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE16_MCP_PARALLEL_A_PRE, &cwd)) - .expect("parallel MCP A is allowed"); - repo.drive(&fixture_at(PROBE16_MCP_PARALLEL_B_PRE, &cwd)) - .expect("parallel MCP B is allowed"); - assert!( - !repo.adapter_state_file_exists(), - "AC9e: neither overlapping MCP execution creates adapter state" - ); - - repo.write("mcp_parallel_a.txt", "a\n"); - repo.write("mcp_parallel_b.txt", "b\n"); - - repo.drive(&fixture_at(PROBE16_MCP_PARALLEL_A_POST, &cwd)) - .expect("parallel MCP A PostToolUse is ignored"); - repo.drive(&fixture_at(PROBE16_MCP_PARALLEL_B_POST, &cwd)) - .expect("parallel MCP B PostToolUse is ignored"); - - assert!(!repo.adapter_state_file_exists()); - - let db = repo.db(); - assert_eq!( - count(&db, "mutation_trace_scopes"), - 0, - "AC9e: overlapping MCP executions produce no scopes and therefore no AiContended" - ); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test13_tracked_scope_overlapping_an_mcp_mutation_is_tracked_exclusivity_ac9f() { - let repo = CodexRepo::new("test13-tracked-plus-mcp"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE01_SHELL_PRE, &cwd)) - .expect("the tracked Bash PreToolUse should Start"); - let scope_id = repo.live_scope_id(); - - repo.drive(&fixture_at(PROBE12_MCP_PRE, &cwd)) - .expect("the overlapping MCP call is allowed and untracked"); - repo.write("mcp_a.txt", "written by the MCP server, not by Bash\n"); - repo.drive(&fixture_at(PROBE12_MCP_POST, &cwd)) - .expect("the MCP PostToolUse is ignored"); - - repo.drive(&fixture_at(PROBE01_SHELL_POST, &cwd)) - .expect("the tracked Bash PostToolUse should close its scope"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!(count(&db, "mutation_trace_scopes"), 1); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![( - "ai_exclusive".to_string(), - Some(scope_id), - "close".to_string(), - )], - "AC9f: ai_exclusive means exactly one TRACKED scope was live in the interval, \ - not that the tracked scope authored every mutation — the MCP call did mutate \ - mcp_a.txt inside this interval and remains unattributed (D14/D23)" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test14_unknown_tool_is_allowed_untracked_ac9b() { - let repo = CodexRepo::new("test14-unknown-tool"); - let cwd = repo.cwd(); - let unknown = TrackedCall { - cwd: &cwd, - session_id: "session-unknown", - turn_id: "turn-1", - tool_name: "some_future_codex_tool", - tool_use_id: "exec-unknown", - agent_id: None, - }; - - assert_eq!( - repo.drive(&unknown.pre()) - .expect("an unknown tool is never denied for being untracked"), - "" - ); - repo.write("unknown_tool_output.txt", "the unknown tool mutated\n"); - assert_eq!( - repo.drive(&unknown.post()) - .expect("an unknown tool's PostToolUse is ignored"), - "" - ); - - assert!(!repo.adapter_state_file_exists()); - - let db = repo.db(); - assert_eq!(count(&db, "mutation_trace_scopes"), 0); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test15_regression_matrix_leaves_raw_agent_trace_tables_untouched_ac20() { - let repo = CodexRepo::new("test15-raw-tables"); - let cwd = repo.cwd(); - - let before = raw_agent_trace_row_counts(&repo.db()); - assert_eq!(before, [0, 0, 0, 0, 0]); - - repo.drive(&fixture_at(PROBE01_SHELL_PRE, &cwd)) - .expect("tracked PreToolUse should succeed"); - repo.write("file.txt", "one\ntwo\n"); - repo.drive(&fixture_at(PROBE01_SHELL_POST, &cwd)) - .expect("tracked PostToolUse should succeed"); - repo.drive(&fixture_at(PROBE12_MCP_PRE, &cwd)) - .expect("MCP PreToolUse should succeed"); - repo.write("mcp_a.txt", "mcp\n"); - repo.drive(&fixture_at(PROBE12_MCP_POST, &cwd)) - .expect("MCP PostToolUse should succeed"); - repo.drive(&fixture_at(PROBE01_APPLY_PATCH_PRE, &cwd)) - .expect("apply_patch PreToolUse should succeed"); - repo.drive(&fixture_at(PROBE01_STOP, &cwd)) - .expect("Stop should succeed"); - - let db = repo.db(); - assert_eq!( - raw_agent_trace_row_counts(&db), - before, - "AC20: mutation-scope-only regressions leave diff_traces, \ - post_commit_patch_intersections, agent_traces, messages and parts unchanged" - ); - assert!(count(&db, "mutation_trace_scopes") > 0); - assert!( - state::adapter_state_dir(&repo.git_dir()).starts_with(repo.git_dir()), - "AC20: adapter state lives only below /sce/" - ); - } - - #[test] - fn test16_arbitrary_blocker_zombie_then_same_lane_successor_ac9a() { - let repo = CodexRepo::new("test16-zombie-successor"); - let cwd = repo.cwd(); - let zombie = bash_call(&cwd, "session-lane", "exec-zombie"); - let successor = bash_call(&cwd, "session-lane", "exec-successor"); - - repo.drive(&zombie.pre()) - .expect("the first tracked PreToolUse should Start"); - let zombie_scope_id = repo.live_scope_id(); - repo.write("file.txt", "one\nzombie-partial\n"); - - repo.drive(&successor.pre()) - .expect("the same-lane successor should sweep, flush, then Start"); - let successor_scope_id = repo.live_scope_id(); - assert_ne!(zombie_scope_id, successor_scope_id); - assert!( - repo.adapter_state().recovery.is_clear(), - "the quiescent flush must clear the barrier before Start(B)" - ); - - repo.write("file.txt", "one\nzombie-partial\nsuccessor\n"); - repo.drive(&successor.post()) - .expect("the successor's PostToolUse should close its scope"); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &zombie_scope_id), - Some(("codex".to_string(), "abandoned".to_string())), - "AC9a: the stale same-lane predecessor is abandoned, never closed" - ); - assert_eq!( - scope_status(&db, &successor_scope_id), - Some(("codex".to_string(), "closed".to_string())) - ); - let worktree_id = repo.worktree_id(); - let events = mutation_events_for(&db, &worktree_id); - assert!( - events.iter().all(|(kind, scope, _)| kind != "ai_contended" - && scope.as_deref() != Some(zombie_scope_id.as_str())), - "AC9a: no false AiContended and nothing attributed to the zombie: {events:?}" - ); - assert_eq!( - events.last(), - Some(&( - "ai_exclusive".to_string(), - Some(successor_scope_id), - "close".to_string() - )), - "the successor is the only live scope at its own Close" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test17_crash_before_start_commit_is_recovered_conservatively_ac21a() { - let repo = CodexRepo::new("test17-crash-before-start"); - let cwd = repo.cwd(); - let git_dir = repo.git_dir(); - let crashed = bash_call(&cwd, "session-crash", "exec-crashed"); - let key = key("session-crash", None, "exec-crashed"); - - let attempt = state::seed_attempt_for_tests( - &git_dir, - &key, - "turn-1", - CODEX_TRACKED_TOOL_BASH, - state::AttemptPhase::PendingStart, - ); - - repo.drive(&crashed.post()) - .expect("D11: a pending_start attempt must abandon, not late-Start"); - - assert!(repo.adapter_state().attempts.is_empty()); - assert!(!repo.adapter_state().recovery.is_clear()); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &attempt.scope_id), - None, - "AC21a: a Start that never committed must never appear as a real scope" - ); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - let fresh = bash_call(&cwd, "session-crash", "exec-fresh"); - repo.drive(&fresh.pre()) - .expect("the next tracked PreToolUse proceeds after the quiescent flush"); - assert!(repo.adapter_state().recovery.is_clear()); - assert_eq!(repo.adapter_state().attempts.len(), 1); - - assert_raw_agent_trace_tables_untouched(&repo.db()); - } - - #[test] - fn test18_start_committed_before_state_settlement_is_abandoned_ac21b() { - let repo = CodexRepo::new("test18-crash-after-start"); - let cwd = repo.cwd(); - let git_dir = repo.git_dir(); - let crashed = bash_call(&cwd, "session-crash", "exec-crashed"); - let key = key("session-crash", None, "exec-crashed"); - - let attempt = state::seed_attempt_for_tests( - &git_dir, - &key, - "turn-1", - CODEX_TRACKED_TOOL_BASH, - state::AttemptPhase::PendingStart, - ); - let scope_id = attempt.scope_id.clone(); - - repo.drive_generic(&scope_boundary_payload( - "start", - &scope_id, - &codex_scope_start_event_id(&scope_id), - )) - .expect("the runtime Start should commit durably"); - assert_eq!( - repo.adapter_state().attempts[0].phase, - state::AttemptPhase::PendingStart - ); - - repo.drive(&crashed.post()) - .expect("D11: a committed Start with unsettled bookkeeping must be abandoned"); - - assert!(repo.adapter_state().attempts.is_empty()); - assert!(!repo.adapter_state().recovery.is_clear()); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "abandoned".to_string())), - "AC21b: the committed Start settles as a real abandonment, not a late Start" - ); - assert!(worktree_row(&db, &repo.worktree_id()) - .is_some_and(|(_, _, needs_rebaseline)| needs_rebaseline)); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test19_close_committed_before_state_cleanup_is_replay_safe_ac21c() { - let repo = CodexRepo::new("test19-crash-after-close"); - let cwd = repo.cwd(); - let call = bash_call(&cwd, "session-close", "exec-close"); - - repo.drive(&call.pre()).expect("PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - repo.write("file.txt", "one\ntwo\n"); - - repo.drive_generic(&scope_boundary_payload( - "close", - &scope_id, - &codex_scope_close_event_id(&scope_id), - )) - .expect("the runtime Close should commit durably"); - assert_eq!(repo.adapter_state().attempts.len(), 1); - - let db = repo.db(); - let (revision_before, events_before) = ( - worktree_row(&db, &repo.worktree_id()) - .map(|(revision, _, _)| revision) - .expect("a worktree row should exist"), - count(&db, "mutation_trace_events"), - ); - - repo.drive(&call.post()) - .expect("a replayed Close against an already-durable commit must be safe"); - - assert!( - repo.adapter_state().attempts.is_empty(), - "the stale bookkeeping is finally cleared" - ); - - let db = repo.db(); - assert_eq!( - worktree_row(&db, &repo.worktree_id()).map(|(revision, _, _)| revision), - Some(revision_before), - "AC21c: a durably completed Close is never re-applied as a second transition" - ); - assert_eq!(count(&db, "mutation_trace_events"), events_before); - assert_eq!( - scope_status(&db, &scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test20_recovery_pending_blocks_a_tracked_successor_until_recovery_succeeds_ac12() { - let repo = CodexRepo::new("test20-recovery-barrier"); - let cwd = repo.cwd(); - let first = bash_call(&cwd, "session-a", "exec-a"); - let second = bash_call(&cwd, "session-b", "exec-b"); - let blocked = bash_call(&cwd, "session-c", "exec-c"); - - repo.drive(&first.pre()).expect("session-a Start"); - repo.drive(&second.pre()).expect("session-b Start"); - assert_eq!(repo.adapter_state().attempts.len(), 2); - - repo.write("file.txt", "one\nabandoned\n"); - repo.drive(&turn_event_json( - HOOK_EVENT_INTERRUPT, - &cwd, - "session-a", - "turn-1", - )) - .expect("Interrupt should retire session-a's attempt"); - assert!(!repo.adapter_state().recovery.is_clear()); - assert_eq!(repo.adapter_state().attempts.len(), 1); - - assert_eq!( - repo.drive(&blocked.pre()) - .expect("a barred PreToolUse still returns Ok with a deny payload"), - pre_tool_use_deny_json(FAIL_CLOSED_DENY_REASON), - "AC12: while recovery is armed and attempts remain, a tracked successor is denied" - ); - assert!( - repo.adapter_state() - .attempts - .iter() - .all(|attempt| attempt.tool_use_id != "exec-c"), - "the denied successor must never be admitted" - ); - - repo.drive(&session_end_json(&cwd, "session-b")) - .expect("SessionEnd should retire session-b's attempt"); - assert!(repo.adapter_state().attempts.is_empty()); - assert!(!repo.adapter_state().recovery.is_clear()); - - repo.drive(&blocked.pre()) - .expect("once quiescent, the flush runs and the successor Starts"); - assert!( - repo.adapter_state().recovery.is_clear(), - "AC12: recovery_pending clears only on durable flush success" - ); - let scope_id = repo.live_scope_id(); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &scope_id).map(|(_, status)| status), - Some("active".to_string()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test21_reused_tool_use_id_after_terminal_gets_a_fresh_scope_id_ac5() { - let repo = CodexRepo::new("test21-reused-identifier"); - let cwd = repo.cwd(); - let call = bash_call(&cwd, "session-reuse", "exec-reused"); - - repo.drive(&call.pre()).expect("first PreToolUse"); - let first_scope_id = repo.live_scope_id(); - repo.write("file.txt", "one\nfirst\n"); - repo.drive(&call.post()).expect("first PostToolUse"); - assert!(repo.adapter_state().attempts.is_empty()); - - repo.drive(&call.pre()) - .expect("a later attempt reusing the same tool_use_id"); - let second_scope_id = repo.live_scope_id(); - assert_ne!( - first_scope_id, second_scope_id, - "AC5: a terminal ScopeId is never reused" - ); - - repo.write("file.txt", "one\nfirst\nsecond\n"); - repo.drive(&call.post()).expect("second PostToolUse"); - - let db = repo.db(); - assert_eq!( - scope_status(&db, &first_scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - assert_eq!( - scope_status(&db, &second_scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test22_self_detaching_descendant_write_is_not_folded_into_the_closed_scope_ac15() { - let repo = CodexRepo::new("test22-detached-descendant"); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(PROBE09_DETACHED_PRE, &cwd)) - .expect("PreToolUse should succeed"); - let scope_id = repo.live_scope_id(); - - repo.write("file.txt", "one\nforeground\n"); - let tree_at_close = repo.working_tree(); - repo.drive(&fixture_at(PROBE09_DETACHED_POST, &cwd)) - .expect("PostToolUse should succeed"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!( - worktree_row(&db, &worktree_id).map(|(_, cursor_tree, _)| cursor_tree), - Some(tree_at_close.clone()) - ); - let events_before_flush = mutation_events_for(&db, &worktree_id); - - repo.write("file.txt", "one\nforeground\ndetached-descendant\n"); - let tree_after_descendant = repo.working_tree(); - assert_ne!(tree_after_descendant, tree_at_close); - - repo.drive_flush() - .expect("a later diagnostic flush should succeed"); - - let db = repo.db(); - let events_after_flush = mutation_events_for(&db, &worktree_id); - assert_eq!(events_after_flush.len(), events_before_flush.len() + 1); - let (attribution_kind, attribution_scope_id, _) = events_after_flush - .last() - .expect("a flush event should exist"); - assert_eq!( - attribution_kind, "ineligible_unscoped", - "AC15/D16: SCE does not supervise self-detaching descendants; \ - a post-terminal write is never folded into the closed tool scope" - ); - assert_ne!(attribution_scope_id.as_deref(), Some(scope_id.as_str())); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test23_denied_tracked_execution_leaves_no_untracked_start_ac7() { - let repo = CodexRepo::new("test23-policy-denied"); - let cwd = repo.cwd(); - fs::create_dir_all(repo.root.join(".sce")).expect(".sce dir should be created"); - fs::write( - repo.root.join(".sce").join("config.json"), - concat!( - r#"{"policies":{"bash":{"custom":[{"id":"no-rm","#, - r#""match":{"argv_prefix":["rm"]},"#, - r#""message":"rm is blocked in this repository"}]}}}"#, - ), - ) - .expect("repo bash policy config should write"); - - let denied = tool_event_json( - &ToolEvent { - event_name: HOOK_EVENT_PRE_TOOL_USE, - cwd: &cwd, - session_id: "session-denied", - turn_id: "turn-1", - tool_name: CODEX_TRACKED_TOOL_BASH, - tool_use_id: "exec-denied", - agent_id: None, - }, - Some(json!({ "command": "rm -rf build" })), - ); - - let response = repo - .drive(&denied) - .expect("a policy-denied Bash command still returns Ok with a deny payload"); - assert!(response.contains(r#""permissionDecision":"deny""#)); - - assert!( - !repo.adapter_state_file_exists(), - "AC7: a denied tracked execution must never leave an untracked Start behind" - ); - - let db = repo.db(); - assert_eq!(count(&db, "mutation_trace_scopes"), 0); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test24_cross_harness_overlap_at_a_non_confirming_boundary_is_ineligible_ac10() { - let repo = CodexRepo::new("test24-cross-harness-ineligible"); - let cwd = repo.cwd(); - let codex_call = bash_call(&cwd, "session-cross", "exec-codex"); - let other_scope_id = "claude-scope-1"; - - repo.drive_generic(&other_harness_payload("start", other_scope_id)) - .expect("the other harness's Start should commit"); - repo.drive(&codex_call.pre()) - .expect("the Codex PreToolUse should Start"); - let codex_scope_id = repo.live_scope_id(); - - repo.write("file.txt", "one\ncontended\n"); - - repo.drive_generic(&other_harness_payload("close", other_scope_id)) - .expect("the other harness's Close should commit"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![("ineligible_unscoped".to_string(), None, "close".to_string())], - "AC10/D14: an unconfirmed live Codex scope forces IneligibleUnscoped at a \ - boundary that does not confirm it — never AiContended" - ); - let mut active = active_scopes_for(&db, &worktree_id); - active.sort(); - let mut expected = vec![codex_scope_id, other_scope_id.to_string()]; - expected.sort(); - assert_eq!( - active, expected, - "active_scopes still records the complete live set; only eligibility changes" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test25_cross_harness_overlap_at_the_codex_close_is_contended_ac10() { - let repo = CodexRepo::new("test25-cross-harness-contended"); - let cwd = repo.cwd(); - let codex_call = bash_call(&cwd, "session-cross", "exec-codex"); - let other_scope_id = "claude-scope-1"; - - repo.drive_generic(&other_harness_payload("start", other_scope_id)) - .expect("the other harness's Start should commit"); - repo.drive(&codex_call.pre()) - .expect("the Codex PreToolUse should Start"); - let codex_scope_id = repo.live_scope_id(); - - repo.write("file.txt", "one\ncontended\n"); - - repo.drive(&codex_call.post()) - .expect("the Codex PostToolUse should close its scope"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![("ai_contended".to_string(), None, "close".to_string())], - "AC10/D14: the Codex scope's own Close confirms it, so the overlap with the \ - other harness's live scope is attributed AiContended" - ); - assert_eq!( - scope_status(&db, &codex_scope_id).map(|(_, status)| status), - Some("closed".to_string()) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test26_a_second_unconfirmed_codex_scope_suppresses_contention_ac10() { - let repo = CodexRepo::new("test26-second-codex-scope"); - let cwd = repo.cwd(); - let confirmed = bash_call(&cwd, "session-one", "exec-one"); - let unconfirmed = bash_call(&cwd, "session-two", "exec-two"); - let other_scope_id = "claude-scope-1"; - - repo.drive_generic(&other_harness_payload("start", other_scope_id)) - .expect("the other harness's Start should commit"); - repo.drive(&confirmed.pre()) - .expect("the first Codex PreToolUse should Start"); - repo.drive(&unconfirmed.pre()) - .expect("a second Codex lane's PreToolUse should Start"); - assert_eq!(repo.adapter_state().attempts.len(), 2); - - repo.write("file.txt", "one\ncontended\n"); - - repo.drive(&confirmed.post()) - .expect("the first Codex scope's Close should commit"); - - let db = repo.db(); - let worktree_id = repo.worktree_id(); - assert_eq!( - mutation_events_for(&db, &worktree_id), - vec![("ineligible_unscoped".to_string(), None, "close".to_string())], - "AC10/D14: a second unconfirmed live Codex scope suppresses attribution back to \ - IneligibleUnscoped even at a confirming Codex Close" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test27_tracked_fixtures_persist_scope_provenance_ac3() { - for (label, fixture) in [ - ("bash", PROBE01_SHELL_PRE), - ("apply-patch", PROBE01_APPLY_PATCH_PRE), - ] { - let repo = CodexRepo::new(&format!("test27-provenance-{label}")); - let cwd = repo.cwd(); - - repo.drive(&fixture_at(fixture, &cwd)) - .expect("a tracked PreToolUse should Start"); - let scope_id = repo.live_scope_id(); - - let db = repo.db(); - assert_eq!( - scope_provenance(&db, &scope_id), - Some(( - "cx_01a07c1e-e08e-7172-8032-cb9d62af21d9".to_string(), - Some("gpt-5.6-sol".to_string()) - )), - "AC3: the {label} fixture must persist its cx_ session and normalized model" - ); - assert_eq!(count(&db, "mutation_trace_scope_provenance"), 1); - assert_eq!( - scope_status(&db, &scope_id), - Some(("codex".to_string(), "active".to_string())) - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - } - - #[test] - fn test28_a_tracked_execution_without_a_model_persists_a_null_model_ac3() { - let repo = CodexRepo::new("test28-provenance-no-model"); - let cwd = repo.cwd(); - let call = bash_call(&cwd, "session-no-model", "exec-no-model"); - - repo.drive(&call.pre()) - .expect("a tracked PreToolUse without a model should still Start"); - let scope_id = repo.live_scope_id(); - - let db = repo.db(); - assert_eq!( - scope_provenance(&db, &scope_id), - Some(("cx_session-no-model".to_string(), None)), - "AC3: a missing model records model_id = NULL without losing the session" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test29_untracked_and_delegation_tools_persist_no_provenance_ac3() { - let repo = CodexRepo::new("test29-untracked-no-provenance"); - let cwd = repo.cwd(); - - for fixture in [ - PROBE12_MCP_PRE, - PROBE08_SPAWN_AGENT_PRE, - PROBE08_WAIT_AGENT_PRE, - ] { - assert_eq!( - repo.drive(&fixture_at(fixture, &cwd)) - .expect("an untracked or delegation PreToolUse should succeed"), - "" - ); - } - - let db = repo.db(); - assert_eq!(count(&db, "mutation_trace_scopes"), 0); - assert_eq!(count(&db, "mutation_trace_scope_provenance"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } -} diff --git a/cli/src/services/hooks/command.rs b/cli/src/services/hooks/command.rs index 5aa0cc6ce..abfebbfd0 100644 --- a/cli/src/services/hooks/command.rs +++ b/cli/src/services/hooks/command.rs @@ -7,8 +7,9 @@ pub struct HooksCommand { } impl HooksCommand { - pub fn execute(&self, context: &C) -> Result { + pub async fn execute(&self, context: &C) -> Result { hooks::run_hooks_subcommand(&self.subcommand, Some(context.logger())) + .await .map_err(CliError::runtime) } } diff --git a/cli/src/services/hooks/commit_hooks.rs b/cli/src/services/hooks/commit_hooks.rs index 960f298de..26d204df5 100644 --- a/cli/src/services/hooks/commit_hooks.rs +++ b/cli/src/services/hooks/commit_hooks.rs @@ -14,7 +14,6 @@ use crate::services::agent_trace_db::{ AgentTraceInsert, PostCommitPatchIntersectionInsert, RecentDiffTracePatches, }; use crate::services::config; -use crate::services::observability::traits::Logger; use crate::services::patch::{ combine_patches as combine_patches_fn, intersect_patches as intersect_patches_fn, parse_patch as parse_patch_from_text, ParsedPatch, @@ -42,10 +41,12 @@ pub(crate) fn run_pre_commit_subcommand(repository_root: &Path) -> Result( repository_root: &Path, message_file: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { let metadata = fs::metadata(message_file).with_context(|| { format!( @@ -71,7 +72,7 @@ pub(crate) fn run_commit_msg_subcommand_in_repo( let gate_passed = commit_msg_policy_gate_passed(&runtime); let ai_contribution_present = if gate_passed { - match staged_diff_has_ai_overlap(repository_root, logger) { + match staged_diff_has_ai_overlap(repository_root, logger).await { StagedDiffAiOverlapResult::Overlap => true, StagedDiffAiOverlapResult::NoOverlap | StagedDiffAiOverlapResult::Error => false, } @@ -99,20 +100,24 @@ pub(crate) fn run_commit_msg_subcommand_in_repo( )) } -pub(crate) fn run_commit_msg_subcommand_with_trace( +pub(crate) async fn run_commit_msg_subcommand_with_trace< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, _: &HookSubcommand, message_file: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { - run_commit_msg_subcommand_in_repo(repository_root, message_file, logger) + run_commit_msg_subcommand_in_repo(repository_root, message_file, logger).await } -pub(crate) fn run_post_commit_subcommand( +pub(crate) async fn run_post_commit_subcommand< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, vcs_type: Option, remote_url: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { run_post_commit_subcommand_with( repository_root, @@ -130,36 +135,44 @@ pub(crate) fn run_post_commit_subcommand( run_post_commit_passive_checkpoint, logger, ) + .await } #[allow(clippy::too_many_arguments)] -pub(crate) fn run_post_commit_subcommand_with( +pub(crate) async fn run_post_commit_subcommand_with< + L: crate::services::observability::traits::Logger, + F, + B, + C, + Launch, + K, +>( repository_root: &Path, vcs_type: Option, remote_url: &str, run_intersection_flow: F, run_agent_trace_flow: B, resolve_auto_sync: C, - launch_auto_sync: L, + launch_auto_sync: Launch, run_passive_checkpoint: K, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result where - F: FnOnce(&Path) -> Result, - B: FnOnce( + F: std::ops::AsyncFnOnce(&Path) -> Result, + B: std::ops::AsyncFnOnce( &Path, &PostCommitIntersectionFlowResult, Option, &str, ) -> Result, C: FnOnce(&Path) -> Result, - L: FnOnce(&Path) -> Result<()>, - K: FnOnce(&Path) -> Result<()>, + Launch: FnOnce(&Path) -> Result<()>, + K: std::ops::AsyncFnOnce(&Path) -> Result<()>, { - let result = run_intersection_flow(repository_root)?; - let _agent_trace = run_agent_trace_flow(repository_root, &result, vcs_type, remote_url)?; + let result = run_intersection_flow(repository_root).await?; + let _agent_trace = run_agent_trace_flow(repository_root, &result, vcs_type, remote_url).await?; - if let Err(error) = run_passive_checkpoint(repository_root) { + if let Err(error) = run_passive_checkpoint(repository_root).await { if let Some(log) = logger { log.warn( "sce.agent_trace_db.passive_checkpoint_failed", @@ -181,25 +194,27 @@ where )) } -pub(crate) fn run_post_commit_passive_checkpoint(repository_root: &Path) -> Result<()> { +pub(crate) async fn run_post_commit_passive_checkpoint(repository_root: &Path) -> Result<()> { let db = open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for post-commit checkpoint.", - )?; + ) + .await?; - db.passive_checkpoint() + db.passive_checkpoint().await } -pub(crate) fn run_post_commit_agent_trace_flow( +pub(crate) async fn run_post_commit_agent_trace_flow( repository_root: &Path, flow_result: &PostCommitIntersectionFlowResult, vcs_type: Option, remote_url: &str, ) -> Result { - let db = open_agent_trace_db_for_hook_runtime( + let mut db = open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for post-commit trace.", - )?; + ) + .await?; let direct_intersection = intersect_patches_fn( &flow_result.combined_recent_patch, @@ -208,10 +223,11 @@ pub(crate) fn run_post_commit_agent_trace_flow( let mutation_ai_patch = crate::services::mutation_trace::runtime::resolve_post_commit_mutation_ai_patch( repository_root, - &db, + &mut db, &direct_intersection, &flow_result.post_commit_data.parsed_patch, - ); + ) + .await; run_post_commit_agent_trace_flow_with( flow_result, @@ -225,16 +241,18 @@ pub(crate) fn run_post_commit_agent_trace_flow( Ok(()) }, - |insert_input| { + async |insert_input| { db.insert_agent_trace(insert_input) + .await .context("Failed to persist built post-commit Agent Trace payload.")?; Ok(()) }, ) + .await } -pub(crate) fn run_post_commit_agent_trace_flow_with( +pub(crate) async fn run_post_commit_agent_trace_flow_with( flow_result: &PostCommitIntersectionFlowResult, vcs_type: Option, remote_url: &str, @@ -244,7 +262,7 @@ pub(crate) fn run_post_commit_agent_trace_flow_with( ) -> Result where V: FnOnce(&Value) -> Result<()>, - I: for<'a> FnOnce(AgentTraceInsert<'a>) -> Result<()>, + I: for<'a> std::ops::AsyncFnOnce(AgentTraceInsert<'a>) -> Result<()>, { let commit_timestamp = DateTime::::from_timestamp_millis(flow_result.post_commit_data.commit_time_ms) @@ -293,36 +311,42 @@ where url: &constructed_url, remote_url, }; - persist_agent_trace(insert_input)?; + persist_agent_trace(insert_input).await?; Ok(agent_trace) } pub(crate) const RECENT_DAYS_MILLIS: i64 = 7 * 24 * 60 * 60 * 1000; -pub(crate) fn run_post_commit_intersection_flow( +pub(crate) async fn run_post_commit_intersection_flow( repository_root: &Path, ) -> Result { let db = open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for post-commit intersection.", - )?; + ) + .await?; run_post_commit_intersection_flow_with( repository_root, capture_post_commit_patch_from_git, current_unix_time_ms, - |cutoff_ms, end_ms| { - db.recent_diff_trace_patches(cutoff_ms, end_ms) - .context("Failed to query recent diff trace patches.") + async |cutoff_ms, end_ms| { + { + db.recent_diff_trace_patches(cutoff_ms, end_ms) + .await + .context("Failed to query recent diff trace patches.") + } }, - |insert_input| { + async |insert_input| { db.insert_post_commit_patch_intersection(insert_input) + .await .context("Failed to persist post-commit patch intersection.")?; Ok(()) }, ) + .await } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -332,14 +356,17 @@ pub(crate) enum StagedDiffAiOverlapResult { Error, } -pub(crate) fn staged_diff_has_ai_overlap( +pub(crate) async fn staged_diff_has_ai_overlap< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> StagedDiffAiOverlapResult { let db_open_result = open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for staged AI-overlap evidence check.", - ); + ) + .await; let db = match db_open_result { Ok(db) => db, @@ -360,8 +387,9 @@ pub(crate) fn staged_diff_has_ai_overlap( repository_root, capture_staged_patch_from_git, current_unix_time_ms, - |cutoff_ms, end_ms| db.recent_diff_trace_patches(cutoff_ms, end_ms), - ); + async |cutoff_ms, end_ms| db.recent_diff_trace_patches(cutoff_ms, end_ms).await, + ) + .await; if result == StagedDiffAiOverlapResult::Error { if let Some(log) = logger { @@ -377,7 +405,7 @@ pub(crate) fn staged_diff_has_ai_overlap( result } -pub(crate) fn staged_diff_has_ai_overlap_with( +pub(crate) async fn staged_diff_has_ai_overlap_with( repository_root: &Path, capture_staged_patch: C, now_ms: N, @@ -386,7 +414,7 @@ pub(crate) fn staged_diff_has_ai_overlap_with( where C: FnOnce(&Path) -> Result, N: FnOnce() -> Result, - Q: FnOnce(i64, i64) -> Result, + Q: std::ops::AsyncFnOnce(i64, i64) -> Result, { let Ok(staged_patch) = capture_staged_patch(repository_root) else { return StagedDiffAiOverlapResult::Error; @@ -401,7 +429,7 @@ where }; let cutoff_ms = now_ms - RECENT_DAYS_MILLIS; - let Ok(recent_patches) = query_recent_patches(cutoff_ms, now_ms) else { + let Ok(recent_patches) = query_recent_patches(cutoff_ms, now_ms).await else { return StagedDiffAiOverlapResult::Error; }; @@ -444,7 +472,7 @@ pub(crate) fn staged_patch_error(detail: &str, context: &str) -> String { format!("Staged patch capture error: {detail} ({context}).") } -pub(crate) fn run_post_commit_intersection_flow_with( +pub(crate) async fn run_post_commit_intersection_flow_with( repository_root: &Path, capture_post_commit_patch: C, now_ms: N, @@ -454,15 +482,15 @@ pub(crate) fn run_post_commit_intersection_flow_with( where C: FnOnce(&Path) -> Result, N: FnOnce() -> Result, - Q: FnOnce(i64, i64) -> Result, - P: for<'a> FnOnce(PostCommitPatchIntersectionInsert<'a>) -> Result<()>, + Q: std::ops::AsyncFnOnce(i64, i64) -> Result, + P: for<'a> std::ops::AsyncFnOnce(PostCommitPatchIntersectionInsert<'a>) -> Result<()>, { let post_commit_data = capture_post_commit_patch(repository_root)?; let now_ms = now_ms()?; let cutoff_ms = now_ms - RECENT_DAYS_MILLIS; - let recent_patches = query_recent_patches(cutoff_ms, now_ms)?; + let recent_patches = query_recent_patches(cutoff_ms, now_ms).await?; #[allow(clippy::cast_possible_wrap)] let loaded_count = recent_patches.loaded_count() as i64; @@ -497,7 +525,7 @@ where intersection_patch: &serialized_intersection, }; - persist_intersection(insert_input)?; + persist_intersection(insert_input).await?; Ok(PostCommitIntersectionFlowResult { combined_recent_patch, @@ -507,11 +535,13 @@ where }) } -pub(crate) fn run_post_commit_subcommand_with_trace( +pub(crate) async fn run_post_commit_subcommand_with_trace< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, vcs_type: Option, remote_url: Option<&str>, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { run_post_commit_subcommand( repository_root, @@ -519,6 +549,7 @@ pub(crate) fn run_post_commit_subcommand_with_trace( remote_url.unwrap_or_default(), logger, ) + .await } pub(crate) fn run_post_rewrite_subcommand( diff --git a/cli/src/services/hooks/conversation_trace.rs b/cli/src/services/hooks/conversation_trace.rs index e784fcfd7..739bbd826 100644 --- a/cli/src/services/hooks/conversation_trace.rs +++ b/cli/src/services/hooks/conversation_trace.rs @@ -6,7 +6,6 @@ use serde_json::{to_string as serialize_to_json, Value}; use crate::services::agent_trace_db::{ InsertMessageInsert, InsertPartInsert, MessageRole, PartType, }; -use crate::services::observability::traits::Logger; use crate::services::patch::{load_patch_from_json, parse_patch as parse_patch_from_text}; use super::claude_transforms::{ @@ -65,9 +64,11 @@ impl ConversationTracePersistenceSummary { ) } } -pub(crate) fn run_conversation_trace_subcommand( +pub(crate) async fn run_conversation_trace_subcommand< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> String { let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, @@ -80,16 +81,20 @@ pub(crate) fn run_conversation_trace_subcommand( &stdin_payload, logger, session_id.as_deref(), - ) { + ) + .await + { Ok(output) => output, Err(error) => log_conversation_trace_fail_open(&error, logger, session_id.as_deref()), } } -pub(crate) fn run_conversation_trace_subcommand_from_payload( +pub(crate) async fn run_conversation_trace_subcommand_from_payload< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, session_id: Option<&str>, ) -> Result { let payload = parse_conversation_trace_payload(stdin_payload)?; @@ -98,12 +103,15 @@ pub(crate) fn run_conversation_trace_subcommand_from_payload( payload, logger, session_id, - )) + ) + .await) } -pub(crate) fn log_conversation_trace_fail_open( +pub(crate) fn log_conversation_trace_fail_open< + L: crate::services::observability::traits::Logger, +>( error: &anyhow::Error, - logger: Option<&dyn Logger>, + logger: Option<&L>, session_id: Option<&str>, ) -> String { if let Some(log) = logger { @@ -118,16 +126,20 @@ pub(crate) fn log_conversation_trace_fail_open( String::from("conversation-trace hook intake failed open; error logged.") } -pub(crate) fn persist_conversation_trace_payload_to_agent_trace_db( +pub(crate) async fn persist_conversation_trace_payload_to_agent_trace_db< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, payload: ConversationTracePayload, - logger: Option<&dyn Logger>, + logger: Option<&L>, session_id: Option<&str>, ) -> String { let db = match open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for conversation-trace persistence.", - ) { + ) + .await + { Ok(db) => db, Err(error) => { if let Some(log) = logger { @@ -146,21 +158,26 @@ pub(crate) fn persist_conversation_trace_payload_to_agent_trace_db( let summary = persist_conversation_trace_payload_to_agent_trace_db_with( payload, logger, - |inserts| db.insert_messages(inserts), - |inserts| db.insert_parts(inserts), - ); + async |inserts| db.insert_messages(inserts).await, + async |inserts| db.insert_parts(inserts).await, + ) + .await; summary.render() } -pub(crate) fn persist_conversation_trace_payload_to_agent_trace_db_with( +pub(crate) async fn persist_conversation_trace_payload_to_agent_trace_db_with< + L: crate::services::observability::traits::Logger, + IM, + IP, +>( payload: ConversationTracePayload, - logger: Option<&dyn Logger>, + logger: Option<&L>, insert_messages: IM, insert_parts: IP, ) -> ConversationTracePersistenceSummary where - IM: FnOnce(Vec) -> Result, - IP: FnOnce(Vec) -> Result, + IM: std::ops::AsyncFnOnce(Vec) -> Result, + IP: std::ops::AsyncFnOnce(Vec) -> Result, { log_skipped_conversation_trace_payloads(logger, "unsupported", &payload.skipped); @@ -168,12 +185,14 @@ where payload.message_updated, logger, insert_messages, - ); + ) + .await; let part_summary = persist_message_part_updated_batch_to_agent_trace_db_with( payload.message_part_updated, logger, insert_parts, - ); + ) + .await; ConversationTracePersistenceSummary { attempted: payload.attempted_count, @@ -188,13 +207,16 @@ pub(crate) struct ConversationTraceEventPersistenceSummary { skipped: usize, } -pub(crate) fn persist_message_updated_batch_to_agent_trace_db_with( +pub(crate) async fn persist_message_updated_batch_to_agent_trace_db_with< + L: crate::services::observability::traits::Logger, + I, +>( batch: ConversationTraceMessageBatch, - logger: Option<&dyn Logger>, + logger: Option<&L>, insert_messages: I, ) -> ConversationTraceEventPersistenceSummary where - I: FnOnce(Vec) -> Result, + I: std::ops::AsyncFnOnce(Vec) -> Result, { const EVENT_TYPE: &str = "message"; @@ -207,7 +229,7 @@ where let persisted = if valid_count == 0 { 0 } else { - match insert_messages(batch.inserts) { + match insert_messages(batch.inserts).await { Ok(affected_rows) => usize::try_from(affected_rows) .unwrap_or(usize::MAX) .min(valid_count), @@ -228,13 +250,16 @@ where ConversationTraceEventPersistenceSummary { persisted, skipped } } -pub(crate) fn persist_message_part_updated_batch_to_agent_trace_db_with( +pub(crate) async fn persist_message_part_updated_batch_to_agent_trace_db_with< + L: crate::services::observability::traits::Logger, + I, +>( batch: ConversationTracePartBatch, - logger: Option<&dyn Logger>, + logger: Option<&L>, insert_parts: I, ) -> ConversationTraceEventPersistenceSummary where - I: FnOnce(Vec) -> Result, + I: std::ops::AsyncFnOnce(Vec) -> Result, { const EVENT_TYPE: &str = "message.part"; @@ -247,7 +272,7 @@ where let persisted = if valid_count == 0 { 0 } else { - match insert_parts(batch.inserts) { + match insert_parts(batch.inserts).await { Ok(affected_rows) => usize::try_from(affected_rows) .unwrap_or(usize::MAX) .min(valid_count), @@ -268,8 +293,10 @@ where ConversationTraceEventPersistenceSummary { persisted, skipped } } -pub(crate) fn log_skipped_conversation_trace_payloads( - logger: Option<&dyn Logger>, +pub(crate) fn log_skipped_conversation_trace_payloads< + L: crate::services::observability::traits::Logger, +>( + logger: Option<&L>, event_type: &str, skipped_payloads: &[SkippedConversationTracePayload], ) { @@ -291,8 +318,10 @@ pub(crate) fn log_skipped_conversation_trace_payloads( } } -pub(crate) fn log_conversation_trace_batch_insert_failure( - logger: Option<&dyn Logger>, +pub(crate) fn log_conversation_trace_batch_insert_failure< + L: crate::services::observability::traits::Logger, +>( + logger: Option<&L>, event_type: &str, valid_count: usize, error: &anyhow::Error, diff --git a/cli/src/services/hooks/diff_trace.rs b/cli/src/services/hooks/diff_trace.rs index 356fd8482..65ee8e892 100644 --- a/cli/src/services/hooks/diff_trace.rs +++ b/cli/src/services/hooks/diff_trace.rs @@ -10,7 +10,6 @@ use crate::services::agent_trace_db::{ ClaudeModelStateObservation, DiffTraceInsert, ObservationKind, PAYLOAD_TYPE_PATCH, PAYLOAD_TYPE_STRUCTURED, }; -use crate::services::observability::traits::Logger; use crate::services::structured_patch::{ derive_claude_structured_patch, ClaudeStructuredPatchDerivationResult, }; @@ -64,9 +63,9 @@ impl StdinPayloadKind { format!("Invalid {} payload from STDIN: {detail}.", self.label()) } } -pub(crate) fn run_diff_trace_subcommand( +pub(crate) async fn run_diff_trace_subcommand( repository_root: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> String { let stdin_payload = match read_hook_stdin() { Ok(payload) => payload, @@ -74,32 +73,30 @@ pub(crate) fn run_diff_trace_subcommand( }; let session_id = diff_trace_fail_open_session_id(&stdin_payload); - match run_diff_trace_subcommand_from_payload(repository_root, &stdin_payload, logger) { + match run_diff_trace_subcommand_from_payload(repository_root, &stdin_payload, logger).await { Ok(output) => output, Err(error) => log_diff_trace_fail_open(&error, logger, session_id.as_deref()), } } -pub(crate) fn run_diff_trace_subcommand_from_payload( +pub(crate) async fn run_diff_trace_subcommand_from_payload< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { let parse_result = parse_diff_trace_payload(stdin_payload)?; let payload = match parse_result { DiffTraceParseResult::Persist(payload) => payload, DiffTraceParseResult::NoOp(message) => return Ok(message), }; - Ok(run_diff_trace_subcommand_from_payload_with( - repository_root, - &payload, - logger, - )) + Ok(run_diff_trace_subcommand_from_payload_with(repository_root, &payload, logger).await) } -pub(crate) fn log_diff_trace_fail_open( +pub(crate) fn log_diff_trace_fail_open( error: &anyhow::Error, - logger: Option<&dyn Logger>, + logger: Option<&L>, session_id: Option<&str>, ) -> String { if let Some(log) = logger { @@ -114,10 +111,12 @@ pub(crate) fn log_diff_trace_fail_open( String::from("diff-trace hook intake failed open; error logged.") } -pub(crate) fn run_diff_trace_subcommand_from_payload_with( +pub(crate) async fn run_diff_trace_subcommand_from_payload_with< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, payload: &DiffTracePayload, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> String { if let Err(error) = diff_trace_db_time_ms(payload.time) { if let Some(log) = logger { @@ -129,21 +128,26 @@ pub(crate) fn run_diff_trace_subcommand_from_payload_with( ); } } - let agent_trace_db_persisted = - match persist_diff_trace_payload_to_agent_trace_db(repository_root, payload, logger) { - Ok(persisted) => persisted, - Err(error) => { - if let Some(log) = logger { - log.warn( - "sce.hooks.diff_trace.agent_trace_db_write_failed", - &error.to_string(), - &[], - Some(&payload.session_id), - ); - } - false + let agent_trace_db_persisted = match persist_diff_trace_payload_to_agent_trace_db( + repository_root, + payload, + logger, + ) + .await + { + Ok(persisted) => persisted, + Err(error) => { + if let Some(log) = logger { + log.warn( + "sce.hooks.diff_trace.agent_trace_db_write_failed", + &error.to_string(), + &[], + Some(&payload.session_id), + ); } - }; + false + } + }; if agent_trace_db_persisted { String::from("diff-trace hook intake persisted payload to AgentTraceDb.") @@ -473,15 +477,19 @@ pub(crate) fn required_u64_millisecond_field( ))) } -pub(crate) fn persist_diff_trace_payload_to_agent_trace_db( +pub(crate) async fn persist_diff_trace_payload_to_agent_trace_db< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, payload: &DiffTracePayload, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { let db = match open_agent_trace_db_for_hook_runtime( repository_root, "Failed to open Agent Trace DB for diff-trace persistence.", - ) { + ) + .await + { Ok(db) => db, Err(error) => { if let Some(log) = logger { @@ -497,15 +505,15 @@ pub(crate) fn persist_diff_trace_payload_to_agent_trace_db( } }; - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, payload)?; + persist_diff_trace_payload_to_agent_trace_db_with_db(&db, payload).await?; Ok(true) } -pub(crate) fn persist_diff_trace_payload_to_agent_trace_db_with_db( +pub(crate) async fn persist_diff_trace_payload_to_agent_trace_db_with_db( db: &RepositoryAgentTraceDb, payload: &DiffTracePayload, ) -> Result<()> { - let model_id = resolve_diff_trace_model_id(db, payload)?; + let model_id = resolve_diff_trace_model_id(db, payload).await?; db.insert_diff_trace(DiffTraceInsert { time_ms: diff_trace_db_time_ms(payload.time)?, session_id: &prefixed_diff_trace_session_id(&payload.tool_name, &payload.session_id), @@ -515,12 +523,13 @@ pub(crate) fn persist_diff_trace_payload_to_agent_trace_db_with_db( tool_version: payload.tool_version.as_deref(), payload_type: &payload.payload_type, }) + .await .context("Failed to persist diff-trace payload to Agent Trace DB.")?; Ok(()) } -pub(crate) fn resolve_diff_trace_model_id( +pub(crate) async fn resolve_diff_trace_model_id( db: &RepositoryAgentTraceDb, payload: &DiffTracePayload, ) -> Result> { @@ -533,19 +542,17 @@ pub(crate) fn resolve_diff_trace_model_id( let session_id = prefixed_diff_trace_session_id(CLAUDE_TOOL_NAME, &payload.session_id); let agent_id = payload.agent_id.as_deref().unwrap_or(""); - if let Some(state) = db.claude_model_state_by_session_and_agent(&session_id, agent_id)? { + if let Some(state) = db + .claude_model_state_by_session_and_agent(&session_id, agent_id) + .await? + { return Ok(Some(state.model_id)); } - Ok(seed_diff_trace_model_from_bridge_chain( - db, - payload, - &session_id, - agent_id, - )) + Ok(seed_diff_trace_model_from_bridge_chain(db, payload, &session_id, agent_id).await) } -pub(crate) fn seed_diff_trace_model_from_bridge_chain( +pub(crate) async fn seed_diff_trace_model_from_bridge_chain( db: &RepositoryAgentTraceDb, payload: &DiffTracePayload, session_id: &str, @@ -556,46 +563,26 @@ pub(crate) fn seed_diff_trace_model_from_bridge_chain( } let transcript_path = payload.transcript_path.as_deref()?; - let model_id = claude_model_state::newest_bridge_chain_model(db, Path::new(transcript_path))?; + let model_id = + claude_model_state::newest_bridge_chain_model(db, Path::new(transcript_path)).await?; let observed_at_ms = current_unix_time_ms().ok()?; - match db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: session_id.to_string(), - agent_id: String::new(), - model_id: model_id.clone(), - observation_kind: ObservationKind::SessionStart, - source: String::from("bridge_inherited"), - observed_at_ms, - }) { + match db + .upsert_claude_model_state(ClaudeModelStateObservation { + session_id: session_id.to_string(), + agent_id: String::new(), + model_id: model_id.clone(), + observation_kind: ObservationKind::SessionStart, + source: String::from("bridge_inherited"), + observed_at_ms, + }) + .await + { Ok(_) => Some(model_id), Err(_) => None, } } -#[cfg(test)] -pub(crate) fn persist_diff_trace_payload_to_agent_trace_db_with( - payload: &DiffTracePayload, - model_id: Option<&str>, - tool_version: Option<&str>, - insert_fn: F, -) -> Result -where - F: FnOnce(DiffTraceInsert<'_>) -> Result, -{ - let time_ms = diff_trace_db_time_ms(payload.time)?; - let session_id = prefixed_diff_trace_session_id(&payload.tool_name, &payload.session_id); - - insert_fn(DiffTraceInsert { - time_ms, - session_id: &session_id, - patch: &payload.diff, - model_id, - tool_name: &payload.tool_name, - tool_version, - payload_type: &payload.payload_type, - }) -} - pub(crate) fn diff_trace_db_time_ms(time: u64) -> Result { i64::try_from(time).map_err(|_| { anyhow!(StdinPayloadKind::DiffTrace.validation_error( diff --git a/cli/src/services/hooks/lifecycle.rs b/cli/src/services/hooks/lifecycle.rs index a8723ec56..437d4c6e3 100644 --- a/cli/src/services/hooks/lifecycle.rs +++ b/cli/src/services/hooks/lifecycle.rs @@ -7,8 +7,8 @@ use anyhow::{anyhow, Context, Result}; use crate::app::HasRepoRoot; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, RequiredHookInstallStatus, - RequiredHooksInstallOutcome, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, RequiredHookInstallStatus, RequiredHooksInstallOutcome, + ServiceLifecycle, SetupOutcome, }; use crate::services::setup::{ hook_merge, install_required_git_hooks, iter_required_hook_assets, @@ -28,11 +28,7 @@ enum HookContentState { } impl ServiceLifecycle for HooksLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::Hooks - } - - fn diagnose(&self, ctx: &C) -> Vec { + async fn diagnose(&self, ctx: &C) -> Vec { let repository_root = match ctx.repo_root() { Some(path) => path.to_path_buf(), None => { @@ -63,7 +59,11 @@ impl ServiceLifecycle for HooksLifecycle { }) } - fn fix(&self, ctx: &C, problems: &[HealthProblem]) -> Vec { + async fn fix( + &self, + ctx: &C, + problems: &[HealthProblem], + ) -> Vec { let should_fix_hooks = problems.iter().any(|problem| { problem.category == HealthCategory::HookRollout && problem.fixability == HealthFixability::AutoFixable @@ -97,7 +97,7 @@ impl ServiceLifecycle for HooksLifecycle { } } - fn setup(&self, ctx: &C) -> Result { + async fn setup(&self, ctx: &C) -> Result { let repository_root = ctx .repo_root() .context("Hooks lifecycle setup requires a resolved repository root")?; diff --git a/cli/src/services/hooks/mod.rs b/cli/src/services/hooks/mod.rs index c7b036aeb..14b114bd2 100644 --- a/cli/src/services/hooks/mod.rs +++ b/cli/src/services/hooks/mod.rs @@ -1,39 +1,12 @@ use anyhow::Context; -#[cfg(test)] -use crate::services::agent_trace::{validate_agent_trace_value, AgentTrace, AgentTraceVcsType}; -#[cfg(test)] -use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; -#[cfg(test)] -use crate::services::agent_trace_db::{ - DiffTraceInsert, MessageRole, PartType, RecentDiffTracePatches, PAYLOAD_TYPE_PATCH, - PAYLOAD_TYPE_STRUCTURED, -}; -#[cfg(test)] -use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, -}; -#[cfg(test)] -use crate::services::observability::traits::Logger; -#[cfg(test)] -use crate::services::patch::{ - intersect_patches as intersect_patches_fn, load_patch_from_json, - parse_patch as parse_patch_from_text, ParsedPatch, -}; -#[cfg(test)] -use anyhow::{anyhow, Result}; -#[cfg(test)] -use serde_json::{json, to_string as serialize_to_json, Value}; - mod claude_transforms; mod commit_hooks; mod conversation_trace; mod diff_trace; -#[cfg(test)] -mod mutation_scope_ingress_conformance; + mod mutation_scope_lock; -#[cfg(test)] -mod mutation_scope_state_conformance; + mod runtime; pub mod claude_bridge_session; @@ -55,9 +28,6 @@ pub(crate) use conversation_trace::*; pub(crate) use diff_trace::*; pub(crate) use runtime::*; -#[cfg(test)] -mod tests; - #[derive(Clone, Debug, Eq, PartialEq)] pub enum HookSubcommand { PreCommit, @@ -86,9 +56,9 @@ pub enum HookSubcommand { pub const NAME: &str = "hooks"; pub const CANONICAL_SCE_COAUTHOR_TRAILER: &str = "Co-authored-by: SCE "; -pub fn run_hooks_subcommand( +pub async fn run_hooks_subcommand( subcommand: &HookSubcommand, - logger: Option<&dyn crate::services::observability::traits::Logger>, + logger: Option<&L>, ) -> anyhow::Result { let repository_root = std::env::current_dir().with_context(|| { format!( @@ -97,56 +67,60 @@ pub fn run_hooks_subcommand( ) })?; - run_hooks_subcommand_in_repo(&repository_root, subcommand, logger) + run_hooks_subcommand_in_repo(&repository_root, subcommand, logger).await } -fn run_hooks_subcommand_in_repo( +async fn run_hooks_subcommand_in_repo( repository_root: &std::path::Path, subcommand: &HookSubcommand, - logger: Option<&dyn crate::services::observability::traits::Logger>, + logger: Option<&L>, ) -> anyhow::Result { match subcommand { HookSubcommand::PreCommit => run_pre_commit_subcommand_with_trace(repository_root), HookSubcommand::CommitMsg { message_file } => { run_commit_msg_subcommand_with_trace(repository_root, subcommand, message_file, logger) + .await } HookSubcommand::PostCommit { vcs_type, remote_url, - } => run_post_commit_subcommand_with_trace( - repository_root, - *vcs_type, - remote_url.as_deref(), - logger, - ), + } => { + run_post_commit_subcommand_with_trace( + repository_root, + *vcs_type, + remote_url.as_deref(), + logger, + ) + .await + } HookSubcommand::PostRewrite { rewrite_method } => { run_post_rewrite_subcommand_with_trace(repository_root, subcommand, rewrite_method) } - HookSubcommand::DiffTrace => Ok(run_diff_trace_subcommand(repository_root, logger)), + HookSubcommand::DiffTrace => Ok(run_diff_trace_subcommand(repository_root, logger).await), HookSubcommand::ConversationTrace => { - Ok(run_conversation_trace_subcommand(repository_root, logger)) + Ok(run_conversation_trace_subcommand(repository_root, logger).await) } - HookSubcommand::Codex => Ok(codex::run_codex_subcommand(repository_root, logger)), + HookSubcommand::Codex => Ok(codex::run_codex_subcommand(repository_root, logger).await), HookSubcommand::ClaudeModelState => Ok( - claude_model_state::run_claude_model_state_subcommand(repository_root, logger), + claude_model_state::run_claude_model_state_subcommand(repository_root, logger).await, ), HookSubcommand::MutationScope => { - mutation_scope::run_mutation_scope_subcommand(repository_root, logger) + mutation_scope::run_mutation_scope_subcommand(repository_root, logger).await } HookSubcommand::ClaudeMutationScope => { - claude_mutation_scope::run_claude_mutation_scope_subcommand(logger) + claude_mutation_scope::run_claude_mutation_scope_subcommand(logger).await } HookSubcommand::CodexMutationScope => { - codex_mutation_scope::run_codex_mutation_scope_subcommand(logger) + codex_mutation_scope::run_codex_mutation_scope_subcommand(logger).await } HookSubcommand::OpenCodeMutationScope => { - opencode_mutation_scope::run_opencode_mutation_scope_subcommand(logger) + opencode_mutation_scope::run_opencode_mutation_scope_subcommand(logger).await } HookSubcommand::PiMutationScope => { - pi_mutation_scope::run_pi_mutation_scope_subcommand(logger) + pi_mutation_scope::run_pi_mutation_scope_subcommand(logger).await } HookSubcommand::ExternalMutationGuard => { - mutation_scope::run_external_mutation_guard_subcommand(repository_root, logger) + mutation_scope::run_external_mutation_guard_subcommand(repository_root, logger).await } } } diff --git a/cli/src/services/hooks/mutation_scope.rs b/cli/src/services/hooks/mutation_scope.rs index 3fa754e9b..67ce7dfc6 100644 --- a/cli/src/services/hooks/mutation_scope.rs +++ b/cli/src/services/hooks/mutation_scope.rs @@ -6,11 +6,11 @@ use serde_json::{json, Map, Value}; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::mutation_trace::runtime::{ - abandon_scope, arm_external_mutation_guard, coordinate, AbandonScopeError, AbandonScopeOutcome, - CoordinateError, CoordinateOutcome, GuardEvent, GuardRequest, RuntimeBoundary, StartProvenance, + abandon_scope, advise_after_completed_boundary, arm_external_mutation_guard, coordinate, + AbandonScopeError, AbandonScopeOutcome, AdvisoryReport, AdvisorySeverity, CoordinateError, + CoordinateOutcome, GuardEvent, GuardRequest, RuntimeBoundary, StartProvenance, }; use crate::services::mutation_trace::types::{ActorKind, EventId, ScopeId}; -use crate::services::observability::traits::Logger; const MUTATION_SCOPE_DB_CONTEXT: &str = "Failed to open Agent Trace DB for mutation-scope runtime."; @@ -241,56 +241,47 @@ fn validation_error(detail: &str) -> String { format!("Invalid mutation-scope payload from STDIN: {detail}.") } -pub(crate) fn run_mutation_scope_subcommand( +pub(crate) async fn run_mutation_scope_subcommand< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { let stdin_payload = super::read_hook_stdin()?; - run_mutation_scope_from_payload(repository_root, &stdin_payload, logger) + run_mutation_scope_from_payload(repository_root, &stdin_payload, logger).await } -pub(crate) fn run_mutation_scope_from_payload( +pub(crate) async fn run_mutation_scope_from_payload< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { run_mutation_scope_from_payload_with( repository_root, stdin_payload, logger, super::open_agent_trace_db_for_hook_runtime, + &mut std::io::stderr(), ) + .await } -#[cfg(test)] -pub(super) fn run_mutation_scope_from_payload_at_state_root( - repository_root: &Path, - state_root: &Path, - stdin_payload: &str, - logger: Option<&dyn Logger>, -) -> Result { - run_mutation_scope_from_payload_with( - repository_root, - stdin_payload, - logger, - |root, context_message| { - super::open_agent_trace_db_for_hook_runtime_at_state_root( - root, - state_root, - context_message, - ) - }, - ) -} - -fn run_mutation_scope_from_payload_with( +async fn run_mutation_scope_from_payload_with< + L: crate::services::observability::traits::Logger, + O, + E, +>( repository_root: &Path, stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, open_db: O, + diagnostics: &mut E, ) -> Result where - O: Fn(&Path, &'static str) -> Result + Copy, + O: std::ops::AsyncFn(&Path, &'static str) -> Result + Copy, + E: Write, { let payload = parse_mutation_scope_payload(stdin_payload)?; @@ -298,21 +289,49 @@ where repository_root, payload, logger, - |root, boundary| coordinate(root, boundary, || open_db(root, MUTATION_SCOPE_DB_CONTEXT)), - |root, scope| abandon_scope(root, scope, || open_db(root, MUTATION_SCOPE_DB_CONTEXT)), + diagnostics, + async |root, boundary| { + coordinate(root, boundary, async || { + open_db(root, MUTATION_SCOPE_DB_CONTEXT).await + }) + .await + }, + async |root, scope| { + abandon_scope(root, scope, async || { + open_db(root, MUTATION_SCOPE_DB_CONTEXT).await + }) + .await + }, + advise_after_completed_boundary, ) + .await } -fn drive_mutation_scope( +pub(crate) async fn drive_mutation_scope< + L: crate::services::observability::traits::Logger, + E: Write, + C, + A, + V, +>( repository_root: &Path, payload: MutationScopePayload, - logger: Option<&dyn Logger>, + logger: Option<&L>, + diagnostics: &mut E, coordinate_boundary: C, abandon: A, + advise: V, ) -> Result where - C: FnOnce(&Path, &RuntimeBoundary) -> std::result::Result, - A: FnOnce(&Path, &ScopeId) -> std::result::Result, + V: FnOnce(&Path) -> AdvisoryReport, + C: std::ops::AsyncFnOnce( + &Path, + &RuntimeBoundary, + ) -> std::result::Result, + A: std::ops::AsyncFnOnce( + &Path, + &ScopeId, + ) -> std::result::Result, { let boundary = match payload { MutationScopePayload::Start { @@ -346,16 +365,78 @@ where }, MutationScopePayload::Flush => RuntimeBoundary::Flush, MutationScopePayload::Abandon { scope_id } => { - return classify_abandon(abandon(repository_root, &ScopeId(scope_id)), logger); + return classify_abandon(abandon(repository_root, &ScopeId(scope_id)).await, logger); } }; - classify_coordinate(coordinate_boundary(repository_root, &boundary), logger) + let advisory_eligible = matches!( + boundary, + RuntimeBoundary::Close { .. } | RuntimeBoundary::Flush + ); + let result = coordinate_boundary(repository_root, &boundary).await; + let durably_completed = matches!( + result, + Ok(_) | Err(CoordinateError::MarkerClearAfterCommit { .. }) + ); + let classified = classify_coordinate(result, logger); + + if advisory_eligible && durably_completed { + report_advisory(logger, diagnostics, &advise(repository_root)); + } + + classified +} + +fn report_advisory( + logger: Option<&L>, + diagnostics: &mut E, + report: &AdvisoryReport, +) { + log_advisory_report(logger, report); + emit_advisory_diagnostic(diagnostics, report); +} + +fn emit_advisory_diagnostic(diagnostics: &mut E, report: &AdvisoryReport) { + let Some(diagnostic) = report.diagnostic.as_deref() else { + return; + }; + let _ = writeln!(diagnostics, "{diagnostic}").and_then(|()| diagnostics.flush()); +} + +fn log_advisory_report( + logger: Option<&L>, + report: &AdvisoryReport, +) { + let Some(log) = logger else { + return; + }; + let message = report + .recommendation + .unwrap_or("Reconciliation advisory check."); + let warning = report.warning.as_deref().unwrap_or(""); + let mut fields = vec![("outcome", report.outcome)]; + if !warning.is_empty() { + fields.push(("persistence_warning", warning)); + } + match report.severity { + AdvisorySeverity::Warn => log.warn( + "sce.hooks.mutation_scope.ref_reconciliation_advisory", + message, + &fields, + None, + ), + AdvisorySeverity::Debug => log.debug( + "sce.hooks.mutation_scope.ref_reconciliation_advisory", + message, + &fields, + None, + ), + } } -fn classify_coordinate( +fn classify_coordinate( result: std::result::Result, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { match result { Ok(_) => Ok(String::new()), @@ -369,9 +450,9 @@ fn classify_coordinate( } } -fn classify_abandon( +fn classify_abandon( result: std::result::Result, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { match result { Ok(_) => Ok(String::new()), @@ -385,8 +466,8 @@ fn classify_abandon( } } -fn log_marker_clear_after_durable_completion( - logger: Option<&dyn Logger>, +fn log_marker_clear_after_durable_completion( + logger: Option<&L>, entrypoint: &str, source: &anyhow::Error, ) { @@ -513,9 +594,11 @@ fn guard_event_json_line(event: &GuardEvent) -> String { } } -pub(crate) fn run_external_mutation_guard_subcommand( +pub(crate) async fn run_external_mutation_guard_subcommand< + L: crate::services::observability::traits::Logger, +>( repository_root: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { let reader = std::io::BufReader::new(std::io::stdin()); let stdout = std::io::stdout(); @@ -524,13 +607,21 @@ pub(crate) fn run_external_mutation_guard_subcommand( logger, reader, stdout.lock(), - |root| super::open_agent_trace_db_for_hook_runtime(root, MUTATION_SCOPE_DB_CONTEXT), + async |root: &Path| { + super::open_agent_trace_db_for_hook_runtime(root, MUTATION_SCOPE_DB_CONTEXT).await + }, ) + .await } -fn run_external_mutation_guard_protocol_with( +async fn run_external_mutation_guard_protocol_with< + L: crate::services::observability::traits::Logger, + R, + W, + O, +>( repository_root: &Path, - logger: Option<&dyn Logger>, + logger: Option<&L>, mut reader: R, mut writer: W, open_db: O, @@ -538,7 +629,7 @@ fn run_external_mutation_guard_protocol_with( where R: std::io::BufRead + Send + 'static, W: Write, - O: Fn(&Path) -> Result, + O: std::ops::AsyncFn(&Path) -> Result, { let mut first_line = String::new(); std::io::BufRead::read_line(&mut reader, &mut first_line) @@ -549,10 +640,11 @@ where let repository_root = repository_root.to_path_buf(); let armed_guard = arm_external_mutation_guard( &repository_root, - || open_db(&repository_root), + async || open_db(&repository_root).await, || write_guard_event(&mut writer, &GuardEvent::Armed), cancel_rx, - )?; + ) + .await?; let mut exec_line = String::new(); match std::io::BufRead::read_line(&mut reader, &mut exec_line) @@ -583,9 +675,11 @@ where } }); - let outcome = armed_guard.exec(&request, |event| { - let _ = write_guard_event(&mut writer, &event); - })?; + let outcome = armed_guard + .exec(&request, |event| { + let _ = write_guard_event(&mut writer, &event); + }) + .await?; if outcome.marker_clear_failed { log_marker_clear_after_durable_completion( @@ -617,1766 +711,447 @@ fn write_guard_line(writer: &mut W, line: &str) -> std::io::Result<()> } #[cfg(test)] -mod tests { - use super::*; +mod advisory_trigger_tests { + use std::cell::Cell; + use std::io::Write; + use std::path::Path; + use std::sync::Mutex; + + use anyhow::anyhow; + + use super::{ + drive_mutation_scope, AbandonScopeError, AbandonScopeOutcome, AdvisoryReport, + AdvisorySeverity, CoordinateError, CoordinateOutcome, MutationScopePayload, + RuntimeBoundary, + }; + use crate::services::error::CliError; + use crate::services::mutation_trace::protocol::CommitEvaluation; + use crate::services::mutation_trace::types::{ActorKind, ScopeId, TreeId, WorktreeId}; + use crate::services::observability::traits::Logger; - fn parse(payload: &str) -> Result { - parse_mutation_scope_payload(payload) + const ADVISORY_EVENT_ID: &str = "sce.hooks.mutation_scope.ref_reconciliation_advisory"; + const RECOMMENDED: &str = "Reconciliation is recommended. Run sce doctor --fix."; + + #[derive(Debug, Eq, PartialEq)] + struct LogRecord { + level: &'static str, + event_id: String, + message: String, + fields: Vec<(String, String)>, } - struct ParseCase { - label: &'static str, - payload: &'static str, - expected: MutationScopePayload, + #[derive(Default)] + struct RecordingLogger { + records: Mutex>, } - struct InvalidCase { - label: &'static str, - payload: &'static str, - expected_fragment: &'static str, + impl RecordingLogger { + fn record(&self, level: &'static str, event_id: &str, message: &str, f: &[(&str, &str)]) { + self.records.lock().unwrap().push(LogRecord { + level, + event_id: event_id.to_string(), + message: message.to_string(), + fields: f + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + }); + } + + fn take(&self) -> Vec { + std::mem::take(&mut self.records.lock().unwrap()) + } } - fn assert_all_rejected(cases: &[InvalidCase]) { - for case in cases { - let error = parse(case.payload) - .expect_err(&format!( - "{}: expected the payload to be rejected", - case.label - )) - .to_string(); - assert!( - error.contains(case.expected_fragment), - "{}: expected error containing {:?}, got: {error}", - case.label, - case.expected_fragment - ); + impl Logger for RecordingLogger { + fn info(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("info", event_id, message, fields); + } + fn debug(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("debug", event_id, message, fields); + } + fn warn(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("warn", event_id, message, fields); + } + fn error(&self, event_id: &str, message: &str, fields: &[(&str, &str)], _: Option<&str>) { + self.record("error", event_id, message, fields); } + fn log_cli_error(&self, _error: &CliError, _session_id: Option<&str>) {} } - #[test] - fn valid_operation_payloads_parse_exactly() { - let cases = [ - ParseCase { - label: "start preserves identity values verbatim", - payload: r#"{"operation":"start","scope_id":" scope-A ","event_id":"event-start","actor_kind":"claude_code"}"#, - expected: MutationScopePayload::Start { - scope_id: " scope-A ".to_string(), - event_id: "event-start".to_string(), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - }, - ParseCase { - label: "advance", - payload: r#"{"operation":"advance","scope_id":"scope-B","event_id":"event-advance","actor_kind":"codex"}"#, - expected: MutationScopePayload::Advance { - scope_id: "scope-B".to_string(), - event_id: "event-advance".to_string(), - actor_kind: ActorKind::Codex, - }, - }, - ParseCase { - label: "close", - payload: r#"{"operation":"close","scope_id":"scope-C","event_id":"event-close","actor_kind":"opencode"}"#, - expected: MutationScopePayload::Close { - scope_id: "scope-C".to_string(), - event_id: "event-close".to_string(), - actor_kind: ActorKind::OpenCode, - }, - }, - ParseCase { - label: "flush takes no identity", - payload: r#"{"operation":"flush"}"#, - expected: MutationScopePayload::Flush, - }, - ParseCase { - label: "abandon takes only scope_id", - payload: r#"{"operation":"abandon","scope_id":"scope-D"}"#, - expected: MutationScopePayload::Abandon { - scope_id: "scope-D".to_string(), - }, - }, - ]; + struct FailingWriter; - for case in cases { - let parsed = parse(case.payload) - .unwrap_or_else(|error| panic!("{}: expected valid payload: {error}", case.label)); - assert_eq!(parsed, case.expected, "{}", case.label); + impl Write for FailingWriter { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + Err(std::io::Error::other("stderr closed")) + } + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::Error::other("stderr closed")) } } - #[test] - fn valid_start_provenance_parses_exactly() { - let cases = [ - ( - "session and model", - r#"{"session_id":"cx_session-1","model_id":"gpt-5-codex"}"#, - StartProvenance { - session_id: "cx_session-1".to_string(), - model_id: Some("gpt-5-codex".to_string()), - }, - ), - ( - "session only", - r#"{"session_id":"cc_session-1"}"#, - StartProvenance { - session_id: "cc_session-1".to_string(), - model_id: None, - }, - ), - ( - "session with a null model", - r#"{"session_id":"cc_session-1","model_id":null}"#, - StartProvenance { - session_id: "cc_session-1".to_string(), - model_id: None, - }, - ), - ]; - - for (label, provenance, expected) in cases { - let payload = format!( - r#"{{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{provenance}}}"# - ); - let parsed = parse(&payload) - .unwrap_or_else(|error| panic!("{label}: expected valid payload: {error}")); - assert_eq!( - parsed, - MutationScopePayload::Start { - scope_id: "A".to_string(), - event_id: "e1".to_string(), - actor_kind: ActorKind::Codex, - provenance: Some(expected), - }, - "{label}" - ); + fn outcome() -> CoordinateOutcome { + CoordinateOutcome { + worktree_id: WorktreeId("main".to_string()), + observed_tree: TreeId("tree".to_string()), + revision: 1, + evaluation: CommitEvaluation::default(), + mutation_event: None, } } - #[test] - #[allow(clippy::too_many_lines)] - fn invalid_provenance_shapes_are_rejected() { - const NOT_ON_THIS_OPERATION: &str = "unexpected field 'provenance'."; - const NOT_AN_OBJECT: &str = "field 'provenance' must be a JSON object"; - const SESSION_NOT_A_STRING: &str = "field 'session_id' must be a string"; - const SESSION_BLANK: &str = "field 'session_id' must be a non-blank string"; - const MODEL_NOT_A_STRING: &str = "field 'model_id' must be a string"; - const MODEL_BLANK: &str = "field 'model_id' must be a non-blank string"; - - assert_all_rejected(&[ - InvalidCase { - label: "advance rejects provenance", - payload: r#"{"operation":"advance","scope_id":"A","event_id":"e2","actor_kind":"codex","provenance":{"session_id":"cx_session-1"}}"#, - expected_fragment: NOT_ON_THIS_OPERATION, - }, - InvalidCase { - label: "close rejects provenance", - payload: r#"{"operation":"close","scope_id":"A","event_id":"e3","actor_kind":"codex","provenance":{"session_id":"cx_session-1"}}"#, - expected_fragment: NOT_ON_THIS_OPERATION, - }, - InvalidCase { - label: "flush rejects provenance", - payload: r#"{"operation":"flush","provenance":{"session_id":"cx_session-1"}}"#, - expected_fragment: NOT_ON_THIS_OPERATION, - }, - InvalidCase { - label: "abandon rejects provenance", - payload: r#"{"operation":"abandon","scope_id":"A","provenance":{"session_id":"cx_session-1"}}"#, - expected_fragment: NOT_ON_THIS_OPERATION, - }, - InvalidCase { - label: "null provenance", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":null}"#, - expected_fragment: NOT_AN_OBJECT, - }, - InvalidCase { - label: "string provenance", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":"cx_session-1"}"#, - expected_fragment: NOT_AN_OBJECT, - }, - InvalidCase { - label: "array provenance", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":[]}"#, - expected_fragment: NOT_AN_OBJECT, - }, - InvalidCase { - label: "numeric provenance", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":5}"#, - expected_fragment: NOT_AN_OBJECT, - }, - InvalidCase { - label: "boolean provenance", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":true}"#, - expected_fragment: NOT_AN_OBJECT, - }, - InvalidCase { - label: "missing session_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"model_id":"gpt-5-codex"}}"#, - expected_fragment: "missing required field 'session_id'", - }, - InvalidCase { - label: "empty session_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":""}}"#, - expected_fragment: SESSION_BLANK, - }, - InvalidCase { - label: "whitespace session_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":" "}}"#, - expected_fragment: SESSION_BLANK, - }, - InvalidCase { - label: "null session_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":null}}"#, - expected_fragment: SESSION_NOT_A_STRING, - }, - InvalidCase { - label: "numeric session_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":7}}"#, - expected_fragment: SESSION_NOT_A_STRING, - }, - InvalidCase { - label: "empty model_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":"cx_session-1","model_id":""}}"#, - expected_fragment: MODEL_BLANK, - }, - InvalidCase { - label: "whitespace model_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":"cx_session-1","model_id":" "}}"#, - expected_fragment: MODEL_BLANK, - }, - InvalidCase { - label: "numeric model_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":"cx_session-1","model_id":42}}"#, - expected_fragment: MODEL_NOT_A_STRING, - }, - InvalidCase { - label: "unexpected provenance key", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"codex","provenance":{"session_id":"cx_session-1","agent_id":"sub"}}"#, - expected_fragment: "unexpected field 'provenance.agent_id'", - }, - ]); + fn report(outcome: &'static str, severity: AdvisorySeverity) -> AdvisoryReport { + AdvisoryReport { + outcome, + severity, + recommendation: None, + warning: None, + diagnostic: None, + } } - #[test] - fn every_actor_kind_wire_string_maps() { - for (wire, expected) in [ - ("claude_code", ActorKind::ClaudeCode), - ("codex", ActorKind::Codex), - ("opencode", ActorKind::OpenCode), - ("pi", ActorKind::Pi), - ] { - let payload = format!( - r#"{{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"{wire}"}}"# - ); - let parsed = parse(&payload) - .unwrap_or_else(|error| panic!("{wire}: expected valid payload: {error}")); - assert_eq!( - parsed, - MutationScopePayload::Start { - scope_id: "A".to_string(), - event_id: "e1".to_string(), - actor_kind: expected, - provenance: None, - }, - "{wire}" - ); + fn close() -> MutationScopePayload { + MutationScopePayload::Close { + scope_id: "scope".to_string(), + event_id: "event".to_string(), + actor_kind: ActorKind::Codex, } } - #[test] - fn invalid_payload_envelopes_are_rejected() { - assert_all_rejected(&[ - InvalidCase { - label: "empty payload", - payload: "", - expected_fragment: "expected a JSON object, got an empty payload", - }, - InvalidCase { - label: "whitespace-only payload", - payload: " \n\t ", - expected_fragment: "expected a JSON object, got an empty payload", - }, - InvalidCase { - label: "unterminated object", - payload: "{", - expected_fragment: "expected valid JSON", - }, - InvalidCase { - label: "incomplete object", - payload: r#"{"operation":"start""#, - expected_fragment: "expected valid JSON", - }, - InvalidCase { - label: "non-JSON text", - payload: "not json at all", - expected_fragment: "expected valid JSON", - }, - InvalidCase { - label: "JSON number", - payload: "123", - expected_fragment: "expected a JSON object", - }, - InvalidCase { - label: "JSON string", - payload: r#""start""#, - expected_fragment: "expected a JSON object", - }, - InvalidCase { - label: "JSON array", - payload: r#"["start"]"#, - expected_fragment: "expected a JSON object", - }, - InvalidCase { - label: "JSON null", - payload: "null", - expected_fragment: "expected a JSON object", - }, - InvalidCase { - label: "missing operation", - payload: r#"{"scope_id":"A","event_id":"e1","actor_kind":"pi"}"#, - expected_fragment: "missing required field 'operation'", - }, - InvalidCase { - label: "unknown operation", - payload: r#"{"operation":"reopen","scope_id":"A","event_id":"e1","actor_kind":"pi"}"#, - expected_fragment: "field 'operation' must be one of", - }, - InvalidCase { - label: "numeric operation", - payload: r#"{"operation":5}"#, - expected_fragment: "field 'operation' must be a string", - }, - ]); + fn start() -> MutationScopePayload { + MutationScopePayload::Start { + scope_id: "scope".to_string(), + event_id: "event".to_string(), + actor_kind: ActorKind::Codex, + provenance: None, + } } - #[test] - fn invalid_required_fields_are_rejected() { - assert_all_rejected(&[ - InvalidCase { - label: "start missing scope_id", - payload: r#"{"operation":"start","event_id":"e1","actor_kind":"pi"}"#, - expected_fragment: "missing required field 'scope_id'", - }, - InvalidCase { - label: "start missing event_id", - payload: r#"{"operation":"start","scope_id":"A","actor_kind":"pi"}"#, - expected_fragment: "missing required field 'event_id'", - }, - InvalidCase { - label: "start missing actor_kind", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1"}"#, - expected_fragment: "missing required field 'actor_kind'", - }, - InvalidCase { - label: "abandon missing scope_id", - payload: r#"{"operation":"abandon"}"#, - expected_fragment: "missing required field 'scope_id'", - }, - InvalidCase { - label: "start empty scope_id", - payload: r#"{"operation":"start","scope_id":"","event_id":"e1","actor_kind":"pi"}"#, - expected_fragment: "field 'scope_id' must be a non-blank string", - }, - InvalidCase { - label: "start whitespace scope_id", - payload: r#"{"operation":"start","scope_id":" ","event_id":"e1","actor_kind":"pi"}"#, - expected_fragment: "field 'scope_id' must be a non-blank string", - }, - InvalidCase { - label: "start empty event_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"","actor_kind":"pi"}"#, - expected_fragment: "field 'event_id' must be a non-blank string", - }, - InvalidCase { - label: "start tab event_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"\t","actor_kind":"pi"}"#, - expected_fragment: "field 'event_id' must be a non-blank string", - }, - InvalidCase { - label: "abandon whitespace scope_id", - payload: r#"{"operation":"abandon","scope_id":" "}"#, - expected_fragment: "field 'scope_id' must be a non-blank string", - }, - InvalidCase { - label: "start numeric scope_id", - payload: r#"{"operation":"start","scope_id":123,"event_id":"e1","actor_kind":"pi"}"#, - expected_fragment: "field 'scope_id' must be a string", - }, - InvalidCase { - label: "start boolean event_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":true,"actor_kind":"pi"}"#, - expected_fragment: "field 'event_id' must be a string", - }, - InvalidCase { - label: "start array actor_kind", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":["pi"]}"#, - expected_fragment: "field 'actor_kind' must be a string", - }, - InvalidCase { - label: "start unknown actor_kind", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"cursor"}"#, - expected_fragment: "field 'actor_kind' must be one of", - }, - ]); + fn advance() -> MutationScopePayload { + MutationScopePayload::Advance { + scope_id: "scope".to_string(), + event_id: "event".to_string(), + actor_kind: ActorKind::Codex, + } } - #[test] - fn operation_schema_rejects_forbidden_fields() { - const WORKTREE: &str = "worktree identity is derived from the invoking checkout"; + #[derive(Default)] + struct Calls { + coordinate: Cell, + abandon: Cell, + advise: Cell, + } - assert_all_rejected(&[ - InvalidCase { - label: "start rejects attempt_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"pi","attempt_id":"x"}"#, - expected_fragment: "unexpected field 'attempt_id'", - }, - InvalidCase { - label: "advance rejects attempt_id", - payload: r#"{"operation":"advance","scope_id":"A","event_id":"e1","actor_kind":"pi","attempt_id":"x"}"#, - expected_fragment: "unexpected field 'attempt_id'", - }, - InvalidCase { - label: "close rejects attempt_id", - payload: r#"{"operation":"close","scope_id":"A","event_id":"e1","actor_kind":"pi","attempt_id":"x"}"#, - expected_fragment: "unexpected field 'attempt_id'", - }, - InvalidCase { - label: "flush rejects scope_id", - payload: r#"{"operation":"flush","scope_id":"A"}"#, - expected_fragment: "unexpected field 'scope_id'", - }, - InvalidCase { - label: "flush rejects event_id", - payload: r#"{"operation":"flush","event_id":"e1"}"#, - expected_fragment: "unexpected field 'event_id'", - }, - InvalidCase { - label: "flush rejects actor_kind", - payload: r#"{"operation":"flush","actor_kind":"pi"}"#, - expected_fragment: "unexpected field 'actor_kind'", - }, - InvalidCase { - label: "abandon rejects event_id", - payload: r#"{"operation":"abandon","scope_id":"A","event_id":"e1"}"#, - expected_fragment: "unexpected field 'event_id'", - }, - InvalidCase { - label: "abandon rejects actor_kind", - payload: r#"{"operation":"abandon","scope_id":"A","actor_kind":"pi"}"#, - expected_fragment: "unexpected field 'actor_kind'", - }, - InvalidCase { - label: "start rejects worktree_id", - payload: r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"pi","worktree_id":"wt"}"#, - expected_fragment: WORKTREE, - }, - InvalidCase { - label: "advance rejects worktree_id", - payload: r#"{"operation":"advance","scope_id":"A","event_id":"e1","actor_kind":"pi","worktree_id":"wt"}"#, - expected_fragment: WORKTREE, - }, - InvalidCase { - label: "close rejects worktree_id", - payload: r#"{"operation":"close","scope_id":"A","event_id":"e1","actor_kind":"pi","worktree_id":"wt"}"#, - expected_fragment: WORKTREE, + async fn drive_with( + payload: MutationScopePayload, + coordinated: Result, + advised: AdvisoryReport, + logger: Option<&L>, + diagnostics: &mut E, + calls: &Calls, + ) -> anyhow::Result { + drive_mutation_scope( + Path::new("."), + payload, + logger, + diagnostics, + async |_root: &Path, _boundary: &RuntimeBoundary| { + calls.coordinate.set(calls.coordinate.get() + 1); + coordinated }, - InvalidCase { - label: "flush rejects worktree_id", - payload: r#"{"operation":"flush","worktree_id":"wt"}"#, - expected_fragment: WORKTREE, + async |_root: &Path, + _scope: &ScopeId| + -> Result { + calls.abandon.set(calls.abandon.get() + 1); + Ok(AbandonScopeOutcome::Abandoned { + worktree_id: WorktreeId("main".to_string()), + scope: ScopeId("scope".to_string()), + revision: 1, + }) }, - InvalidCase { - label: "abandon rejects worktree_id", - payload: r#"{"operation":"abandon","scope_id":"A","worktree_id":"wt"}"#, - expected_fragment: WORKTREE, + |_root: &Path| { + calls.advise.set(calls.advise.get() + 1); + advised }, - ]); + ) + .await } - mod runtime_dispatch { - use std::cell::{Cell, RefCell}; - - use super::*; - use crate::services::mutation_trace::protocol; - use crate::services::mutation_trace::types::{TreeId, WorktreeId}; - - fn committed_outcome() -> CoordinateOutcome { - CoordinateOutcome { - worktree_id: WorktreeId("wt-1".to_string()), - observed_tree: TreeId("tree-1".to_string()), - revision: 1, - evaluation: protocol::CommitEvaluation::default(), - mutation_event: None, - } - } - - fn abandoned_outcome() -> AbandonScopeOutcome { - AbandonScopeOutcome::Abandoned { - worktree_id: WorktreeId("wt-1".to_string()), - scope: ScopeId("A".to_string()), - revision: 1, - } - } - - fn unreachable_coordinate( - _root: &Path, - _boundary: &RuntimeBoundary, - ) -> std::result::Result { - panic!("coordinate must not be invoked for this payload"); - } - - fn unreachable_abandon( - _root: &Path, - _scope: &ScopeId, - ) -> std::result::Result { - panic!("abandon_scope must not be invoked for this payload"); - } - - fn ids(scope: &str, event: &str) -> (ScopeId, EventId) { - (ScopeId(scope.to_string()), EventId(event.to_string())) - } - - fn assert_runtime_boundary_eq( - actual: &RuntimeBoundary, - expected: &RuntimeBoundary, - label: &str, - ) { - match (actual, expected) { - ( - RuntimeBoundary::Start { - scope: actual_scope, - event: actual_event, - actor_kind: actual_actor, - provenance: actual_provenance, - }, - RuntimeBoundary::Start { - scope: expected_scope, - event: expected_event, - actor_kind: expected_actor, - provenance: expected_provenance, - }, - ) => { - assert_eq!(actual_scope, expected_scope, "{label}: scope"); - assert_eq!(actual_event, expected_event, "{label}: event"); - assert_eq!(actual_actor, expected_actor, "{label}: actor"); - assert_eq!( - actual_provenance, expected_provenance, - "{label}: provenance" - ); - } - ( - RuntimeBoundary::Advance { - scope: actual_scope, - event: actual_event, - actor_kind: actual_actor, - }, - RuntimeBoundary::Advance { - scope: expected_scope, - event: expected_event, - actor_kind: expected_actor, - }, - ) - | ( - RuntimeBoundary::Close { - scope: actual_scope, - event: actual_event, - actor_kind: actual_actor, - }, - RuntimeBoundary::Close { - scope: expected_scope, - event: expected_event, - actor_kind: expected_actor, - }, - ) => { - assert_eq!(actual_scope, expected_scope, "{label}: scope"); - assert_eq!(actual_event, expected_event, "{label}: event"); - assert_eq!(actual_actor, expected_actor, "{label}: actor"); - } - (RuntimeBoundary::Flush, RuntimeBoundary::Flush) => {} - _ => panic!("{label}: runtime boundary variant mismatch"), - } - } - - #[test] - #[allow(clippy::too_many_lines)] - fn coordinate_payloads_forward_exact_runtime_boundaries() { - let (scope_a, event_a) = ids(" scope-A ", "event-start"); - let (scope_b, event_b) = ids("scope-B", "event-advance"); - let (scope_c, event_c) = ids("scope-C", "event-close"); - let (scope_d, event_d) = ids("scope-D", "event-start-model"); - let (scope_e, event_e) = ids("scope-E", "event-start-session"); - let with_model = StartProvenance { - session_id: "cx_session-1".to_string(), - model_id: Some("gpt-5-codex".to_string()), - }; - let without_model = StartProvenance { - session_id: "cc_session-1".to_string(), - model_id: None, - }; - - let cases = [ - ( - "start without provenance", - MutationScopePayload::Start { - scope_id: scope_a.0.clone(), - event_id: event_a.0.clone(), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - RuntimeBoundary::Start { - scope: scope_a, - event: event_a, - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ), - ( - "start with session and model provenance", - MutationScopePayload::Start { - scope_id: scope_d.0.clone(), - event_id: event_d.0.clone(), - actor_kind: ActorKind::Codex, - provenance: Some(with_model.clone()), - }, - RuntimeBoundary::Start { - scope: scope_d, - event: event_d, - actor_kind: ActorKind::Codex, - provenance: Some(with_model), - }, - ), - ( - "start with session-only provenance", - MutationScopePayload::Start { - scope_id: scope_e.0.clone(), - event_id: event_e.0.clone(), - actor_kind: ActorKind::Pi, - provenance: Some(without_model.clone()), - }, - RuntimeBoundary::Start { - scope: scope_e, - event: event_e, - actor_kind: ActorKind::Pi, - provenance: Some(without_model), - }, - ), - ( - "advance", - MutationScopePayload::Advance { - scope_id: scope_b.0.clone(), - event_id: event_b.0.clone(), - actor_kind: ActorKind::Codex, - }, - RuntimeBoundary::Advance { - scope: scope_b, - event: event_b, - actor_kind: ActorKind::Codex, - }, - ), - ( - "close", - MutationScopePayload::Close { - scope_id: scope_c.0.clone(), - event_id: event_c.0.clone(), - actor_kind: ActorKind::OpenCode, - }, - RuntimeBoundary::Close { - scope: scope_c, - event: event_c, - actor_kind: ActorKind::OpenCode, - }, - ), - ("flush", MutationScopePayload::Flush, RuntimeBoundary::Flush), - ]; - - for (label, payload, expected) in cases { - let calls = Cell::new(0_u32); - let result = drive_mutation_scope( - Path::new("/unused"), - payload, - None, - |_root, boundary| { - assert_runtime_boundary_eq(boundary, &expected, label); - calls.set(calls.get() + 1); - Ok(committed_outcome()) - }, - unreachable_abandon, - ); - - assert_eq!( - result.unwrap_or_else(|error| panic!("{label}: expected success: {error}")), - "", - "{label}" - ); - assert_eq!(calls.get(), 1, "{label}: coordinate call count"); - } - } - - #[test] - fn abandon_dispatches_only_to_abandon_scope() { - let seen = RefCell::new(Vec::new()); - let result = drive_mutation_scope( - Path::new("/unused"), - MutationScopePayload::Abandon { - scope_id: "A".to_string(), - }, - None, - unreachable_coordinate, - |_root, scope| { - seen.borrow_mut().push(scope.0.clone()); - Ok(abandoned_outcome()) - }, - ); - - assert_eq!(result.expect("abandon should succeed"), ""); - assert_eq!(seen.into_inner(), vec!["A".to_string()]); - } - - #[test] - fn marker_clear_after_commit_is_durable_success_without_reexecution() { - let calls = Cell::new(0_u32); - let result = drive_mutation_scope( - Path::new("/unused"), - MutationScopePayload::Advance { - scope_id: "A".to_string(), - event_id: "e2".to_string(), - actor_kind: ActorKind::ClaudeCode, - }, - None, - |_root, _boundary| { - calls.set(calls.get() + 1); - Err(CoordinateError::MarkerClearAfterCommit { - source: anyhow!("external-taint marker cleanup failed"), - committed: Box::new(committed_outcome()), - }) - }, - unreachable_abandon, - ); - - assert_eq!(result.expect("carried outcome is durable success"), ""); - assert_eq!(calls.get(), 1); - } - - #[test] - fn marker_clear_after_completion_is_durable_success_without_reexecution() { - let calls = Cell::new(0_u32); - let result = drive_mutation_scope( - Path::new("/unused"), - MutationScopePayload::Abandon { - scope_id: "A".to_string(), - }, - None, - unreachable_coordinate, - |_root, _scope| { - calls.set(calls.get() + 1); - Err(AbandonScopeError::MarkerClearAfterCompletion { - source: anyhow!("external-taint marker cleanup failed"), - completed: Box::new(abandoned_outcome()), - }) - }, - ); - - assert_eq!(result.expect("carried outcome is durable success"), ""); - assert_eq!(calls.get(), 1); - } + async fn drive( + payload: MutationScopePayload, + coordinated: Result, + calls: &Calls, + ) -> anyhow::Result { + drive_with( + payload, + coordinated, + report("no_action", AdvisorySeverity::Debug), + None::<&RecordingLogger>, + &mut Vec::new(), + calls, + ) + .await + } - #[test] - fn pre_completion_coordinate_error_propagates() { - let result = drive_mutation_scope( - Path::new("/unused"), - MutationScopePayload::Close { - scope_id: "A".to_string(), - event_id: "e3".to_string(), - actor_kind: ActorKind::ClaudeCode, - }, - None, - |_root, _boundary| Err(CoordinateError::Other(anyhow!("snapshot capture failed"))), - unreachable_abandon, - ); + async fn deliver(advised: AdvisoryReport) -> (Vec, String) { + let logger = RecordingLogger::default(); + let mut sink = Vec::new(); + let result = drive_with( + close(), + Ok(outcome()), + advised, + Some(&logger), + &mut sink, + &Calls::default(), + ) + .await; + assert_eq!(result.unwrap(), ""); + (logger.take(), String::from_utf8(sink).unwrap()) + } - assert!(result.is_err()); - } + fn field<'a>(record: &'a LogRecord, key: &str) -> Option<&'a str> { + record + .fields + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value.as_str()) + } - #[test] - fn pre_completion_abandon_error_propagates() { - let result = drive_mutation_scope( - Path::new("/unused"), - MutationScopePayload::Abandon { - scope_id: "A".to_string(), - }, - None, - unreachable_coordinate, - |_root, _scope| Err(AbandonScopeError::Other(anyhow!("lock acquisition failed"))), - ); + #[tokio::test] + async fn advisory_runs_once_after_completed_close_and_flush() { + let calls = Calls::default(); + assert_eq!(drive(close(), Ok(outcome()), &calls).await.unwrap(), ""); + assert_eq!(calls.advise.get(), 1); + assert_eq!( + drive(MutationScopePayload::Flush, Ok(outcome()), &calls) + .await + .unwrap(), + "" + ); + assert_eq!(calls.advise.get(), 2); + assert_eq!(calls.coordinate.get(), 2); + assert_eq!(calls.abandon.get(), 0); + } - assert!(result.is_err()); - } + #[tokio::test] + async fn advisory_never_runs_for_start_or_advance() { + let calls = Calls::default(); + drive(start(), Ok(outcome()), &calls).await.unwrap(); + drive(advance(), Ok(outcome()), &calls).await.unwrap(); + assert_eq!(calls.advise.get(), 0); } - mod real_git_db_ingress { - use std::cell::Cell; - use std::fs; - use std::io::{Cursor, Error, ErrorKind, Write}; - use std::path::{Path, PathBuf}; - use std::process::Command; + #[tokio::test] + async fn advisory_never_runs_after_failed_coordinate() { + let calls = Calls::default(); + let logger = RecordingLogger::default(); + let mut sink = Vec::new(); + let result = drive_with( + close(), + Err(CoordinateError::Other(anyhow!("boom"))), + AdvisoryReport { + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }, + Some(&logger), + &mut sink, + &calls, + ) + .await; + assert!(result.is_err()); + assert_eq!(calls.advise.get(), 0); + assert!(sink.is_empty()); + assert!(logger.take().is_empty()); + } - use super::*; - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, + #[tokio::test] + async fn advisory_never_runs_for_abandon() { + let calls = Calls::default(); + let payload = MutationScopePayload::Abandon { + scope_id: "scope".to_string(), }; - use crate::services::mutation_trace::runtime::resolve_git_dir; - use crate::services::mutation_trace::store::decode_revision; - - fn git(dir: &Path, args: &[&str]) -> String { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).expect("git output should be UTF-8") - } - - struct IngressRepo { - _temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, - } - - impl IngressRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-mutation-scope-ingress-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); - - Self { - _temp: temp, - root, - state_root, - } - } - - fn drive(&self, payload: &str) -> Result { - run_mutation_scope_from_payload_at_state_root( - &self.root, - &self.state_root, - payload, - None, - ) - } - - fn db(&self) -> RepositoryAgentTraceDb { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &self.root, - &self.state_root, - "mutation-scope ingress test assertions", - ) - .expect("assertion DB should open") - } - - fn working_tree(&self) -> String { - git(&self.root, &["add", "-A"]); - git(&self.root, &["write-tree"]).trim().to_owned() - } - - fn marker_path(&self) -> PathBuf { - resolve_git_dir(&self.root) - .expect("git dir should resolve") - .join("sce") - .join("mutation-cursor-tainted") - } - } - - fn assert_raw_agent_trace_tables_untouched(db: &RepositoryAgentTraceDb) { - assert_eq!(count(db, "diff_traces"), 0); - assert_eq!(count(db, "post_commit_patch_intersections"), 0); - assert_eq!(count(db, "agent_traces"), 0); - } - - fn count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("a count row should exist") - } - - fn worktree_revision(db: &RepositoryAgentTraceDb) -> u64 { - db.query_map("SELECT revision FROM mutation_trace_worktrees", (), |row| { - let blob: Vec = row.get(0).map_err(anyhow::Error::from)?; - decode_revision(&blob) - }) - .expect("worktree revision query should succeed") - .into_iter() - .next() - .expect("a worktree row should exist") - } - - fn cursor_tree(db: &RepositoryAgentTraceDb) -> String { - db.query_map( - "SELECT cursor_tree FROM mutation_trace_worktrees", - (), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("cursor_tree query should succeed") - .into_iter() - .next() - .expect("a worktree row should exist") - } - - fn needs_rebaseline(db: &RepositoryAgentTraceDb) -> bool { - db.query_map( - "SELECT needs_rebaseline FROM mutation_trace_worktrees", - (), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("needs_rebaseline query should succeed") - .into_iter() - .next() - .expect("a worktree row should exist") - != 0 - } - - fn processed_events(db: &RepositoryAgentTraceDb) -> Vec<(String, String)> { - db.query_map( - "SELECT scope_id, event_id FROM mutation_trace_processed_events \ - ORDER BY scope_id, event_id", - (), - |row| { - let scope_id = row.get::(0).map_err(anyhow::Error::from)?; - let event_id = row.get::(1).map_err(anyhow::Error::from)?; - Ok((scope_id, event_id)) - }, - ) - .expect("processed-events query should succeed") - } - - fn scope_status(db: &RepositoryAgentTraceDb, scope_id: &str) -> Option<(String, String)> { - db.query_map( - "SELECT actor_kind, status FROM mutation_trace_scopes WHERE scope_id = ?1", - (scope_id,), - |row| { - let actor_kind = row.get::(0).map_err(anyhow::Error::from)?; - let status = row.get::(1).map_err(anyhow::Error::from)?; - Ok((actor_kind, status)) - }, - ) - .expect("scope query should succeed") - .into_iter() - .next() - } - - fn scope_provenance( - db: &RepositoryAgentTraceDb, - scope_id: &str, - ) -> Option<(String, Option)> { - db.query_map( - "SELECT session_id, model_id FROM mutation_trace_scope_provenance \ - WHERE scope_id = ?1", - (scope_id,), - |row| { - let session_id = row.get::(0).map_err(anyhow::Error::from)?; - let model_id = row.get::>(1).map_err(anyhow::Error::from)?; - Ok((session_id, model_id)) - }, - ) - .expect("scope-provenance query should succeed") - .into_iter() - .next() - } - - fn mutation_events(db: &RepositoryAgentTraceDb) -> Vec<(String, Option, String)> { - db.query_map( - "SELECT attribution_kind, attribution_scope_id, boundary_kind \ - FROM mutation_trace_events ORDER BY revision", - (), - |row| { - let attribution_kind = row.get::(0).map_err(anyhow::Error::from)?; - let attribution_scope_id = - row.get::>(1).map_err(anyhow::Error::from)?; - let boundary_kind = row.get::(2).map_err(anyhow::Error::from)?; - Ok((attribution_kind, attribution_scope_id, boundary_kind)) - }, - ) - .expect("mutation-events query should succeed") - } - - struct LostArmedWriter; - - impl Write for LostArmedWriter { - fn write(&mut self, _buffer: &[u8]) -> std::io::Result { - Err(Error::new( - ErrorKind::BrokenPipe, - "injected lost Armed delivery", - )) - } - - fn flush(&mut self) -> std::io::Result<()> { - Err(Error::new( - ErrorKind::BrokenPipe, - "injected lost Armed delivery", - )) - } - } - - #[test] - fn hidden_guard_lost_armed_ack_never_runs_the_exec_command() { - let repo = IngressRepo::new("guard-lost-armed-ack"); - let target = repo.root.join("lost-armed-side-effect"); - let command = format!("touch '{}'", target.display()); - let input = format!( - "{{\"operation\":\"arm\"}}\n{{\"operation\":\"exec\",\"command\":{command:?}}}\n" - ); - - let result = run_external_mutation_guard_protocol_with( - &repo.root, - None, - Cursor::new(input.into_bytes()), - LostArmedWriter, - |root| { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - root, - &repo.state_root, - "lost Armed guard transport test", - ) - }, - ); - - assert!(result.is_err()); - assert!(!target.exists(), "lost Armed must not run the exec command"); - assert!( - repo.marker_path().exists(), - "ambiguous establishment remains conservatively tainted" - ); - } - - #[test] - fn hidden_guard_cancel_after_armed_exits_without_running_a_shell() { - let repo = IngressRepo::new("guard-cancel-before-exec"); - let target = repo.root.join("cancel-side-effect"); - let mut output = Vec::new(); - - run_external_mutation_guard_protocol_with( - &repo.root, - None, - Cursor::new(b"{\"operation\":\"arm\"}\n{\"operation\":\"cancel\"}\n".to_vec()), - &mut output, - |root| { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - root, - &repo.state_root, - "cancel before exec transport test", - ) - }, - ) - .expect("cancel before exec should terminate cleanly"); - - assert!(!target.exists()); - assert!(repo.marker_path().exists()); - assert_eq!( - String::from_utf8(output) - .expect("guard output should be UTF-8") - .lines() - .count(), - 1, - "only Armed should be emitted" - ); - } - - #[test] - fn hidden_guard_eof_after_armed_exits_without_running_a_shell() { - let repo = IngressRepo::new("guard-arm-without-exec"); - let target = repo.root.join("eof-side-effect"); - let mut output = Vec::new(); - - run_external_mutation_guard_protocol_with( - &repo.root, - None, - Cursor::new(b"{\"operation\":\"arm\"}\n".to_vec()), - &mut output, - |root| { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - root, - &repo.state_root, - "arm without exec transport test", - ) - }, - ) - .expect("EOF before exec should terminate cleanly"); - - assert!(!target.exists()); - assert!(repo.marker_path().exists()); - assert_eq!( - String::from_utf8(output) - .expect("guard output should be UTF-8") - .lines() - .count(), - 1, - "only Armed should be emitted" - ); - repo.drive(FLUSH) - .expect("next boundary should self-heal marker"); - assert!(!repo.marker_path().exists()); - } - - #[test] - fn hidden_guard_transport_arms_then_executes_only_the_explicit_exec_command() { - let repo = IngressRepo::new("guard-two-phase-transport"); - let target = repo.root.join("exec-side-effect"); - let duplicate_target = repo.root.join("duplicate-exec-side-effect"); - let command = format!("touch '{}'", target.display()); - let duplicate_command = format!("touch '{}'", duplicate_target.display()); - let input = format!( - "{{\"operation\":\"arm\"}}\n{{\"operation\":\"exec\",\"command\":{command:?}}}\n{{\"operation\":\"exec\",\"command\":{duplicate_command:?}}}\n" - ); - let mut output = Vec::new(); - - run_external_mutation_guard_protocol_with( - &repo.root, - None, - Cursor::new(input.into_bytes()), - &mut output, - |root| { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - root, - &repo.state_root, - "two-phase guard transport test", - ) - }, - ) - .expect("the hidden guard transport should complete"); - - assert!(target.exists(), "the side effect must occur after exec"); - assert!( - !duplicate_target.exists(), - "a duplicate exec must not launch a second shell" - ); - let output = String::from_utf8(output).expect("guard output should be UTF-8"); - let lines: Vec = output - .lines() - .map(|line| serde_json::from_str(line).expect("guard output line should be JSON")) - .collect(); - assert_eq!( - lines.first().and_then(|line| line.get("status")), - Some(&json!("armed")) - ); - assert_eq!( - lines.last().and_then(|line| line.get("status")), - Some(&json!("result")) - ); - assert_eq!( - lines.last().and_then(|line| line.get("exit_code")), - Some(&json!(0)) - ); - } - - const START_A_E1: &str = - r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"claude_code"}"#; - const ADVANCE_A_E2: &str = - r#"{"operation":"advance","scope_id":"A","event_id":"e2","actor_kind":"claude_code"}"#; - const CLOSE_A_E3: &str = - r#"{"operation":"close","scope_id":"A","event_id":"e3","actor_kind":"claude_code"}"#; - const START_A_E1_WITH_PROVENANCE: &str = r#"{"operation":"start","scope_id":"A","event_id":"e1","actor_kind":"claude_code","provenance":{"session_id":"cc_session-1","model_id":"claude/opus"}}"#; - const FLUSH: &str = r#"{"operation":"flush"}"#; - const ABANDON_A: &str = r#"{"operation":"abandon","scope_id":"A"}"#; - - #[test] - #[allow(clippy::too_many_lines)] - fn test1_observed_start_advance_close_lifecycle_persists_durable_rows() { - let repo = IngressRepo::new("observed-lifecycle"); - - assert_eq!(repo.drive(START_A_E1).expect("start should succeed"), ""); - fs::write(repo.root.join("file.txt"), "one\ntwo\n") - .expect("the scoped edit should write"); - assert_eq!( - repo.drive(ADVANCE_A_E2).expect("advance should succeed"), - "" - ); - assert_eq!(repo.drive(CLOSE_A_E3).expect("close should succeed"), ""); - - let db = repo.db(); - assert_eq!( - scope_status(&db, "A").map(|(_, status)| status), - Some("closed".to_string()) - ); - assert_eq!( - processed_events(&db), - vec![ - ("A".to_string(), "e1".to_string()), - ("A".to_string(), "e2".to_string()), - ("A".to_string(), "e3".to_string()), - ] - ); - assert_eq!( - mutation_events(&db), - vec![( - "ai_exclusive".to_string(), - Some("A".to_string()), - "advance".to_string(), - )] - ); - assert_eq!(cursor_tree(&db), repo.working_tree()); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn test2_replayed_advance_is_fully_idempotent() { - let repo = IngressRepo::new("replay-idempotent"); - - repo.drive(START_A_E1).expect("start should succeed"); - fs::write(repo.root.join("file.txt"), "one\ntwo\n") - .expect("the scoped edit should write"); - repo.drive(ADVANCE_A_E2) - .expect("the first advance should succeed"); - - let (revision_before, events_before, processed_before) = { - let db = repo.db(); - ( - worktree_revision(&db), - count(&db, "mutation_trace_events"), - count(&db, "mutation_trace_processed_events"), - ) - }; - - assert_eq!( - repo.drive(ADVANCE_A_E2) - .expect("the replayed advance should succeed"), - "" - ); - - let db = repo.db(); - assert_eq!(worktree_revision(&db), revision_before); - assert_eq!(count(&db, "mutation_trace_events"), events_before); - assert_eq!( - count(&db, "mutation_trace_processed_events"), - processed_before - ); - assert_eq!( - processed_events(&db) - .into_iter() - .filter(|(scope_id, event_id)| scope_id == "A" && event_id == "e2") - .count(), - 1 - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - #[allow(clippy::too_many_lines)] - fn test3_conflicting_actor_kind_commits_no_second_boundary() { - let repo = IngressRepo::new("actor-conflict"); - - repo.drive(START_A_E1).expect("start should succeed"); - - let (revision_before, processed_before, scope_before, events_before) = { - let db = repo.db(); - ( - worktree_revision(&db), - processed_events(&db), - scope_status(&db, "A"), - count(&db, "mutation_trace_events"), - ) - }; - - let error = repo - .drive( - r#"{"operation":"advance","scope_id":"A","event_id":"e2","actor_kind":"codex"}"#, - ) - .expect_err( - "a conflicting actor_kind must fail the ingress, not commit a boundary", - ); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("is already registered to actor"), - "the ingress error must carry the scope/actor identity mismatch diagnostic, \ - got: {rendered}" - ); - - let db = repo.db(); - assert_eq!(worktree_revision(&db), revision_before); - assert_eq!(processed_events(&db), processed_before); - assert!(!processed_events(&db) - .into_iter() - .any(|(scope_id, event_id)| scope_id == "A" && event_id == "e2")); - assert_eq!(scope_status(&db, "A"), scope_before); - assert_eq!( - scope_status(&db, "A").map(|(actor_kind, _)| actor_kind), - Some("claude_code".to_string()) - ); - assert_eq!(count(&db, "mutation_trace_events"), events_before); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - #[allow(clippy::too_many_lines)] - fn test4_abandonment_keeps_no_snapshot_semantics_for_an_unobserved_edit() { - let repo = IngressRepo::new("abandon-unobserved-edit"); - - repo.drive(START_A_E1).expect("start should succeed"); - - let (revision_after_start, cursor_after_start) = { - let db = repo.db(); - (worktree_revision(&db), cursor_tree(&db)) - }; - - fs::write(repo.root.join("file.txt"), "one\nunobserved\n") - .expect("the unobserved edit should write"); - let edited_tree = repo.working_tree(); - assert_ne!( - edited_tree, cursor_after_start, - "the unobserved edit must move the Git tree" - ); - - assert_eq!(repo.drive(ABANDON_A).expect("abandon should succeed"), ""); - - let db = repo.db(); - assert_eq!( - scope_status(&db, "A").map(|(_, status)| status), - Some("abandoned".to_string()) - ); - assert_eq!(worktree_revision(&db), revision_after_start + 1); - assert!(needs_rebaseline(&db)); - assert_eq!(cursor_tree(&db), cursor_after_start); - assert_ne!(cursor_tree(&db), edited_tree); - assert_eq!(count(&db, "mutation_trace_events"), 0); - assert_eq!( - processed_events(&db), - vec![("A".to_string(), "e1".to_string())] - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - #[allow(clippy::too_many_lines)] - fn test5_adversarial_flush_drives_real_observed_flush_behavior() { - let repo = IngressRepo::new("adversarial-flush"); - - assert_eq!( - repo.drive(FLUSH) - .expect("the baseline flush should succeed"), - "" - ); - let revision_after_baseline = { - let db = repo.db(); - worktree_revision(&db) - }; - - fs::write(repo.root.join("file.txt"), "one\nunscoped\n") - .expect("the unscoped edit should write"); - let edited_tree = repo.working_tree(); - - assert_eq!(repo.drive(FLUSH).expect("the flush should succeed"), ""); - - let db = repo.db(); - assert_eq!(cursor_tree(&db), edited_tree); - assert_eq!(worktree_revision(&db), revision_after_baseline + 1); - assert_eq!( - mutation_events(&db), - vec![("ineligible_unscoped".to_string(), None, "flush".to_string())] - ); - assert_eq!(count(&db, "mutation_trace_scopes"), 0); - assert_eq!(count(&db, "mutation_trace_processed_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - #[allow(clippy::too_many_lines)] - fn test6_marker_clear_after_commit_is_durable_success_through_the_ingress() { - let repo = IngressRepo::new("marker-clear-after-commit"); - - repo.drive(START_A_E1).expect("start should succeed"); - fs::write(repo.root.join("file.txt"), "one\nattributable\n") - .expect("the scoped edit should write"); - - let marker = repo.marker_path(); - let calls = Cell::new(0_u32); - let resolver = - |root: &Path, context_message: &'static str| -> Result { - calls.set(calls.get() + 1); - fs::remove_file(&marker) - .expect("the armed marker file should be present mid-invocation"); - fs::create_dir_all(marker.join("nested")) - .expect("planting a non-empty directory at the marker path should succeed"); - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - root, - &repo.state_root, - context_message, - ) - }; - - let result = - run_mutation_scope_from_payload_with(&repo.root, ADVANCE_A_E2, None, resolver); - - assert_eq!( - result.expect("a post-commit marker-clear failure is durable success"), - "" - ); - assert_eq!( - calls.get(), - 1, - "the runtime entrypoint must run exactly once, with no retried transition" - ); - - let db = repo.db(); - assert_eq!( - mutation_events(&db), - vec![( - "ai_exclusive".to_string(), - Some("A".to_string()), - "advance".to_string(), - )] - ); - assert_eq!( - processed_events(&db) - .into_iter() - .filter(|(scope_id, event_id)| scope_id == "A" && event_id == "e2") - .count(), - 1 - ); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - #[allow(clippy::too_many_lines)] - fn test7_marker_clear_after_abandon_is_durable_success_through_the_ingress() { - let repo = IngressRepo::new("marker-clear-after-abandon"); - - repo.drive(START_A_E1).expect("start should succeed"); - let revision_after_start = { - let db = repo.db(); - worktree_revision(&db) - }; - - fs::write(repo.root.join("file.txt"), "one\nunobserved\n") - .expect("the unobserved edit should write"); - - let marker = repo.marker_path(); - let calls = Cell::new(0_u32); - let resolver = - |root: &Path, context_message: &'static str| -> Result { - calls.set(calls.get() + 1); - fs::remove_file(&marker) - .expect("the armed marker file should be present mid-invocation"); - fs::create_dir_all(marker.join("nested")) - .expect("planting a non-empty directory at the marker path should succeed"); - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - root, - &repo.state_root, - context_message, - ) - }; - - let result = - run_mutation_scope_from_payload_with(&repo.root, ABANDON_A, None, resolver); - - assert_eq!( - result.expect("a post-completion marker-clear failure is durable success"), - "" - ); - assert_eq!(calls.get(), 1, "abandon_scope must run exactly once"); - - let db = repo.db(); - assert_eq!( - scope_status(&db, "A").map(|(_, status)| status), - Some("abandoned".to_string()) - ); - assert_eq!(worktree_revision(&db), revision_after_start + 1); - assert!(needs_rebaseline(&db)); - assert_eq!(count(&db, "mutation_trace_events"), 0); - - assert_raw_agent_trace_tables_untouched(&db); - } - - #[test] - fn start_registers_provenance_before_committing_protocol_start() { - let repo = IngressRepo::new("provenance-start"); - - assert_eq!( - repo.drive(START_A_E1_WITH_PROVENANCE) - .expect("a start carrying provenance should succeed"), - "" - ); - - let db = repo.db(); - assert_eq!( - scope_status(&db, "A"), - Some(("claude_code".to_string(), "active".to_string())), - "the owning scope row must exist before provenance is registered" - ); - assert_eq!( - scope_provenance(&db, "A"), - Some(("cc_session-1".to_string(), Some("claude/opus".to_string()))) - ); - assert_eq!( - processed_events(&db), - vec![("A".to_string(), "e1".to_string())], - "the pure protocol Start must commit after provenance is durably registered" - ); - - assert_raw_agent_trace_tables_untouched(&db); - } + drive(payload, Ok(outcome()), &calls).await.unwrap(); + assert_eq!(calls.advise.get(), 0); + assert_eq!(calls.abandon.get(), 1); + } - #[test] - fn start_without_provenance_creates_no_provenance() { - let repo = IngressRepo::new("provenance-absent"); - - repo.drive(START_A_E1) - .expect("a start without provenance should succeed"); - - let db = repo.db(); - assert_eq!( - scope_status(&db, "A").map(|(_, status)| status), - Some("active".to_string()) - ); - assert_eq!( - count(&db, "mutation_trace_scope_provenance"), - 0, - "a start that carried no provenance must not reach provenance persistence" - ); - assert_eq!( - processed_events(&db), - vec![("A".to_string(), "e1".to_string())] - ); - - assert_raw_agent_trace_tables_untouched(&db); + #[tokio::test] + async fn quiet_outcomes_log_debug_and_emit_no_diagnostic() { + for name in ["anchored", "no_action", "busy"] { + let (records, stderr) = deliver(report(name, AdvisorySeverity::Debug)).await; + assert_eq!(stderr, "", "{name}"); + assert_eq!(records.len(), 1, "{name}"); + assert_eq!(records[0].level, "debug", "{name}"); + assert_eq!(records[0].event_id, ADVISORY_EVENT_ID); + assert_eq!(field(&records[0], "outcome"), Some(name)); + assert_eq!(field(&records[0], "persistence_warning"), None); } + } - #[test] - fn replayed_start_with_provenance_commits_nothing_new() { - let repo = IngressRepo::new("provenance-replay"); - - repo.drive(START_A_E1_WITH_PROVENANCE) - .expect("the first start should succeed"); - let (revision_before, processed_before, provenance_before) = { - let db = repo.db(); - ( - worktree_revision(&db), - processed_events(&db), - scope_provenance(&db, "A"), - ) - }; - - assert_eq!( - repo.drive(START_A_E1_WITH_PROVENANCE) - .expect("an identical replay should succeed"), - "" - ); - - let db = repo.db(); - assert_eq!(scope_provenance(&db, "A"), provenance_before); - assert_eq!(worktree_revision(&db), revision_before); - assert_eq!(processed_events(&db), processed_before); - - assert_raw_agent_trace_tables_untouched(&db); - } + #[tokio::test] + async fn advised_logs_warn_and_writes_recommendation_to_diagnostics() { + let diagnostic = format!("SCE: {RECOMMENDED}"); + let (records, stderr) = deliver(AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(diagnostic.clone()), + ..report("advised", AdvisorySeverity::Warn) + }) + .await; + assert_eq!(stderr, format!("{diagnostic}\n")); + assert_eq!(records.len(), 1); + assert_eq!(records[0].level, "warn"); + assert_eq!(records[0].event_id, ADVISORY_EVENT_ID); + assert_eq!(records[0].message, RECOMMENDED); + assert_eq!(field(&records[0], "outcome"), Some("advised")); + assert_eq!(field(&records[0], "persistence_warning"), None); + } - #[test] - fn provenance_conflict_prevents_start_commit() { - let repo = IngressRepo::new("provenance-session-conflict"); - - repo.drive(START_A_E1_WITH_PROVENANCE) - .expect("the first start should succeed"); - - let (revision_before, processed_before, provenance_before) = { - let db = repo.db(); - ( - worktree_revision(&db), - processed_events(&db), - scope_provenance(&db, "A"), - ) - }; - - let error = repo - .drive( - r#"{"operation":"start","scope_id":"A","event_id":"e9","actor_kind":"claude_code","provenance":{"session_id":"cc_session-2","model_id":"claude/opus"}}"#, - ) - .expect_err("a conflicting provenance session must fail the start"); - - let rendered = format!("{error:#}"); - assert!( - rendered.contains("already has provenance for session"), - "the ingress error must carry the provenance session conflict diagnostic, \ - got: {rendered}" - ); - - let db = repo.db(); - assert_eq!( - processed_events(&db), - processed_before, - "the fresh event e9 must not be processed once provenance registration fails" - ); - assert_eq!(worktree_revision(&db), revision_before); - assert_eq!(scope_provenance(&db, "A"), provenance_before); - - assert_raw_agent_trace_tables_untouched(&db); - } + #[tokio::test] + async fn durability_uncertain_keeps_persistence_warning_in_log_and_diagnostics() { + let diagnostic = format!( + "SCE: {RECOMMENDED}\nSCE: Advisory-state durability could not be confirmed: sync failed." + ); + let (records, stderr) = deliver(AdvisoryReport { + recommendation: Some(RECOMMENDED), + warning: Some("sync failed".to_string()), + diagnostic: Some(diagnostic.clone()), + ..report("advised_durability_uncertain", AdvisorySeverity::Warn) + }) + .await; + assert_eq!(stderr, format!("{diagnostic}\n")); + assert_eq!(records[0].level, "warn"); + assert_eq!( + field(&records[0], "outcome"), + Some("advised_durability_uncertain") + ); + assert_eq!( + field(&records[0], "persistence_warning"), + Some("sync failed") + ); + } - #[test] - fn admitted_scope_is_never_backfilled_with_replay_provenance() { - let repo = IngressRepo::new("provenance-no-late-backfill"); - - repo.drive(START_A_E1) - .expect("a start without provenance should succeed"); - - let (revision_before, processed_before) = { - let db = repo.db(); - assert_eq!( - scope_status(&db, "A").map(|(_, status)| status), - Some("active".to_string()) - ); - assert_eq!(scope_provenance(&db, "A"), None); - (worktree_revision(&db), processed_events(&db)) - }; - - assert_eq!( - repo.drive(START_A_E1_WITH_PROVENANCE) - .expect("replaying the start with provenance should follow replay semantics"), - "" - ); - - let db = repo.db(); - assert_eq!( - scope_provenance(&db, "A"), + #[tokio::test] + async fn state_failures_log_warn_and_write_diagnostic_without_claiming_advice() { + for (name, warning, diagnostic) in [ + ( + "state_write_failed", + Some("rename failed"), + "SCE: Reconciliation advisory state could not be written: rename failed. Run sce doctor to check reconciliation status.", + ), + ( + "state_unavailable", None, - "provenance may only be created while the scope is never_seen" - ); - assert_eq!(count(&db, "mutation_trace_scope_provenance"), 0); - assert_eq!( - scope_status(&db, "A").map(|(_, status)| status), - Some("active".to_string()) - ); - assert_eq!(worktree_revision(&db), revision_before); - assert_eq!(processed_events(&db), processed_before); - - assert_raw_agent_trace_tables_untouched(&db); + "SCE: Reconciliation advisory state is unavailable. Run sce doctor to check reconciliation status.", + ), + ] { + let (records, stderr) = deliver(AdvisoryReport { + warning: warning.map(str::to_string), + diagnostic: Some(diagnostic.to_string()), + ..report(name, AdvisorySeverity::Warn) + }) + .await; + assert_eq!(stderr, format!("{diagnostic}\n"), "{name}"); + assert!(!stderr.contains(RECOMMENDED), "{name}"); + assert_eq!(records[0].level, "warn", "{name}"); + assert_eq!(field(&records[0], "outcome"), Some(name)); + assert_eq!(field(&records[0], "persistence_warning"), warning); } } - mod guard_protocol { - use super::*; - - #[test] - fn arm_has_no_execution_fields() { - assert!(parse_guard_arm(r#"{"operation":"arm"}"#).is_ok()); - assert!(parse_guard_arm(r#"{"operation":"arm","command":"true"}"#).is_err()); - assert!(parse_guard_arm(r#"{"operation":"guard","command":"true"}"#).is_err()); - } - - #[test] - fn malformed_arm_is_rejected() { - assert!(parse_guard_arm("").is_err()); - assert!(parse_guard_arm("{").is_err()); - assert!(parse_guard_arm(r#"{"operation":"start"}"#).is_err()); - } - - #[test] - fn exec_requires_a_non_blank_command() { - assert!(parse_guard_exec(r#"{"operation":"exec","command":"pwd"}"#).is_ok()); - assert!(parse_guard_exec(r#"{"operation":"exec","command":" "}"#).is_err()); - assert!(parse_guard_exec(r#"{"operation":"exec"}"#).is_err()); - assert!(parse_guard_exec(r#"{"operation":"guard","command":"pwd"}"#).is_err()); - } - - #[test] - fn exec_parses_cwd_and_environment() { - let request = parse_guard_exec( - r#"{"operation":"exec","command":"pwd","cwd":"/repo/crates/foo","env":{"FOO":"bar","BAZ":"qux"}}"#, - ) - .unwrap(); - assert_eq!(request.command, "pwd"); - assert_eq!(request.cwd, Some("/repo/crates/foo".to_string())); - let mut env = request.env; - env.sort(); - assert_eq!( - env, - vec![("BAZ".into(), "qux".into()), ("FOO".into(), "bar".into())] - ); - } - - #[test] - fn invalid_exec_fields_are_rejected() { - assert!( - parse_guard_exec(r#"{"operation":"exec","command":"pwd","cwd":" "}"#).is_err() - ); - assert!( - parse_guard_exec(r#"{"operation":"exec","command":"pwd","env":{"FOO":1}}"#) - .is_err() - ); - assert!( - parse_guard_exec(r#"{"operation":"exec","command":"pwd","env":"nope"}"#).is_err() - ); - assert!( - parse_guard_exec(r#"{"operation":"exec","command":"pwd","extra":true}"#).is_err() - ); - } - - #[test] - fn cancel_is_a_strict_control_frame() { - assert!(parse_guard_cancel(r#"{"operation":"cancel"}"#).is_ok()); - assert!(parse_guard_cancel(r#"{"operation":"cancel","command":"pwd"}"#).is_err()); - assert_eq!( - guard_operation(r#"{"operation":"cancel"}"#).as_deref(), - Some("cancel") - ); - assert_eq!(guard_operation("not json"), None); - } + #[tokio::test] + async fn missing_logger_still_delivers_actionable_diagnostic() { + let mut sink = Vec::new(); + let result = drive_with( + close(), + Ok(outcome()), + AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }, + None::<&RecordingLogger>, + &mut sink, + &Calls::default(), + ) + .await; + assert_eq!(result.unwrap(), ""); + assert_eq!( + String::from_utf8(sink).unwrap(), + format!("SCE: {RECOMMENDED}\n") + ); + } - #[test] - fn armed_event_serializes_without_a_stream_field() { - let line = guard_event_json_line(&GuardEvent::Armed); - let parsed: Value = serde_json::from_str(&line).unwrap(); - assert_eq!(parsed["status"], "armed"); - } + #[tokio::test] + async fn diagnostic_write_failure_does_not_change_mutation_result() { + let calls = Calls::default(); + let advised = AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }; + let logger = RecordingLogger::default(); + let result = drive_with( + close(), + Ok(outcome()), + advised.clone(), + Some(&logger), + &mut FailingWriter, + &calls, + ) + .await; + assert_eq!(result.unwrap(), ""); + assert_eq!(logger.take().len(), 1); + + let marker_clear = CoordinateError::MarkerClearAfterCommit { + source: anyhow!("clear failed"), + committed: Box::new(outcome()), + }; + let result = drive_with( + MutationScopePayload::Flush, + Err(marker_clear), + advised, + None::<&RecordingLogger>, + &mut FailingWriter, + &calls, + ) + .await; + assert_eq!(result.unwrap(), ""); + } - #[test] - fn stdout_and_stderr_events_tag_their_stream() { - let stdout_line = guard_event_json_line(&GuardEvent::Stdout(b"hello".to_vec())); - let parsed: Value = serde_json::from_str(&stdout_line).unwrap(); - assert_eq!(parsed["stream"], "stdout"); - assert_eq!(parsed["data"], "hello"); - - let stderr_line = guard_event_json_line(&GuardEvent::Stderr(b"oops".to_vec())); - let parsed: Value = serde_json::from_str(&stderr_line).unwrap(); - assert_eq!(parsed["stream"], "stderr"); - assert_eq!(parsed["data"], "oops"); - } + #[tokio::test] + async fn marker_clear_after_commit_logs_taint_warning_and_still_advises_once() { + let logger = RecordingLogger::default(); + let mut sink = Vec::new(); + let calls = Calls::default(); + let error = CoordinateError::MarkerClearAfterCommit { + source: anyhow!("clear failed"), + committed: Box::new(outcome()), + }; + let result = drive_with( + close(), + Err(error), + AdvisoryReport { + recommendation: Some(RECOMMENDED), + diagnostic: Some(format!("SCE: {RECOMMENDED}")), + ..report("advised", AdvisorySeverity::Warn) + }, + Some(&logger), + &mut sink, + &calls, + ) + .await; + assert_eq!(result.unwrap(), ""); + let records = logger.take(); + assert_eq!(records.len(), 2); + assert_eq!( + records[0].event_id, + "sce.hooks.mutation_scope.marker_clear_after_durable_completion" + ); + assert_eq!(records[1].event_id, ADVISORY_EVENT_ID); + assert_eq!( + String::from_utf8(sink).unwrap(), + format!("SCE: {RECOMMENDED}\n") + ); + assert_eq!((calls.coordinate.get(), calls.advise.get()), (1, 1)); } } diff --git a/cli/src/services/hooks/mutation_scope_health.rs b/cli/src/services/hooks/mutation_scope_health.rs index 122628d98..6ea7e77cd 100644 --- a/cli/src/services/hooks/mutation_scope_health.rs +++ b/cli/src/services/hooks/mutation_scope_health.rs @@ -1,7 +1,6 @@ use crate::services::mutation_trace::types::ActorKind; #[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) enum MutationScopeHealthStatus { Healthy, Recovering, @@ -10,14 +9,12 @@ pub(crate) enum MutationScopeHealthStatus { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) enum Repairability { AutoFixable, ManualOnly, } #[derive(Clone, Debug, Eq, PartialEq)] -#[allow(dead_code)] pub(crate) struct MutationScopeAdapterHealth { pub(crate) adapter: ActorKind, pub(crate) status: MutationScopeHealthStatus, @@ -25,7 +22,6 @@ pub(crate) struct MutationScopeAdapterHealth { pub(crate) detail: Option, } -#[allow(dead_code)] impl MutationScopeAdapterHealth { pub(crate) fn new( adapter: ActorKind, diff --git a/cli/src/services/hooks/mutation_scope_ingress_conformance.rs b/cli/src/services/hooks/mutation_scope_ingress_conformance.rs index efe4b5110..0731762bf 100644 --- a/cli/src/services/hooks/mutation_scope_ingress_conformance.rs +++ b/cli/src/services/hooks/mutation_scope_ingress_conformance.rs @@ -7,10 +7,6 @@ use serde_json::{json, Map, Value}; use crate::services::observability::traits::Logger; -pub(crate) type GitDirResolver<'a> = &'a dyn Fn(&str) -> Result; - -pub(crate) type IngressSeam<'a> = &'a dyn Fn(&Path, &str, Option<&dyn Logger>) -> Result; - static NEXT_CONFORMANCE_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); pub(crate) trait IngressConformance { @@ -24,12 +20,12 @@ pub(crate) trait IngressConformance { fn parse(payload: &str) -> Result<()>; - fn run(payload: &str) -> Result; + async fn run(payload: &str) -> Result; - fn run_with_seams( + async fn run_with_seams( payload: &str, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result; } @@ -212,28 +208,28 @@ fn unresolved_git_dir(contract: &str) -> PathBuf { )) } -fn assert_fails_closed_before_dispatch(rejections: &[Rejection]) { +async fn assert_fails_closed_before_dispatch(rejections: &[Rejection]) { assert!(!rejections.is_empty()); for rejection in rejections { assert_validation_error::( "production entrypoint", rejection, - A::run(&rejection.payload), + A::run(&rejection.payload).await, ); let git_dir = unresolved_git_dir::("fails-closed"); let resolver_calls = Cell::new(0_usize); let seam_calls = Cell::new(0_usize); - let resolver = |_cwd: &str| { + let resolver = async |_cwd: &str| { resolver_calls.set(resolver_calls.get() + 1); Ok(git_dir.clone()) }; - let seam = |_root: &Path, _payload: &str, _logger: Option<&dyn Logger>| { + let seam = async |_root: &Path, _payload: &str, _logger: Option<&crate::services::observability::traits::NoopLogger>| { seam_calls.set(seam_calls.get() + 1); Ok(String::new()) }; - let result = A::run_with_seams(&rejection.payload, &resolver, &seam); + let result = A::run_with_seams(&rejection.payload, &resolver, &seam).await; let fabricated_state = git_dir.exists(); let _ = std::fs::remove_dir_all(&git_dir); @@ -269,10 +265,12 @@ pub(crate) fn non_object_json_is_rejected() { assert_parser_rejects::(&non_object_json_rejections()); } -pub(crate) fn missing_required_identity_is_rejected_without_fabrication() { +pub(crate) async fn missing_required_identity_is_rejected_without_fabrication< + A: IngressConformance, +>() { let rejections = missing_required_rejections::(); assert_parser_rejects::(&rejections); - assert_fails_closed_before_dispatch::(&rejections); + assert_fails_closed_before_dispatch::(&rejections).await; } pub(crate) fn blank_required_identifiers_are_rejected() { @@ -316,17 +314,17 @@ pub(crate) fn wrong_typed_or_blank_optional_fields_are_rejected(&invalid_optional_rejections::()); } -pub(crate) fn malformed_runtime_input_fails_closed_before_dispatch() { +pub(crate) async fn malformed_runtime_input_fails_closed_before_dispatch() { let mut rejections = malformed_json_rejections(); rejections.extend(non_object_json_rejections()); rejections.extend(blank_required_rejections::()); rejections.extend(wrong_typed_required_rejections::()); rejections.extend(unsupported_event_rejections::()); rejections.extend(invalid_optional_rejections::()); - assert_fails_closed_before_dispatch::(&rejections); + assert_fails_closed_before_dispatch::(&rejections).await; } -pub(crate) fn tracked_start_fails_closed_when_its_checkout_cannot_be_resolved< +pub(crate) async fn tracked_start_fails_closed_when_its_checkout_cannot_be_resolved< A: CheckoutResolutionConformance, >() { let unresolvable = start_with::( @@ -334,6 +332,7 @@ pub(crate) fn tracked_start_fails_closed_when_its_checkout_cannot_be_resolved< json!(format!("/nonexistent/sce/{}/checkout", A::ADAPTER)), ); let error = A::run(&unresolvable) + .await .expect_err("a tracked start that cannot resolve its checkout must fail closed"); assert!( error.to_string().contains(A::FAIL_CLOSED_MESSAGE), @@ -342,17 +341,21 @@ pub(crate) fn tracked_start_fails_closed_when_its_checkout_cannot_be_resolved< let resolver_calls = Cell::new(0_usize); let seam_calls = Cell::new(0_usize); - let resolver = |_cwd: &str| { + let resolver = async |_cwd: &str| { resolver_calls.set(resolver_calls.get() + 1); Err(anyhow!("checkout resolution failure injected by test")) }; - let seam = |_root: &Path, _payload: &str, _logger: Option<&dyn Logger>| { - seam_calls.set(seam_calls.get() + 1); - Ok(String::new()) - }; + let seam = + async |_root: &Path, + _payload: &str, + _logger: Option<&crate::services::observability::traits::NoopLogger>| { + seam_calls.set(seam_calls.get() + 1); + Ok(String::new()) + }; let payload = Value::Object(A::tracked_start()).to_string(); let error = A::run_with_seams(&payload, &resolver, &seam) + .await .expect_err("a failed checkout resolution must surface as a hard failure"); assert!( error.to_string().contains(A::FAIL_CLOSED_MESSAGE), @@ -379,9 +382,9 @@ macro_rules! mutation_scope_ingress_conformance_tests { }; ($adapter:ty => $($contract:ident),+ $(,)?) => { $( - #[test] - fn $contract() { - $crate::services::hooks::mutation_scope_ingress_conformance::$contract::<$adapter>(); + #[tokio::test] + async fn $contract() { + $crate::services::hooks::mutation_scope_ingress_conformance::$contract::<$adapter>().await; } )+ }; diff --git a/cli/src/services/hooks/mutation_scope_lock.rs b/cli/src/services/hooks/mutation_scope_lock.rs index 78d6b20eb..05b7b62be 100644 --- a/cli/src/services/hooks/mutation_scope_lock.rs +++ b/cli/src/services/hooks/mutation_scope_lock.rs @@ -1,5 +1,6 @@ use std::fs::{File, OpenOptions, TryLockError}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{anyhow, Context}; @@ -14,6 +15,11 @@ const BOUNDARY_LOCK_WHAT: &str = "adapter-boundary"; const STATE_LOCK_FAILURE_CONTEXT: &str = "Failed to acquire adapter-state lock"; const BOUNDARY_LOCK_FAILURE_CONTEXT: &str = "Failed to acquire adapter boundary lock"; +const LOCKED_OPERATION_WORKER_FAILURE_CONTEXT: &str = "The locked operation worker failed"; + +tokio::task_local! { + static BOUNDARY_LEASE: Arc; +} #[derive(Debug)] pub(crate) enum AdvisoryLockError { @@ -23,6 +29,10 @@ pub(crate) enum AdvisoryLockError { what: &'static str, }, Io(anyhow::Error), + WorkerFailed { + what: &'static str, + source: tokio::task::JoinError, + }, } impl std::fmt::Display for AdvisoryLockError { @@ -38,6 +48,9 @@ impl std::fmt::Display for AdvisoryLockError { path.display() ), AdvisoryLockError::Io(source) => write!(f, "{source}"), + AdvisoryLockError::WorkerFailed { what, source } => { + write!(f, "The {what} lock acquisition worker failed: {source}") + } } } } @@ -134,11 +147,67 @@ impl AdapterLockSpec { adapter_state_dir.join(self.file_name) } - pub(crate) fn acquire(&self, adapter_state_dir: &Path) -> anyhow::Result { - self.acquire_with_timeout(adapter_state_dir, self.default_timeout) + pub(crate) async fn run_locked_blocking( + &self, + adapter_state_dir: &Path, + operation: F, + ) -> anyhow::Result + where + T: Send + 'static, + F: FnOnce() -> anyhow::Result + Send + 'static, + { + let spec = *self; + let adapter_state_dir = adapter_state_dir.to_owned(); + let boundary_lease = BOUNDARY_LEASE.try_with(Arc::clone).ok(); + tokio::task::spawn_blocking(move || { + let _boundary_lease = boundary_lease; + let _lock = spec + .acquire_with_timeout(&adapter_state_dir, spec.default_timeout) + .map_err(|error| anyhow!("{}: {error}", spec.failure_context))?; + operation() + }) + .await + .map_err(|source| { + anyhow!( + "{LOCKED_OPERATION_WORKER_FAILURE_CONTEXT} for the {} lock: {source}", + spec.what + ) + })? + } + + pub(crate) async fn run_under_boundary( + &self, + adapter_state_dir: &Path, + operation: impl std::ops::AsyncFnOnce() -> anyhow::Result, + ) -> anyhow::Result { + let lease = Arc::new(self.acquire_async(adapter_state_dir).await?); + BOUNDARY_LEASE.scope(lease, operation()).await + } + + pub(crate) async fn acquire_async( + &self, + adapter_state_dir: &Path, + ) -> anyhow::Result { + self.acquire_with_timeout_async(adapter_state_dir, self.default_timeout) + .await .map_err(|error| anyhow!("{}: {error}", self.failure_context)) } + pub(crate) async fn acquire_with_timeout_async( + &self, + adapter_state_dir: &Path, + timeout: Duration, + ) -> Result { + let spec = *self; + let adapter_state_dir = adapter_state_dir.to_owned(); + tokio::task::spawn_blocking(move || spec.acquire_with_timeout(&adapter_state_dir, timeout)) + .await + .map_err(|source| AdvisoryLockError::WorkerFailed { + what: spec.what, + source, + })? + } + pub(crate) fn acquire_with_timeout( &self, adapter_state_dir: &Path, @@ -155,242 +224,62 @@ impl AdapterLockSpec { #[cfg(test)] mod tests { - use std::io::Write as _; - use std::process::Command; - use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; - use std::thread; - - use super::*; - use crate::services::hooks::{ - codex_mutation_scope, opencode_mutation_scope, pi_mutation_scope, - }; - - const TEST_LOCK: AdapterLockSpec = AdapterLockSpec::boundary("test-mutation-scope.lock"); - - static NEXT_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_lock_dir(label: &str) -> PathBuf { - let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-mutation-scope-lock-{label}-{}-{id}", - std::process::id() - )) - } - - const CHILD_ENV_LOCK_DIR: &str = "SCE_MUTATION_SCOPE_LOCK_CHILD_DIR"; - const CHILD_ENV_TIMEOUT_MS: &str = "SCE_MUTATION_SCOPE_LOCK_CHILD_TIMEOUT_MS"; - const CHILD_HELPER_PATH: &str = - "services::hooks::mutation_scope_lock::tests::mutation_scope_lock_child_helper"; - - #[test] - #[ignore = "subprocess helper, driven by process_death_releases_the_lock"] - fn mutation_scope_lock_child_helper() { - let lock_dir = PathBuf::from( - std::env::var(CHILD_ENV_LOCK_DIR).expect("child helper needs a lock dir in the env"), - ); - let timeout = Duration::from_millis( - std::env::var(CHILD_ENV_TIMEOUT_MS) - .expect("child helper needs a timeout in the env") - .parse() - .expect("timeout must parse"), - ); - match TEST_LOCK.acquire_with_timeout(&lock_dir, timeout) { - Ok(lock) => { - print!("ACQUIRED"); - std::io::stdout().flush().expect("flush stdout"); - std::mem::forget(lock); + use std::time::Duration; + + use super::{AdapterLockSpec, AdvisoryLockError}; + + const TEST_BOUNDARY: AdapterLockSpec = AdapterLockSpec::boundary("test-boundary.lock"); + const TEST_STATE: AdapterLockSpec = AdapterLockSpec::state("test-state.lock"); + const PROBE_TIMEOUT: Duration = Duration::from_millis(200); + const WORKER_TIMEOUT: Duration = Duration::from_secs(10); + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn cancelled_boundary_keeps_its_lock_until_the_started_state_transition_finishes() { + let dir = tempfile::tempdir().expect("temp dir"); + let state_dir = dir.path().to_path_buf(); + let (started_tx, started_rx) = mpsc::channel::<()>(); + let (release_tx, release_rx) = mpsc::channel::<()>(); + + let boundary_task = tokio::spawn({ + let state_dir = state_dir.clone(); + async move { + TEST_BOUNDARY + .run_under_boundary(&state_dir.clone(), async || { + TEST_STATE + .run_locked_blocking(&state_dir, move || { + started_tx.send(()).expect("started signal"); + release_rx + .recv_timeout(WORKER_TIMEOUT) + .expect("release signal"); + Ok(()) + }) + .await + }) + .await } - Err(AdvisoryLockError::TimedOut { .. }) => { - print!("TIMEOUT"); - std::io::stdout().flush().expect("flush stdout"); - } - Err(other) => panic!("unexpected child lock error: {other}"), - } - } - - fn run_child(lock_dir: &Path, timeout: Duration) -> String { - let exe = std::env::current_exe().expect("test executable path should resolve"); - let output = Command::new(exe) - .args(["--exact", "--ignored", "--nocapture", CHILD_HELPER_PATH]) - .env(CHILD_ENV_LOCK_DIR, lock_dir) - .env(CHILD_ENV_TIMEOUT_MS, timeout.as_millis().to_string()) - .output() - .expect("the child test process should spawn"); - assert!( - output.status.success(), - "child process failed: stdout={:?} stderr={:?}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - ); - let stdout = String::from_utf8_lossy(&output.stdout); - if stdout.contains("ACQUIRED") { - "ACQUIRED".to_string() - } else if stdout.contains("TIMEOUT") { - "TIMEOUT".to_string() - } else { - panic!("child produced no lock verdict: {stdout}"); - } - } - - #[test] - fn process_death_releases_the_lock() { - let lock_dir = unique_lock_dir("process-death"); - - let held = TEST_LOCK - .acquire_with_timeout(&lock_dir, Duration::from_secs(5)) - .expect("the parent should acquire the lock immediately"); - - assert_eq!( - run_child(&lock_dir, Duration::from_millis(300)), - "TIMEOUT", - "a second OS process must not acquire the lock while the parent holds it", - ); - - drop(held); - - assert_eq!( - run_child(&lock_dir, Duration::from_millis(300)), - "ACQUIRED", - "once the parent releases, a child process acquires it", - ); - - let reacquired = TEST_LOCK - .acquire_with_timeout(&lock_dir, Duration::from_secs(2)) - .expect("the child's process exit must have released the lock"); - drop(reacquired); - - let _ = std::fs::remove_dir_all(&lock_dir); - } - - #[test] - fn a_leftover_lock_file_alone_does_not_block_acquisition_and_is_never_truncated_or_removed() { - let lock_dir = unique_lock_dir("leftover-file"); - std::fs::create_dir_all(&lock_dir).expect("lock dir should be created"); - let lock_path = TEST_LOCK.path(&lock_dir); - std::fs::write(&lock_path, b"leftover").expect("leftover lock file should be writable"); - - let lock = TEST_LOCK - .acquire_with_timeout(&lock_dir, Duration::from_millis(200)) - .expect("a lock file with no live OS owner must not block a new acquirer"); - drop(lock); - - assert_eq!( - std::fs::read(&lock_path).expect("the lock file must survive release"), - b"leftover", - "ownership must never be expressed by truncating or removing the lock file", - ); - - let _ = std::fs::remove_dir_all(&lock_dir); - } + }); - #[test] - fn a_second_in_process_acquirer_blocks_until_the_first_releases() { - let lock_dir = unique_lock_dir("in-process-contention"); + tokio::task::spawn_blocking(move || started_rx.recv_timeout(WORKER_TIMEOUT)) + .await + .expect("started wait joins") + .expect("state transition starts"); - let holder = TEST_LOCK - .acquire_with_timeout(&lock_dir, Duration::from_secs(5)) - .expect("first acquirer should succeed immediately"); - - let (tx, rx) = mpsc::channel(); - let lock_dir_clone = lock_dir.clone(); - let handle = thread::spawn(move || { - let result = TEST_LOCK.acquire_with_timeout(&lock_dir_clone, Duration::from_secs(5)); - let _ = tx.send(()); - result.is_ok() - }); + boundary_task.abort(); + assert!(boundary_task.await.expect_err("aborted").is_cancelled()); + let overtaken = TEST_BOUNDARY + .acquire_with_timeout_async(&state_dir, PROBE_TIMEOUT) + .await; assert!( - rx.recv_timeout(Duration::from_millis(300)).is_err(), - "the second acquirer must not proceed while the first holds the lock", + matches!(overtaken, Err(AdvisoryLockError::TimedOut { .. })), + "a second boundary must not overtake the paused state transition" ); - drop(holder); - - rx.recv_timeout(Duration::from_secs(5)) - .expect("the second acquirer should complete once the first releases"); - assert!(handle - .join() - .expect("second acquirer thread should not panic")); - - let _ = std::fs::remove_dir_all(&lock_dir); - } - - #[test] - fn a_contended_acquire_times_out_with_a_role_specific_diagnostic() { - let lock_dir = unique_lock_dir("timeout"); - let state_lock = AdapterLockSpec::state("test-mutation-scope-state.lock"); - - let _boundary_holder = TEST_LOCK - .acquire_with_timeout(&lock_dir, Duration::from_secs(5)) - .expect("boundary holder should acquire immediately"); - let _state_holder = state_lock - .acquire_with_timeout(&lock_dir, Duration::from_secs(5)) - .expect("state holder should acquire immediately"); - - for (spec, what) in [ - (TEST_LOCK, "adapter-boundary"), - (state_lock, "adapter-state"), - ] { - let timeout = Duration::from_millis(100); - let started = Instant::now(); - let Err(error) = spec.acquire_with_timeout(&lock_dir, timeout) else { - panic!("a held {what} lock must not be acquired a second time"); - }; - assert!( - started.elapsed() < Duration::from_secs(5), - "a contended acquire must give up at its timeout", - ); - let expected = format!( - "Timed out after {timeout:?} waiting for the {what} lock '{}'", - spec.path(&lock_dir).display() - ); - assert_eq!(error.to_string(), expected); - assert!(matches!( - &error, - AdvisoryLockError::TimedOut { path, timeout: t, what: w } - if *path == spec.path(&lock_dir) && *t == timeout && *w == what - )); - } - - let _ = std::fs::remove_dir_all(&lock_dir); - } - - #[test] - fn adapter_lock_identities_are_exact_distinct_and_below_the_adapter_state_dir() { - let git_dir = unique_lock_dir("adapter-identities"); - let rows = [ - ( - codex_mutation_scope::state::adapter_state_dir(&git_dir), - codex_mutation_scope::state::STATE_LOCK, - codex_mutation_scope::state::BOUNDARY_LOCK, - "codex-mutation-scope-state.lock", - "codex-mutation-scope-boundary.lock", - ), - ( - opencode_mutation_scope::state::adapter_state_dir(&git_dir), - opencode_mutation_scope::state::STATE_LOCK, - opencode_mutation_scope::state::BOUNDARY_LOCK, - "opencode-mutation-scope-state.lock", - "opencode-mutation-scope-boundary.lock", - ), - ( - pi_mutation_scope::state::adapter_state_dir(&git_dir), - pi_mutation_scope::state::STATE_LOCK, - pi_mutation_scope::state::BOUNDARY_LOCK, - "pi-mutation-scope-state.lock", - "pi-mutation-scope-boundary.lock", - ), - ]; - - let sce_dir = git_dir.join("sce"); - for (state_dir, state_lock, boundary_lock, state_file, boundary_file) in rows { - assert_eq!(state_dir, sce_dir); - assert_eq!(state_lock, AdapterLockSpec::state(state_file)); - assert_eq!(boundary_lock, AdapterLockSpec::boundary(boundary_file)); - assert_eq!(state_lock.path(&state_dir), sce_dir.join(state_file)); - assert_eq!(boundary_lock.path(&state_dir), sce_dir.join(boundary_file)); - assert_ne!(state_lock.path(&state_dir), boundary_lock.path(&state_dir)); - } + release_tx.send(()).expect("release worker"); + TEST_BOUNDARY + .acquire_with_timeout_async(&state_dir, WORKER_TIMEOUT) + .await + .expect("boundary is free once the transition finished"); } } diff --git a/cli/src/services/hooks/mutation_scope_owner.rs b/cli/src/services/hooks/mutation_scope_owner.rs index 73f466406..8197c4f95 100644 --- a/cli/src/services/hooks/mutation_scope_owner.rs +++ b/cli/src/services/hooks/mutation_scope_owner.rs @@ -76,98 +76,3 @@ fn parse_start_ticks(stat: &str) -> Option { let after_comm = stat.rsplit_once(')')?.1; after_comm.split_whitespace().nth(19)?.parse().ok() } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn no_ttl_or_elapsed_time_primitive_is_used_by_this_module() { - let source = include_str!("mutation_scope_owner.rs"); - let production_source = source - .split_once("#[cfg(test)]") - .expect("this module has a #[cfg(test)] boundary") - .0; - let forbidden_tokens = ["Instant", "SystemTime", "time::Duration"]; - for forbidden in forbidden_tokens { - assert!( - !production_source.contains(forbidden), - "D10 forbids TTL/elapsed-time staleness evidence, found {forbidden:?}" - ); - } - } - - #[test] - fn the_current_process_owner_is_never_reported_dead() { - let owner = process_owner_for(std::process::id().cast_signed()); - assert!(!is_definitely_dead(&owner)); - } - - #[test] - fn a_reaped_child_process_is_positively_dead() { - let mut child = std::process::Command::new("true") - .spawn() - .expect("spawning 'true' should succeed"); - let pid = i32::try_from(child.id()).expect("pid fits in i32"); - child.wait().expect("child should exit and be reaped"); - - let owner = ProcessOwner { - pid, - instance_token: None, - }; - assert!( - is_definitely_dead(&owner), - "a reaped child's pid must be positively proven dead, not merely assumed" - ); - } - - #[test] - fn a_live_process_is_never_abandoned_merely_because_no_instance_token_is_recorded() { - let owner = ProcessOwner { - pid: std::process::id().cast_signed(), - instance_token: None, - }; - assert!(!is_definitely_dead(&owner)); - } - - #[cfg(target_os = "linux")] - #[test] - fn a_mismatched_instance_token_proves_death_even_though_the_pid_is_alive() { - let real_owner = process_owner_for(std::process::id().cast_signed()); - let recorded_ticks = real_owner - .instance_token - .expect("this process's own /proc/self/stat starttime must be readable on Linux"); - - let stale_owner = ProcessOwner { - pid: real_owner.pid, - instance_token: Some(recorded_ticks.wrapping_add(1)), - }; - assert!( - is_definitely_dead(&stale_owner), - "a live pid whose recorded start time no longer matches must be treated as a \ - different, dead process (PID reuse), never as the still-live original owner" - ); - } - - #[cfg(target_os = "linux")] - #[test] - fn a_matching_instance_token_is_never_reported_dead() { - let real_owner = process_owner_for(std::process::id().cast_signed()); - assert!(real_owner.instance_token.is_some()); - assert!(!is_definitely_dead(&real_owner)); - } - - #[cfg(target_os = "linux")] - #[test] - fn parse_start_ticks_reads_field_twenty_two_after_the_parenthesized_comm() { - let stat = "1234 (my comm) S 1 1234 1234 0 -1 4194304 100 0 0 0 5 3 0 0 20 0 4 0 987654 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 2 0 0 0 0 0 0 0 0 0 0 0 0 0"; - assert_eq!(parse_start_ticks(stat), Some(987_654)); - } - - #[cfg(target_os = "linux")] - #[test] - fn parse_start_ticks_handles_a_comm_containing_spaces_and_parens() { - let stat = "1234 (weird ) comm)) S 1 1234 1234 0 -1 4194304 100 0 0 0 5 3 0 0 20 0 4 0 42 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 2 0 0 0 0 0 0 0 0 0 0 0 0 0"; - assert_eq!(parse_start_ticks(stat), Some(42)); - } -} diff --git a/cli/src/services/hooks/mutation_scope_state_conformance.rs b/cli/src/services/hooks/mutation_scope_state_conformance.rs index 22486a6c5..eea89ac1f 100644 --- a/cli/src/services/hooks/mutation_scope_state_conformance.rs +++ b/cli/src/services/hooks/mutation_scope_state_conformance.rs @@ -482,7 +482,9 @@ pub(crate) fn relinquish_recovery_flush_returns_only_the_claimed_generation_to_p assert_eq!(canonical_bytes::(git_dir.path()), relinquished_bytes); } -pub(crate) fn only_one_concurrent_caller_claims_the_flush_for_a_generation() { +pub(crate) async fn only_one_concurrent_caller_claims_the_flush_for_a_generation< + A: StateConformance, +>() { let git_dir = ConformanceGitDir::create::("one-flush-owner"); seed::( git_dir.path(), diff --git a/cli/src/services/hooks/opencode_mutation_scope/health.rs b/cli/src/services/hooks/opencode_mutation_scope/health.rs index 4eb00cf52..b6b293266 100644 --- a/cli/src/services/hooks/opencode_mutation_scope/health.rs +++ b/cli/src/services/hooks/opencode_mutation_scope/health.rs @@ -14,8 +14,8 @@ pub(crate) enum Repairability { ManualOnly, } -pub(crate) fn assess_repairability(git_dir: &Path) -> Repairability { - let Ok(state) = state::read_state(git_dir) else { +pub(crate) async fn assess_repairability(git_dir: &Path) -> Repairability { + let Ok(state) = state::read_state(git_dir).await else { return Repairability::ManualOnly; }; @@ -40,8 +40,8 @@ pub(crate) fn assess_repairability(git_dir: &Path) -> Repairability { } } -pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { - let state = match state::read_state(git_dir) { +pub(crate) async fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { + let state = match state::read_state(git_dir).await { Ok(state) => state, Err(error) => { return MutationScopeAdapterHealth::new( @@ -90,991 +90,3 @@ pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { ), } } - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::Mutex; - - use serde_json::{json, Value}; - - use super::super::events::AttemptKey; - use super::super::lifecycle::run_opencode_mutation_scope_from_payload_with_seams; - use super::*; - use crate::services::observability::traits::Logger; - - const FAIL_CLOSED_MESSAGE: &str = - "SCE could not establish OpenCode mutation attribution for this tool execution."; - const CWD: &str = "/repo/opencode-checkout"; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-opencode-mutation-scope-health-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(call_id: &str) -> AttemptKey { - AttemptKey { - session_id: "ses-main".to_string(), - call_id: call_id.to_string(), - } - } - - fn tool_before(tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecuteBefore", - "session_id": "ses-main", - "call_id": call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() - } - - fn tool_error(tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolError", - "session_id": "ses-main", - "call_id": call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() - } - - struct RecordingSeam { - calls: Mutex>, - fail_operations: Vec, - fail_operation_occurrence: Option<(String, usize)>, - } - - impl RecordingSeam { - fn new() -> Self { - Self { - calls: Mutex::new(Vec::new()), - fail_operations: Vec::new(), - fail_operation_occurrence: None, - } - } - - fn failing_on(operations: &[&str]) -> Self { - Self { - fail_operations: operations.iter().map(|op| (*op).to_string()).collect(), - ..Self::new() - } - } - - fn failing_on_nth_occurrence(operation: &str, occurrence: usize) -> Self { - Self { - fail_operation_occurrence: Some((operation.to_string(), occurrence)), - ..Self::new() - } - } - - fn handle(&self, payload: &str) -> anyhow::Result { - let operation = operation_of(payload); - let occurrence = { - let mut calls = self.calls.lock().expect("seam mutex"); - calls.push(operation.clone()); - calls - .iter() - .filter(|candidate| *candidate == &operation) - .count() - }; - if self.fail_operations.contains(&operation) { - anyhow::bail!("seam failure injected by test for '{operation}'"); - } - if let Some((target, target_occurrence)) = &self.fail_operation_occurrence { - if target == &operation && *target_occurrence == occurrence { - anyhow::bail!( - "seam failure injected by test for '{operation}' occurrence {occurrence}" - ); - } - } - Ok(String::new()) - } - } - - fn operation_of(payload: &str) -> String { - let value: Value = serde_json::from_str(payload).expect("seam payload is JSON"); - value - .get("operation") - .and_then(Value::as_str) - .expect("seam payload has an operation") - .to_string() - } - - fn drive(git_dir: &Path, seam: &RecordingSeam, payload: &str) -> anyhow::Result { - let resolver = |_cwd: &str| Ok(git_dir.to_path_buf()); - let seam_fn = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| seam.handle(payload); - run_opencode_mutation_scope_from_payload_with_seams(payload, None, &resolver, &seam_fn) - } - - #[test] - fn absent_state_file_is_healthy() { - let git_dir = unique_test_git_dir("absent"); - - let health = classify_health(&git_dir); - - assert_eq!(health.adapter, ActorKind::OpenCode); - assert_eq!(health.status, MutationScopeHealthStatus::Healthy); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_start_with_clear_recovery_is_blocked_and_denies_repeated_unrelated_admissions_ac4() { - let git_dir = unique_test_git_dir("pending-start-blocked"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let failing_start = RecordingSeam::failing_on(&["start"]); - drive(&git_dir, &failing_start, &tool_before("write", "call-1")) - .expect_err("a failed Start seam must fail closed, leaving the attempt PendingStart"); - assert_eq!( - state::read_state(&git_dir) - .expect("state readable") - .attempts[0] - .phase, - AttemptPhase::PendingStart, - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "a stale PendingStart with no recovery in progress has no automatic sweep for unrelated admissions" - ); - - let healthy = RecordingSeam::new(); - for (index, call_id) in ["call-2", "call-3"].into_iter().enumerate() { - let error = - drive(&git_dir, &healthy, &tool_before("write", call_id)).expect_err(&format!( - "unrelated admission #{} must be denied without self-clearing", - index + 1 - )); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - } - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "the classifier must still report Blocked after repeated unrelated denial" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_recovery_with_a_pending_start_attempt_is_blocked_even_though_an_unrelated_pending_abandon_can_still_clear_ac4( - ) { - let git_dir = unique_test_git_dir("pending-recovery-with-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("write", "call-a")).expect("A starts"); - } - - let failing_start = RecordingSeam::failing_on(&["start"]); - drive(&git_dir, &failing_start, &tool_before("write", "call-b")) - .expect_err("B's Start seam fails, leaving B PendingStart"); - assert_eq!( - state::read_state(&git_dir) - .expect("state readable") - .attempts - .iter() - .find(|a| a.call_id == "call-b") - .expect("B is tracked") - .phase, - AttemptPhase::PendingStart, - ); - - let failing_flush = RecordingSeam::failing_on(&["flush"]); - drive(&git_dir, &failing_flush, &tool_error("write", "call-a")) - .expect("A's terminal cleanup returns best-effort even though its flush fails"); - - let seeded = state::read_state(&git_dir).expect("state readable"); - assert_eq!(seeded.attempts.len(), 2, "both A and B are still tracked"); - assert_eq!( - seeded - .attempts - .iter() - .find(|a| a.call_id == "call-a") - .expect("A is tracked") - .phase, - AttemptPhase::PendingAbandon, - ); - assert_eq!( - seeded - .attempts - .iter() - .find(|a| a.call_id == "call-b") - .expect("B is tracked") - .phase, - AttemptPhase::PendingStart, - ); - assert!( - matches!(seeded.recovery, RecoveryState::Pending { .. }), - "the failed ambiguity flush relinquishes Flushing back to Pending: {:?}", - seeded.recovery - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "B's PendingStart has no automatic sweep, so this state has no future path back to \ - normal admission that doesn't depend on B's own missing terminal event, even though \ - A's PendingAbandon under the same Pending recovery generation could still self-heal \ - on its own" - ); - - let healthy = RecordingSeam::new(); - let error = drive(&git_dir, &healthy, &tool_before("write", "call-c")) - .expect_err("C is unrelated to both A and B and must still be denied"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!( - resolved.recovery.is_clear(), - "A's recovery generation advanced to completion via C's admission attempt: {:?}", - resolved.recovery - ); - assert!( - resolved.attempts.iter().all(|a| a.call_id != "call-a"), - "A was cleaned up by the same recovery resolution that denied C" - ); - assert_eq!( - resolved - .attempts - .iter() - .find(|a| a.call_id == "call-b") - .expect("B is still tracked") - .phase, - AttemptPhase::PendingStart, - "B's PendingStart survives the recovery generation that cleared A" - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "recovery for A fully resolved to Clear, but B's PendingStart durably wedges the \ - adapter: this is the reachable Pending+PendingStart -> Clear+PendingStart counterexample" - ); - - let error = drive(&git_dir, &healthy, &tool_before("write", "call-d")) - .expect_err("D is denied again; the adapter never self-clears without B's own event"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "repeated unrelated denial must not change the classification" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn orphaned_flushing_with_a_pending_start_attempt_is_blocked_not_recovering_ac4() { - let git_dir = unique_test_git_dir("orphaned-flushing-with-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("write", "call-a")).expect("A starts"); - } - - let failing_start = RecordingSeam::failing_on(&["start"]); - drive(&git_dir, &failing_start, &tool_before("write", "call-b")) - .expect_err("B's Start seam fails, leaving B PendingStart"); - - let doomed = state::read_state(&git_dir) - .expect("state readable") - .attempts - .into_iter() - .find(|a| a.call_id == "call-a") - .expect("A is tracked"); - - state::begin_terminal_cleanup(&git_dir, std::slice::from_ref(&doomed.scope_id)) - .expect("seeding an orphaned flush should succeed"); - let seeded = state::read_state(&git_dir).expect("state readable"); - assert!(matches!(seeded.recovery, RecoveryState::Flushing { .. })); - assert_eq!( - seeded - .attempts - .iter() - .find(|a| a.call_id == "call-b") - .expect("B is tracked") - .phase, - AttemptPhase::PendingStart, - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "the next boundary reclaims Flushing -> Pending and can advance A's recovery, but \ - B's PendingStart has no reclaim path, so admission stays durably wedged" - ); - - let healthy = RecordingSeam::new(); - let error = drive(&git_dir, &healthy, &tool_before("write", "call-c")) - .expect_err("C is unrelated to both A and B and must still be denied"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!( - resolved.recovery.is_clear(), - "the orphaned flush was reclaimed and A's recovery generation completed: {:?}", - resolved.recovery - ); - assert!(resolved.attempts.iter().all(|a| a.call_id != "call-a")); - assert_eq!( - resolved - .attempts - .iter() - .find(|a| a.call_id == "call-b") - .expect("B is still tracked") - .phase, - AttemptPhase::PendingStart, - ); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "A's orphaned-flush recovery reaching Clear does not rescue B's PendingStart" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_recovery_with_pending_abandon_attempts_is_recovering_and_an_unrelated_admission_clears_it( - ) { - let git_dir = unique_test_git_dir("pending-non-empty-recovering"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("write", "call-1")).expect("Start"); - } - - let failing = RecordingSeam::failing_on(&["abandon"]); - drive(&git_dir, &failing, &tool_error("write", "call-1")) - .expect("a terminal failure whose abandon fails still returns best-effort"); - - let seeded = state::read_state(&git_dir).expect("state readable"); - assert_eq!( - seeded.attempts.len(), - 1, - "the doomed attempt is not forgotten" - ); - assert_eq!(seeded.attempts[0].phase, AttemptPhase::PendingAbandon); - assert!(matches!(seeded.recovery, RecoveryState::Pending { .. })); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "a proven self-healing path exists even though this state currently denies admission" - ); - - for attempt_number in 1..=2 { - let error = drive( - &git_dir, - &failing, - &tool_before("write", &format!("call-retry-{attempt_number}")), - ) - .expect_err("admission while recovery is unresolved must stay fail-closed"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - } - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "repeated denial under a still-failing seam must not change the classification" - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_before("write", "call-2")) - .expect("an unrelated call's admission resolves recovery once the seam succeeds"); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert!(resolved.attempts.iter().all(|a| a.call_id != "call-1")); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_recovery_with_empty_attempts_is_recovering_and_the_next_admission_clears_it() { - let git_dir = unique_test_git_dir("pending-empty-recovering"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("edit", "call-1")).expect("Start"); - } - - let rebaseline_failing = RecordingSeam::failing_on_nth_occurrence("flush", 2); - drive(&git_dir, &rebaseline_failing, &tool_error("edit", "call-1")) - .expect("a terminal failure whose rebaseline flush fails still returns"); - - let seeded = state::read_state(&git_dir).expect("state readable"); - assert!( - seeded.attempts.is_empty(), - "the abandon succeeded so the attempt was removed before the rebaseline flush failed" - ); - assert!(matches!(seeded.recovery, RecoveryState::Pending { .. })); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_before("edit", "call-2")).expect("retry admits new work"); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn orphaned_flushing_with_pending_abandon_attempts_is_recovering_and_reclaimed_by_the_next_boundary( - ) { - let git_dir = unique_test_git_dir("orphaned-flushing-recovering"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let attempt = state::seed_attempt_for_tests( - &git_dir, - &key("call-stuck"), - "write", - AttemptPhase::Active, - ); - state::begin_terminal_cleanup(&git_dir, std::slice::from_ref(&attempt.scope_id)) - .expect("seeding an orphaned flush should succeed"); - assert!(matches!( - state::read_state(&git_dir) - .expect("state readable") - .recovery, - RecoveryState::Flushing { .. } - )); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "a crash mid-flush leaves an orphaned Flushing that the next boundary reclaims" - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_before("write", "call-new")) - .expect("the orphaned flush is reclaimed and retried, then the new call is admitted"); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert!(resolved.attempts.iter().all(|a| a.call_id != "call-stuck")); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn malformed_state_file_is_invalid_with_the_read_error_surfaced() { - let git_dir = unique_test_git_dir("malformed"); - let dir = state::adapter_state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write(state::state_path(&git_dir), b"not json") - .expect("malformed file should be writable"); - - let health = classify_health(&git_dir); - - assert_eq!(health.status, MutationScopeHealthStatus::Invalid); - assert!( - health - .detail - .as_deref() - .unwrap_or_default() - .contains("malformed"), - "the read error must be surfaced in the detail: {:?}", - health.detail - ); - - remove_test_git_dir(&git_dir); - } - - fn matrix_attempt(call_id: &str, phase: AttemptPhase) -> state::AdapterAttempt { - let key = AttemptKey { - session_id: "ses-main".to_string(), - call_id: call_id.to_string(), - }; - - state::AdapterAttempt { - scope_id: super::super::format_opencode_scope_id(&key), - session_id: key.session_id, - call_id: key.call_id, - tool_name: "write".to_string(), - phase, - owner: None, - } - } - - fn write_matrix_state( - git_dir: &Path, - recovery: RecoveryState, - has_active: bool, - has_pending_abandon: bool, - has_pending_start: bool, - ) { - std::fs::create_dir_all(state::adapter_state_dir(git_dir)) - .expect("adapter state dir should be created"); - - let mut attempts = Vec::new(); - if has_active { - attempts.push(matrix_attempt("call-active", AttemptPhase::Active)); - } - if has_pending_abandon { - attempts.push(matrix_attempt("call-abandon", AttemptPhase::PendingAbandon)); - } - if has_pending_start { - attempts.push(matrix_attempt("call-start", AttemptPhase::PendingStart)); - } - - let state = state::AdapterState { - version: 1, - next_recovery_generation: 2, - recovery, - attempts, - }; - std::fs::write( - state::state_path(git_dir), - serde_json::to_string(&state).expect("matrix state serializes"), - ) - .expect("hand-built matrix state should be writable"); - } - - #[test] - fn health_classification_matrix_covers_all_twelve_recovery_and_attempt_phase_combinations() { - use MutationScopeHealthStatus::{Blocked, Healthy, Invalid, Recovering}; - - let clear = RecoveryState::Clear; - let pending = RecoveryState::Pending { generation: 1 }; - let flushing = RecoveryState::Flushing { generation: 1 }; - - let rows = [ - (clear, false, false, Healthy), - (clear, false, true, Blocked), - (clear, true, false, Invalid), - (clear, true, true, Invalid), - (pending, false, false, Recovering), - (pending, false, true, Blocked), - (pending, true, false, Recovering), - (pending, true, true, Blocked), - (flushing, false, false, Recovering), - (flushing, false, true, Blocked), - (flushing, true, false, Recovering), - (flushing, true, true, Blocked), - ]; - - for (index, (recovery, has_pending_abandon, has_pending_start, expected)) in - rows.into_iter().enumerate() - { - for has_active in [false, true] { - let git_dir = unique_test_git_dir(&format!("matrix-{index}-{has_active}")); - write_matrix_state( - &git_dir, - recovery, - has_active, - has_pending_abandon, - has_pending_start, - ); - - assert_eq!( - classify_health(&git_dir).status, - expected, - "row {index}: recovery={recovery:?} has_active={has_active} \ - has_pending_abandon={has_pending_abandon} \ - has_pending_start={has_pending_start}", - ); - - remove_test_git_dir(&git_dir); - } - } - } - - fn dead_owner() -> crate::services::hooks::mutation_scope_owner::ProcessOwner { - let mut child = std::process::Command::new("true") - .spawn() - .expect("spawning 'true' should succeed"); - let pid = i32::try_from(child.id()).expect("pid fits in i32"); - child.wait().expect("child should exit and be reaped"); - crate::services::hooks::mutation_scope_owner::ProcessOwner { - pid, - instance_token: None, - } - } - - fn live_owner() -> crate::services::hooks::mutation_scope_owner::ProcessOwner { - crate::services::hooks::mutation_scope_owner::current_process_owner() - } - - fn seed_pending_start_with_owner( - git_dir: &Path, - call_id: &str, - owner: Option, - ) -> state::AdapterAttempt { - let attempt = state::seed_attempt_for_tests( - git_dir, - &key(call_id), - "write", - AttemptPhase::PendingStart, - ); - state::set_attempt_owner_for_tests(git_dir, &attempt.scope_id, owner) - } - - #[test] - fn assess_repairability_is_manual_only_when_the_adapter_is_not_blocked() { - let git_dir = unique_test_git_dir("assess-not-blocked"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - assert_eq!(assess_repairability(&git_dir), Repairability::ManualOnly); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn assess_repairability_is_manual_only_for_a_legacy_pending_start_attempt_with_no_recorded_owner( - ) { - let git_dir = unique_test_git_dir("assess-legacy-no-owner"); - let dir = state::adapter_state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write( - state::state_path(&git_dir), - serde_json::json!({ - "version": 1, - "next_recovery_generation": 1, - "recovery": { "phase": "clear" }, - "attempts": [{ - "scope_id": "oc-tool-v1|s=8:ses-main|c=6:call-1", - "session_id": "ses-main", - "call_id": "call-1", - "tool_name": "write", - "phase": "pending_start", - }], - }) - .to_string(), - ) - .expect("legacy state file with no owner field should be writable"); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "a legacy state file predating owner evidence must still classify Blocked" - ); - assert_eq!( - assess_repairability(&git_dir), - Repairability::ManualOnly, - "no recorded owner is never treated as proof of death" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn assess_repairability_is_manual_only_when_the_pending_start_owner_is_live() { - let git_dir = unique_test_git_dir("assess-live-owner"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_pending_start_with_owner(&git_dir, "call-1", Some(live_owner())); - - assert_eq!(assess_repairability(&git_dir), Repairability::ManualOnly); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn assess_repairability_is_auto_fixable_when_every_pending_start_owner_is_positively_dead() { - let git_dir = unique_test_git_dir("assess-dead-owner"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_pending_start_with_owner(&git_dir, "call-1", Some(dead_owner())); - - assert_eq!(assess_repairability(&git_dir), Repairability::AutoFixable); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn assess_repairability_is_manual_only_when_one_of_several_pending_start_owners_is_live() { - let git_dir = unique_test_git_dir("assess-mixed-owners"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_pending_start_with_owner(&git_dir, "call-dead", Some(dead_owner())); - seed_pending_start_with_owner(&git_dir, "call-live", Some(live_owner())); - - assert_eq!( - assess_repairability(&git_dir), - Repairability::ManualOnly, - "every contributing PendingStart attempt must have a proven-dead owner" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_clears_a_dead_owner_pending_start_end_to_end() { - let git_dir = unique_test_git_dir("repair-end-to-end"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_pending_start_with_owner(&git_dir, "call-1", Some(dead_owner())); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked - ); - - let healthy = RecordingSeam::new(); - let repository_root = Path::new(CWD); - let outcome = super::super::lifecycle::repair_blocked( - &git_dir, - repository_root, - None, - &|_root: &Path, payload: &str, _logger: Option<&dyn Logger>| healthy.handle(payload), - ) - .expect("repair should not error"); - - assert_eq!(outcome, super::super::lifecycle::RepairOutcome::Repaired); - let final_status = classify_health(&git_dir).status; - assert!( - matches!( - final_status, - MutationScopeHealthStatus::Healthy | MutationScopeHealthStatus::Recovering - ), - "a reported repair must never leave the final health Blocked: {final_status:?}" - ); - assert!(state::read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_is_a_safe_no_op_when_the_pending_start_owner_is_live() { - let git_dir = unique_test_git_dir("repair-live-owner-noop"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt = seed_pending_start_with_owner(&git_dir, "call-1", Some(live_owner())); - - let seam = RecordingSeam::new(); - let repository_root = Path::new(CWD); - let outcome = super::super::lifecycle::repair_blocked( - &git_dir, - repository_root, - None, - &|_root: &Path, payload: &str, _logger: Option<&dyn Logger>| seam.handle(payload), - ) - .expect("repair should not error"); - - assert_eq!(outcome, super::super::lifecycle::RepairOutcome::NoOp); - assert!( - seam.calls.lock().expect("seam mutex").is_empty(), - "a live owner must never trigger any seam call" - ); - let state = state::read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].scope_id, attempt.scope_id); - assert_eq!(state.attempts[0].phase, AttemptPhase::PendingStart); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_refuses_to_abandon_an_attempt_a_concurrent_process_already_started_before_the_lock_is_acquired( - ) { - let git_dir = unique_test_git_dir("repair-concurrent-race"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt = seed_pending_start_with_owner(&git_dir, "call-1", Some(dead_owner())); - - assert_eq!(assess_repairability(&git_dir), Repairability::AutoFixable); - - state::mark_active(&git_dir, &attempt.scope_id) - .expect("simulating the concurrent owning process completing its own Start"); - - let seam = RecordingSeam::new(); - let repository_root = Path::new(CWD); - let outcome = super::super::lifecycle::repair_blocked( - &git_dir, - repository_root, - None, - &|_root: &Path, payload: &str, _logger: Option<&dyn Logger>| seam.handle(payload), - ) - .expect("repair should not error"); - - assert_eq!( - outcome, - super::super::lifecycle::RepairOutcome::NoOp, - "the fresh, lock-protected re-proof must refuse to act on state assessed before it changed" - ); - assert!( - seam.calls.lock().expect("seam mutex").is_empty(), - "no seam call may fire once the attempt is no longer PendingStart" - ); - let state = state::read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy, - "the concurrently-started attempt must be left exactly as the owning process left it" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_is_all_or_nothing_when_auto_fixable_assessment_becomes_stale() { - let git_dir = unique_test_git_dir("repair-all-or-nothing-stale"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let attempt0 = seed_pending_start_with_owner(&git_dir, "call-0", Some(dead_owner())); - let attempt1 = seed_pending_start_with_owner(&git_dir, "call-1", Some(dead_owner())); - - assert_eq!( - assess_repairability(&git_dir), - Repairability::AutoFixable, - "doctor's initial, unlocked assessment sees both owners positively dead" - ); - - state::set_attempt_owner_for_tests(&git_dir, &attempt1.scope_id, Some(live_owner())); - - let seam = RecordingSeam::new(); - let repository_root = Path::new(CWD); - let outcome = super::super::lifecycle::repair_blocked( - &git_dir, - repository_root, - None, - &|_root: &Path, payload: &str, _logger: Option<&dyn Logger>| seam.handle(payload), - ) - .expect("repair should not error"); - - assert_eq!( - outcome, - super::super::lifecycle::RepairOutcome::NoOp, - "the fresh repair-time re-proof must reject the whole batch once any current \ - PendingStart owner is no longer positively dead" - ); - - let state = state::read_state(&git_dir).expect("state readable"); - assert_eq!( - state - .attempts - .iter() - .find(|a| a.scope_id == attempt0.scope_id) - .expect("attempt0 is still tracked") - .phase, - AttemptPhase::PendingStart, - "the still-dead attempt0 must not be transitioned to PendingAbandon when another \ - current blocker fails the all-dead proof" - ); - assert_eq!( - state - .attempts - .iter() - .find(|a| a.scope_id == attempt1.scope_id) - .expect("attempt1 is still tracked") - .phase, - AttemptPhase::PendingStart, - ); - - assert!( - state.recovery.is_clear(), - "a failed re-proof must perform no durable recovery transition: {:?}", - state.recovery - ); - - assert!( - seam.calls.lock().expect("seam mutex").is_empty(), - "the state transaction must return None before resolve_recovery is entered, so no \ - seam call (flush/abandon) may fire" - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Blocked, - "both attempts remain PendingStart, so the adapter stays Blocked" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn repair_blocked_interrupted_before_the_seam_resolves_leaves_state_the_ordinary_recovery_path_completes_without_duplication( - ) { - let git_dir = unique_test_git_dir("repair-interrupted-resume"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_pending_start_with_owner(&git_dir, "call-1", Some(dead_owner())); - - let failing_abandon = RecordingSeam::failing_on(&["abandon"]); - let repository_root = Path::new(CWD); - let outcome = super::super::lifecycle::repair_blocked( - &git_dir, - repository_root, - None, - &|_root: &Path, payload: &str, _logger: Option<&dyn Logger>| { - failing_abandon.handle(payload) - }, - ) - .expect("repair should not error even though the seam abandon call fails"); - - assert_eq!(outcome, super::super::lifecycle::RepairOutcome::NoOp); - let interrupted = state::read_state(&git_dir).expect("state readable"); - assert_eq!(interrupted.attempts.len(), 1); - assert_eq!(interrupted.attempts[0].phase, AttemptPhase::PendingAbandon); - assert!( - matches!(interrupted.recovery, RecoveryState::Pending { .. }), - "a failed seam call during repair must relinquish Flushing back to Pending: {:?}", - interrupted.recovery - ); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "an interrupted repair must never remain Blocked" - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_before("write", "call-2")) - .expect("the ordinary recovery path resumes the interrupted repair automatically"); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert_eq!( - resolved.attempts.len(), - 1, - "the interrupted repair's attempt must not be duplicated" - ); - assert_eq!(resolved.attempts[0].call_id, "call-2"); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } -} diff --git a/cli/src/services/hooks/opencode_mutation_scope/lifecycle.rs b/cli/src/services/hooks/opencode_mutation_scope/lifecycle.rs index a1d5215a7..b6a7c3b2f 100644 --- a/cli/src/services/hooks/opencode_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/opencode_mutation_scope/lifecycle.rs @@ -4,7 +4,6 @@ use anyhow::{bail, Result}; use crate::services::hooks; use crate::services::mutation_trace::runtime::resolve_git_dir; -use crate::services::observability::traits::Logger; use super::events::{ opencode_scope_close_event_id, opencode_scope_provenance, opencode_scope_start_event_id, @@ -14,20 +13,25 @@ use super::events::{ use super::payload::{abandon_payload, flush_payload, scope_boundary_payload, scope_start_payload}; use super::state::{self, AdmitDecision, RecoveryFlushCompletion}; -pub(crate) fn run_opencode_mutation_scope_subcommand( - logger: Option<&dyn Logger>, +pub(crate) async fn run_opencode_mutation_scope_subcommand< + L: crate::services::observability::traits::Logger, +>( + logger: Option<&L>, ) -> Result { let stdin_payload = hooks::read_hook_stdin()?; - run_opencode_mutation_scope_from_payload(&stdin_payload, logger) + run_opencode_mutation_scope_from_payload(&stdin_payload, logger).await } -pub(crate) fn run_opencode_mutation_scope_from_payload( +pub(crate) async fn run_opencode_mutation_scope_from_payload< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { + let resolve_git_dir_fn = async |cwd: &str| resolve_git_dir(Path::new(cwd)).await; + let seam_fn = async |repository_root: &Path, payload: &str, logger: Option<&L>| { hooks::mutation_scope::run_mutation_scope_from_payload(repository_root, payload, logger) + .await }; run_opencode_mutation_scope_from_payload_with_seams( @@ -36,42 +40,19 @@ pub(crate) fn run_opencode_mutation_scope_from_payload( &resolve_git_dir_fn, &seam_fn, ) + .await } -#[cfg(test)] -pub(crate) fn run_opencode_mutation_scope_from_payload_at_state_root( - state_root: &Path, - stdin_payload: &str, - logger: Option<&dyn Logger>, -) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { - hooks::mutation_scope::run_mutation_scope_from_payload_at_state_root( - repository_root, - state_root, - payload, - logger, - ) - }; - - run_opencode_mutation_scope_from_payload_with_seams( - stdin_payload, - logger, - &resolve_git_dir_fn, - &seam_fn, - ) -} - -pub(super) type GitDirResolver<'a> = &'a dyn Fn(&str) -> Result; - -pub(super) type IngressSeam<'a> = &'a dyn Fn(&Path, &str, Option<&dyn Logger>) -> Result; - pub(super) const FAIL_CLOSED_MESSAGE: &str = "SCE could not establish OpenCode mutation attribution for this tool execution."; pub(super) const FAIL_CLOSED_EVENT: &str = "sce.hooks.opencode_mutation_scope.start_fail_closed"; -pub(super) fn log_fail_closed(logger: Option<&dyn Logger>, context: &str, error: &anyhow::Error) { +pub(super) fn log_fail_closed( + logger: Option<&L>, + context: &str, + error: &anyhow::Error, +) { if let Some(log) = logger { log.warn( FAIL_CLOSED_EVENT, @@ -82,21 +63,25 @@ pub(super) fn log_fail_closed(logger: Option<&dyn Logger>, context: &str, error: } } -pub(super) fn run_opencode_mutation_scope_from_payload_with_seams( +pub(super) async fn run_opencode_mutation_scope_from_payload_with_seams< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { let event = parse_opencode_hook_event(stdin_payload)?; - dispatch_opencode_hook_event(event, logger, resolve_git_dir, seam) + dispatch_opencode_hook_event(event, logger, resolve_git_dir, seam).await } -pub(super) fn dispatch_opencode_hook_event( +pub(super) async fn dispatch_opencode_hook_event< + L: crate::services::observability::traits::Logger, +>( event: OpenCodeHookEvent, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { match event { OpenCodeHookEvent::ToolExecuteBefore(execution) => { @@ -118,6 +103,7 @@ pub(super) fn dispatch_opencode_hook_event( resolve_git_dir, seam, ) + .await } ToolClassification::Delegation | ToolClassification::Untracked => Ok(String::new()), } @@ -133,6 +119,7 @@ pub(super) fn dispatch_opencode_hook_event( resolve_git_dir, seam, ) + .await } OpenCodeHookEvent::ToolExecuteAfter(identity) => { if !matches!( @@ -141,27 +128,30 @@ pub(super) fn dispatch_opencode_hook_event( ) { return Ok(String::new()); } - let git_dir = resolve_git_dir(&identity.cwd)?; + let git_dir = resolve_git_dir(&identity.cwd).await?; let repository_root = Path::new(&identity.cwd); let key = identity.attempt_key(); - with_boundary_lock(&git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(&git_dir)?; - handle_close(&git_dir, repository_root, &key, logger, seam) + with_boundary_lock(&git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(&git_dir).await?; + handle_close(&git_dir, repository_root, &key, logger, seam).await }) + .await } OpenCodeHookEvent::ToolError(call) => { if !matches!(call.classification(), ToolClassification::TrackedMutation) { return Ok(String::new()); } - let git_dir = resolve_git_dir(&call.cwd)?; + let git_dir = resolve_git_dir(&call.cwd).await?; let repository_root = Path::new(&call.cwd); let key = call.attempt_key(); - with_boundary_lock(&git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(&git_dir)?; + with_boundary_lock(&git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(&git_dir).await?; abandon_and_consume(&git_dir, repository_root, logger, seam, |attempt| { attempt.session_id == key.session_id && attempt.call_id == key.call_id }) + .await }) + .await } OpenCodeHookEvent::SessionIdle(_) | OpenCodeHookEvent::SessionError(_) @@ -176,33 +166,36 @@ pub(crate) enum RepairOutcome { NoOp, } -pub(crate) fn repair_blocked( +pub(crate) async fn repair_blocked( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - with_boundary_lock(git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(git_dir)?; + with_boundary_lock(git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(git_dir).await?; - let Some(generation) = state::reprove_dead_owner_pending_start_and_begin_repair(git_dir)? + let Some(generation) = + state::reprove_dead_owner_pending_start_and_begin_repair(git_dir).await? else { return Ok(RepairOutcome::NoOp); }; - match resolve_recovery(git_dir, repository_root, generation, logger, seam)? { + match resolve_recovery(git_dir, repository_root, generation, logger, seam).await? { RecoveryResolution::Cleared => Ok(RepairOutcome::Repaired), RecoveryResolution::Unresolved => Ok(RepairOutcome::NoOp), } }) + .await } -pub(super) fn with_boundary_lock( +pub(super) async fn with_boundary_lock( git_dir: &Path, - operation: impl FnOnce() -> Result, + operation: impl std::ops::AsyncFnOnce() -> Result, ) -> Result { - let _boundary = state::BOUNDARY_LOCK.acquire(&state::adapter_state_dir(git_dir))?; - operation() + state::BOUNDARY_LOCK + .run_under_boundary(&state::adapter_state_dir(git_dir), operation) + .await } pub(super) enum Admission { @@ -215,16 +208,16 @@ pub(super) enum StartOutcome { Denied, } -pub(super) fn establish_tracked_start( +pub(super) async fn establish_tracked_start( cwd: &str, key: &AttemptKey, tool_name: &str, provenance: &OpenCodeScopeProvenance, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let git_dir = match resolve_git_dir(cwd) { + let git_dir = match resolve_git_dir(cwd).await { Ok(git_dir) => git_dir, Err(error) => { log_fail_closed(logger, "resolve_git_dir", &error); @@ -233,10 +226,10 @@ pub(super) fn establish_tracked_start( }; let repository_root = Path::new(cwd); - let outcome = with_boundary_lock(&git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(&git_dir)?; + let outcome = with_boundary_lock(&git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(&git_dir).await?; - match admit_or_recover(&git_dir, repository_root, key, tool_name, logger, seam)? { + match admit_or_recover(&git_dir, repository_root, key, tool_name, logger, seam).await? { Admission::Admitted(allocated) => { establish_start( &git_dir, @@ -245,12 +238,14 @@ pub(super) fn establish_tracked_start( provenance, logger, seam, - )?; + ) + .await?; Ok(StartOutcome::Established) } Admission::Denied => Ok(StartOutcome::Denied), } - }); + }) + .await; match outcome { Ok(StartOutcome::Established) => Ok(String::new()), @@ -262,37 +257,37 @@ pub(super) fn establish_tracked_start( } } -pub(super) fn admit_or_recover( +pub(super) async fn admit_or_recover( git_dir: &Path, repository_root: &Path, key: &AttemptKey, tool_name: &str, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - match state::admit_tracked_attempt(git_dir, key, tool_name)? { + match state::admit_tracked_attempt(git_dir, key, tool_name).await? { AdmitDecision::Admitted(allocated) => Ok(Admission::Admitted(allocated)), AdmitDecision::RecoveryBlocked | AdmitDecision::UncertainAttemptBlocked | AdmitDecision::TerminalAttemptBlocked => Ok(Admission::Denied), AdmitDecision::FlushClaimed { generation } => { - match resolve_recovery(git_dir, repository_root, generation, logger, seam)? { - RecoveryResolution::Cleared => readmit_after_flush(git_dir, key, tool_name), + match resolve_recovery(git_dir, repository_root, generation, logger, seam).await? { + RecoveryResolution::Cleared => readmit_after_flush(git_dir, key, tool_name).await, RecoveryResolution::Unresolved => Ok(Admission::Denied), } } } } -pub(super) fn readmit_after_flush( +pub(super) async fn readmit_after_flush( git_dir: &Path, key: &AttemptKey, tool_name: &str, ) -> Result { - match state::admit_tracked_attempt(git_dir, key, tool_name)? { + match state::admit_tracked_attempt(git_dir, key, tool_name).await? { AdmitDecision::Admitted(allocated) => Ok(Admission::Admitted(allocated)), AdmitDecision::FlushClaimed { generation } => { - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; Ok(Admission::Denied) } AdmitDecision::RecoveryBlocked @@ -301,13 +296,13 @@ pub(super) fn readmit_after_flush( } } -pub(super) fn establish_start( +pub(super) async fn establish_start( git_dir: &Path, repository_root: &Path, allocated: &state::AllocatedAttempt, provenance: &OpenCodeScopeProvenance, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { let scope_id = &allocated.attempt.scope_id; @@ -321,19 +316,19 @@ pub(super) fn establish_start( provenance, ); - seam(repository_root, &start_payload, logger)?; - state::mark_active(git_dir, scope_id)?; + seam(repository_root, &start_payload, logger).await?; + state::mark_active(git_dir, scope_id).await?; Ok(()) } -pub(super) fn handle_close( +pub(super) async fn handle_close( git_dir: &Path, repository_root: &Path, key: &AttemptKey, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let Some(attempt) = current .attempts .iter() @@ -351,7 +346,8 @@ pub(super) fn handle_close( ) { return abandon_and_consume(git_dir, repository_root, logger, seam, move |candidate| { candidate.scope_id == doomed_scope_id - }); + }) + .await; } let close_payload = scope_boundary_payload( @@ -360,13 +356,14 @@ pub(super) fn handle_close( &opencode_scope_close_event_id(&attempt.scope_id), ); - if seam(repository_root, &close_payload, logger).is_ok() { - state::remove_attempt(git_dir, &attempt.scope_id)?; + if seam(repository_root, &close_payload, logger).await.is_ok() { + state::remove_attempt(git_dir, &attempt.scope_id).await?; Ok(String::new()) } else { abandon_and_consume(git_dir, repository_root, logger, seam, move |candidate| { candidate.scope_id == doomed_scope_id }) + .await } } @@ -375,14 +372,15 @@ pub(super) enum RecoveryResolution { Unresolved, } -pub(super) fn abandon_and_consume( +pub(super) async fn abandon_and_consume( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, doomed: impl Fn(&state::AdapterAttempt) -> bool, ) -> Result { - let doomed_scope_ids: Vec = state::read_state(git_dir)? + let doomed_scope_ids: Vec = state::read_state(git_dir) + .await? .attempts .into_iter() .filter(|attempt| doomed(attempt)) @@ -392,46 +390,48 @@ pub(super) fn abandon_and_consume( return Ok(String::new()); } - let generation = state::begin_terminal_cleanup(git_dir, &doomed_scope_ids)?; - resolve_recovery(git_dir, repository_root, generation, logger, seam)?; + let generation = state::begin_terminal_cleanup(git_dir, &doomed_scope_ids).await?; + resolve_recovery(git_dir, repository_root, generation, logger, seam).await?; Ok(String::new()) } -pub(super) fn resolve_recovery( +pub(super) async fn resolve_recovery( git_dir: &Path, repository_root: &Path, generation: u64, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let pending_abandon: Vec = state::read_state(git_dir)? + let pending_abandon: Vec = state::read_state(git_dir) + .await? .attempts .into_iter() .filter(|attempt| attempt.phase == state::AttemptPhase::PendingAbandon) .collect(); - if let Err(error) = seam(repository_root, &flush_payload(), logger) { + if let Err(error) = seam(repository_root, &flush_payload(), logger).await { log_fail_closed(logger, "recovery_ambiguity_flush", &error); - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; return Ok(RecoveryResolution::Unresolved); } for attempt in &pending_abandon { - if let Err(error) = seam(repository_root, &abandon_payload(&attempt.scope_id), logger) { + if let Err(error) = seam(repository_root, &abandon_payload(&attempt.scope_id), logger).await + { log_fail_closed(logger, "recovery_abandon", &error); - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; return Ok(RecoveryResolution::Unresolved); } - state::remove_attempt(git_dir, &attempt.scope_id)?; + state::remove_attempt(git_dir, &attempt.scope_id).await?; } - if let Err(error) = seam(repository_root, &flush_payload(), logger) { + if let Err(error) = seam(repository_root, &flush_payload(), logger).await { log_fail_closed(logger, "recovery_rebaseline_flush", &error); - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; return Ok(RecoveryResolution::Unresolved); } - match state::complete_recovery_flush(git_dir, generation)? { + match state::complete_recovery_flush(git_dir, generation).await? { RecoveryFlushCompletion::Cleared => Ok(RecoveryResolution::Cleared), RecoveryFlushCompletion::Superseded => Ok(RecoveryResolution::Unresolved), } diff --git a/cli/src/services/hooks/opencode_mutation_scope/mod.rs b/cli/src/services/hooks/opencode_mutation_scope/mod.rs index cca8c28b0..c5c5160e3 100644 --- a/cli/src/services/hooks/opencode_mutation_scope/mod.rs +++ b/cli/src/services/hooks/opencode_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; @@ -10,11 +8,6 @@ pub(crate) use events::{format_opencode_scope_id, AttemptKey}; #[allow(unused_imports)] pub(crate) use health::{assess_repairability, Repairability}; -#[cfg(test)] -pub(crate) use lifecycle::run_opencode_mutation_scope_from_payload_at_state_root; pub(crate) use lifecycle::run_opencode_mutation_scope_subcommand; #[allow(unused_imports)] pub(crate) use lifecycle::{repair_blocked, RepairOutcome}; - -#[cfg(test)] -mod tests; diff --git a/cli/src/services/hooks/opencode_mutation_scope/state.rs b/cli/src/services/hooks/opencode_mutation_scope/state.rs index 7fcc93dd1..e19403105 100644 --- a/cli/src/services/hooks/opencode_mutation_scope/state.rs +++ b/cli/src/services/hooks/opencode_mutation_scope/state.rs @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, Context, Result}; use serde::{Deserialize, Serialize}; -use crate::services::hooks::mutation_scope_lock::{AdapterLockSpec, OsAdvisoryLock}; +use crate::services::hooks::mutation_scope_lock::AdapterLockSpec; use super::{format_opencode_scope_id, AttemptKey}; use crate::services::hooks::mutation_scope_owner::{ @@ -43,12 +43,6 @@ pub(crate) enum RecoveryState { }, } -impl RecoveryState { - pub(crate) fn is_clear(&self) -> bool { - matches!(self, RecoveryState::Clear) - } -} - #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct AdapterAttempt { pub scope_id: String, @@ -120,7 +114,14 @@ pub(crate) fn state_path(git_dir: &Path) -> PathBuf { adapter_state_dir(git_dir).join(ADAPTER_STATE_FILE) } -pub(crate) fn read_state(git_dir: &Path) -> Result { +pub(crate) async fn read_state(git_dir: &Path) -> Result { + let git_dir = git_dir.to_owned(); + tokio::task::spawn_blocking(move || read_state_sync(&git_dir)) + .await + .context("Adapter state read worker failed")? +} + +fn read_state_sync(git_dir: &Path) -> Result { let path = state_path(git_dir); if !path.exists() { return Ok(AdapterState::default()); @@ -145,11 +146,11 @@ fn parse_adapter_state(content: &str, path: &Path) -> Result { Ok(state) } -fn write_state_durably(git_dir: &Path, state: &AdapterState) -> Result<()> { - write_state_durably_inner(git_dir, state, |_, _| Ok(())) +fn write_state_durably_sync(git_dir: &Path, state: &AdapterState) -> Result<()> { + write_state_durably_sync_inner(git_dir, state, |_, _| Ok(())) } -fn write_state_durably_inner( +fn write_state_durably_sync_inner( git_dir: &Path, state: &AdapterState, before_rename: F, @@ -216,8 +217,29 @@ where Ok(()) } -fn acquire_lock(git_dir: &Path) -> Result { - STATE_LOCK.acquire(&adapter_state_dir(git_dir)) +enum StateTransaction { + Unchanged(T), + Persist(T), +} + +async fn with_locked_state(git_dir: &Path, operation: F) -> Result +where + T: Send + 'static, + F: FnOnce(&mut AdapterState) -> Result> + Send + 'static, +{ + let git_dir = git_dir.to_owned(); + STATE_LOCK + .run_locked_blocking(&adapter_state_dir(&git_dir), move || { + let mut state = read_state_sync(&git_dir)?; + match operation(&mut state)? { + StateTransaction::Unchanged(result) => Ok(result), + StateTransaction::Persist(result) => { + write_state_durably_sync(&git_dir, &state)?; + Ok(result) + } + } + }) + .await } fn allocate_pending_start( @@ -237,108 +259,105 @@ fn allocate_pending_start( attempt } -pub(crate) fn admit_tracked_attempt( +pub(crate) async fn admit_tracked_attempt( git_dir: &Path, key: &AttemptKey, tool_name: &str, ) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let Some(existing) = state - .attempts - .iter() - .find(|attempt| attempt.matches_key(key)) - { - if existing.phase == AttemptPhase::PendingAbandon { - return Ok(AdmitDecision::TerminalAttemptBlocked); + let key = key.clone(); + let tool_name = tool_name.to_string(); + with_locked_state(git_dir, move |state| { + if let Some(existing) = state + .attempts + .iter() + .find(|attempt| attempt.matches_key(&key)) + { + if existing.phase == AttemptPhase::PendingAbandon { + return Ok(StateTransaction::Unchanged( + AdmitDecision::TerminalAttemptBlocked, + )); + } + return Ok(StateTransaction::Unchanged(AdmitDecision::Admitted( + AllocatedAttempt { + attempt: existing.clone(), + reused: true, + }, + ))); } - return Ok(AdmitDecision::Admitted(AllocatedAttempt { - attempt: existing.clone(), - reused: true, - })); - } - match state.recovery { - RecoveryState::Flushing { .. } => return Ok(AdmitDecision::RecoveryBlocked), - RecoveryState::Pending { generation } => { - state.recovery = RecoveryState::Flushing { generation }; - write_state_durably(git_dir, &state)?; - return Ok(AdmitDecision::FlushClaimed { generation }); + match state.recovery { + RecoveryState::Flushing { .. } => { + return Ok(StateTransaction::Unchanged(AdmitDecision::RecoveryBlocked)) + } + RecoveryState::Pending { generation } => { + state.recovery = RecoveryState::Flushing { generation }; + return Ok(StateTransaction::Persist(AdmitDecision::FlushClaimed { + generation, + })); + } + RecoveryState::Clear => {} } - RecoveryState::Clear => {} - } - - if state.attempts.iter().any(|attempt| { - matches!( - attempt.phase, - AttemptPhase::PendingStart | AttemptPhase::PendingAbandon - ) - }) { - return Ok(AdmitDecision::UncertainAttemptBlocked); - } - - let attempt = allocate_pending_start(&mut state, key, tool_name); - write_state_durably(git_dir, &state)?; - Ok(AdmitDecision::Admitted(AllocatedAttempt { - attempt, - reused: false, - })) -} -pub(crate) fn mark_active(git_dir: &Path, scope_id: &str) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - - let mut state = read_state(git_dir)?; - let attempt = state - .attempts - .iter_mut() - .find(|attempt| attempt.scope_id == scope_id) - .ok_or_else(|| anyhow!("No adapter-state attempt found for scope_id '{scope_id}'"))?; - attempt.phase = AttemptPhase::Active; - write_state_durably(git_dir, &state) -} - -pub(crate) fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { - let _lock = acquire_lock(git_dir)?; + if state.attempts.iter().any(|attempt| { + matches!( + attempt.phase, + AttemptPhase::PendingStart | AttemptPhase::PendingAbandon + ) + }) { + return Ok(StateTransaction::Unchanged( + AdmitDecision::UncertainAttemptBlocked, + )); + } - let mut state = read_state(git_dir)?; - let before = state.attempts.len(); - state - .attempts - .retain(|attempt| attempt.scope_id != scope_id); - if state.attempts.len() == before { - return Ok(()); - } - write_state_durably(git_dir, &state) + let attempt = allocate_pending_start(state, &key, &tool_name); + Ok(StateTransaction::Persist(AdmitDecision::Admitted( + AllocatedAttempt { + attempt, + reused: false, + }, + ))) + }) + .await } -pub(crate) fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let RecoveryState::Flushing { generation } = state.recovery { - state.recovery = RecoveryState::Pending { generation }; - write_state_durably(git_dir, &state)?; - } - Ok(()) +pub(crate) async fn mark_active(git_dir: &Path, scope_id: &str) -> Result<()> { + let scope_id = scope_id.to_string(); + with_locked_state(git_dir, move |state| { + let attempt = state + .attempts + .iter_mut() + .find(|attempt| attempt.scope_id == scope_id) + .ok_or_else(|| anyhow!("No adapter-state attempt found for scope_id '{scope_id}'"))?; + attempt.phase = AttemptPhase::Active; + Ok(StateTransaction::Persist(())) + }) + .await +} + +pub(crate) async fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { + let scope_id = scope_id.to_string(); + with_locked_state(git_dir, move |state| { + let before = state.attempts.len(); + state + .attempts + .retain(|attempt| attempt.scope_id != scope_id); + if state.attempts.len() == before { + return Ok(StateTransaction::Unchanged(())); + } + Ok(StateTransaction::Persist(())) + }) + .await } -pub(crate) fn arm_recovery(git_dir: &Path) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - let generation = match state.recovery { - RecoveryState::Pending { generation } => generation, - RecoveryState::Clear | RecoveryState::Flushing { .. } => { - let generation = state.next_recovery_generation; - state.next_recovery_generation += 1; - generation +pub(crate) async fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { + with_locked_state(git_dir, |state| { + if let RecoveryState::Flushing { generation } = state.recovery { + state.recovery = RecoveryState::Pending { generation }; + return Ok(StateTransaction::Persist(())); } - }; - state.recovery = RecoveryState::Pending { generation }; - write_state_durably(git_dir, &state)?; - Ok(generation) + Ok(StateTransaction::Unchanged(())) + }) + .await } fn transition_to_pending_abandon_and_arm_flush( @@ -368,653 +387,73 @@ fn transition_to_pending_abandon_and_arm_flush( generation } -pub(crate) fn begin_terminal_cleanup(git_dir: &Path, scope_ids: &[String]) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - let generation = transition_to_pending_abandon_and_arm_flush(&mut state, scope_ids); - write_state_durably(git_dir, &state)?; - Ok(generation) +pub(crate) async fn begin_terminal_cleanup(git_dir: &Path, scope_ids: &[String]) -> Result { + let scope_ids = scope_ids.to_vec(); + with_locked_state(git_dir, move |state| { + let generation = transition_to_pending_abandon_and_arm_flush(state, &scope_ids); + Ok(StateTransaction::Persist(generation)) + }) + .await } -pub(crate) fn reprove_dead_owner_pending_start_and_begin_repair( +pub(crate) async fn reprove_dead_owner_pending_start_and_begin_repair( git_dir: &Path, ) -> Result> { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - let pending_start: Vec<&AdapterAttempt> = state - .attempts - .iter() - .filter(|attempt| attempt.phase == AttemptPhase::PendingStart) - .collect(); - - if pending_start.is_empty() { - return Ok(None); - } - - let every_owner_is_positively_dead = pending_start - .iter() - .all(|attempt| attempt.owner.as_ref().is_some_and(is_definitely_dead)); + with_locked_state(git_dir, |state| { + let pending_start: Vec<&AdapterAttempt> = state + .attempts + .iter() + .filter(|attempt| attempt.phase == AttemptPhase::PendingStart) + .collect(); - if !every_owner_is_positively_dead { - return Ok(None); - } + if pending_start.is_empty() { + return Ok(StateTransaction::Unchanged(None)); + } - let dead_scope_ids: Vec = pending_start - .iter() - .map(|attempt| attempt.scope_id.clone()) - .collect(); + let every_owner_is_positively_dead = pending_start + .iter() + .all(|attempt| attempt.owner.as_ref().is_some_and(is_definitely_dead)); - let generation = transition_to_pending_abandon_and_arm_flush(&mut state, &dead_scope_ids); - write_state_durably(git_dir, &state)?; - Ok(Some(generation)) -} + if !every_owner_is_positively_dead { + return Ok(StateTransaction::Unchanged(None)); + } -pub(crate) fn arm_and_begin_recovery_flush(git_dir: &Path) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; + let dead_scope_ids: Vec = pending_start + .iter() + .map(|attempt| attempt.scope_id.clone()) + .collect(); - let generation = match state.recovery { - RecoveryState::Pending { generation } | RecoveryState::Flushing { generation } => { - generation - } - RecoveryState::Clear => { - let generation = state.next_recovery_generation; - state.next_recovery_generation += 1; - generation - } - }; - state.recovery = RecoveryState::Flushing { generation }; - write_state_durably(git_dir, &state)?; - Ok(generation) + let generation = transition_to_pending_abandon_and_arm_flush(state, &dead_scope_ids); + Ok(StateTransaction::Persist(Some(generation))) + }) + .await } -pub(crate) fn complete_recovery_flush( +pub(crate) async fn complete_recovery_flush( git_dir: &Path, generation: u64, ) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - match state.recovery { + with_locked_state(git_dir, move |state| match state.recovery { RecoveryState::Flushing { generation: owned } if owned == generation => { state.recovery = RecoveryState::Clear; - write_state_durably(git_dir, &state)?; - Ok(RecoveryFlushCompletion::Cleared) - } - _ => Ok(RecoveryFlushCompletion::Superseded), - } -} - -pub(crate) fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let RecoveryState::Flushing { generation: owned } = state.recovery { - if owned == generation { - state.recovery = RecoveryState::Pending { generation: owned }; - write_state_durably(git_dir, &state)?; - } - } - Ok(()) -} - -#[cfg(test)] -pub(crate) fn seed_attempt_for_tests( - git_dir: &Path, - key: &AttemptKey, - tool_name: &str, - phase: AttemptPhase, -) -> AdapterAttempt { - let _lock = acquire_lock(git_dir).expect("test seed lock"); - let mut state = read_state(git_dir).expect("test seed read"); - allocate_pending_start(&mut state, key, tool_name); - let seeded = state.attempts.last_mut().expect("attempt was just pushed"); - seeded.phase = phase; - let attempt = seeded.clone(); - write_state_durably(git_dir, &state).expect("test seed write"); - attempt -} - -#[cfg(test)] -pub(crate) fn set_attempt_owner_for_tests( - git_dir: &Path, - scope_id: &str, - owner: Option, -) -> AdapterAttempt { - let _lock = acquire_lock(git_dir).expect("test owner-override lock"); - let mut state = read_state(git_dir).expect("test owner-override read"); - let attempt = state - .attempts - .iter_mut() - .find(|attempt| attempt.scope_id == scope_id) - .expect("attempt to override must already exist"); - attempt.owner = owner; - let updated = attempt.clone(); - write_state_durably(git_dir, &state).expect("test owner-override write"); - updated -} - -#[cfg(test)] -mod state_conformance { - use super::*; - use crate::services::hooks::mutation_scope_state_conformance::{ - mutation_scope_state_conformance_tests, CompletionView, FlushClaimView, RecoveryView, - StateConformance, - }; - - const FIXTURE_SESSION: &str = "ses-conformance"; - const FIXTURE_PHASES: [AttemptPhase; 3] = [ - AttemptPhase::Active, - AttemptPhase::PendingAbandon, - AttemptPhase::PendingStart, - ]; - - struct OpenCodeStateConformance; - - fn fixture_key(index: usize) -> AttemptKey { - AttemptKey { - session_id: FIXTURE_SESSION.to_string(), - call_id: format!("call-{index}"), - } - } - - impl StateConformance for OpenCodeStateConformance { - type State = AdapterState; - - const ADAPTER: &'static str = "opencode"; - const SUPPORTED_VERSION: u32 = ADAPTER_STATE_VERSION; - - fn state_path(git_dir: &Path) -> PathBuf { - state_path(git_dir) - } - - fn fixture_state( - recovery: RecoveryView, - next_recovery_generation: u64, - attempt_count: usize, - ) -> AdapterState { - let mut state = AdapterState { - next_recovery_generation, - recovery: match recovery { - RecoveryView::Clear => RecoveryState::Clear, - RecoveryView::Pending(generation) => RecoveryState::Pending { generation }, - RecoveryView::Flushing(generation) => RecoveryState::Flushing { generation }, - }, - ..AdapterState::default() - }; - for index in 0..attempt_count { - allocate_pending_start(&mut state, &fixture_key(index), "apply_patch"); - state.attempts[index].phase = FIXTURE_PHASES[index % FIXTURE_PHASES.len()]; - if index % 2 == 1 { - state.attempts[index].owner = None; - } - } - state - } - - fn read_state(git_dir: &Path) -> Result { - read_state(git_dir) - } - - fn persist(git_dir: &Path, state: &AdapterState) -> Result<()> { - write_state_durably(git_dir, state) - } - - fn persist_with_before_rename_hook( - git_dir: &Path, - state: &AdapterState, - before_rename: F, - ) -> Result<()> - where - F: FnOnce(&Path, &Path) -> Result<()>, - { - write_state_durably_inner(git_dir, state, before_rename) - } - - fn recovery_view(state: &AdapterState) -> RecoveryView { - match state.recovery { - RecoveryState::Clear => RecoveryView::Clear, - RecoveryState::Pending { generation } => RecoveryView::Pending(generation), - RecoveryState::Flushing { generation } => RecoveryView::Flushing(generation), - } - } - - fn next_recovery_generation(state: &AdapterState) -> u64 { - state.next_recovery_generation + Ok(StateTransaction::Persist(RecoveryFlushCompletion::Cleared)) } - - fn scope_ids(state: &AdapterState) -> Vec { - state - .attempts - .iter() - .map(|attempt| attempt.scope_id.clone()) - .collect() - } - - fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { - remove_attempt(git_dir, scope_id) - } - - fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { - normalize_recovery_after_boundary_lock_acquired(git_dir) - } - - fn complete_recovery_flush(git_dir: &Path, generation: u64) -> Result { - Ok(match complete_recovery_flush(git_dir, generation)? { - RecoveryFlushCompletion::Cleared => CompletionView::Cleared, - RecoveryFlushCompletion::Superseded => CompletionView::Superseded, - }) - } - - fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { - relinquish_recovery_flush(git_dir, generation) - } - - fn claim_flush_by_admitting_a_new_attempt( - git_dir: &Path, - contender: usize, - ) -> Result { - let decision = admit_tracked_attempt(git_dir, &fixture_key(contender), "bash")?; - Ok(match decision { - AdmitDecision::FlushClaimed { generation } => FlushClaimView::Claimed(generation), - AdmitDecision::RecoveryBlocked => FlushClaimView::Blocked, - other => FlushClaimView::Other(format!("{other:?}")), - }) - } - } - - mutation_scope_state_conformance_tests!(OpenCodeStateConformance); + _ => Ok(StateTransaction::Unchanged( + RecoveryFlushCompletion::Superseded, + )), + }) + .await } -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - use std::thread; - use std::time::Duration; - - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-opencode-mutation-scope-state-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(session_id: &str, call_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - call_id: call_id.to_string(), - } - } - - fn admit(git_dir: &Path, key: &AttemptKey, tool_name: &str) -> AdmitDecision { - admit_tracked_attempt(git_dir, key, tool_name).expect("admit should not error") - } - - fn admit_and_activate(git_dir: &Path, key: &AttemptKey, tool_name: &str) -> AdapterAttempt { - match admit(git_dir, key, tool_name) { - AdmitDecision::Admitted(allocated) => { - mark_active(git_dir, &allocated.attempt.scope_id) - .expect("mark_active should succeed"); - allocated.attempt +pub(crate) async fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { + with_locked_state(git_dir, move |state| { + if let RecoveryState::Flushing { generation: owned } = state.recovery { + if owned == generation { + state.recovery = RecoveryState::Pending { generation: owned }; + return Ok(StateTransaction::Persist(())); } - other => panic!("expected Admitted, got {other:?}"), } - } - - #[test] - fn admit_persists_the_pending_start_attempt_before_returning() { - let git_dir = unique_test_git_dir("admit-persists-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let AdmitDecision::Admitted(allocated) = admit(&git_dir, &key("ses-1", "call-1"), "write") - else { - panic!("expected Admitted"); - }; - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0], allocated.attempt); - assert_eq!(state.attempts[0].phase, AttemptPhase::PendingStart); - assert_eq!(state.attempts[0].session_id, "ses-1"); - assert_eq!(state.attempts[0].call_id, "call-1"); - assert_eq!(state.attempts[0].tool_name, "write"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_state_is_checkout_local_to_its_git_dir() { - let git_dir_a = unique_test_git_dir("checkout-local-a"); - let git_dir_b = unique_test_git_dir("checkout-local-b"); - std::fs::create_dir_all(&git_dir_a).expect("git dir A should be created"); - std::fs::create_dir_all(&git_dir_b).expect("git dir B should be created"); - - admit_and_activate(&git_dir_a, &key("ses-1", "call-1"), "bash"); - admit_and_activate(&git_dir_a, &key("ses-1", "call-2"), "bash"); - admit_and_activate(&git_dir_b, &key("ses-1", "call-1"), "bash"); - - assert_eq!( - read_state(&git_dir_a) - .expect("state A readable") - .attempts - .len(), - 2 - ); - assert_eq!( - read_state(&git_dir_b) - .expect("state B readable") - .attempts - .len(), - 1 - ); - - remove_test_git_dir(&git_dir_a); - remove_test_git_dir(&git_dir_b); - } - - #[test] - fn duplicate_live_delivery_reuses_the_same_attempt_and_scope_id() { - let git_dir = unique_test_git_dir("duplicate-reuse"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("ses-1", "call-1"); - - let AdmitDecision::Admitted(first) = admit(&git_dir, &attempt_key, "bash") else { - panic!("first admission should be Admitted"); - }; - assert!(!first.reused); - - let AdmitDecision::Admitted(second) = admit(&git_dir, &attempt_key, "bash") else { - panic!("duplicate delivery should still be Admitted"); - }; - assert!(second.reused); - assert_eq!(first.attempt.scope_id, second.attempt.scope_id); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts.len(), 1); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_allows_a_new_key_alongside_an_active_attempt_in_the_same_session() { - let git_dir = unique_test_git_dir("admit-alongside-active"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - admit_and_activate(&git_dir, &key("ses-1", "call-a"), "bash"); - - let AdmitDecision::Admitted(second) = admit(&git_dir, &key("ses-1", "call-b"), "bash") - else { - panic!("D9: a distinct concurrent call may run alongside an Active attempt"); - }; - assert!(!second.reused); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts.len(), - 2 - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_blocks_a_successor_while_an_unresolved_pending_start_exists() { - let git_dir = unique_test_git_dir("admit-blocks-on-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - seed_attempt_for_tests( - &git_dir, - &key("ses-1", "call-a"), - "bash", - AttemptPhase::PendingStart, - ); - - assert_eq!( - admit(&git_dir, &key("ses-1", "call-b"), "bash"), - AdmitDecision::UncertainAttemptBlocked, - "a lingering unconfirmed PendingStart from a crashed invocation must fail closed" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_never_treats_duplicate_delivery_of_a_pending_start_as_uncertain() { - let git_dir = unique_test_git_dir("admit-duplicate-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("ses-1", "call-1"); - - seed_attempt_for_tests(&git_dir, &attempt_key, "bash", AttemptPhase::PendingStart); - - let AdmitDecision::Admitted(again) = admit(&git_dir, &attempt_key, "bash") else { - panic!("re-delivery of the same key must reuse, never UncertainAttemptBlocked"); - }; - assert!(again.reused); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_claims_the_flush_even_while_attempts_remain_outstanding() { - let git_dir = unique_test_git_dir("admit-flush-with-outstanding"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - seed_attempt_for_tests( - &git_dir, - &key("ses-1", "call-live"), - "bash", - AttemptPhase::Active, - ); - let generation = arm_recovery(&git_dir).expect("arming recovery should succeed"); - - assert_eq!( - admit(&git_dir, &key("ses-1", "call-new"), "bash"), - AdmitDecision::FlushClaimed { generation }, - "a recovery/flush operation must be allowed to consume the ambiguous interval \ - even while other attempts remain live", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn arm_and_begin_recovery_flush_claims_flushing_regardless_of_pending_generation() { - let git_dir = unique_test_git_dir("arm-and-begin-flush"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let clear_gen = arm_and_begin_recovery_flush(&git_dir).expect("arm from clear"); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Flushing { - generation: clear_gen - }, - ); - - relinquish_recovery_flush(&git_dir, clear_gen).expect("relinquish"); - let pending_gen = arm_and_begin_recovery_flush(&git_dir).expect("arm from pending"); - assert_eq!(pending_gen, clear_gen, "a pending generation is reused"); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Flushing { - generation: pending_gen - }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_stale_flush_completion_never_clears_a_newer_recovery_generation() { - let git_dir = unique_test_git_dir("stale-flush-completion"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let generation_a = arm_and_begin_recovery_flush(&git_dir).expect("claim generation A"); - assert_eq!(generation_a, 1); - - let generation_b = arm_recovery(&git_dir).expect("a newer abandonment arms generation B"); - assert_eq!(generation_b, 2); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 2 }, - ); - - assert_eq!( - complete_recovery_flush(&git_dir, generation_a).expect("the stale completion returns"), - RecoveryFlushCompletion::Superseded, - ); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 2 }, - "generation A's completion must not clear generation B's recovery requirement", - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_claims_the_flush_when_recovery_is_quiescent() { - let git_dir = unique_test_git_dir("admit-claims-flush"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let generation = arm_recovery(&git_dir).expect("arming recovery should succeed"); - - assert_eq!( - admit(&git_dir, &key("ses-1", "call-new"), "bash"), - AdmitDecision::FlushClaimed { generation }, - ); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Flushing { generation }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn arm_recovery_moves_clear_to_pending_with_a_fresh_generation() { - let git_dir = unique_test_git_dir("arm-clear-to-pending"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let generation = arm_recovery(&git_dir).expect("arming should succeed"); - assert_eq!(generation, 1); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.recovery, RecoveryState::Pending { generation: 1 }); - assert_eq!(state.next_recovery_generation, 2); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn arm_recovery_keeps_the_same_generation_when_already_pending() { - let git_dir = unique_test_git_dir("arm-pending-idempotent"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - assert_eq!(arm_recovery(&git_dir).expect("first arm"), 1); - assert_eq!(arm_recovery(&git_dir).expect("second arm"), 1); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn arm_recovery_supersedes_a_flushing_generation_with_a_newer_one() { - let git_dir = unique_test_git_dir("arm-supersedes-flushing"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - arm_recovery(&git_dir).expect("arm g1"); - admit(&git_dir, &key("ses-1", "call-new"), "bash"); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Flushing { generation: 1 }, - ); - - let generation = arm_recovery(&git_dir).expect("re-arm during flush"); - assert_eq!(generation, 2); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 2 }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_active_transitions_phase_and_rejects_unknown_scope_ids() { - let git_dir = unique_test_git_dir("mark-active"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let AdmitDecision::Admitted(allocated) = admit(&git_dir, &key("ses-1", "call-1"), "bash") - else { - panic!("expected Admitted"); - }; - - mark_active(&git_dir, &allocated.attempt.scope_id).expect("mark_active should succeed"); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::Active, - ); - - let error = mark_active(&git_dir, "oc-tool-v1|s=1:x|c=1:y") - .expect_err("marking an unknown scope active must be rejected"); - assert!(error.to_string().contains("No adapter-state attempt found")); - - remove_test_git_dir(&git_dir); - } - - const PARALLEL_ADMISSION_COUNT: usize = 6; - - #[test] - fn parallel_admissions_serialize_and_converge_without_lost_updates() { - let git_dir = unique_test_git_dir("parallel-admissions"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let handles: Vec<_> = (0..PARALLEL_ADMISSION_COUNT) - .map(|index| { - let git_dir = git_dir.clone(); - thread::spawn(move || { - let attempt_key = key("ses-1", &format!("call-{index}")); - loop { - match admit_tracked_attempt(&git_dir, &attempt_key, "bash") - .expect("admit should not error") - { - AdmitDecision::Admitted(allocated) => { - mark_active(&git_dir, &allocated.attempt.scope_id) - .expect("mark_active should succeed"); - break allocated.attempt.scope_id; - } - AdmitDecision::UncertainAttemptBlocked => { - thread::sleep(Duration::from_millis(5)); - } - other => panic!("unexpected admission decision: {other:?}"), - } - } - }) - }) - .collect(); - - let mut scope_ids: Vec = handles - .into_iter() - .map(|handle| handle.join().expect("thread should not panic")) - .collect(); - scope_ids.sort_unstable(); - scope_ids.dedup(); - assert_eq!(scope_ids.len(), PARALLEL_ADMISSION_COUNT); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts.len(), PARALLEL_ADMISSION_COUNT); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::Active)); - - remove_test_git_dir(&git_dir); - } + Ok(StateTransaction::Unchanged(())) + }) + .await } diff --git a/cli/src/services/hooks/opencode_mutation_scope/tests.rs b/cli/src/services/hooks/opencode_mutation_scope/tests.rs deleted file mode 100644 index 469f6394f..000000000 --- a/cli/src/services/hooks/opencode_mutation_scope/tests.rs +++ /dev/null @@ -1,1929 +0,0 @@ -use super::events::{ - classify_tool, format_opencode_scope_id, opencode_scope_close_event_id, - opencode_scope_provenance, opencode_scope_start_event_id, parse_opencode_hook_event, - AttemptKey, OpenCodeHookEvent, OpenCodeToolExecution, OpenCodeToolIdentity, ToolClassification, - CALL_ID_FIELD, CWD_FIELD, HOOK_EVENT_NAME_FIELD, HOOK_EVENT_SERVER_DISPOSED, - HOOK_EVENT_SESSION_DELETED, HOOK_EVENT_SESSION_ERROR, HOOK_EVENT_SESSION_IDLE, - HOOK_EVENT_SHELL_ENV, HOOK_EVENT_TOOL_ERROR, HOOK_EVENT_TOOL_EXECUTE_AFTER, - HOOK_EVENT_TOOL_EXECUTE_BEFORE, MODEL_FIELD, SESSION_ID_FIELD, TOOL_NAME_FIELD, -}; -use super::lifecycle::{run_opencode_mutation_scope_from_payload, FAIL_CLOSED_MESSAGE}; -use serde_json::{Map, Value}; - -fn tool_event_json(hook_event_name: &str, overrides: &[(&str, Value)]) -> String { - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(hook_event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("ses_main".to_string()), - ); - object.insert( - CALL_ID_FIELD.to_string(), - Value::String("call_1".to_string()), - ); - object.insert( - CWD_FIELD.to_string(), - Value::String("/repo/checkout".to_string()), - ); - object.insert( - TOOL_NAME_FIELD.to_string(), - Value::String("write".to_string()), - ); - for (field, value) in overrides { - object.insert((*field).to_string(), value.clone()); - } - Value::Object(object).to_string() -} - -fn key(session_id: &str, call_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - call_id: call_id.to_string(), - } -} - -fn tool_execution(payload: &str) -> OpenCodeToolExecution { - match parse_opencode_hook_event(payload).expect("valid ToolExecuteBefore parses") { - OpenCodeHookEvent::ToolExecuteBefore(execution) => execution, - other => panic!("expected ToolExecuteBefore, got {other:?}"), - } -} - -mod ingress_conformance { - use anyhow::Result; - - use super::super::lifecycle::run_opencode_mutation_scope_from_payload_with_seams; - use super::*; - use crate::services::hooks::mutation_scope_ingress_conformance::{ - self as conformance, mutation_scope_ingress_conformance_tests, - CheckoutResolutionConformance, IngressConformance, - }; - - struct OpenCodeIngressConformance; - - impl IngressConformance for OpenCodeIngressConformance { - const ADAPTER: &'static str = "OpenCode"; - const EVENT_NAME_FIELD: &'static str = HOOK_EVENT_NAME_FIELD; - const REQUIRED_START_FIELDS: &'static [&'static str] = - &[SESSION_ID_FIELD, CALL_ID_FIELD, CWD_FIELD, TOOL_NAME_FIELD]; - const OPTIONAL_START_FIELDS: &'static [(&'static str, &'static str)] = - &[(MODEL_FIELD, "opencode/big-pickle")]; - const UNSUPPORTED_EVENT_NAMES: &'static [&'static str] = - &["PreToolUse", "ToolExecute", "chat.params"]; - - fn tracked_start() -> Map { - serde_json::from_str(&tool_event_json(HOOK_EVENT_TOOL_EXECUTE_BEFORE, &[])) - .expect("the tracked ToolExecuteBefore fixture is a JSON object") - } - - fn parse(payload: &str) -> Result<()> { - parse_opencode_hook_event(payload).map(|_| ()) - } - - fn run(payload: &str) -> Result { - run_opencode_mutation_scope_from_payload(payload, None) - } - - fn run_with_seams( - payload: &str, - resolve_git_dir: conformance::GitDirResolver, - seam: conformance::IngressSeam, - ) -> Result { - run_opencode_mutation_scope_from_payload_with_seams( - payload, - None, - resolve_git_dir, - seam, - ) - } - } - - impl CheckoutResolutionConformance for OpenCodeIngressConformance { - const CWD_FIELD: &'static str = CWD_FIELD; - const FAIL_CLOSED_MESSAGE: &'static str = FAIL_CLOSED_MESSAGE; - } - - mutation_scope_ingress_conformance_tests!(OpenCodeIngressConformance); - mutation_scope_ingress_conformance_tests!( - OpenCodeIngressConformance => - tracked_start_fails_closed_when_its_checkout_cannot_be_resolved - ); -} - -#[test] -fn tool_execute_before_parses_identity_and_model() { - let execution = tool_execution(&tool_event_json( - HOOK_EVENT_TOOL_EXECUTE_BEFORE, - &[ - (TOOL_NAME_FIELD, Value::String("edit".to_string())), - ( - MODEL_FIELD, - Value::String("opencode/big-pickle".to_string()), - ), - ], - )); - assert_eq!(execution.identity.session_id, "ses_main"); - assert_eq!(execution.identity.call_id, "call_1"); - assert_eq!(execution.identity.tool_name, "edit"); - assert_eq!(execution.model.as_deref(), Some("opencode/big-pickle")); - assert_eq!( - execution.identity.classification(), - ToolClassification::TrackedMutation - ); -} - -#[test] -fn tool_execute_before_model_is_optional() { - let execution = tool_execution(&tool_event_json(HOOK_EVENT_TOOL_EXECUTE_BEFORE, &[])); - assert_eq!(execution.model, None); -} - -#[test] -fn shell_env_parses_without_a_tool_name() { - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_SHELL_ENV.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("ses_main".to_string()), - ); - object.insert( - CALL_ID_FIELD.to_string(), - Value::String("call_bash".to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String("/repo".to_string())); - object.insert( - MODEL_FIELD.to_string(), - Value::String("opencode/big-pickle".to_string()), - ); - let payload = Value::Object(object).to_string(); - - let OpenCodeHookEvent::ShellEnv(shell) = parse_opencode_hook_event(&payload).unwrap() else { - panic!("expected ShellEnv"); - }; - assert_eq!(shell.call_id, "call_bash"); - assert_eq!(shell.model.as_deref(), Some("opencode/big-pickle")); - assert_eq!(shell.attempt_key(), key("ses_main", "call_bash")); -} - -#[test] -fn tool_execute_after_parses_tool_identity() { - let OpenCodeHookEvent::ToolExecuteAfter(identity) = - parse_opencode_hook_event(&tool_event_json(HOOK_EVENT_TOOL_EXECUTE_AFTER, &[])).unwrap() - else { - panic!("expected ToolExecuteAfter"); - }; - assert_eq!(identity.attempt_key(), key("ses_main", "call_1")); - assert_eq!(identity.tool_name, "write"); -} - -#[test] -fn terminal_events_parse_their_minimal_identity() { - for name in [ - HOOK_EVENT_SESSION_IDLE, - HOOK_EVENT_SESSION_ERROR, - HOOK_EVENT_SESSION_DELETED, - ] { - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("ses_main".to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String("/repo".to_string())); - let payload = Value::Object(object).to_string(); - - let event = parse_opencode_hook_event(&payload).unwrap(); - let identity = match event { - OpenCodeHookEvent::SessionIdle(identity) - | OpenCodeHookEvent::SessionError(identity) - | OpenCodeHookEvent::SessionDeleted(identity) => identity, - other => panic!("expected a session-identity event, got {other:?}"), - }; - assert_eq!(identity.session_id, "ses_main"); - assert_eq!(identity.cwd, "/repo"); - } - - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_TOOL_ERROR.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("ses_main".to_string()), - ); - object.insert( - CALL_ID_FIELD.to_string(), - Value::String("call_1".to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String("/repo".to_string())); - object.insert( - TOOL_NAME_FIELD.to_string(), - Value::String("write".to_string()), - ); - let OpenCodeHookEvent::ToolError(identity) = - parse_opencode_hook_event(&Value::Object(object).to_string()).unwrap() - else { - panic!("expected ToolError"); - }; - assert_eq!(identity.attempt_key(), key("ses_main", "call_1")); - assert_eq!( - identity.classification(), - ToolClassification::TrackedMutation - ); - - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(HOOK_EVENT_SERVER_DISPOSED.to_string()), - ); - object.insert(CWD_FIELD.to_string(), Value::String("/repo".to_string())); - let OpenCodeHookEvent::ServerDisposed(workspace) = - parse_opencode_hook_event(&Value::Object(object).to_string()).unwrap() - else { - panic!("expected ServerDisposed"); - }; - assert_eq!(workspace.cwd, "/repo"); -} - -#[test] -fn classification_table() { - let cases: &[(&str, ToolClassification)] = &[ - ("bash", ToolClassification::TrackedMutation), - ("write", ToolClassification::TrackedMutation), - ("edit", ToolClassification::TrackedMutation), - ("apply_patch", ToolClassification::TrackedMutation), - ("task", ToolClassification::Delegation), - ("read", ToolClassification::Untracked), - ("glob", ToolClassification::Untracked), - ("grep", ToolClassification::Untracked), - ("webfetch", ToolClassification::Untracked), - ("websearch", ToolClassification::Untracked), - ("todowrite", ToolClassification::Untracked), - ("probe_mutate", ToolClassification::Untracked), - ( - "brave-search_brave_web_search", - ToolClassification::Untracked, - ), - ("Bash", ToolClassification::Untracked), - ("some_future_opencode_tool", ToolClassification::Untracked), - ("", ToolClassification::Untracked), - ]; - for (tool_name, expected) in cases { - assert_eq!( - classify_tool(tool_name), - *expected, - "classify_tool({tool_name:?})" - ); - } -} - -#[test] -fn classification_is_total_and_single_valued() { - for tool_name in ["bash", "write", "edit", "apply_patch", "task", "read", "x"] { - let _: ToolClassification = classify_tool(tool_name); - } -} - -#[test] -fn scope_id_is_deterministic_for_the_same_key() { - let k = key("ses_main", "call_1"); - assert_eq!(format_opencode_scope_id(&k), format_opencode_scope_id(&k)); - - let scope_id = format_opencode_scope_id(&k); - assert_eq!(scope_id, "oc-tool-v1|s=8:ses_main|c=6:call_1"); - assert_eq!( - opencode_scope_start_event_id(&scope_id), - format!("{scope_id}|start") - ); - assert_eq!( - opencode_scope_close_event_id(&scope_id), - format!("{scope_id}|close") - ); - assert_ne!( - opencode_scope_start_event_id(&scope_id), - opencode_scope_close_event_id(&scope_id) - ); -} - -#[test] -fn duplicate_events_reuse_the_same_scope_id() { - let payload = tool_event_json( - HOOK_EVENT_TOOL_EXECUTE_BEFORE, - &[(TOOL_NAME_FIELD, Value::String("bash".to_string()))], - ); - let first = tool_execution(&payload).identity.attempt_key(); - let second = tool_execution(&payload).identity.attempt_key(); - assert_eq!( - format_opencode_scope_id(&first), - format_opencode_scope_id(&second) - ); -} - -#[test] -fn length_prefix_disambiguates_delimiter_collisions() { - let a = key("s|c=1:x", "y"); - let b = key("s", "1:x|y"); - assert_ne!(format_opencode_scope_id(&a), format_opencode_scope_id(&b)); - - let tricky = key("ses|c=0:x", "call:with:colons"); - assert_eq!( - format_opencode_scope_id(&tricky), - format!( - "oc-tool-v1|s={}:{}|c={}:{}", - tricky.session_id.len(), - tricky.session_id, - tricky.call_id.len(), - tricky.call_id, - ) - ); -} - -#[test] -fn parallel_call_ids_in_one_session_stay_distinguishable() { - let a = tool_execution(&tool_event_json( - HOOK_EVENT_TOOL_EXECUTE_BEFORE, - &[ - (TOOL_NAME_FIELD, Value::String("bash".to_string())), - (CALL_ID_FIELD, Value::String("call_a".to_string())), - ], - )); - let b = tool_execution(&tool_event_json( - HOOK_EVENT_TOOL_EXECUTE_BEFORE, - &[ - (TOOL_NAME_FIELD, Value::String("bash".to_string())), - (CALL_ID_FIELD, Value::String("call_b".to_string())), - ], - )); - assert_ne!(a.identity.attempt_key(), b.identity.attempt_key()); - assert_ne!( - format_opencode_scope_id(&a.identity.attempt_key()), - format_opencode_scope_id(&b.identity.attempt_key()) - ); -} - -#[test] -fn task_child_session_identity_flows_through_the_attempt_key() { - let child = tool_execution(&tool_event_json( - HOOK_EVENT_TOOL_EXECUTE_BEFORE, - &[ - (TOOL_NAME_FIELD, Value::String("bash".to_string())), - (SESSION_ID_FIELD, Value::String("ses_child".to_string())), - (CALL_ID_FIELD, Value::String("call_child".to_string())), - ], - )); - assert_eq!(child.identity.attempt_key(), key("ses_child", "call_child")); - assert_ne!( - format_opencode_scope_id(&child.identity.attempt_key()), - format_opencode_scope_id(&key("ses_main", "call_child")) - ); -} - -#[test] -fn attempt_key_projects_only_session_and_call() { - let identity_a = OpenCodeToolIdentity { - session_id: "ses_main".to_string(), - call_id: "call_1".to_string(), - cwd: "/repo".to_string(), - tool_name: "write".to_string(), - }; - let identity_b = OpenCodeToolIdentity { - tool_name: "bash".to_string(), - cwd: "/other".to_string(), - ..identity_a.clone() - }; - assert_eq!(identity_a.attempt_key(), identity_b.attempt_key()); -} - -#[test] -fn provenance_canonicalizes_the_session_and_normalizes_the_model() { - let provenance = opencode_scope_provenance("ses_main", Some("opencode/big-pickle")); - assert_eq!(provenance.session_id, "oc_ses_main"); - assert_eq!(provenance.model_id.as_deref(), Some("opencode/big-pickle")); -} - -#[test] -fn provenance_keeps_an_already_prefixed_session_id() { - let provenance = opencode_scope_provenance("oc_ses_main", None); - assert_eq!(provenance.session_id, "oc_ses_main"); -} - -#[test] -fn provenance_without_model_evidence_is_null() { - for model in [None, Some(""), Some(" ")] { - let provenance = opencode_scope_provenance("ses_main", model); - assert_eq!(provenance.model_id, None, "model {model:?}"); - assert_eq!(provenance.session_id, "oc_ses_main", "model {model:?}"); - } -} - -#[test] -fn provenance_is_built_from_a_parsed_start_event() { - let execution = tool_execution(&tool_event_json( - HOOK_EVENT_TOOL_EXECUTE_BEFORE, - &[ - (TOOL_NAME_FIELD, Value::String("write".to_string())), - ( - MODEL_FIELD, - Value::String("opencode/big-pickle".to_string()), - ), - ], - )); - let provenance = - opencode_scope_provenance(&execution.identity.session_id, execution.model.as_deref()); - assert_eq!(provenance.session_id, "oc_ses_main"); - assert_eq!(provenance.model_id.as_deref(), Some("opencode/big-pickle")); -} - -#[test] -fn run_from_payload_is_neutral_for_untracked_and_delegation_events() { - for tool_name in ["read", "task", "probe_mutate"] { - let payload = tool_event_json( - HOOK_EVENT_TOOL_EXECUTE_BEFORE, - &[(TOOL_NAME_FIELD, Value::String(tool_name.to_string()))], - ); - assert_eq!( - run_opencode_mutation_scope_from_payload(&payload, None).unwrap(), - String::new() - ); - } -} - -mod lifecycle_tests { - use std::path::{Path, PathBuf}; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::Mutex; - - use anyhow::{bail, Result}; - use serde_json::{json, Value}; - - use crate::services::observability::traits::Logger; - - use super::super::lifecycle::{ - run_opencode_mutation_scope_from_payload_with_seams, FAIL_CLOSED_MESSAGE, - }; - use super::super::state::{read_state, AttemptPhase, RecoveryState}; - - static NEXT_ID: AtomicU64 = AtomicU64::new(0); - - fn temp_git_dir(label: &str) -> PathBuf { - let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-opencode-mutation-scope-lifecycle-{label}-{}-{id}", - std::process::id() - )) - } - - const CWD: &str = "/repo/opencode-checkout"; - - struct RecordingSeam { - calls: Mutex>, - fail_operations: Vec, - fail_once_operations: Mutex>, - fail_operation_occurrence: Option<(String, usize)>, - } - - impl RecordingSeam { - fn new() -> Self { - Self { - calls: Mutex::new(Vec::new()), - fail_operations: Vec::new(), - fail_once_operations: Mutex::new(Vec::new()), - fail_operation_occurrence: None, - } - } - - fn failing_on(operations: &[&str]) -> Self { - Self { - fail_operations: operations.iter().map(|op| (*op).to_string()).collect(), - ..Self::new() - } - } - - fn failing_once_on(operations: &[&str]) -> Self { - Self { - fail_once_operations: Mutex::new( - operations.iter().map(|op| (*op).to_string()).collect(), - ), - ..Self::new() - } - } - - fn failing_on_nth_occurrence(operation: &str, occurrence: usize) -> Self { - Self { - fail_operation_occurrence: Some((operation.to_string(), occurrence)), - ..Self::new() - } - } - - fn handle(&self, payload: &str) -> Result { - let operation = operation_of(payload); - let occurrence = { - let mut calls = self.calls.lock().expect("seam mutex"); - calls.push(operation.clone()); - calls - .iter() - .filter(|candidate| *candidate == &operation) - .count() - }; - if self.fail_operations.contains(&operation) { - bail!("seam failure injected by test for '{operation}'"); - } - if let Some((target, target_occurrence)) = &self.fail_operation_occurrence { - if target == &operation && *target_occurrence == occurrence { - bail!( - "seam failure injected by test for '{operation}' occurrence {occurrence}" - ); - } - } - { - let mut once = self.fail_once_operations.lock().expect("seam mutex"); - if let Some(position) = once.iter().position(|candidate| candidate == &operation) { - once.remove(position); - bail!("transient seam failure injected once by test for '{operation}'"); - } - } - Ok(String::new()) - } - - fn operations(&self) -> Vec { - self.calls.lock().expect("seam mutex").clone() - } - } - - fn operation_of(payload: &str) -> String { - let value: Value = serde_json::from_str(payload).expect("seam payload is JSON"); - value - .get("operation") - .and_then(Value::as_str) - .expect("seam payload has an operation") - .to_string() - } - - fn drive(git_dir: &Path, seam: &RecordingSeam, payload: &str) -> Result { - let resolver = |_cwd: &str| Ok(git_dir.to_path_buf()); - let seam_fn = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| seam.handle(payload); - run_opencode_mutation_scope_from_payload_with_seams(payload, None, &resolver, &seam_fn) - } - - fn tool_before(tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecuteBefore", - "session_id": "ses_main", - "call_id": call_id, - "cwd": CWD, - "tool_name": tool_name, - "model": "opencode/big-pickle", - }) - .to_string() - } - - fn shell_env(call_id: &str) -> String { - json!({ - "hook_event_name": "ShellEnv", - "session_id": "ses_main", - "call_id": call_id, - "cwd": CWD, - "model": "opencode/big-pickle", - }) - .to_string() - } - - fn tool_after(tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecuteAfter", - "session_id": "ses_main", - "call_id": call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() - } - - fn tool_error(tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolError", - "session_id": "ses_main", - "call_id": call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() - } - - fn session_event(hook_event_name: &str, session_id: &str) -> String { - json!({ - "hook_event_name": hook_event_name, - "session_id": session_id, - "cwd": CWD, - }) - .to_string() - } - - fn server_disposed() -> String { - json!({ "hook_event_name": "ServerDisposed", "cwd": CWD }).to_string() - } - - fn cleanup(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - #[test] - fn file_tool_before_establishes_a_write_ahead_start_and_replays_idempotently() { - let git_dir = temp_git_dir("write-ahead-start"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_before("write", "call_1")).expect("first Start"); - drive(&git_dir, &seam, &tool_before("write", "call_1")) - .expect("duplicate Start is a no-op"); - - assert_eq!(seam.operations(), vec!["start"]); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - assert_eq!(state.attempts[0].tool_name, "write"); - - cleanup(&git_dir); - } - - #[test] - fn bash_start_is_anchored_to_shell_env_not_tool_execute_before() { - let git_dir = temp_git_dir("bash-shell-env"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_before("bash", "call_bash")).expect("bash before is inert"); - assert!(seam.operations().is_empty()); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - drive(&git_dir, &seam, &shell_env("call_bash")).expect("shell.env establishes Start"); - assert_eq!(seam.operations(), vec!["start"]); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::Active, - ); - - cleanup(&git_dir); - } - - #[test] - fn concurrent_bash_calls_in_one_session_stay_separate_live_scopes() { - let git_dir = temp_git_dir("concurrent-bash"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &shell_env("call_a")).expect("A Start"); - drive(&git_dir, &seam, &shell_env("call_b")).expect("B Start must not retire A"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 2); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::Active)); - assert_eq!(seam.operations(), vec!["start", "start"]); - - cleanup(&git_dir); - } - - #[test] - fn successful_after_closes_exactly_that_attempt_and_replays_as_a_no_op() { - let git_dir = temp_git_dir("close-replay"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_before("edit", "call_1")).expect("Start"); - drive(&git_dir, &seam, &tool_after("edit", "call_1")).expect("Close"); - drive(&git_dir, &seam, &tool_after("edit", "call_1")).expect("duplicate Close is a no-op"); - - assert_eq!(seam.operations(), vec!["start", "close"]); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - cleanup(&git_dir); - } - - #[test] - fn tool_error_retires_the_named_attempt_and_consumes_the_ambiguous_interval() { - let git_dir = temp_git_dir("tool-error-consume"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &shell_env("call_a")).expect("A Start"); - drive(&git_dir, &seam, &shell_env("call_b")).expect("B Start"); - drive(&git_dir, &seam, &tool_error("bash", "call_a")).expect("A terminal failure"); - - assert_eq!( - seam.operations(), - vec!["start", "start", "flush", "abandon", "flush"], - "the ambiguous interval is flushed before A is abandoned, then the abandon \ - rebaseline is flushed away so B keeps its future intervals", - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].call_id, "call_b"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - assert!( - state.recovery.is_clear(), - "a successful ambiguity flush clears the recovery barrier while B stays live", - ); - - cleanup(&git_dir); - } - - #[test] - fn exact_error_retires_only_the_named_sibling() { - let git_dir = temp_git_dir("exact-error-siblings"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &shell_env("call_a")).expect("A Start"); - drive(&git_dir, &seam, &shell_env("call_b")).expect("B Start"); - drive(&git_dir, &seam, &shell_env("call_c")).expect("C Start"); - drive(&git_dir, &seam, &tool_error("bash", "call_b")).expect("B terminal failure"); - - let state = read_state(&git_dir).expect("state readable"); - let mut remaining: Vec<&str> = state - .attempts - .iter() - .map(|attempt| attempt.call_id.as_str()) - .collect(); - remaining.sort_unstable(); - assert_eq!(remaining, vec!["call_a", "call_c"]); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::Active)); - assert!(state.recovery.is_clear()); - assert_eq!( - seam.operations() - .iter() - .filter(|op| *op == "abandon") - .count(), - 1, - "abandoning B must not sweep A or C", - ); - - cleanup(&git_dir); - } - - #[test] - fn a_close_before_start_confirmation_consumes_rather_than_closes() { - let git_dir = temp_git_dir("pending-start-close"); - let seam = RecordingSeam::failing_on(&["start"]); - - let error = drive(&git_dir, &seam, &tool_before("write", "call_1")) - .expect_err("a failed Start seam must fail closed"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::PendingStart, - ); - - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_after("write", "call_1")).expect("After on a PendingStart"); - assert_eq!(ok_seam.operations(), vec!["flush", "abandon", "flush"]); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - cleanup(&git_dir); - } - - #[test] - fn session_idle_is_non_destructive() { - let git_dir = temp_git_dir("session-idle-noop"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &shell_env("call_a")).expect("A Start"); - drive( - &git_dir, - &seam, - &json!({ - "hook_event_name": "ShellEnv", - "session_id": "ses_other", - "call_id": "call_c", - "cwd": CWD, - }) - .to_string(), - ) - .expect("other-session Start"); - - for name in ["SessionIdle", "SessionError", "SessionDeleted"] { - drive(&git_dir, &seam, &session_event(name, "ses_main")).expect("broad event is inert"); - } - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!( - state.attempts.len(), - 2, - "no attempt is retired by a broad event" - ); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::Active)); - assert!(state.recovery.is_clear()); - assert_eq!(seam.operations(), vec!["start", "start"]); - - cleanup(&git_dir); - } - - #[test] - fn delayed_session_idle_cannot_retire_a_newer_call() { - let git_dir = temp_git_dir("delayed-session-idle"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &shell_env("call_b")).expect("newer call B Start"); - drive(&git_dir, &seam, &session_event("SessionIdle", "ses_main")) - .expect("a delayed SessionIdle for the same session arrives after B started"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].call_id, "call_b"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - assert!(state.recovery.is_clear()); - assert!(!seam.operations().iter().any(|op| op == "abandon")); - - cleanup(&git_dir); - } - - #[test] - fn server_disposed_cannot_sweep_another_processes_attempt() { - let git_dir = temp_git_dir("server-disposed-noop"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &shell_env("call_a")).expect("process P1 owns call A"); - drive( - &git_dir, - &seam, - &json!({ - "hook_event_name": "ShellEnv", - "session_id": "ses_other", - "call_id": "call_b", - "cwd": CWD, - }) - .to_string(), - ) - .expect("process P2 owns call B in the same checkout"); - - drive(&git_dir, &seam, &server_disposed()).expect("P1 server disposal is inert"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!( - state.attempts.len(), - 2, - "one process's disposal must not retire another process's live attempt", - ); - assert!(state.recovery.is_clear()); - assert_eq!(seam.operations(), vec!["start", "start"]); - - cleanup(&git_dir); - } - - #[test] - fn a_failed_sibling_does_not_retire_survivors_or_block_new_starts() { - let git_dir = temp_git_dir("failed-sibling-survivors"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &shell_env("call_a")).expect("A Start"); - drive(&git_dir, &seam, &shell_env("call_b")).expect("B Start"); - drive(&git_dir, &seam, &tool_error("bash", "call_a")).expect("A fails"); - - drive(&git_dir, &seam, &shell_env("call_c")) - .expect("a new Start is admitted normally once the ambiguity flush cleared recovery"); - - let state = read_state(&git_dir).expect("state readable"); - let mut remaining: Vec<&str> = state - .attempts - .iter() - .map(|attempt| attempt.call_id.as_str()) - .collect(); - remaining.sort_unstable(); - assert_eq!(remaining, vec!["call_b", "call_c"]); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::Active)); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); - } - - #[test] - fn a_terminal_failure_consumes_the_interval_and_the_next_start_proceeds() { - let git_dir = temp_git_dir("terminal-then-start"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_before("write", "call_1")).expect("Start"); - drive(&git_dir, &seam, &tool_error("write", "call_1")).expect("terminal failure"); - assert!( - read_state(&git_dir) - .expect("state readable") - .recovery - .is_clear(), - "the ambiguity flush is done at abandon time, not deferred to the next Start", - ); - - drive(&git_dir, &seam, &tool_before("write", "call_2")).expect("Start after consume"); - - assert_eq!( - seam.operations(), - vec!["start", "flush", "abandon", "flush", "start"], - ); - let state = read_state(&git_dir).expect("state readable"); - assert!(state.recovery.is_clear()); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].call_id, "call_2"); - - cleanup(&git_dir); - } - - #[test] - fn a_failed_ambiguity_flush_stays_recovery_pending_and_fails_closed_starts() { - let git_dir = temp_git_dir("failed-ambiguity-flush"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("write", "call_1")).expect("Start"); - } - - let failing = RecordingSeam::failing_on(&["flush"]); - drive(&git_dir, &failing, &tool_error("write", "call_1")) - .expect("a terminal failure whose flush fails still returns (best-effort)"); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 1 }, - "a failed ambiguity flush retains a recovery-required state", - ); - - let error = drive(&git_dir, &failing, &tool_before("write", "call_2")) - .expect_err("a new Start while recovery is unresolved must stay fail-closed"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 1 }, - ); - - cleanup(&git_dir); - } - - #[test] - fn untracked_and_delegation_events_are_zero_footprint() { - let git_dir = temp_git_dir("zero-footprint"); - let seam = RecordingSeam::new(); - - for payload in [ - tool_before("read", "call_r"), - tool_before("task", "call_t"), - tool_before("some_future_tool", "call_f"), - tool_after("read", "call_r"), - tool_error("read", "call_r"), - tool_error("task", "call_t"), - ] { - drive(&git_dir, &seam, &payload).expect("untracked event is neutral"); - } - - assert!(seam.operations().is_empty()); - assert!( - !git_dir.join("sce").exists(), - "no state directory is created" - ); - - cleanup(&git_dir); - } - - #[test] - fn close_seam_failure_falls_back_to_consume() { - let git_dir = temp_git_dir("close-seam-failure"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("edit", "call_1")).expect("Start"); - } - - let failing = RecordingSeam::failing_on(&["close"]); - drive(&git_dir, &failing, &tool_after("edit", "call_1")).expect("Close seam failure"); - - assert_eq!( - failing.operations(), - vec!["close", "flush", "abandon", "flush"], - ); - let state = read_state(&git_dir).expect("state readable"); - assert!(state.attempts.is_empty()); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); - } - - #[test] - fn regression_a_abandon_failure_preserves_terminal_intent_then_recovers() { - let git_dir = temp_git_dir("regression-a-abandon-failure"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("write", "call_1")).expect("A Start"); - } - - let failing = RecordingSeam::failing_on(&["abandon"]); - drive(&git_dir, &failing, &tool_error("write", "call_1")) - .expect("a terminal failure whose abandon fails still returns best-effort"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1, "A is not forgotten"); - assert_eq!(state.attempts[0].call_id, "call_1"); - assert_eq!( - state.attempts[0].phase, - AttemptPhase::PendingAbandon, - "the exact terminal attempt is durably marked PendingAbandon", - ); - assert!( - !state.recovery.is_clear(), - "recovery stays unresolved while Abandon has not succeeded", - ); - assert_eq!( - failing.operations(), - vec!["flush", "abandon"], - "the ambiguity flush ran, then the abandon that failed; no rebaseline flush, \ - no removal", - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_before("write", "call_2")) - .expect("a healthy retry boundary resolves recovery and admits the new Start"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].call_id, "call_2"); - assert!(state.recovery.is_clear()); - assert_eq!( - healthy.operations(), - vec!["flush", "abandon", "flush", "start"], - "recovery replays flush + abandon + rebaseline flush before the new Start", - ); - - cleanup(&git_dir); - } - - #[test] - fn regression_b_ambiguity_flush_failure_blocks_new_starts_then_recovers() { - let git_dir = temp_git_dir("regression-b-ambiguity-flush-failure"); - let live = RecordingSeam::new(); - - drive(&git_dir, &live, &shell_env("call_a")).expect("A Start"); - drive(&git_dir, &live, &shell_env("call_b")).expect("B Start"); - - let failing = RecordingSeam::failing_on(&["flush"]); - drive(&git_dir, &failing, &tool_error("bash", "call_a")).expect("A terminal failure"); - - let state = read_state(&git_dir).expect("state readable"); - let pending_abandon: Vec<&str> = state - .attempts - .iter() - .filter(|attempt| attempt.phase == AttemptPhase::PendingAbandon) - .map(|attempt| attempt.call_id.as_str()) - .collect(); - let active: Vec<&str> = state - .attempts - .iter() - .filter(|attempt| attempt.phase == AttemptPhase::Active) - .map(|attempt| attempt.call_id.as_str()) - .collect(); - assert_eq!(pending_abandon, vec!["call_a"], "A is PendingAbandon"); - assert_eq!(active, vec!["call_b"], "B stays Active"); - assert!(!state.recovery.is_clear(), "recovery pending"); - - let error = drive(&git_dir, &failing, &shell_env("call_c")) - .expect_err("a new tracked Start must fail closed while recovery is unresolved"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .iter() - .all(|attempt| attempt.call_id != "call_c")); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &shell_env("call_c")) - .expect("once recovery succeeds a new Start is admitted normally"); - - let state = read_state(&git_dir).expect("state readable"); - let mut remaining: Vec<&str> = state - .attempts - .iter() - .map(|attempt| attempt.call_id.as_str()) - .collect(); - remaining.sort_unstable(); - assert_eq!( - remaining, - vec!["call_b", "call_c"], - "A removed, B kept, C admitted" - ); - assert!(state - .attempts - .iter() - .all(|attempt| attempt.phase == AttemptPhase::Active)); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); - } - - #[test] - fn regression_c_rebaseline_flush_failure_is_recoverable_without_poison() { - let git_dir = temp_git_dir("regression-c-rebaseline-flush-failure"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("edit", "call_1")).expect("Start"); - } - - let rebaseline_failing = RecordingSeam::failing_on_nth_occurrence("flush", 2); - drive(&git_dir, &rebaseline_failing, &tool_error("edit", "call_1")) - .expect("terminal failure whose rebaseline flush fails still returns"); - - assert_eq!( - rebaseline_failing.operations(), - vec!["flush", "abandon", "flush"], - "the ambiguity flush and the abandon succeeded; the rebaseline flush failed", - ); - let state = read_state(&git_dir).expect("state readable"); - assert!( - state.attempts.is_empty(), - "the abandon succeeded so the attempt is removed", - ); - assert!( - !state.recovery.is_clear(), - "recovery state still carries the outstanding rebaseline", - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_before("edit", "call_2")).expect("retry admits new work"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].call_id, "call_2"); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); - } - - #[test] - fn regression_e_duplicate_tool_error_is_idempotent() { - let git_dir = temp_git_dir("regression-e-duplicate-tool-error"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("write", "call_1")).expect("Start"); - } - - let stuck = RecordingSeam::failing_on(&["abandon"]); - drive(&git_dir, &stuck, &tool_error("write", "call_1")).expect("first terminal failure"); - drive(&git_dir, &stuck, &tool_error("write", "call_1")) - .expect("a duplicate ToolError while PendingAbandon is idempotent"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1, "no second attempt is created"); - assert_eq!(state.attempts[0].phase, AttemptPhase::PendingAbandon); - assert_eq!( - state.next_recovery_generation, 2, - "the recovery generation is not incremented by the duplicate", - ); - - cleanup(&git_dir); - } - - #[test] - fn regression_f_start_replay_for_a_pending_abandon_identity_never_reactivates() { - let git_dir = temp_git_dir("regression-f-start-replay-pending-abandon"); - - { - let ok_seam = RecordingSeam::new(); - drive(&git_dir, &ok_seam, &tool_before("write", "call_1")).expect("Start"); - } - - let stuck = RecordingSeam::failing_on(&["abandon"]); - drive(&git_dir, &stuck, &tool_error("write", "call_1")).expect("terminal failure"); - - let error = drive(&git_dir, &stuck, &tool_before("write", "call_1")) - .expect_err("a replayed Start for a PendingAbandon identity must fail closed"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!( - state.attempts[0].phase, - AttemptPhase::PendingAbandon, - "the replayed Start does not return the attempt to Active", - ); - - cleanup(&git_dir); - } - - #[test] - fn regression_g_late_tool_error_after_close_is_a_harmless_no_op() { - let git_dir = temp_git_dir("regression-g-late-tool-error"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_before("write", "call_1")).expect("Start"); - drive(&git_dir, &seam, &tool_after("write", "call_1")).expect("Close"); - drive(&git_dir, &seam, &tool_error("write", "call_1")) - .expect("a late ToolError after a completed Close is inert"); - - assert_eq!(seam.operations(), vec!["start", "close"]); - let state = read_state(&git_dir).expect("state readable"); - assert!(state.attempts.is_empty()); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); - } - - #[test] - fn regression_h_siblings_stay_active_through_a_transient_cleanup_failure() { - let git_dir = temp_git_dir("regression-h-siblings-preserved"); - let live = RecordingSeam::new(); - - drive(&git_dir, &live, &shell_env("call_a")).expect("A Start"); - drive(&git_dir, &live, &shell_env("call_b")).expect("B Start"); - drive(&git_dir, &live, &shell_env("call_c")).expect("C Start"); - - let transient = RecordingSeam::failing_once_on(&["abandon"]); - drive(&git_dir, &transient, &tool_error("bash", "call_b")) - .expect("B terminal failure with a transient abandon failure"); - - let snapshot = read_state(&git_dir).expect("state readable"); - let mut siblings: Vec<&str> = snapshot - .attempts - .iter() - .filter(|attempt| attempt.phase == AttemptPhase::Active) - .map(|attempt| attempt.call_id.as_str()) - .collect(); - siblings.sort_unstable(); - assert_eq!( - siblings, - vec!["call_a", "call_c"], - "A and C stay Active mid-failure" - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &shell_env("call_d")) - .expect("the retry boundary resolves recovery and admits D"); - - let state = read_state(&git_dir).expect("state readable"); - let mut remaining: Vec<&str> = state - .attempts - .iter() - .map(|attempt| attempt.call_id.as_str()) - .collect(); - remaining.sort_unstable(); - assert_eq!( - remaining, - vec!["call_a", "call_c", "call_d"], - "B gone, A/C/D active" - ); - assert!(state - .attempts - .iter() - .all(|attempt| attempt.phase == AttemptPhase::Active)); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); - } -} - -mod runtime_seam_tests { - use std::fs; - use std::path::{Path, PathBuf}; - use std::process::Command; - - use anyhow::{anyhow, Result}; - use serde_json::json; - - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - use crate::services::hooks::opencode_mutation_scope::lifecycle::{ - run_opencode_mutation_scope_from_payload_at_state_root, - run_opencode_mutation_scope_from_payload_with_seams, - }; - use crate::services::mutation_trace::runtime::resolve_git_dir; - use crate::services::observability::traits::Logger; - - use super::super::state::{adapter_state_dir, read_state, AttemptPhase}; - - fn git(dir: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - struct OpenCodeRepo { - temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, - } - - impl OpenCodeRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-opencode-mutation-scope-seam-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); - - Self { - temp, - root, - state_root, - } - } - - fn cwd(&self) -> String { - self.root.to_string_lossy().into_owned() - } - - fn drive(&self, payload: &str) -> Result { - run_opencode_mutation_scope_from_payload_at_state_root(&self.state_root, payload, None) - } - - fn drive_failing_seam_operation_once( - &self, - payload: &str, - fail_operation: &str, - remaining_failures: &std::cell::Cell, - ) -> Result { - let resolver = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let seam_fn = |root: &Path, seam_payload: &str, logger: Option<&dyn Logger>| { - let operation = serde_json::from_str::(seam_payload) - .ok() - .and_then(|value| { - value - .get("operation") - .and_then(|op| op.as_str()) - .map(str::to_owned) - }) - .unwrap_or_default(); - if operation == fail_operation && remaining_failures.get() > 0 { - remaining_failures.set(remaining_failures.get() - 1); - return Err(anyhow!("injected transient '{operation}' seam failure")); - } - crate::services::hooks::mutation_scope::run_mutation_scope_from_payload_at_state_root( - root, - &self.state_root, - seam_payload, - logger, - ) - }; - run_opencode_mutation_scope_from_payload_with_seams(payload, None, &resolver, &seam_fn) - } - - fn write(&self, name: &str, contents: &str) { - fs::write(self.root.join(name), contents).expect("write should succeed"); - } - - fn git_dir(&self) -> PathBuf { - resolve_git_dir(&self.root).expect("git dir should resolve") - } - - fn db(&self) -> RepositoryAgentTraceDb { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &self.root, - &self.state_root, - "opencode mutation-scope seam test assertions", - ) - .expect("assertion DB should open") - } - - fn scope_status(&self, scope_id: &str) -> Option<(String, String)> { - self.db() - .query_map( - "SELECT actor_kind, status FROM mutation_trace_scopes WHERE scope_id = ?1", - (scope_id,), - |row| { - let actor_kind = row.get::(0).map_err(anyhow::Error::from)?; - let status = row.get::(1).map_err(anyhow::Error::from)?; - Ok((actor_kind, status)) - }, - ) - .expect("scope query should succeed") - .into_iter() - .next() - } - - fn scope_count(&self) -> i64 { - self.db() - .query_map("SELECT COUNT(*) FROM mutation_trace_scopes", (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("a count row should exist") - } - - fn mutation_events(&self) -> Vec<(String, Option)> { - self.db() - .query_map( - "SELECT attribution_kind, attribution_scope_id \ - FROM mutation_trace_events ORDER BY revision", - (), - |row| { - let attribution_kind = row.get::(0).map_err(anyhow::Error::from)?; - let attribution_scope_id = - row.get::>(1).map_err(anyhow::Error::from)?; - Ok((attribution_kind, attribution_scope_id)) - }, - ) - .expect("mutation-events query should succeed") - } - } - - fn error(repo: &OpenCodeRepo, tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolError", - "session_id": "ses_seam", - "call_id": call_id, - "cwd": repo.cwd(), - "tool_name": tool_name, - }) - .to_string() - } - - fn before(repo: &OpenCodeRepo, tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecuteBefore", - "session_id": "ses_seam", - "call_id": call_id, - "cwd": repo.cwd(), - "tool_name": tool_name, - "model": "opencode/big-pickle", - }) - .to_string() - } - - fn after(repo: &OpenCodeRepo, tool_name: &str, call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecuteAfter", - "session_id": "ses_seam", - "call_id": call_id, - "cwd": repo.cwd(), - "tool_name": tool_name, - }) - .to_string() - } - - #[test] - fn a_write_start_then_after_closes_the_scope_through_the_real_runtime() { - let repo = OpenCodeRepo::new("write-start-close"); - - assert_eq!( - repo.drive(&before(&repo, "write", "call_1")) - .expect("Start"), - "" - ); - let scope_id = { - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - assert_eq!(state.attempts.len(), 1); - state.attempts[0].scope_id.clone() - }; - - repo.write("file.txt", "one\ntwo\n"); - assert_eq!( - repo.drive(&after(&repo, "write", "call_1")).expect("Close"), - "" - ); - - assert!(read_state(&repo.git_dir()) - .expect("adapter state readable") - .attempts - .is_empty()); - assert_eq!( - repo.scope_status(&scope_id), - Some(("opencode".to_string(), "closed".to_string())), - ); - } - - #[test] - fn linked_worktree_uses_its_worktree_specific_git_dir_for_mutation_scope_state() { - let repo = OpenCodeRepo::new("linked-worktree"); - let linked_root = repo.temp.path().join("linked"); - git( - &repo.root, - &[ - "worktree", - "add", - "--quiet", - linked_root - .to_str() - .expect("linked worktree path should be UTF-8"), - ], - ); - - let main_git_dir = resolve_git_dir(&repo.root).expect("main git dir should resolve"); - let linked_git_dir = resolve_git_dir(&linked_root).expect("linked git dir should resolve"); - assert_ne!( - main_git_dir, linked_git_dir, - "linked worktrees must use distinct Git state locations", - ); - - let before = json!({ - "hook_event_name": "ToolExecuteBefore", - "session_id": "ses_linked", - "call_id": "call_linked", - "cwd": linked_root, - "tool_name": "write", - "model": "opencode/big-pickle", - }) - .to_string(); - run_opencode_mutation_scope_from_payload_at_state_root(&repo.state_root, &before, None) - .expect("linked-worktree Start"); - - let scope_id = read_state(&linked_git_dir) - .expect("linked adapter state should be readable") - .attempts[0] - .scope_id - .clone(); - fs::write(linked_root.join("file.txt"), "one\nlinked\n") - .expect("linked worktree file should be writable"); - - let after = json!({ - "hook_event_name": "ToolExecuteAfter", - "session_id": "ses_linked", - "call_id": "call_linked", - "cwd": linked_root, - "tool_name": "write", - }) - .to_string(); - run_opencode_mutation_scope_from_payload_at_state_root(&repo.state_root, &after, None) - .expect("linked-worktree Close"); - - assert!(read_state(&linked_git_dir) - .expect("linked adapter state should be readable") - .attempts - .is_empty()); - assert_eq!( - repo.scope_status(&scope_id), - Some(("opencode".to_string(), "closed".to_string())), - ); - } - - #[test] - fn a_tool_error_abandons_the_scope_through_the_real_runtime() { - let repo = OpenCodeRepo::new("tool-error-abandon"); - - repo.drive(&before(&repo, "edit", "call_1")).expect("Start"); - let scope_id = read_state(&repo.git_dir()) - .expect("adapter state readable") - .attempts[0] - .scope_id - .clone(); - - repo.write("file.txt", "one\nabandoned-a\n"); - repo.drive(&error(&repo, "edit", "call_1")) - .expect("terminal failure"); - - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - assert!(state.attempts.is_empty()); - assert!( - state.recovery.is_clear(), - "a successful ambiguity flush clears the recovery barrier", - ); - assert_eq!( - repo.scope_status(&scope_id).map(|(_, status)| status), - Some("abandoned".to_string()), - ); - } - - #[test] - fn regression_a_failed_concurrent_scope_cannot_contaminate_a_survivor() { - let repo = OpenCodeRepo::new("regression-a-no-contamination"); - - repo.drive(&before(&repo, "write", "call_a")) - .expect("A Start"); - repo.drive(&before(&repo, "write", "call_b")) - .expect("B Start"); - let scope_b = read_state(&repo.git_dir()) - .expect("adapter state readable") - .attempts - .iter() - .find(|attempt| attempt.call_id == "call_b") - .expect("B is tracked") - .scope_id - .clone(); - - repo.write("file_a.txt", "a mutated\n"); - repo.write("file_b.txt", "b mutated\n"); - - repo.drive(&error(&repo, "write", "call_a")) - .expect("A terminal failure"); - - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - assert_eq!(state.attempts.len(), 1, "B stays tracked after A's cleanup"); - assert_eq!(state.attempts[0].call_id, "call_b"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - - repo.drive(&after(&repo, "write", "call_b")) - .expect("B Close"); - - let events = repo.mutation_events(); - assert!( - events.iter().any(|(kind, _)| kind == "ineligible_unscoped"), - "the ambiguous interval containing A's mutations is consumed as \ - IneligibleUnscoped: {events:?}", - ); - assert!( - !events.iter().any(|(kind, scope)| kind == "ai_exclusive" - && scope.as_deref() == Some(scope_b.as_str())), - "B must never be attributed the interval that could contain A's changes: {events:?}", - ); - assert_eq!( - repo.scope_status(&scope_b).map(|(_, status)| status), - Some("closed".to_string()), - ); - } - - #[test] - fn regression_b_survivor_still_attributes_its_later_mutations() { - let repo = OpenCodeRepo::new("regression-b-survivor-liveness"); - - repo.drive(&before(&repo, "write", "call_a")) - .expect("A Start"); - repo.drive(&before(&repo, "write", "call_b")) - .expect("B Start"); - let scope_b = read_state(&repo.git_dir()) - .expect("adapter state readable") - .attempts - .iter() - .find(|attempt| attempt.call_id == "call_b") - .expect("B is tracked") - .scope_id - .clone(); - - repo.write("file_a.txt", "a mutated\n"); - repo.drive(&error(&repo, "write", "call_a")) - .expect("A terminal failure consumes the ambiguous interval"); - - repo.write("file_c.txt", "b's own later work\n"); - repo.drive(&after(&repo, "write", "call_b")) - .expect("B Close"); - - let events = repo.mutation_events(); - assert!( - events.iter().any(|(kind, scope)| kind == "ai_exclusive" - && scope.as_deref() == Some(scope_b.as_str())), - "after the ambiguity flush, B may legitimately attribute its own later \ - mutations: {events:?}", - ); - assert_eq!( - repo.scope_status(&scope_b).map(|(_, status)| status), - Some("closed".to_string()), - ); - } - - #[test] - fn regression_c_exact_error_does_not_sweep_siblings_through_the_real_runtime() { - let repo = OpenCodeRepo::new("regression-c-no-sibling-sweep"); - - repo.drive(&before(&repo, "write", "call_a")) - .expect("A Start"); - repo.drive(&before(&repo, "write", "call_b")) - .expect("B Start"); - repo.drive(&before(&repo, "write", "call_c")) - .expect("C Start"); - - repo.write("file.txt", "one\nmutated\n"); - repo.drive(&error(&repo, "write", "call_b")) - .expect("B terminal failure"); - - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - let mut remaining: Vec<&str> = state - .attempts - .iter() - .map(|attempt| attempt.call_id.as_str()) - .collect(); - remaining.sort_unstable(); - assert_eq!(remaining, vec!["call_a", "call_c"], "A and C remain active"); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::Active)); - assert!(state.recovery.is_clear()); - } - - #[test] - fn regression_d_delayed_session_idle_cannot_kill_a_newer_call() { - let repo = OpenCodeRepo::new("regression-d-delayed-session-idle"); - - repo.drive(&before(&repo, "write", "call_b")) - .expect("newer call B Start"); - - repo.drive( - &json!({ - "hook_event_name": "SessionIdle", - "session_id": "ses_seam", - "cwd": repo.cwd(), - }) - .to_string(), - ) - .expect("a delayed SessionIdle for the same session is inert"); - - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].call_id, "call_b"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - assert!(state.recovery.is_clear()); - } - - #[test] - fn regression_e_server_disposed_cannot_sweep_another_process() { - let repo = OpenCodeRepo::new("regression-e-server-disposed"); - - repo.drive(&before(&repo, "write", "call_a")) - .expect("P1 owns call A"); - repo.drive( - &json!({ - "hook_event_name": "ToolExecuteBefore", - "session_id": "ses_p2", - "call_id": "call_b", - "cwd": repo.cwd(), - "tool_name": "write", - }) - .to_string(), - ) - .expect("P2 owns call B in the same checkout"); - - repo.drive(&json!({ "hook_event_name": "ServerDisposed", "cwd": repo.cwd() }).to_string()) - .expect("P1 server disposal is inert"); - - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - assert_eq!( - state.attempts.len(), - 2, - "one process's disposal must not retire another process's live attempt", - ); - assert!(state.recovery.is_clear()); - } - - #[test] - fn regression_f_untracked_tool_error_is_zero_footprint() { - let repo = OpenCodeRepo::new("regression-f-untracked-tool-error"); - - for tool_name in ["read", "task", "brave-search_brave_web_search"] { - repo.drive(&error(&repo, tool_name, "call_x")) - .expect("an untracked ToolError is neutral"); - } - - assert_eq!(repo.scope_count(), 0); - assert!(!adapter_state_dir(&repo.git_dir()) - .join("opencode-mutation-scope-state.json") - .exists()); - } - - #[test] - fn untracked_events_never_reach_the_runtime_or_touch_adapter_state() { - let repo = OpenCodeRepo::new("untracked-zero-footprint"); - - assert_eq!( - repo.drive(&before(&repo, "read", "call_r")).expect("read"), - "" - ); - assert_eq!( - repo.drive(&before(&repo, "task", "call_t")).expect("task"), - "" - ); - - assert_eq!(repo.scope_count(), 0); - assert!(!adapter_state_dir(&repo.git_dir()) - .join("opencode-mutation-scope-state.json") - .exists()); - } - - #[test] - fn regression_d_concurrent_survivor_stays_usable_after_a_transient_cleanup_failure() { - let repo = OpenCodeRepo::new("regression-d-transient-cleanup-failure"); - - repo.drive(&before(&repo, "write", "call_a")) - .expect("A Start"); - repo.drive(&before(&repo, "write", "call_b")) - .expect("B Start"); - let scoped = read_state(&repo.git_dir()).expect("adapter state readable"); - let scope_a = scoped - .attempts - .iter() - .find(|attempt| attempt.call_id == "call_a") - .expect("A is tracked") - .scope_id - .clone(); - let scope_b = scoped - .attempts - .iter() - .find(|attempt| attempt.call_id == "call_b") - .expect("B is tracked") - .scope_id - .clone(); - - repo.write("file_a.txt", "a mutated\n"); - repo.write("file_b.txt", "b mutated\n"); - - let remaining_failures = std::cell::Cell::new(1_u32); - repo.drive_failing_seam_operation_once( - &error(&repo, "write", "call_a"), - "abandon", - &remaining_failures, - ) - .expect("A terminal failure with a transient abandon failure returns best-effort"); - - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - assert_eq!( - state - .attempts - .iter() - .find(|attempt| attempt.call_id == "call_a") - .map(|attempt| attempt.phase), - Some(AttemptPhase::PendingAbandon), - "A's terminal intent survives the transient failure", - ); - assert_eq!( - state - .attempts - .iter() - .find(|attempt| attempt.call_id == "call_b") - .map(|attempt| attempt.phase), - Some(AttemptPhase::Active), - "B is untouched", - ); - assert!(!state.recovery.is_clear()); - - repo.drive(&error(&repo, "write", "call_a")) - .expect("a healthy duplicate ToolError retries and completes cleanup"); - - let state = read_state(&repo.git_dir()).expect("adapter state readable"); - let remaining: Vec<&str> = state - .attempts - .iter() - .map(|attempt| attempt.call_id.as_str()) - .collect(); - assert_eq!(remaining, vec!["call_b"], "A retired, B still live"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Active); - assert!(state.recovery.is_clear()); - assert_eq!( - repo.scope_status(&scope_a).map(|(_, status)| status), - Some("abandoned".to_string()), - ); - - repo.write("file_c.txt", "b's own later work\n"); - repo.drive(&after(&repo, "write", "call_b")) - .expect("B Close"); - - let events = repo.mutation_events(); - assert!( - events.iter().any(|(kind, _)| kind == "ineligible_unscoped"), - "the ambiguous A/B interval is consumed as IneligibleUnscoped: {events:?}", - ); - assert!( - events.iter().any(|(kind, scope)| kind == "ai_exclusive" - && scope.as_deref() == Some(scope_b.as_str())), - "B still attributes its own later mutation once recovery completed: {events:?}", - ); - } -} diff --git a/cli/src/services/hooks/pi_mutation_scope/guard_reconciliation_tests.rs b/cli/src/services/hooks/pi_mutation_scope/guard_reconciliation_tests.rs deleted file mode 100644 index 8611a81d1..000000000 --- a/cli/src/services/hooks/pi_mutation_scope/guard_reconciliation_tests.rs +++ /dev/null @@ -1,606 +0,0 @@ -use std::fs; -use std::path::PathBuf; -use std::process::Command; -use std::sync::mpsc; -use std::time::Duration; - -use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; -use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, -}; -use crate::services::mutation_trace::runtime::{ - coordinate, run_external_mutation_guard, GuardRequest, RuntimeBoundary, -}; -use crate::services::mutation_trace::types::{ActorKind, EventId, ScopeId}; - -use super::*; - -fn git(dir: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); -} - -struct GuardRepo { - _temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, -} - -impl GuardRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-pi-guard-reconciliation-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); - - Self { - _temp: temp, - root, - state_root, - } - } - - fn drive(&self, payload: &str) -> Result { - run_pi_mutation_scope_from_payload_at_state_root(&self.state_root, payload, None) - } - - fn open_db(&self) -> anyhow::Result { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &self.root, - &self.state_root, - "Pi guard-reconciliation test assertions", - ) - } - - fn db(&self) -> RepositoryAgentTraceDb { - self.open_db().expect("assertion DB should open") - } - - fn scope_status(&self, scope_id: &str) -> Option<(String, String)> { - self.db() - .query_map( - "SELECT actor_kind, status FROM mutation_trace_scopes WHERE scope_id = ?1", - (scope_id,), - |row| { - let actor_kind = row.get::(0).map_err(anyhow::Error::from)?; - let status = row.get::(1).map_err(anyhow::Error::from)?; - Ok((actor_kind, status)) - }, - ) - .expect("scope query should succeed") - .into_iter() - .next() - } - - fn write(&self, name: &str, contents: &str) { - fs::write(self.root.join(name), contents).expect("write should succeed"); - } - - fn mutation_events(&self) -> Vec<(String, Option)> { - self.db() - .query_map( - "SELECT attribution_kind, attribution_scope_id \ - FROM mutation_trace_events ORDER BY revision", - (), - |row| { - let attribution_kind = row.get::(0).map_err(anyhow::Error::from)?; - let attribution_scope_id = - row.get::>(1).map_err(anyhow::Error::from)?; - Ok((attribution_kind, attribution_scope_id)) - }, - ) - .expect("mutation-events query should succeed") - } -} - -fn tool_call(repo: &GuardRepo, tool_call_id: &str, session_id: &str) -> String { - json!({ - "hook_event_name": "ToolCall", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": repo.root.to_string_lossy(), - "tool_name": "bash", - "model": "openai-codex/gpt-5.5", - }) - .to_string() -} - -fn tool_execution_end(repo: &GuardRepo, tool_call_id: &str, session_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecutionEnd", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": repo.root.to_string_lossy(), - "tool_name": "bash", - }) - .to_string() -} - -fn tool_result(repo: &GuardRepo, tool_call_id: &str, session_id: &str) -> String { - json!({ - "hook_event_name": "ToolResult", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": repo.root.to_string_lossy(), - "tool_name": "bash", - }) - .to_string() -} - -#[test] -fn a_guard_triggered_worktree_abandonment_reconciles_with_the_pi_adapters_own_state() { - let repo = GuardRepo::new("reconcile"); - let session = "01a091f4-guard-session"; - let key_a = AttemptKey { - session_id: session.to_string(), - tool_call_id: "call_a".to_string(), - }; - let key_b = AttemptKey { - session_id: session.to_string(), - tool_call_id: "call_b".to_string(), - }; - let scope_a = format_pi_scope_id(&key_a, 1); - let scope_b = format_pi_scope_id(&key_b, 2); - - repo.drive(&tool_call(&repo, "call_a", session)) - .expect("A's Start should reach the real runtime"); - repo.drive(&tool_call(&repo, "call_b", session)) - .expect("B's Start should reach the real runtime"); - assert_eq!( - repo.scope_status(&scope_a), - Some(("pi".to_string(), "active".to_string())) - ); - assert_eq!( - repo.scope_status(&scope_b), - Some(("pi".to_string(), "active".to_string())) - ); - - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let root = repo.root.clone(); - let outcome = run_external_mutation_guard( - &root, - &GuardRequest { - command: "printf changed >> file.txt".to_string(), - cwd: None, - env: Vec::new(), - }, - || repo.open_db(), - |_event| {}, - cancel_rx, - ) - .expect("the guard should finish successfully"); - assert_eq!(outcome.exit_code, Some(0)); - assert!(!outcome.marker_clear_failed); - - assert_eq!( - repo.scope_status(&scope_a), - Some(("pi".to_string(), "abandoned".to_string())), - "the guard's finish-time forced recovery must abandon every scope live during the \ - guarded interval, regardless of which harness's boundary happened to observe \ - user_bash" - ); - assert_eq!( - repo.scope_status(&scope_b), - Some(("pi".to_string(), "abandoned".to_string())) - ); - - repo.drive(&tool_execution_end(&repo, "call_a", session)) - .expect( - "the adapter's next interaction for an already-abandoned scope must reconcile \ - safely (falling back through the existing Close-failure-to-abandon path) rather \ - than erroring or resurrecting the scope", - ); - repo.drive(&tool_execution_end(&repo, "call_b", session)) - .expect("the same reconciliation must hold for every sibling abandoned by the guard"); - - assert!( - state::read_state(&resolve_git_dir(&repo.root).expect("git dir resolves")) - .expect("state readable") - .attempts - .is_empty(), - "the Pi adapter's own durable local attempt state must converge to empty once it \ - observes the terminal event for a scope the generic runtime already abandoned out \ - from under it" - ); -} - -#[test] -fn a_guard_abandons_a_live_pi_scope_alongside_a_live_scope_from_another_harness() { - let repo = GuardRepo::new("cross-harness"); - let key = AttemptKey { - session_id: "01a091f4-guard-cross-session".to_string(), - tool_call_id: "call_pi".to_string(), - }; - let pi_scope_id = format_pi_scope_id(&key, 1); - let claude_scope = ScopeId("claude-scope-under-guard".to_string()); - - repo.drive(&tool_call(&repo, "call_pi", "01a091f4-guard-cross-session")) - .expect("Pi's Start should reach the real runtime"); - coordinate( - &repo.root, - &RuntimeBoundary::Start { - scope: claude_scope.clone(), - event: EventId("claude-evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - || repo.open_db(), - ) - .expect("Claude's Start should reach the real runtime"); - - assert_eq!( - repo.scope_status(&pi_scope_id), - Some(("pi".to_string(), "active".to_string())) - ); - assert_eq!( - repo.scope_status(&claude_scope.0), - Some(("claude_code".to_string(), "active".to_string())) - ); - - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let root = repo.root.clone(); - run_external_mutation_guard( - &root, - &GuardRequest { - command: "printf changed >> file.txt".to_string(), - cwd: None, - env: Vec::new(), - }, - || repo.open_db(), - |_event| {}, - cancel_rx, - ) - .expect("the guard should finish successfully"); - - assert_eq!( - repo.scope_status(&pi_scope_id), - Some(("pi".to_string(), "abandoned".to_string())), - "the guard's forced recovery must abandon the Pi scope even though a different \ - harness's boundary is the one that happened to observe user_bash" - ); - assert_eq!( - repo.scope_status(&claude_scope.0), - Some(("claude_code".to_string(), "abandoned".to_string())), - "the guard's forced recovery must abandon every live scope on the worktree \ - regardless of which harness owns it" - ); - - repo.drive(&tool_execution_end( - &repo, - "call_pi", - "01a091f4-guard-cross-session", - )) - .expect( - "the Pi adapter must still reconcile cleanly with its own scope even when a \ - sibling scope belonging to a different harness was abandoned by the same guard", - ); - assert!( - state::read_state(&resolve_git_dir(&repo.root).expect("git dir resolves")) - .expect("state readable") - .attempts - .is_empty() - ); -} - -#[test] -fn a_foreign_pi_start_racing_an_active_guard_fails_closed_touching_no_state_then_succeeds_on_retry() -{ - let repo = GuardRepo::new("race"); - let ready = repo.root.join("ready"); - let release = repo.root.join("release"); - let command = format!( - "touch '{}'; while [ ! -f '{}' ]; do sleep 0.02; done", - ready.display(), - release.display(), - ); - - let root = repo.root.clone(); - let state_root = repo.state_root.clone(); - let guard_thread = std::thread::spawn(move || { - let (_cancel_tx, cancel_rx) = mpsc::channel(); - run_external_mutation_guard( - &root, - &GuardRequest { - command, - cwd: None, - env: Vec::new(), - }, - || { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &root, - &state_root, - "guard race test", - ) - }, - |_event| {}, - cancel_rx, - ) - }); - - let deadline = std::time::Instant::now() + Duration::from_secs(5); - while !ready.exists() { - assert!( - std::time::Instant::now() < deadline, - "the guarded shell never reported ready" - ); - std::thread::sleep(Duration::from_millis(20)); - } - - let key = AttemptKey { - session_id: "01a091f4-guard-race-session".to_string(), - tool_call_id: "call_race".to_string(), - }; - let scope_id = format_pi_scope_id(&key, 1); - - let error = repo - .drive(&tool_call( - &repo, - "call_race", - "01a091f4-guard-race-session", - )) - .expect_err( - "a Pi Start racing an active external-mutation guard must block then fail \ - closed with CoordinateError::LockAcquisition, never proceed", - ); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - assert!( - repo.scope_status(&scope_id).is_none(), - "a boundary that fails closed on lock acquisition must touch no protocol state" - ); - - fs::write(&release, "go").expect("release file should write"); - let outcome = guard_thread - .join() - .expect("guard thread should not panic") - .expect("the guard should finish successfully once released"); - assert_eq!(outcome.exit_code, Some(0)); - - repo.drive(&tool_call( - &repo, - "call_race", - "01a091f4-guard-race-session", - )) - .expect("retrying the same Start after the guard finishes must succeed normally"); - assert_eq!( - repo.scope_status(&scope_id), - Some(("pi".to_string(), "active".to_string())) - ); -} - -#[test] -#[allow(clippy::too_many_lines)] -fn a_foreign_harnesss_boundary_racing_the_guard_fails_closed_then_succeeds_normally_on_retry() { - let repo = GuardRepo::new("cross-harness-race"); - let claude_scope = ScopeId("claude-scope-racing-guard".to_string()); - let key = AttemptKey { - session_id: "01a091f4-guard-cross-race-session".to_string(), - tool_call_id: "call_pi_race".to_string(), - }; - let pi_scope_id = format_pi_scope_id(&key, 1); - - repo.drive(&tool_call( - &repo, - "call_pi_race", - "01a091f4-guard-cross-race-session", - )) - .expect("Pi's Start should reach the real runtime"); - coordinate( - &repo.root, - &RuntimeBoundary::Start { - scope: claude_scope.clone(), - event: EventId("claude-evt-race-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - || repo.open_db(), - ) - .expect("Claude's Start should reach the real runtime"); - - let ready = repo.root.join("ready"); - let release = repo.root.join("release"); - let command = format!( - "touch '{}'; printf w1 >> file.txt; while [ ! -f '{}' ]; do sleep 0.02; done; printf w2 >> file.txt", - ready.display(), - release.display(), - ); - - let root = repo.root.clone(); - let state_root = repo.state_root.clone(); - let guard_thread = std::thread::spawn(move || { - let (_cancel_tx, cancel_rx) = mpsc::channel(); - run_external_mutation_guard( - &root, - &GuardRequest { - command, - cwd: None, - env: Vec::new(), - }, - || { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &root, - &state_root, - "guard cross-harness race test", - ) - }, - |_event| {}, - cancel_rx, - ) - }); - - let deadline = std::time::Instant::now() + Duration::from_secs(5); - while !ready.exists() { - assert!( - std::time::Instant::now() < deadline, - "the guarded shell never reported ready" - ); - std::thread::sleep(Duration::from_millis(20)); - } - - let close_while_active = coordinate( - &repo.root, - &RuntimeBoundary::Close { - scope: claude_scope.clone(), - event: EventId("claude-evt-race-close".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - || repo.open_db(), - ); - assert!( - matches!( - close_while_active, - Err(crate::services::mutation_trace::runtime::CoordinateError::LockAcquisition(_)) - ), - "a foreign harness's boundary racing an active external-mutation guard must fail \ - closed with LockAcquisition, never proceed while the guard still holds the \ - worktree: {close_while_active:?}" - ); - assert!( - repo.mutation_events().is_empty(), - "a boundary that fails closed on lock acquisition must touch no protocol state" - ); - - fs::write(&release, "go").expect("release file should write"); - let outcome = guard_thread - .join() - .expect("guard thread should not panic") - .expect("the guard should finish successfully once released"); - assert_eq!(outcome.exit_code, Some(0)); - - assert_eq!( - repo.scope_status(&pi_scope_id), - Some(("pi".to_string(), "abandoned".to_string())) - ); - assert_eq!( - repo.scope_status(&claude_scope.0), - Some(("claude_code".to_string(), "abandoned".to_string())) - ); - - coordinate( - &repo.root, - &RuntimeBoundary::Close { - scope: claude_scope.clone(), - event: EventId("claude-evt-race-close-retry".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - || repo.open_db(), - ) - .expect( - "Claude's deferred boundary must succeed normally once retried against the \ - recovered worktree, rather than continuing to fail closed", - ); - repo.drive(&tool_execution_end( - &repo, - "call_pi_race", - "01a091f4-guard-cross-race-session", - )) - .expect("the Pi adapter must reconcile its own already-abandoned scope cleanly too"); - - assert!( - repo.mutation_events().is_empty(), - "both human writes made under the guard must remain excluded from positive AI \ - attribution for the Pi scope and for the racing foreign-harness scope alike" - ); -} - -#[test] -fn a_guard_triggered_abandonment_does_not_poison_the_checkout_for_a_fresh_pi_scope() { - let repo = GuardRepo::new("post-recovery-fresh-start"); - let session = "01a091f4-guard-fresh-session"; - let key_a = AttemptKey { - session_id: session.to_string(), - tool_call_id: "call_doomed".to_string(), - }; - let scope_a = format_pi_scope_id(&key_a, 1); - - repo.drive(&tool_call(&repo, "call_doomed", session)) - .expect("A's Start should reach the real runtime"); - assert_eq!( - repo.scope_status(&scope_a), - Some(("pi".to_string(), "active".to_string())) - ); - - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let root = repo.root.clone(); - let outcome = run_external_mutation_guard( - &root, - &GuardRequest { - command: "printf changed >> file.txt".to_string(), - cwd: None, - env: Vec::new(), - }, - || repo.open_db(), - |_event| {}, - cancel_rx, - ) - .expect("the guard should finish successfully"); - assert_eq!(outcome.exit_code, Some(0)); - - assert_eq!( - repo.scope_status(&scope_a), - Some(("pi".to_string(), "abandoned".to_string())) - ); - repo.drive(&tool_execution_end(&repo, "call_doomed", session)) - .expect("the adapter must reconcile the guard-abandoned scope cleanly"); - - let key_c = AttemptKey { - session_id: session.to_string(), - tool_call_id: "call_clean".to_string(), - }; - let scope_c = format_pi_scope_id(&key_c, 2); - - repo.drive(&tool_call(&repo, "call_clean", session)) - .expect("a fresh Start on the same worktree after recovery must succeed normally"); - assert_eq!( - repo.scope_status(&scope_c), - Some(("pi".to_string(), "active".to_string())) - ); - - repo.write("file.txt", "one\nchanged\nclean\n"); - repo.drive(&tool_result(&repo, "call_clean", session)) - .expect("tool_result should mark Executed"); - repo.drive(&tool_execution_end(&repo, "call_clean", session)) - .expect("Close should reach the real runtime"); - - assert_eq!( - repo.scope_status(&scope_c), - Some(("pi".to_string(), "closed".to_string())) - ); - assert_eq!( - repo.mutation_events(), - vec![("ai_exclusive".to_string(), Some(scope_c))], - "a clean Pi mutation after guard-triggered recovery must still reach AiExclusive; \ - recovery must not permanently poison the checkout for later, uninterfered-with work" - ); -} diff --git a/cli/src/services/hooks/pi_mutation_scope/health.rs b/cli/src/services/hooks/pi_mutation_scope/health.rs index 84d6abddd..035850ace 100644 --- a/cli/src/services/hooks/pi_mutation_scope/health.rs +++ b/cli/src/services/hooks/pi_mutation_scope/health.rs @@ -8,8 +8,8 @@ use crate::services::mutation_trace::types::ActorKind; use super::state::{self, AttemptPhase, RecoveryState}; use crate::services::hooks::mutation_scope_owner::is_definitely_dead; -pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { - let state = match state::read_state(git_dir) { +pub(crate) async fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { + let state = match state::read_state(git_dir).await { Ok(state) => state, Err(error) => { return MutationScopeAdapterHealth::new( @@ -60,712 +60,3 @@ pub(crate) fn classify_health(git_dir: &Path) -> MutationScopeAdapterHealth { ), } } - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::Mutex; - - use serde_json::{json, Value}; - - use super::super::{ - force_attempt_owner_dead_for_tests, run_pi_mutation_scope_from_payload_with_seams, - AttemptKey, - }; - use super::*; - use crate::services::hooks::mutation_scope_owner::ProcessOwner; - use crate::services::observability::traits::Logger; - - const FAIL_CLOSED_MESSAGE: &str = - "SCE could not establish Pi mutation attribution for this tool execution."; - const CWD: &str = "/repo/pi-checkout"; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-pi-mutation-scope-health-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(session_id: &str, tool_call_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - tool_call_id: tool_call_id.to_string(), - } - } - - struct RecordingSeam { - calls: Mutex>, - fail_operations: Vec, - fail_once_operations: Mutex>, - } - - impl RecordingSeam { - fn new() -> Self { - Self { - calls: Mutex::new(Vec::new()), - fail_operations: Vec::new(), - fail_once_operations: Mutex::new(Vec::new()), - } - } - - fn failing_on(operations: &[&str]) -> Self { - Self { - fail_operations: operations.iter().map(|op| (*op).to_string()).collect(), - ..Self::new() - } - } - - fn failing_once_on(operations: &[&str]) -> Self { - Self { - fail_once_operations: Mutex::new( - operations.iter().map(|op| (*op).to_string()).collect(), - ), - ..Self::new() - } - } - - fn handle(&self, payload: &str) -> anyhow::Result { - let operation = operation_of(payload); - self.calls - .lock() - .expect("seam mutex") - .push(operation.clone()); - if self.fail_operations.contains(&operation) { - anyhow::bail!("seam failure injected by test for '{operation}'"); - } - let mut once = self.fail_once_operations.lock().expect("seam mutex"); - if let Some(position) = once.iter().position(|candidate| candidate == &operation) { - once.remove(position); - anyhow::bail!("transient seam failure injected once by test for '{operation}'"); - } - Ok(String::new()) - } - } - - fn operation_of(payload: &str) -> String { - let value: Value = serde_json::from_str(payload).expect("seam payload is JSON"); - value - .get("operation") - .and_then(Value::as_str) - .expect("seam payload has an operation") - .to_string() - } - - fn drive(git_dir: &Path, seam: &RecordingSeam, payload: &str) -> anyhow::Result { - let resolver = |_cwd: &str| Ok(git_dir.to_path_buf()); - let seam_fn = - |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| seam.handle(payload); - run_pi_mutation_scope_from_payload_with_seams(payload, None, &resolver, &seam_fn) - } - - fn tool_call_event(session_id: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolCall", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": "bash", - }) - .to_string() - } - - fn tool_result_event(session_id: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolResult", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": "bash", - }) - .to_string() - } - - fn tool_execution_end_event(session_id: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecutionEnd", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": "bash", - }) - .to_string() - } - - #[test] - fn absent_state_file_is_healthy() { - let git_dir = unique_test_git_dir("absent"); - - let health = classify_health(&git_dir); - - assert_eq!(health.adapter, ActorKind::Pi); - assert_eq!(health.status, MutationScopeHealthStatus::Healthy); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_is_healthy_with_a_live_owner_pending_start_attempt_never_swept_by_an_unrelated_start( - ) { - let git_dir = unique_test_git_dir("clear-live-owner"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt = state::seed_attempt_for_tests( - &git_dir, - &key("ses-1", "call-1"), - "bash", - AttemptPhase::PendingStart, - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - let seam = RecordingSeam::new(); - drive(&git_dir, &seam, &tool_call_event("ses-c", "call-c")) - .expect("an unrelated session's Start must proceed alongside a live-owner attempt"); - assert_eq!( - seam.calls.lock().expect("seam mutex").clone(), - vec!["start".to_string()], - "no D10 sweep may fire for an owner that is still alive" - ); - let after = state::read_state(&git_dir).expect("state readable"); - assert_eq!( - after - .attempts - .iter() - .find(|a| a.scope_id == attempt.scope_id) - .expect("the live-owner attempt is still tracked") - .phase, - AttemptPhase::PendingStart, - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_is_healthy_with_an_uncertain_owner_pending_start_attempt_never_swept_by_an_unrelated_start( - ) { - let git_dir = unique_test_git_dir("clear-uncertain-owner"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt = state::seed_attempt_for_tests( - &git_dir, - &key("ses-a", "call-a"), - "bash", - AttemptPhase::PendingStart, - ); - state::set_attempt_owner_for_tests( - &git_dir, - &attempt.scope_id, - ProcessOwner { - pid: std::process::id().cast_signed(), - instance_token: None, - }, - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy, - "a live pid with no instance-token evidence is conservatively treated as alive, never dead" - ); - - let seam = RecordingSeam::new(); - drive(&git_dir, &seam, &tool_call_event("ses-c", "call-c")).expect( - "an unrelated session's Start must proceed without touching an uncertain-owner attempt", - ); - assert_eq!( - seam.calls.lock().expect("seam mutex").clone(), - vec!["start".to_string()], - "no D10 sweep may fire for an owner that cannot be positively proven dead" - ); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_with_a_dead_owner_pending_start_attempt_is_recovering_and_an_unrelated_session_start_sweeps_it_ac4( - ) { - let git_dir = unique_test_git_dir("clear-dead-owner-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("ses-a", "call-a")).expect("A starts"); - let scope_a = state::read_state(&git_dir) - .expect("state readable") - .attempts[0] - .scope_id - .clone(); - force_attempt_owner_dead_for_tests(&git_dir, &scope_a); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "a dead-owner PendingStart attempt is unfinished recovery work with a proven \ - automatic sweep, not a durable wedge" - ); - - drive(&git_dir, &seam, &tool_call_event("ses-b", "call-b")) - .expect("B's Start must recover A's stale owner without ever replaying A's own key"); - - assert_eq!( - seam.calls.lock().expect("seam mutex").clone(), - vec!["start", "flush", "abandon", "flush", "start"], - "D10: the dead-owner attempt must be retired through the ordinary flush/abandon/flush \ - sequence before B's own triggering Start is admitted" - ); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy, - "once the sweep completes and B is admitted as an ordinary live attempt, no recovery \ - condition remains" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn clear_recovery_with_a_dead_owner_executed_attempt_is_recovering_and_is_swept_without_a_synthetic_close( - ) { - let git_dir = unique_test_git_dir("clear-dead-owner-executed"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("ses-a", "call-a")).expect("A starts"); - drive(&git_dir, &seam, &tool_result_event("ses-a", "call-a")) - .expect("A's tool_result marks Executed"); - let state = state::read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Executed); - force_attempt_owner_dead_for_tests(&git_dir, &state.attempts[0].scope_id); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering - ); - - drive(&git_dir, &seam, &tool_call_event("ses-b", "call-b")) - .expect("B's Start must recover A's dead Executed attempt"); - - assert_eq!( - seam.calls.lock().expect("seam mutex").clone(), - vec!["start", "flush", "abandon", "flush", "start"], - "ToolResult marks Executed locally without touching the seam" - ); - assert!( - !seam - .calls - .lock() - .expect("seam mutex") - .contains(&"close".to_string()), - "a dead Executed attempt must never be given a synthetic delayed Close" - ); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_recovery_from_a_failed_terminal_abandon_is_recovering_and_self_heals_on_the_next_start( - ) { - let git_dir = unique_test_git_dir("pending-failed-abandon"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let crashing = RecordingSeam::failing_once_on(&["abandon"]); - - drive(&git_dir, &crashing, &tool_call_event("ses-1", "call-1")).expect("Start"); - drive( - &git_dir, - &crashing, - &tool_execution_end_event("ses-1", "call-1"), - ) - .expect("a transient abandon failure mid-recovery must not surface an error"); - - let seeded = state::read_state(&git_dir).expect("state readable"); - assert_eq!(seeded.recovery, RecoveryState::Pending { generation: 1 }); - assert_eq!(seeded.attempts[0].phase, AttemptPhase::PendingAbandon); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "a proven self-healing path exists even though this state currently denies admission" - ); - - let still_failing = RecordingSeam::failing_on(&["abandon"]); - for attempt_number in 1..=2 { - let error = drive( - &git_dir, - &still_failing, - &tool_call_event("ses-other", &format!("call-retry-{attempt_number}")), - ) - .expect_err("admission while recovery is unresolved must stay fail-closed"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - } - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "repeated denial under a still-failing seam must not change the classification" - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_call_event("ses-2", "call-2")) - .expect("recovery must self-heal and complete on the next successful invocation"); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn pending_recovery_from_an_interrupted_dead_owner_sweep_is_recovering_and_denies_the_triggering_start_until_resumed( - ) { - let git_dir = unique_test_git_dir("pending-interrupted-sweep"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("ses-a", "call-a")).expect("A's Start"); - let scope_a = state::read_state(&git_dir) - .expect("state readable") - .attempts[0] - .scope_id - .clone(); - force_attempt_owner_dead_for_tests(&git_dir, &scope_a); - - let crashing = RecordingSeam::failing_once_on(&["abandon"]); - drive(&git_dir, &crashing, &tool_call_event("ses-b", "call-b")).expect_err( - "a Start that triggers a stale-owner recovery which fails mid-way must not commit", - ); - - let seeded = state::read_state(&git_dir).expect("state readable"); - assert_eq!(seeded.recovery, RecoveryState::Pending { generation: 1 }); - assert!(!seeded.attempts.iter().any(|a| a.session_id == "ses-b")); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "an interrupted dead-owner sweep still has a proven resumption path on the next \ - tracked Start" - ); - - drive(&git_dir, &crashing, &tool_call_event("ses-b", "call-b")).expect( - "the next boundary-lock acquisition must resume and complete the pending recovery, \ - then admit B", - ); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn orphaned_flushing_is_recovering_and_reclaimed_by_the_next_boundary() { - let git_dir = unique_test_git_dir("orphaned-flushing"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let attempt = state::seed_attempt_for_tests( - &git_dir, - &key("ses-stuck", "call-stuck"), - "bash", - AttemptPhase::PendingStart, - ); - state::begin_terminal_cleanup(&git_dir, std::slice::from_ref(&attempt.scope_id)) - .expect("seeding an orphaned flush should succeed"); - assert!(matches!( - state::read_state(&git_dir) - .expect("state readable") - .recovery, - RecoveryState::Flushing { .. } - )); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "a crash mid-flush leaves an orphaned Flushing that the next boundary reclaims" - ); - - let healthy = RecordingSeam::new(); - drive(&git_dir, &healthy, &tool_call_event("ses-new", "call-new")) - .expect("the orphaned flush is reclaimed and retried, then the new call is admitted"); - - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert!(resolved - .attempts - .iter() - .all(|a| a.session_id != "ses-stuck")); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn malformed_state_file_is_invalid_with_the_read_error_surfaced() { - let git_dir = unique_test_git_dir("malformed"); - let dir = state::adapter_state_dir(&git_dir); - std::fs::create_dir_all(&dir).expect("state dir should be created"); - std::fs::write(state::state_path(&git_dir), b"not json") - .expect("malformed file should be writable"); - - let health = classify_health(&git_dir); - - assert_eq!(health.status, MutationScopeHealthStatus::Invalid); - assert!( - health - .detail - .as_deref() - .unwrap_or_default() - .contains("malformed"), - "the read error must be surfaced in the detail: {:?}", - health.detail - ); - - remove_test_git_dir(&git_dir); - } - - fn matrix_attempt( - call_id: &str, - phase: AttemptPhase, - owner: ProcessOwner, - ) -> state::AdapterAttempt { - let attempt_key = key("ses-main", call_id); - state::AdapterAttempt { - attempt_seq: 1, - scope_id: super::super::format_pi_scope_id(&attempt_key, 1), - session_id: attempt_key.session_id, - tool_call_id: attempt_key.tool_call_id, - tool_name: "bash".to_string(), - phase, - owner, - } - } - - fn write_matrix_state( - git_dir: &Path, - recovery: RecoveryState, - bystander_owner: Option, - has_pending_abandon: bool, - has_dead_owner_attempt: bool, - dead_owner: ProcessOwner, - ) { - std::fs::create_dir_all(state::adapter_state_dir(git_dir)) - .expect("adapter state dir should be created"); - - let mut attempts = Vec::new(); - if let Some(owner) = bystander_owner { - attempts.push(matrix_attempt( - "call-bystander", - AttemptPhase::PendingStart, - owner, - )); - } - if has_pending_abandon { - attempts.push(matrix_attempt( - "call-abandon", - AttemptPhase::PendingAbandon, - crate::services::hooks::mutation_scope_owner::current_process_owner(), - )); - } - if has_dead_owner_attempt { - attempts.push(matrix_attempt( - "call-dead", - AttemptPhase::Executed, - dead_owner, - )); - } - - let state = state::AdapterState { - version: 2, - next_attempt_seq: 2, - next_recovery_generation: 2, - recovery, - attempts, - }; - std::fs::write( - state::state_path(git_dir), - serde_json::to_string(&state).expect("matrix state serializes"), - ) - .expect("hand-built matrix state should be writable"); - } - - #[test] - fn health_classification_matrix_covers_all_twelve_recovery_and_attempt_condition_combinations() - { - use MutationScopeHealthStatus::{Healthy, Invalid, Recovering}; - - let mut dead_child = std::process::Command::new("true") - .spawn() - .expect("spawning 'true' should succeed"); - let dead_pid = i32::try_from(dead_child.id()).expect("pid fits in i32"); - dead_child.wait().expect("child should exit and be reaped"); - let dead_owner = ProcessOwner { - pid: dead_pid, - instance_token: None, - }; - let live_owner = crate::services::hooks::mutation_scope_owner::current_process_owner(); - let uncertain_owner = ProcessOwner { - pid: std::process::id().cast_signed(), - instance_token: None, - }; - let bystanders = [ - ("none", None), - ("live-owner", Some(live_owner)), - ("uncertain-owner", Some(uncertain_owner)), - ]; - - let clear = RecoveryState::Clear; - let pending = RecoveryState::Pending { generation: 1 }; - let flushing = RecoveryState::Flushing { generation: 1 }; - - let rows = [ - (clear, false, false, Healthy), - (clear, false, true, Recovering), - (clear, true, false, Invalid), - (clear, true, true, Invalid), - (pending, false, false, Recovering), - (pending, false, true, Recovering), - (pending, true, false, Recovering), - (pending, true, true, Recovering), - (flushing, false, false, Recovering), - (flushing, false, true, Recovering), - (flushing, true, false, Recovering), - (flushing, true, true, Recovering), - ]; - - for (index, (recovery, has_pending_abandon, has_dead_owner_attempt, expected)) in - rows.into_iter().enumerate() - { - for (bystander, bystander_owner) in bystanders { - let git_dir = unique_test_git_dir(&format!("matrix-{index}-{bystander}")); - write_matrix_state( - &git_dir, - recovery, - bystander_owner, - has_pending_abandon, - has_dead_owner_attempt, - dead_owner, - ); - - assert_eq!( - classify_health(&git_dir).status, - expected, - "row {index}: recovery={recovery:?} bystander={bystander} \ - has_pending_abandon={has_pending_abandon} \ - has_dead_owner_attempt={has_dead_owner_attempt}", - ); - - remove_test_git_dir(&git_dir); - } - } - } - - #[test] - fn pending_recovery_reuses_a_duplicate_start_for_an_existing_nonterminal_key_without_advancing_recovery_then_a_fresh_start_recovers( - ) { - let git_dir = unique_test_git_dir("pending-duplicate-start-vs-fresh-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let setup = RecordingSeam::new(); - drive(&git_dir, &setup, &tool_call_event("ses-b", "call-b")).expect("B's Start"); - drive(&git_dir, &setup, &tool_call_event("ses-a", "call-a")).expect("A's Start"); - - let crashing_abandon = RecordingSeam::failing_once_on(&["abandon"]); - drive( - &git_dir, - &crashing_abandon, - &tool_execution_end_event("ses-a", "call-a"), - ) - .expect("a transient abandon failure mid-recovery must not surface an error"); - - let seeded = state::read_state(&git_dir).expect("state readable"); - assert_eq!(seeded.recovery, RecoveryState::Pending { generation: 1 }); - let phase_by_session = |session_id: &str| { - seeded - .attempts - .iter() - .find(|attempt| attempt.session_id == session_id) - .expect("attempt for session must exist") - .phase - }; - assert_eq!(phase_by_session("ses-a"), AttemptPhase::PendingAbandon); - assert_eq!(phase_by_session("ses-b"), AttemptPhase::PendingStart); - - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering - ); - - let duplicate = RecordingSeam::new(); - drive(&git_dir, &duplicate, &tool_call_event("ses-b", "call-b")) - .expect("a duplicate Start for an already-tracked nonterminal key stays idempotent"); - - assert_eq!( - duplicate.calls.lock().expect("seam mutex").clone(), - vec!["start".to_string()], - "a duplicate Start for B's own key must be reused without touching recovery" - ); - let after_duplicate = state::read_state(&git_dir).expect("state readable"); - assert_eq!( - after_duplicate.recovery, - RecoveryState::Pending { generation: 1 }, - "recovery must remain Pending: the duplicate Start never reached the recovery-state gate" - ); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Recovering, - "reusing B's key must not change the classification" - ); - - let fresh = RecordingSeam::new(); - drive(&git_dir, &fresh, &tool_call_event("ses-c", "call-c")).expect( - "C's fresh Start must claim and complete the pending recovery, then be admitted", - ); - - assert_eq!( - fresh.calls.lock().expect("seam mutex").clone(), - vec!["flush", "abandon", "flush", "start"], - "C claims the Pending generation, resolve_recovery retires A, then C's own Start is admitted" - ); - let resolved = state::read_state(&git_dir).expect("state readable"); - assert!(resolved.recovery.is_clear()); - assert!(!resolved.attempts.iter().any(|a| a.session_id == "ses-a")); - assert!(resolved.attempts.iter().any(|a| a.session_id == "ses-b")); - assert!(resolved.attempts.iter().any(|a| a.session_id == "ses-c")); - assert_eq!( - classify_health(&git_dir).status, - MutationScopeHealthStatus::Healthy - ); - - remove_test_git_dir(&git_dir); - } -} diff --git a/cli/src/services/hooks/pi_mutation_scope/lifecycle.rs b/cli/src/services/hooks/pi_mutation_scope/lifecycle.rs index d02dc258f..50e71cf00 100644 --- a/cli/src/services/hooks/pi_mutation_scope/lifecycle.rs +++ b/cli/src/services/hooks/pi_mutation_scope/lifecycle.rs @@ -4,7 +4,6 @@ use anyhow::{bail, Result}; use crate::services::hooks; use crate::services::mutation_trace::runtime::resolve_git_dir; -use crate::services::observability::traits::Logger; use super::state::{self, AdmitDecision, RecoveryFlushCompletion}; use super::{ @@ -13,18 +12,25 @@ use super::{ AttemptKey, PiHookEvent, PiScopeProvenance, ToolClassification, }; -pub(crate) fn run_pi_mutation_scope_subcommand(logger: Option<&dyn Logger>) -> Result { +pub(crate) async fn run_pi_mutation_scope_subcommand< + L: crate::services::observability::traits::Logger, +>( + logger: Option<&L>, +) -> Result { let stdin_payload = hooks::read_hook_stdin()?; - run_pi_mutation_scope_from_payload(&stdin_payload, logger) + run_pi_mutation_scope_from_payload(&stdin_payload, logger).await } -pub(crate) fn run_pi_mutation_scope_from_payload( +pub(crate) async fn run_pi_mutation_scope_from_payload< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, + logger: Option<&L>, ) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { + let resolve_git_dir_fn = async |cwd: &str| resolve_git_dir(Path::new(cwd)).await; + let seam_fn = async |repository_root: &Path, payload: &str, logger: Option<&L>| { hooks::mutation_scope::run_mutation_scope_from_payload(repository_root, payload, logger) + .await }; run_pi_mutation_scope_from_payload_with_seams( @@ -33,42 +39,19 @@ pub(crate) fn run_pi_mutation_scope_from_payload( &resolve_git_dir_fn, &seam_fn, ) + .await } -#[cfg(test)] -pub(crate) fn run_pi_mutation_scope_from_payload_at_state_root( - state_root: &Path, - stdin_payload: &str, - logger: Option<&dyn Logger>, -) -> Result { - let resolve_git_dir_fn = |cwd: &str| resolve_git_dir(Path::new(cwd)); - let seam_fn = |repository_root: &Path, payload: &str, logger: Option<&dyn Logger>| { - hooks::mutation_scope::run_mutation_scope_from_payload_at_state_root( - repository_root, - state_root, - payload, - logger, - ) - }; - - run_pi_mutation_scope_from_payload_with_seams( - stdin_payload, - logger, - &resolve_git_dir_fn, - &seam_fn, - ) -} - -pub(super) type GitDirResolver<'a> = &'a dyn Fn(&str) -> Result; - -pub(super) type IngressSeam<'a> = &'a dyn Fn(&Path, &str, Option<&dyn Logger>) -> Result; - pub(super) const FAIL_CLOSED_MESSAGE: &str = "SCE could not establish Pi mutation attribution for this tool execution."; pub(super) const FAIL_CLOSED_EVENT: &str = "sce.hooks.pi_mutation_scope.start_fail_closed"; -pub(super) fn log_fail_closed(logger: Option<&dyn Logger>, context: &str, error: &anyhow::Error) { +pub(super) fn log_fail_closed( + logger: Option<&L>, + context: &str, + error: &anyhow::Error, +) { if let Some(log) = logger { log.warn( FAIL_CLOSED_EVENT, @@ -79,21 +62,23 @@ pub(super) fn log_fail_closed(logger: Option<&dyn Logger>, context: &str, error: } } -pub(super) fn run_pi_mutation_scope_from_payload_with_seams( +pub(super) async fn run_pi_mutation_scope_from_payload_with_seams< + L: crate::services::observability::traits::Logger, +>( stdin_payload: &str, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { let event = parse_pi_hook_event(stdin_payload)?; - dispatch_pi_hook_event(event, logger, resolve_git_dir, seam) + dispatch_pi_hook_event(event, logger, resolve_git_dir, seam).await } -pub(super) fn dispatch_pi_hook_event( +pub(super) async fn dispatch_pi_hook_event( event: PiHookEvent, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { match event { PiHookEvent::ExecutionStart(_identity) => Ok(String::new()), @@ -110,6 +95,7 @@ pub(super) fn dispatch_pi_hook_event( resolve_git_dir, seam, ) + .await } ToolClassification::Untracked => Ok(String::new()), }, @@ -120,8 +106,8 @@ pub(super) fn dispatch_pi_hook_event( ) { return Ok(String::new()); } - let git_dir = resolve_git_dir(&identity.cwd)?; - state::mark_executed(&git_dir, &identity.attempt_key())?; + let git_dir = resolve_git_dir(&identity.cwd).await?; + state::mark_executed(&git_dir, &identity.attempt_key()).await?; Ok(String::new()) } PiHookEvent::ExecutionEnd(identity) => { @@ -131,13 +117,14 @@ pub(super) fn dispatch_pi_hook_event( ) { return Ok(String::new()); } - let git_dir = resolve_git_dir(&identity.cwd)?; + let git_dir = resolve_git_dir(&identity.cwd).await?; let repository_root = Path::new(&identity.cwd); let key = identity.attempt_key(); - with_boundary_lock(&git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(&git_dir)?; - handle_tool_execution_end(&git_dir, repository_root, &key, logger, seam) + with_boundary_lock(&git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(&git_dir).await?; + handle_tool_execution_end(&git_dir, repository_root, &key, logger, seam).await }) + .await } PiHookEvent::ExecutionAbandon(identity) => { if !matches!( @@ -146,23 +133,25 @@ pub(super) fn dispatch_pi_hook_event( ) { return Ok(String::new()); } - let git_dir = resolve_git_dir(&identity.cwd)?; + let git_dir = resolve_git_dir(&identity.cwd).await?; let repository_root = Path::new(&identity.cwd); let key = identity.attempt_key(); - with_boundary_lock(&git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(&git_dir)?; - force_abandon_attempt(&git_dir, repository_root, &key, logger, seam) + with_boundary_lock(&git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(&git_dir).await?; + force_abandon_attempt(&git_dir, repository_root, &key, logger, seam).await }) + .await } } } -pub(super) fn with_boundary_lock( +pub(super) async fn with_boundary_lock( git_dir: &Path, - operation: impl FnOnce() -> Result, + operation: impl std::ops::AsyncFnOnce() -> Result, ) -> Result { - let _boundary = state::BOUNDARY_LOCK.acquire(&state::adapter_state_dir(git_dir))?; - operation() + state::BOUNDARY_LOCK + .run_under_boundary(&state::adapter_state_dir(git_dir), operation) + .await } pub(super) enum Admission { @@ -175,16 +164,16 @@ pub(super) enum StartOutcome { Denied, } -pub(super) fn establish_tracked_start( +pub(super) async fn establish_tracked_start( cwd: &str, key: &AttemptKey, tool_name: &str, provenance: &PiScopeProvenance, - logger: Option<&dyn Logger>, - resolve_git_dir: GitDirResolver, - seam: IngressSeam, + logger: Option<&L>, + resolve_git_dir: &impl std::ops::AsyncFn(&str) -> Result, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let git_dir = match resolve_git_dir(cwd) { + let git_dir = match resolve_git_dir(cwd).await { Ok(git_dir) => git_dir, Err(error) => { log_fail_closed(logger, "resolve_git_dir", &error); @@ -193,10 +182,10 @@ pub(super) fn establish_tracked_start( }; let repository_root = Path::new(cwd); - let outcome = with_boundary_lock(&git_dir, || { - state::normalize_recovery_after_boundary_lock_acquired(&git_dir)?; + let outcome = with_boundary_lock(&git_dir, async || { + state::normalize_recovery_after_boundary_lock_acquired(&git_dir).await?; - match admit_or_recover(&git_dir, repository_root, key, tool_name, logger, seam)? { + match admit_or_recover(&git_dir, repository_root, key, tool_name, logger, seam).await? { Admission::Admitted(allocated) => { establish_start( &git_dir, @@ -205,12 +194,14 @@ pub(super) fn establish_tracked_start( provenance, logger, seam, - )?; + ) + .await?; Ok(StartOutcome::Established) } Admission::Denied => Ok(StartOutcome::Denied), } - }); + }) + .await; match outcome { Ok(StartOutcome::Established) => Ok(String::new()), @@ -222,48 +213,48 @@ pub(super) fn establish_tracked_start( } } -pub(super) fn admit_or_recover( +pub(super) async fn admit_or_recover( git_dir: &Path, repository_root: &Path, key: &AttemptKey, tool_name: &str, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - match reconcile_stale_owners(git_dir, repository_root, logger, seam)? { + match reconcile_stale_owners(git_dir, repository_root, logger, seam).await? { RecoveryResolution::Cleared => {} RecoveryResolution::Unresolved => return Ok(Admission::Denied), } - match state::admit_tracked_attempt(git_dir, key, tool_name)? { + match state::admit_tracked_attempt(git_dir, key, tool_name).await? { AdmitDecision::Admitted(allocated) => Ok(Admission::Admitted(allocated)), AdmitDecision::RecoveryBlocked | AdmitDecision::UncertainAttemptBlocked | AdmitDecision::TerminalAttemptBlocked => Ok(Admission::Denied), AdmitDecision::FlushClaimed { generation } => { - match resolve_recovery(git_dir, repository_root, generation, logger, seam)? { - RecoveryResolution::Cleared => readmit_after_flush(git_dir, key, tool_name), + match resolve_recovery(git_dir, repository_root, generation, logger, seam).await? { + RecoveryResolution::Cleared => readmit_after_flush(git_dir, key, tool_name).await, RecoveryResolution::Unresolved => Ok(Admission::Denied), } } } } -pub(super) fn reconcile_stale_owners( +pub(super) async fn reconcile_stale_owners( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { loop { - let dead_scope_ids = state::find_definitely_dead_attempts(git_dir)?; + let dead_scope_ids = state::find_definitely_dead_attempts(git_dir).await?; if dead_scope_ids.is_empty() { return Ok(RecoveryResolution::Cleared); } - let generation = state::begin_terminal_cleanup(git_dir, &dead_scope_ids)?; + let generation = state::begin_terminal_cleanup(git_dir, &dead_scope_ids).await?; if matches!( - resolve_recovery(git_dir, repository_root, generation, logger, seam)?, + resolve_recovery(git_dir, repository_root, generation, logger, seam).await?, RecoveryResolution::Unresolved ) { return Ok(RecoveryResolution::Unresolved); @@ -271,15 +262,15 @@ pub(super) fn reconcile_stale_owners( } } -pub(super) fn readmit_after_flush( +pub(super) async fn readmit_after_flush( git_dir: &Path, key: &AttemptKey, tool_name: &str, ) -> Result { - match state::admit_tracked_attempt(git_dir, key, tool_name)? { + match state::admit_tracked_attempt(git_dir, key, tool_name).await? { AdmitDecision::Admitted(allocated) => Ok(Admission::Admitted(allocated)), AdmitDecision::FlushClaimed { generation } => { - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; Ok(Admission::Denied) } AdmitDecision::RecoveryBlocked @@ -288,31 +279,31 @@ pub(super) fn readmit_after_flush( } } -pub(super) fn establish_start( +pub(super) async fn establish_start( _git_dir: &Path, repository_root: &Path, allocated: &state::AllocatedAttempt, provenance: &PiScopeProvenance, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result<()> { let scope_id = &allocated.attempt.scope_id; let start_payload = scope_start_payload(scope_id, &pi_scope_start_event_id(scope_id), provenance); - seam(repository_root, &start_payload, logger)?; + seam(repository_root, &start_payload, logger).await?; Ok(()) } -pub(super) fn handle_tool_execution_end( +pub(super) async fn handle_tool_execution_end( git_dir: &Path, repository_root: &Path, key: &AttemptKey, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let Some(attempt) = current .attempts .iter() @@ -329,7 +320,8 @@ pub(super) fn handle_tool_execution_end( if !matches!(attempt.phase, state::AttemptPhase::Executed) { return abandon_and_consume(git_dir, repository_root, logger, seam, move |candidate| { candidate.scope_id == doomed_scope_id - }); + }) + .await; } let close_payload = scope_boundary_payload( @@ -338,24 +330,25 @@ pub(super) fn handle_tool_execution_end( &pi_scope_close_event_id(&attempt.scope_id), ); - if seam(repository_root, &close_payload, logger).is_ok() { - state::remove_attempt(git_dir, &attempt.scope_id)?; + if seam(repository_root, &close_payload, logger).await.is_ok() { + state::remove_attempt(git_dir, &attempt.scope_id).await?; Ok(String::new()) } else { abandon_and_consume(git_dir, repository_root, logger, seam, move |candidate| { candidate.scope_id == doomed_scope_id }) + .await } } -pub(super) fn force_abandon_attempt( +pub(super) async fn force_abandon_attempt( git_dir: &Path, repository_root: &Path, key: &AttemptKey, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let current = state::read_state(git_dir)?; + let current = state::read_state(git_dir).await?; let Some(attempt) = current .attempts .iter() @@ -371,6 +364,7 @@ pub(super) fn force_abandon_attempt( abandon_and_consume(git_dir, repository_root, logger, seam, move |candidate| { candidate.scope_id == doomed_scope_id }) + .await } pub(super) enum RecoveryResolution { @@ -378,14 +372,15 @@ pub(super) enum RecoveryResolution { Unresolved, } -pub(super) fn abandon_and_consume( +pub(super) async fn abandon_and_consume( git_dir: &Path, repository_root: &Path, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, doomed: impl Fn(&state::AdapterAttempt) -> bool, ) -> Result { - let doomed_scope_ids: Vec = state::read_state(git_dir)? + let doomed_scope_ids: Vec = state::read_state(git_dir) + .await? .attempts .into_iter() .filter(|attempt| doomed(attempt)) @@ -395,64 +390,49 @@ pub(super) fn abandon_and_consume( return Ok(String::new()); } - let generation = state::begin_terminal_cleanup(git_dir, &doomed_scope_ids)?; - resolve_recovery(git_dir, repository_root, generation, logger, seam)?; + let generation = state::begin_terminal_cleanup(git_dir, &doomed_scope_ids).await?; + resolve_recovery(git_dir, repository_root, generation, logger, seam).await?; Ok(String::new()) } -pub(super) fn resolve_recovery( +pub(super) async fn resolve_recovery( git_dir: &Path, repository_root: &Path, generation: u64, - logger: Option<&dyn Logger>, - seam: IngressSeam, + logger: Option<&L>, + seam: &impl std::ops::AsyncFn(&Path, &str, Option<&L>) -> Result, ) -> Result { - let pending_abandon: Vec = state::read_state(git_dir)? + let pending_abandon: Vec = state::read_state(git_dir) + .await? .attempts .into_iter() .filter(|attempt| attempt.phase == state::AttemptPhase::PendingAbandon) .collect(); - if let Err(error) = seam(repository_root, &flush_payload(), logger) { + if let Err(error) = seam(repository_root, &flush_payload(), logger).await { log_fail_closed(logger, "recovery_ambiguity_flush", &error); - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; return Ok(RecoveryResolution::Unresolved); } for attempt in &pending_abandon { - if let Err(error) = seam(repository_root, &abandon_payload(&attempt.scope_id), logger) { + if let Err(error) = seam(repository_root, &abandon_payload(&attempt.scope_id), logger).await + { log_fail_closed(logger, "recovery_abandon", &error); - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; return Ok(RecoveryResolution::Unresolved); } - state::remove_attempt(git_dir, &attempt.scope_id)?; + state::remove_attempt(git_dir, &attempt.scope_id).await?; } - if let Err(error) = seam(repository_root, &flush_payload(), logger) { + if let Err(error) = seam(repository_root, &flush_payload(), logger).await { log_fail_closed(logger, "recovery_rebaseline_flush", &error); - state::relinquish_recovery_flush(git_dir, generation)?; + state::relinquish_recovery_flush(git_dir, generation).await?; return Ok(RecoveryResolution::Unresolved); } - match state::complete_recovery_flush(git_dir, generation)? { + match state::complete_recovery_flush(git_dir, generation).await? { RecoveryFlushCompletion::Cleared => Ok(RecoveryResolution::Cleared), RecoveryFlushCompletion::Superseded => Ok(RecoveryResolution::Unresolved), } } - -#[cfg(test)] -pub(crate) fn force_attempt_owner_dead_for_tests(git_dir: &Path, scope_id: &str) { - let mut dead_child = std::process::Command::new("true") - .spawn() - .expect("spawning 'true' should succeed"); - let dead_pid = i32::try_from(dead_child.id()).expect("pid fits in i32"); - dead_child.wait().expect("child should exit and be reaped"); - state::set_attempt_owner_for_tests( - git_dir, - scope_id, - crate::services::hooks::mutation_scope_owner::ProcessOwner { - pid: dead_pid, - instance_token: None, - }, - ); -} diff --git a/cli/src/services/hooks/pi_mutation_scope/lifecycle_tests.rs b/cli/src/services/hooks/pi_mutation_scope/lifecycle_tests.rs deleted file mode 100644 index 6d3be6601..000000000 --- a/cli/src/services/hooks/pi_mutation_scope/lifecycle_tests.rs +++ /dev/null @@ -1,1155 +0,0 @@ -use std::path::PathBuf; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Mutex; - -use serde_json::Value; - -use super::state::{read_state, AdapterAttempt, AdapterState, AttemptPhase, RecoveryState}; -use super::*; - -static NEXT_ID: AtomicU64 = AtomicU64::new(0); - -fn temp_git_dir(label: &str) -> PathBuf { - let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-pi-mutation-scope-lifecycle-{label}-{}-{id}", - std::process::id() - )) -} - -const CWD: &str = "/repo/pi-checkout"; - -struct RecordingSeam { - calls: Mutex>, - fail_operations: Vec, - fail_once_operations: Mutex>, -} - -impl RecordingSeam { - fn new() -> Self { - Self { - calls: Mutex::new(Vec::new()), - fail_operations: Vec::new(), - fail_once_operations: Mutex::new(Vec::new()), - } - } - - fn failing_on(operations: &[&str]) -> Self { - Self { - fail_operations: operations.iter().map(|op| (*op).to_string()).collect(), - ..Self::new() - } - } - - fn failing_once_on(operations: &[&str]) -> Self { - Self { - fail_once_operations: Mutex::new( - operations.iter().map(|op| (*op).to_string()).collect(), - ), - ..Self::new() - } - } - - fn handle(&self, payload: &str) -> Result { - let operation = operation_of(payload); - { - let mut calls = self.calls.lock().expect("seam mutex"); - calls.push(operation.clone()); - } - if self.fail_operations.contains(&operation) { - bail!("seam failure injected by test for '{operation}'"); - } - { - let mut once = self.fail_once_operations.lock().expect("seam mutex"); - if let Some(position) = once.iter().position(|candidate| candidate == &operation) { - once.remove(position); - bail!("transient seam failure injected once by test for '{operation}'"); - } - } - Ok(String::new()) - } - - fn operations(&self) -> Vec { - self.calls.lock().expect("seam mutex").clone() - } -} - -fn operation_of(payload: &str) -> String { - let value: Value = serde_json::from_str(payload).expect("seam payload is JSON"); - value - .get("operation") - .and_then(Value::as_str) - .expect("seam payload has an operation") - .to_string() -} - -fn drive(git_dir: &Path, seam: &RecordingSeam, payload: &str) -> Result { - let resolver = |_cwd: &str| Ok(git_dir.to_path_buf()); - let seam_fn = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| seam.handle(payload); - run_pi_mutation_scope_from_payload_with_seams(payload, None, &resolver, &seam_fn) -} - -fn tool_call_event(tool_name: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolCall", - "session_id": "ses-main", - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": tool_name, - "model": "openai-codex/gpt-5.5", - }) - .to_string() -} - -fn tool_result_event(tool_name: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolResult", - "session_id": "ses-main", - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() -} - -fn tool_execution_end_event(tool_name: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecutionEnd", - "session_id": "ses-main", - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() -} - -fn tool_execution_abandon_event(tool_name: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecutionAbandon", - "session_id": "ses-main", - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() -} - -fn tool_execution_abandon_event_for_session( - tool_name: &str, - session_id: &str, - tool_call_id: &str, -) -> String { - json!({ - "hook_event_name": "ToolExecutionAbandon", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() -} - -fn cleanup(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); -} - -fn tool_call_event_for_session(tool_name: &str, session_id: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolCall", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": tool_name, - "model": "openai-codex/gpt-5.5", - }) - .to_string() -} - -fn tool_result_event_for_session(tool_name: &str, session_id: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolResult", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": CWD, - "tool_name": tool_name, - }) - .to_string() -} - -fn dead_process_owner() -> crate::services::hooks::mutation_scope_owner::ProcessOwner { - let mut dead_child = std::process::Command::new("true") - .spawn() - .expect("spawning 'true' should succeed"); - let dead_pid = i32::try_from(dead_child.id()).expect("pid fits in i32"); - dead_child.wait().expect("child should exit and be reaped"); - crate::services::hooks::mutation_scope_owner::ProcessOwner { - pid: dead_pid, - instance_token: None, - } -} - -fn attempt_owned_by(state: &AdapterState, session_id: &str) -> AdapterAttempt { - state - .attempts - .iter() - .find(|attempt| attempt.session_id == session_id) - .expect("attempt for session must exist") - .clone() -} - -#[test] -fn tool_call_establishes_a_write_ahead_start_and_replays_idempotently() { - let git_dir = temp_git_dir("write-ahead-start"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("write", "call_1")).expect("first Start"); - drive(&git_dir, &seam, &tool_call_event("write", "call_1")) - .expect("duplicate Start is idempotent"); - - assert_eq!(seam.operations(), vec!["start", "start"]); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].phase, AttemptPhase::PendingStart); - assert_eq!(state.attempts[0].tool_name, "write"); - - cleanup(&git_dir); -} - -#[test] -fn concurrent_bash_calls_in_one_session_stay_separate_live_scopes() { - let git_dir = temp_git_dir("concurrent-bash"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_a")).expect("A Start"); - drive(&git_dir, &seam, &tool_call_event("bash", "call_b")).expect("B Start must not retire A"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 2); - assert!(state - .attempts - .iter() - .all(|attempt| attempt.phase == AttemptPhase::PendingStart)); - - cleanup(&git_dir); -} - -#[test] -fn full_success_lifecycle_start_result_close() { - let git_dir = temp_git_dir("success-lifecycle"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::PendingStart - ); - - drive(&git_dir, &seam, &tool_result_event("bash", "call_1")).expect("tool_result"); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::Executed, - "D5/D6: tool_result is the sole Executed-transition evidence" - ); - - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")) - .expect("tool_execution_end closes an Executed attempt"); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - assert_eq!(seam.operations(), vec!["start", "close"]); - - cleanup(&git_dir); -} - -#[test] -fn tool_execution_end_without_a_preceding_tool_result_abandons_never_closes() { - let git_dir = temp_git_dir("d7-abandon"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")) - .expect("D7: a terminal event with no preceding tool_result must abandon"); - - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - assert!( - !seam.operations().contains(&"close".to_string()), - "D7: an unexecuted attempt must never be closed" - ); - assert!(seam.operations().contains(&"abandon".to_string())); - assert!(read_state(&git_dir) - .expect("state readable") - .recovery - .is_clear()); - - cleanup(&git_dir); -} - -#[test] -fn a_failed_close_falls_back_to_abandon_recovery() { - let git_dir = temp_git_dir("close-failure-falls-back"); - let seam = RecordingSeam::failing_on(&["close"]); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - drive(&git_dir, &seam, &tool_result_event("bash", "call_1")).expect("tool_result"); - - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")) - .expect("a Close failure must recover via abandon, not surface an error"); - - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - assert!(seam.operations().contains(&"abandon".to_string())); - - cleanup(&git_dir); -} - -#[test] -fn execution_abandon_forces_abandon_even_when_the_attempt_is_already_executed() { - let git_dir = temp_git_dir("d9-execution-abandon-executed"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - drive(&git_dir, &seam, &tool_result_event("bash", "call_1")).expect("tool_result"); - - drive( - &git_dir, - &seam, - &tool_execution_abandon_event("bash", "call_1"), - ) - .expect("ExecutionAbandon must recover via abandon, not surface an error"); - - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - assert!( - !seam.operations().contains(&"close".to_string()), - "D9: an explicit abandon request must never be treated as a Close, \ - even for an attempt already marked Executed" - ); - assert!(seam.operations().contains(&"abandon".to_string())); - - cleanup(&git_dir); -} - -#[test] -fn execution_abandon_on_a_pending_start_attempt_abandons() { - let git_dir = temp_git_dir("d9-execution-abandon-pending-start"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - - drive( - &git_dir, - &seam, - &tool_execution_abandon_event("bash", "call_1"), - ) - .expect("ExecutionAbandon must recover a PendingStart attempt via abandon"); - - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - assert!(seam.operations().contains(&"abandon".to_string())); - - cleanup(&git_dir); -} - -#[test] -fn execution_abandon_for_an_unknown_attempt_is_a_safe_no_op() { - let git_dir = temp_git_dir("d9-execution-abandon-unknown"); - let seam = RecordingSeam::new(); - - let result = drive( - &git_dir, - &seam, - &tool_execution_abandon_event("bash", "call_1"), - ) - .expect("an unknown attempt must be a safe no-op, never an error"); - - assert_eq!(result, ""); - assert!(seam.operations().is_empty()); - - cleanup(&git_dir); -} - -#[test] -fn execution_abandon_for_an_untracked_tool_is_a_no_op() { - let git_dir = temp_git_dir("d9-execution-abandon-untracked"); - let seam = RecordingSeam::new(); - - let result = drive( - &git_dir, - &seam, - &tool_execution_abandon_event("read", "call_1"), - ) - .expect("untracked tools are never adapter-relevant"); - - assert_eq!(result, ""); - assert!(seam.operations().is_empty()); - - cleanup(&git_dir); -} - -#[test] -fn duplicate_execution_abandon_on_an_already_abandoned_attempt_is_a_safe_no_op() { - let git_dir = temp_git_dir("d9-execution-abandon-duplicate"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - - drive( - &git_dir, - &seam, - &tool_execution_abandon_event("bash", "call_1"), - ) - .expect("first ExecutionAbandon retires the attempt"); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - let result = drive( - &git_dir, - &seam, - &tool_execution_abandon_event("bash", "call_1"), - ) - .expect("a duplicate ExecutionAbandon for an already-retired attempt must be a safe no-op"); - - assert_eq!(result, ""); - assert_eq!( - seam.operations(), - vec!["start", "flush", "abandon", "flush"], - "a duplicate ExecutionAbandon must never issue a second abandon or a close" - ); - - cleanup(&git_dir); -} - -#[test] -fn execution_abandon_for_one_session_never_touches_another_sessions_attempt_with_the_same_tool_call_id( -) { - let git_dir = temp_git_dir("d9-execution-abandon-cross-session"); - let seam = RecordingSeam::new(); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "ses-a", "call_1"), - ) - .expect("session A Start"); - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "ses-b", "call_1"), - ) - .expect("session B Start with the same tool_call_id"); - - assert_eq!( - read_state(&git_dir).expect("state readable").attempts.len(), - 2 - ); - - drive( - &git_dir, - &seam, - &tool_execution_abandon_event_for_session("bash", "ses-a", "call_1"), - ) - .expect("ExecutionAbandon for session A must not error"); - - let remaining = read_state(&git_dir).expect("state readable").attempts; - assert_eq!( - remaining.len(), - 1, - "abandoning session A's attempt must leave session B's untouched" - ); - assert_eq!(remaining[0].session_id, "ses-b"); - assert_eq!(remaining[0].tool_call_id, "call_1"); - assert_eq!(remaining[0].phase, AttemptPhase::PendingStart); - - drive( - &git_dir, - &seam, - &tool_result_event_for_session("bash", "ses-b", "call_1"), - ) - .expect("session B must still be able to progress normally after A's abandon"); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::Executed - ); - - cleanup(&git_dir); -} - -#[test] -fn a_terminal_recovery_flush_failure_leaves_a_pending_recovery_and_denies_new_admission() { - let git_dir = temp_git_dir("recovery-flush-failure"); - let persistently_failing = RecordingSeam::failing_on(&["flush"]); - - drive( - &git_dir, - &persistently_failing, - &tool_call_event("bash", "call_1"), - ) - .expect("Start"); - drive( - &git_dir, - &persistently_failing, - &tool_execution_end_event("bash", "call_1"), - ) - .expect("abandon path swallows the flush failure rather than surfacing an error"); - - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 1 }, - "a failed ambiguity flush must leave recovery Pending, not Clear" - ); - - let error = drive( - &git_dir, - &persistently_failing, - &tool_call_event("bash", "call_2"), - ) - .expect_err("a new admission must fail closed while recovery remains unresolved"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - - let recovered = RecordingSeam::new(); - drive(&git_dir, &recovered, &tool_call_event("bash", "call_3")) - .expect("a new admission must self-heal once recovery can complete"); - assert!(read_state(&git_dir) - .expect("state readable") - .recovery - .is_clear()); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].tool_call_id, "call_3"); - - cleanup(&git_dir); -} - -#[test] -fn start_provenance_carries_the_prefixed_session_and_normalized_model_to_the_seam() { - let git_dir = temp_git_dir("provenance-present"); - let captured: Mutex> = Mutex::new(Vec::new()); - let resolver = |_cwd: &str| Ok(git_dir.clone()); - let seam_fn = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| { - captured - .lock() - .expect("capture mutex") - .push(payload.to_string()); - Ok(String::new()) - }; - - run_pi_mutation_scope_from_payload_with_seams( - &tool_call_event("bash", "call_model"), - None, - &resolver, - &seam_fn, - ) - .expect("Start should succeed"); - - let payloads = captured.into_inner().expect("capture mutex"); - assert_eq!(payloads.len(), 1); - let sent: Value = serde_json::from_str(&payloads[0]).expect("seam payload is JSON"); - assert_eq!( - sent["provenance"]["session_id"].as_str(), - Some("pi_ses-main") - ); - assert_eq!( - sent["provenance"]["model_id"].as_str(), - Some("openai-codex/gpt-5.5") - ); - - cleanup(&git_dir); -} - -#[test] -fn start_provenance_is_null_model_when_the_event_carries_no_model() { - let git_dir = temp_git_dir("provenance-absent"); - let captured: Mutex> = Mutex::new(Vec::new()); - let resolver = |_cwd: &str| Ok(git_dir.clone()); - let seam_fn = |_root: &Path, payload: &str, _logger: Option<&dyn Logger>| { - captured - .lock() - .expect("capture mutex") - .push(payload.to_string()); - Ok(String::new()) - }; - - let payload = json!({ - "hook_event_name": "ToolCall", - "session_id": "ses-main", - "tool_call_id": "call_no_model", - "cwd": CWD, - "tool_name": "bash", - }) - .to_string(); - - run_pi_mutation_scope_from_payload_with_seams(&payload, None, &resolver, &seam_fn) - .expect("Start should succeed"); - - let payloads = captured.into_inner().expect("capture mutex"); - let sent: Value = serde_json::from_str(&payloads[0]).expect("seam payload is JSON"); - assert!(sent["provenance"]["model_id"].is_null()); - - cleanup(&git_dir); -} - -#[test] -fn untracked_tool_call_never_admits_an_attempt() { - let git_dir = temp_git_dir("untracked-no-admit"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("read", "call_ro")).expect("untracked is inert"); - assert!(seam.operations().is_empty()); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - drive(&git_dir, &seam, &tool_result_event("read", "call_ro")).expect("untracked result inert"); - drive( - &git_dir, - &seam, - &tool_execution_end_event("read", "call_ro"), - ) - .expect("untracked terminal inert"); - assert!(seam.operations().is_empty()); - - cleanup(&git_dir); -} - -#[test] -fn a_pending_start_attempt_owned_by_a_dead_process_is_abandoned_not_replayed() { - let git_dir = temp_git_dir("d10-dead-owner-abandon"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("first Start"); - let scope_id = read_state(&git_dir).expect("state readable").attempts[0] - .scope_id - .clone(); - - let mut dead_child = std::process::Command::new("true") - .spawn() - .expect("spawning 'true' should succeed"); - let dead_pid = i32::try_from(dead_child.id()).expect("pid fits in i32"); - dead_child.wait().expect("child should exit and be reaped"); - state::set_attempt_owner_for_tests( - &git_dir, - &scope_id, - crate::services::hooks::mutation_scope_owner::ProcessOwner { - pid: dead_pid, - instance_token: None, - }, - ); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")) - .expect("a replay whose recorded owner is positively dead must abandon, not reuse"); - - assert_eq!( - seam.operations(), - vec!["start", "flush", "abandon", "flush", "start"], - "D10: a dead-owner PendingStart must be abandoned via the existing D8 flush/abandon/\ - flush pattern, then the triggering event admitted as a fresh attempt" - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_ne!( - state.attempts[0].scope_id, scope_id, - "the fresh attempt must never reuse the abandoned attempt's ScopeId" - ); - assert_eq!(state.attempts[0].attempt_seq, 2); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); -} - -#[test] -fn a_pending_start_attempt_owned_by_a_live_process_is_never_abandoned_by_a_replay() { - let git_dir = temp_git_dir("d10-live-owner-no-abandon"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("first Start"); - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")) - .expect("a replay owned by a still-live process must be treated as a normal replay"); - - assert_eq!( - seam.operations(), - vec!["start", "start"], - "no TTL and no elapsed time may ever cause an abandon here: the owner is this test \ - process's own live parent for the whole test" - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].phase, AttemptPhase::PendingStart); - - cleanup(&git_dir); -} - -#[test] -fn a_dead_pending_start_attempt_is_recovered_by_an_unrelated_fresh_session_start() { - let git_dir = temp_git_dir("d10-fresh-session-dead-pending-start"); - let seam = RecordingSeam::new(); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's Start"); - let scope_a = - attempt_owned_by(&read_state(&git_dir).expect("state readable"), "sess-a").scope_id; - state::set_attempt_owner_for_tests(&git_dir, &scope_a, dead_process_owner()); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-b", "call-b"), - ) - .expect( - "B's Start must recover A's stale owner without ever replaying A's \ - (session_id, tool_call_id) key", - ); - - assert_eq!( - seam.operations(), - vec!["start", "flush", "abandon", "flush", "start"], - "D10: a dead PendingStart owner discovered by an unrelated fresh-session Start must \ - be retired through the existing D8 flush/abandon/flush sequence before the \ - triggering Start is admitted" - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].session_id, "sess-b"); - assert_ne!(state.attempts[0].scope_id, scope_a); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); -} - -#[test] -fn a_dead_executed_attempt_is_recovered_by_a_fresh_session_start_without_a_synthetic_close() { - let git_dir = temp_git_dir("d10-fresh-session-dead-executed"); - let seam = RecordingSeam::new(); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's Start"); - drive( - &git_dir, - &seam, - &tool_result_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's tool_result marks Executed"); - let state = read_state(&git_dir).expect("state readable"); - let scope_a = attempt_owned_by(&state, "sess-a").scope_id; - assert_eq!( - attempt_owned_by(&state, "sess-a").phase, - AttemptPhase::Executed - ); - state::set_attempt_owner_for_tests(&git_dir, &scope_a, dead_process_owner()); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-b", "call-b"), - ) - .expect("B's Start must recover A's dead Executed attempt"); - - assert_eq!( - seam.operations(), - vec!["start", "flush", "abandon", "flush", "start"], - "a dead Executed attempt must be abandoned/rebaselined via D8, never given a \ - synthetic delayed Close" - ); - assert!( - !seam.operations().contains(&"close".to_string()), - "D9: the current Git tree no longer represents the original terminal observation \ - time, so a dead Executed attempt must never be closed" - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].session_id, "sess-b"); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); -} - -#[test] -fn a_dead_owner_scope_is_recovered_while_a_live_owner_sibling_survives_untouched() { - let git_dir = temp_git_dir("d10-dead-live-sibling-isolation"); - let seam = RecordingSeam::new(); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's Start (owner will die)"); - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-b", "call-b"), - ) - .expect("B's Start (owner stays live)"); - - let state = read_state(&git_dir).expect("state readable"); - let scope_a = attempt_owned_by(&state, "sess-a").scope_id; - let scope_b = attempt_owned_by(&state, "sess-b").scope_id; - state::set_attempt_owner_for_tests(&git_dir, &scope_a, dead_process_owner()); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-c", "call-c"), - ) - .expect("C's Start must recover only A"); - - assert_eq!( - seam.operations(), - vec!["start", "start", "flush", "abandon", "flush", "start"], - "exactly one abandon must occur, and only for A's own positively dead owner" - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 2); - assert!(!state.attempts.iter().any(|a| a.scope_id == scope_a)); - let b = attempt_owned_by(&state, "sess-b"); - assert_eq!(b.scope_id, scope_b); - assert_eq!( - b.phase, - AttemptPhase::PendingStart, - "B must survive reconciliation exactly as it was, untouched" - ); - assert_eq!( - attempt_owned_by(&state, "sess-c").phase, - AttemptPhase::PendingStart - ); - - cleanup(&git_dir); -} - -#[test] -fn multiple_dead_owner_scopes_are_retired_in_one_recovery_generation_while_a_live_sibling_survives() -{ - let git_dir = temp_git_dir("d10-multiple-dead-owner-scopes"); - let seam = RecordingSeam::new(); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's Start"); - drive( - &git_dir, - &seam, - &tool_result_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's tool_result marks Executed"); - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-b", "call-b"), - ) - .expect("B's Start"); - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-q", "call-q"), - ) - .expect("Q's Start (owner stays live)"); - - let state = read_state(&git_dir).expect("state readable"); - let scope_a = attempt_owned_by(&state, "sess-a").scope_id; - let scope_b = attempt_owned_by(&state, "sess-b").scope_id; - let scope_q = attempt_owned_by(&state, "sess-q").scope_id; - let dead_owner = dead_process_owner(); - state::set_attempt_owner_for_tests(&git_dir, &scope_a, dead_owner); - state::set_attempt_owner_for_tests(&git_dir, &scope_b, dead_owner); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-c", "call-c"), - ) - .expect("C's Start must recover both A and B, grouped into one recovery generation"); - - assert_eq!( - seam.operations(), - vec!["start", "start", "start", "flush", "abandon", "abandon", "flush", "start"], - "a single flush/abandon.../flush recovery generation must retire every \ - independently-proven-dead scope owned by the same dead process together" - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 2); - assert!(!state.attempts.iter().any(|a| a.scope_id == scope_a)); - assert!(!state.attempts.iter().any(|a| a.scope_id == scope_b)); - assert_eq!(attempt_owned_by(&state, "sess-q").scope_id, scope_q); - assert!(state.recovery.is_clear()); - - cleanup(&git_dir); -} - -#[test] -fn an_owner_that_cannot_be_positively_proven_dead_is_never_abandoned_by_an_unrelated_start() { - let git_dir = temp_git_dir("d10-uncertain-owner-preserved"); - let seam = RecordingSeam::new(); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's Start"); - let scope_a = - attempt_owned_by(&read_state(&git_dir).expect("state readable"), "sess-a").scope_id; - state::set_attempt_owner_for_tests( - &git_dir, - &scope_a, - crate::services::hooks::mutation_scope_owner::ProcessOwner { - pid: std::process::id().cast_signed(), - instance_token: None, - }, - ); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-c", "call-c"), - ) - .expect( - "C's Start must proceed without touching A, whose owner cannot be positively \ - proven dead", - ); - - assert_eq!( - seam.operations(), - vec!["start", "start"], - "a live pid with no instance-token evidence must never be converted into proof of \ - death: uncertain identity is conservatively treated as alive" - ); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 2); - assert!(state - .attempts - .iter() - .all(|attempt| attempt.phase == AttemptPhase::PendingStart)); - assert!(state.attempts.iter().any(|a| a.scope_id == scope_a)); - - cleanup(&git_dir); -} - -#[test] -fn an_interrupted_stale_owner_recovery_remains_pending_and_denies_the_triggering_start_until_resumed( -) { - let git_dir = temp_git_dir("d10-interrupted-stale-recovery"); - let seam = RecordingSeam::new(); - - drive( - &git_dir, - &seam, - &tool_call_event_for_session("bash", "sess-a", "call-a"), - ) - .expect("A's Start"); - let scope_a = - attempt_owned_by(&read_state(&git_dir).expect("state readable"), "sess-a").scope_id; - state::set_attempt_owner_for_tests(&git_dir, &scope_a, dead_process_owner()); - - let crashing = RecordingSeam::failing_once_on(&["abandon"]); - let error = drive( - &git_dir, - &crashing, - &tool_call_event_for_session("bash", "sess-b", "call-b"), - ) - .expect_err("a Start that triggers a stale-owner recovery which fails mid-way must not commit"); - assert!(error.to_string().contains(FAIL_CLOSED_MESSAGE)); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.recovery, RecoveryState::Pending { generation: 1 }); - assert!( - !state.attempts.iter().any(|a| a.session_id == "sess-b"), - "B must never be admitted while A's stale-owner recovery is still pending" - ); - assert_eq!( - attempt_owned_by(&state, "sess-a").phase, - AttemptPhase::PendingAbandon - ); - - drive( - &git_dir, - &crashing, - &tool_call_event_for_session("bash", "sess-b", "call-b"), - ) - .expect( - "the next boundary-lock acquisition must resume and complete the pending recovery, \ - and only then admit B", - ); - - let state = read_state(&git_dir).expect("state readable"); - assert!(state.recovery.is_clear()); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].session_id, "sess-b"); - assert!(!state.attempts.iter().any(|a| a.scope_id == scope_a)); - - cleanup(&git_dir); -} - -#[test] -fn duplicate_tool_result_after_close_is_a_safe_no_op() { - let git_dir = temp_git_dir("duplicate-tool-result-after-close"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - drive(&git_dir, &seam, &tool_result_event("bash", "call_1")).expect("tool_result"); - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")).expect("Close"); - - drive(&git_dir, &seam, &tool_result_event("bash", "call_1")) - .expect("a late duplicate tool_result after Close must be a safe no-op"); - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")) - .expect("a late duplicate tool_execution_end after Close must be a safe no-op"); - - assert_eq!( - seam.operations(), - vec!["start", "close"], - "a resurrected attempt must never re-enter the runtime seam after its own Close" - ); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - cleanup(&git_dir); -} - -#[test] -fn duplicate_tool_execution_end_after_abandon_is_a_safe_no_op() { - let git_dir = temp_git_dir("duplicate-terminal-after-abandon"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("Start"); - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")).expect("D7 abandon"); - - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")) - .expect("a late duplicate terminal event after abandon must be a safe no-op"); - - assert_eq!( - seam.operations(), - vec!["start", "flush", "abandon", "flush"], - "a duplicate terminal delivery for an already-abandoned attempt must never issue a \ - second abandon" - ); - - cleanup(&git_dir); -} - -#[test] -fn abandoning_one_sibling_never_touches_a_concurrent_sibling_in_the_same_session() { - let git_dir = temp_git_dir("sibling-abandon-isolation"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_a")).expect("A Start"); - drive(&git_dir, &seam, &tool_call_event("bash", "call_b")).expect("B Start"); - drive(&git_dir, &seam, &tool_result_event("bash", "call_b")).expect("B tool_result"); - - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_a")) - .expect("A's terminal event with no tool_result must abandon only A"); - - let state = read_state(&git_dir).expect("state readable"); - assert_eq!( - state.attempts.len(), - 1, - "abandoning A must never remove or block sibling B" - ); - assert_eq!(state.attempts[0].tool_call_id, "call_b"); - assert_eq!(state.attempts[0].phase, AttemptPhase::Executed); - - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_b")) - .expect("B must still close normally after A's abandonment and recovery"); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - assert_eq!( - seam.operations(), - vec!["start", "start", "flush", "abandon", "flush", "close"] - ); - - cleanup(&git_dir); -} - -#[test] -fn a_crash_mid_abandon_loop_is_resumed_and_completed_on_the_next_boundary_lock_acquisition() { - let git_dir = temp_git_dir("crash-mid-abandon-loop"); - let crashing = RecordingSeam::failing_once_on(&["abandon"]); - - drive(&git_dir, &crashing, &tool_call_event("bash", "call_1")).expect("Start"); - drive( - &git_dir, - &crashing, - &tool_execution_end_event("bash", "call_1"), - ) - .expect( - "a transient abandon failure mid-recovery must leave recovery Pending, not surface \ - an error, simulating a crash between marking PendingAbandon and completing the \ - flush/abandon/flush sequence", - ); - - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Pending { generation: 1 }, - "the interrupted abandon loop must leave recovery durably Pending for the next \ - boundary-lock acquisition to resume, never Clear and never lost" - ); - assert_eq!( - read_state(&git_dir) - .expect("state readable") - .attempts - .first() - .expect("the doomed attempt must still be recorded") - .phase, - AttemptPhase::PendingAbandon - ); - - drive(&git_dir, &crashing, &tool_call_event("bash", "call_2")) - .expect("recovery must self-heal and complete on the very next invocation"); - - assert!(read_state(&git_dir) - .expect("state readable") - .recovery - .is_clear()); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].tool_call_id, "call_2"); - - cleanup(&git_dir); -} - -#[test] -fn a_reused_tool_call_id_after_terminal_cleanup_gets_a_distinct_scope_id() { - let git_dir = temp_git_dir("terminal-scope-id-non-reuse"); - let seam = RecordingSeam::new(); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("first Start"); - drive(&git_dir, &seam, &tool_result_event("bash", "call_1")).expect("first tool_result"); - drive(&git_dir, &seam, &tool_execution_end_event("bash", "call_1")).expect("first Close"); - assert!(read_state(&git_dir) - .expect("state readable") - .attempts - .is_empty()); - - drive(&git_dir, &seam, &tool_call_event("bash", "call_1")).expect("reused toolCallId Start"); - let state = read_state(&git_dir).expect("state readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0].attempt_seq, 2); - - cleanup(&git_dir); -} diff --git a/cli/src/services/hooks/pi_mutation_scope/mod.rs b/cli/src/services/hooks/pi_mutation_scope/mod.rs index cc35fafa2..3ee6f8559 100644 --- a/cli/src/services/hooks/pi_mutation_scope/mod.rs +++ b/cli/src/services/hooks/pi_mutation_scope/mod.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - mod events; pub(crate) mod health; mod lifecycle; @@ -31,11 +29,7 @@ use events::{ HOOK_EVENT_TOOL_EXECUTION_START, HOOK_EVENT_TOOL_RESULT, MODEL_FIELD, SESSION_ID_FIELD, TOOL_CALL_ID_FIELD, TOOL_NAME_FIELD, }; -#[cfg(test)] -#[allow(unused_imports)] -pub(crate) use lifecycle::force_attempt_owner_dead_for_tests; -#[cfg(test)] -pub(crate) use lifecycle::run_pi_mutation_scope_from_payload_at_state_root; + #[allow(unused_imports)] pub(crate) use lifecycle::{run_pi_mutation_scope_from_payload, run_pi_mutation_scope_subcommand}; #[allow(unused_imports)] @@ -44,15 +38,3 @@ use lifecycle::{ }; #[allow(unused_imports)] use payload::{abandon_payload, flush_payload, scope_boundary_payload, scope_start_payload}; - -#[cfg(test)] -mod tests; - -#[cfg(test)] -mod lifecycle_tests; - -#[cfg(test)] -mod runtime_seam_tests; - -#[cfg(all(unix, test))] -mod guard_reconciliation_tests; diff --git a/cli/src/services/hooks/pi_mutation_scope/runtime_seam_tests.rs b/cli/src/services/hooks/pi_mutation_scope/runtime_seam_tests.rs deleted file mode 100644 index a35d65657..000000000 --- a/cli/src/services/hooks/pi_mutation_scope/runtime_seam_tests.rs +++ /dev/null @@ -1,240 +0,0 @@ -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; -use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, -}; - -use super::*; - -fn git(dir: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); -} - -struct PiRepo { - _temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, -} - -impl PiRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-pi-mutation-scope-seam-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); - - Self { - _temp: temp, - root, - state_root, - } - } - - fn cwd(&self) -> String { - self.root.to_string_lossy().into_owned() - } - - fn drive(&self, payload: &str) -> Result { - run_pi_mutation_scope_from_payload_at_state_root(&self.state_root, payload, None) - } - - fn write(&self, name: &str, contents: &str) { - fs::write(self.root.join(name), contents).expect("write should succeed"); - } - - fn db(&self) -> RepositoryAgentTraceDb { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &self.root, - &self.state_root, - "Pi mutation-scope seam test assertions", - ) - .expect("assertion DB should open") - } - - fn scope_status(&self, scope_id: &str) -> Option<(String, String)> { - self.db() - .query_map( - "SELECT actor_kind, status FROM mutation_trace_scopes WHERE scope_id = ?1", - (scope_id,), - |row| { - let actor_kind = row.get::(0).map_err(anyhow::Error::from)?; - let status = row.get::(1).map_err(anyhow::Error::from)?; - Ok((actor_kind, status)) - }, - ) - .expect("scope query should succeed") - .into_iter() - .next() - } - - fn scope_provenance(&self, scope_id: &str) -> Option<(String, Option)> { - self.db() - .query_map( - "SELECT session_id, model_id FROM mutation_trace_scope_provenance \ - WHERE scope_id = ?1", - (scope_id,), - |row| { - let session_id = row.get::(0).map_err(anyhow::Error::from)?; - let model_id = row.get::>(1).map_err(anyhow::Error::from)?; - Ok((session_id, model_id)) - }, - ) - .expect("scope-provenance query should succeed") - .into_iter() - .next() - } - - fn mutation_events(&self) -> Vec<(String, Option)> { - self.db() - .query_map( - "SELECT attribution_kind, attribution_scope_id \ - FROM mutation_trace_events ORDER BY revision", - (), - |row| { - let attribution_kind = row.get::(0).map_err(anyhow::Error::from)?; - let attribution_scope_id = - row.get::>(1).map_err(anyhow::Error::from)?; - Ok((attribution_kind, attribution_scope_id)) - }, - ) - .expect("mutation-events query should succeed") - } -} - -fn tool_call(repo: &PiRepo, tool_name: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolCall", - "session_id": "01a091f4-seam-session", - "tool_call_id": tool_call_id, - "cwd": repo.cwd(), - "tool_name": tool_name, - "model": "openai-codex/gpt-5.5", - }) - .to_string() -} - -fn tool_result(repo: &PiRepo, tool_name: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolResult", - "session_id": "01a091f4-seam-session", - "tool_call_id": tool_call_id, - "cwd": repo.cwd(), - "tool_name": tool_name, - }) - .to_string() -} - -fn tool_execution_end(repo: &PiRepo, tool_name: &str, tool_call_id: &str) -> String { - json!({ - "hook_event_name": "ToolExecutionEnd", - "session_id": "01a091f4-seam-session", - "tool_call_id": tool_call_id, - "cwd": repo.cwd(), - "tool_name": tool_name, - }) - .to_string() -} - -#[test] -fn a_write_start_result_close_lands_a_real_ai_exclusive_event_with_pi_provenance() { - let repo = PiRepo::new("real-lifecycle"); - let key = AttemptKey { - session_id: "01a091f4-seam-session".to_string(), - tool_call_id: "call_write".to_string(), - }; - let scope_id = format_pi_scope_id(&key, 1); - - repo.drive(&tool_call(&repo, "write", "call_write")) - .expect("Start should reach the real runtime"); - assert_eq!( - repo.scope_status(&scope_id), - Some(("pi".to_string(), "active".to_string())) - ); - assert_eq!( - repo.scope_provenance(&scope_id), - Some(( - "pi_01a091f4-seam-session".to_string(), - Some("openai-codex/gpt-5.5".to_string()) - )) - ); - - repo.write("file.txt", "one\ntwo\n"); - repo.drive(&tool_result(&repo, "write", "call_write")) - .expect("tool_result should mark Executed"); - - repo.drive(&tool_execution_end(&repo, "write", "call_write")) - .expect("Close should reach the real runtime"); - - assert_eq!( - repo.scope_status(&scope_id), - Some(("pi".to_string(), "closed".to_string())) - ); - assert_eq!( - repo.mutation_events(), - vec![("ai_exclusive".to_string(), Some(scope_id))] - ); - assert!( - state::read_state(&resolve_git_dir(&repo.root).expect("git dir resolves")) - .expect("state readable") - .attempts - .is_empty() - ); -} - -#[test] -fn a_start_followed_by_no_execution_abandons_through_the_real_runtime() { - let repo = PiRepo::new("real-abandon"); - let key = AttemptKey { - session_id: "01a091f4-seam-session".to_string(), - tool_call_id: "call_blocked".to_string(), - }; - let scope_id = format_pi_scope_id(&key, 1); - - repo.drive(&tool_call(&repo, "bash", "call_blocked")) - .expect("Start should reach the real runtime"); - - repo.drive(&tool_execution_end(&repo, "bash", "call_blocked")) - .expect("the terminal event must resolve via abandon, not surface an error"); - - assert_eq!( - repo.scope_status(&scope_id), - Some(("pi".to_string(), "abandoned".to_string())) - ); - assert!(repo.mutation_events().is_empty()); -} diff --git a/cli/src/services/hooks/pi_mutation_scope/state.rs b/cli/src/services/hooks/pi_mutation_scope/state.rs index 923d0d891..05759bc45 100644 --- a/cli/src/services/hooks/pi_mutation_scope/state.rs +++ b/cli/src/services/hooks/pi_mutation_scope/state.rs @@ -5,7 +5,7 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, Context, Result}; use serde::{Deserialize, Serialize}; -use crate::services::hooks::mutation_scope_lock::{AdapterLockSpec, OsAdvisoryLock}; +use crate::services::hooks::mutation_scope_lock::AdapterLockSpec; use super::{format_pi_scope_id, AttemptKey}; use crate::services::hooks::mutation_scope_owner::{ @@ -43,12 +43,6 @@ pub(crate) enum RecoveryState { }, } -impl RecoveryState { - pub(crate) fn is_clear(&self) -> bool { - matches!(self, RecoveryState::Clear) - } -} - #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub(crate) struct AdapterAttempt { pub attempt_seq: u64, @@ -127,7 +121,14 @@ pub(crate) fn state_path(git_dir: &Path) -> PathBuf { adapter_state_dir(git_dir).join(ADAPTER_STATE_FILE) } -pub(crate) fn read_state(git_dir: &Path) -> Result { +pub(crate) async fn read_state(git_dir: &Path) -> Result { + let git_dir = git_dir.to_owned(); + tokio::task::spawn_blocking(move || read_state_sync(&git_dir)) + .await + .context("Adapter state read worker failed")? +} + +fn read_state_sync(git_dir: &Path) -> Result { let path = state_path(git_dir); if !path.exists() { return Ok(AdapterState::default()); @@ -152,11 +153,11 @@ fn parse_adapter_state(content: &str, path: &Path) -> Result { Ok(state) } -fn write_state_durably(git_dir: &Path, state: &AdapterState) -> Result<()> { - write_state_durably_inner(git_dir, state, |_, _| Ok(())) +fn write_state_durably_sync(git_dir: &Path, state: &AdapterState) -> Result<()> { + write_state_durably_sync_inner(git_dir, state, |_, _| Ok(())) } -fn write_state_durably_inner( +fn write_state_durably_sync_inner( git_dir: &Path, state: &AdapterState, before_rename: F, @@ -223,8 +224,29 @@ where Ok(()) } -fn acquire_lock(git_dir: &Path) -> Result { - STATE_LOCK.acquire(&adapter_state_dir(git_dir)) +enum StateTransaction { + Unchanged(T), + Persist(T), +} + +async fn with_locked_state(git_dir: &Path, operation: F) -> Result +where + T: Send + 'static, + F: FnOnce(&mut AdapterState) -> Result> + Send + 'static, +{ + let git_dir = git_dir.to_owned(); + STATE_LOCK + .run_locked_blocking(&adapter_state_dir(&git_dir), move || { + let mut state = read_state_sync(&git_dir)?; + match operation(&mut state)? { + StateTransaction::Unchanged(result) => Ok(result), + StateTransaction::Persist(result) => { + write_state_durably_sync(&git_dir, &state)?; + Ok(result) + } + } + }) + .await } fn allocate_pending_start( @@ -248,610 +270,184 @@ fn allocate_pending_start( attempt } -pub(crate) fn admit_tracked_attempt( +pub(crate) async fn admit_tracked_attempt( git_dir: &Path, key: &AttemptKey, tool_name: &str, ) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let Some(existing) = state - .attempts - .iter() - .find(|attempt| attempt.matches_key(key)) - { - if existing.phase == AttemptPhase::PendingAbandon { - return Ok(AdmitDecision::TerminalAttemptBlocked); - } - return Ok(AdmitDecision::Admitted(AllocatedAttempt { - attempt: existing.clone(), - reused: true, - })); - } - - match state.recovery { - RecoveryState::Flushing { .. } => return Ok(AdmitDecision::RecoveryBlocked), - RecoveryState::Pending { generation } => { - state.recovery = RecoveryState::Flushing { generation }; - write_state_durably(git_dir, &state)?; - return Ok(AdmitDecision::FlushClaimed { generation }); - } - RecoveryState::Clear => {} - } - - if state - .attempts - .iter() - .any(|attempt| attempt.phase == AttemptPhase::PendingAbandon) - { - return Ok(AdmitDecision::UncertainAttemptBlocked); - } - - let attempt = allocate_pending_start(&mut state, key, tool_name); - write_state_durably(git_dir, &state)?; - Ok(AdmitDecision::Admitted(AllocatedAttempt { - attempt, - reused: false, - })) -} - -pub(crate) fn find_definitely_dead_attempts(git_dir: &Path) -> Result> { - let _lock = acquire_lock(git_dir)?; - let state = read_state(git_dir)?; - Ok(state - .attempts - .iter() - .filter(|attempt| { - matches!( - attempt.phase, - AttemptPhase::PendingStart | AttemptPhase::Executed - ) && is_definitely_dead(&attempt.owner) - }) - .map(|attempt| attempt.scope_id.clone()) - .collect()) -} - -pub(crate) fn mark_executed(git_dir: &Path, key: &AttemptKey) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - - let mut state = read_state(git_dir)?; - let Some(attempt) = state - .attempts - .iter_mut() - .find(|attempt| attempt.matches_key(key)) - else { - return Ok(()); - }; - - if attempt.phase == AttemptPhase::PendingStart { - attempt.phase = AttemptPhase::Executed; - write_state_durably(git_dir, &state)?; - } - Ok(()) -} - -pub(crate) fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - - let mut state = read_state(git_dir)?; - let before = state.attempts.len(); - state - .attempts - .retain(|attempt| attempt.scope_id != scope_id); - if state.attempts.len() == before { - return Ok(()); - } - write_state_durably(git_dir, &state) -} - -pub(crate) fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let RecoveryState::Flushing { generation } = state.recovery { - state.recovery = RecoveryState::Pending { generation }; - write_state_durably(git_dir, &state)?; - } - Ok(()) -} - -pub(crate) fn begin_terminal_cleanup(git_dir: &Path, scope_ids: &[String]) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - for attempt in &mut state.attempts { - if scope_ids + let key = key.clone(); + let tool_name = tool_name.to_string(); + with_locked_state(git_dir, move |state| { + if let Some(existing) = state + .attempts .iter() - .any(|scope_id| scope_id == &attempt.scope_id) + .find(|attempt| attempt.matches_key(&key)) { - attempt.phase = AttemptPhase::PendingAbandon; - } - } - - let generation = match state.recovery { - RecoveryState::Pending { generation } | RecoveryState::Flushing { generation } => { - generation - } - RecoveryState::Clear => { - let generation = state.next_recovery_generation; - state.next_recovery_generation += 1; - generation - } - }; - state.recovery = RecoveryState::Flushing { generation }; - write_state_durably(git_dir, &state)?; - Ok(generation) -} - -pub(crate) fn complete_recovery_flush( - git_dir: &Path, - generation: u64, -) -> Result { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - match state.recovery { - RecoveryState::Flushing { generation: owned } if owned == generation => { - state.recovery = RecoveryState::Clear; - write_state_durably(git_dir, &state)?; - Ok(RecoveryFlushCompletion::Cleared) - } - _ => Ok(RecoveryFlushCompletion::Superseded), - } -} - -pub(crate) fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { - let _lock = acquire_lock(git_dir)?; - let mut state = read_state(git_dir)?; - - if let RecoveryState::Flushing { generation: owned } = state.recovery { - if owned == generation { - state.recovery = RecoveryState::Pending { generation: owned }; - write_state_durably(git_dir, &state)?; - } - } - Ok(()) -} - -#[cfg(test)] -pub(crate) fn seed_attempt_for_tests( - git_dir: &Path, - key: &AttemptKey, - tool_name: &str, - phase: AttemptPhase, -) -> AdapterAttempt { - let _lock = acquire_lock(git_dir).expect("test seed lock"); - let mut state = read_state(git_dir).expect("test seed read"); - allocate_pending_start(&mut state, key, tool_name); - let seeded = state.attempts.last_mut().expect("attempt was just pushed"); - seeded.phase = phase; - let attempt = seeded.clone(); - write_state_durably(git_dir, &state).expect("test seed write"); - attempt -} - -#[cfg(test)] -pub(crate) fn set_attempt_owner_for_tests( - git_dir: &Path, - scope_id: &str, - owner: ProcessOwner, -) -> AdapterAttempt { - let _lock = acquire_lock(git_dir).expect("test owner-override lock"); - let mut state = read_state(git_dir).expect("test owner-override read"); - let attempt = state - .attempts - .iter_mut() - .find(|attempt| attempt.scope_id == scope_id) - .expect("attempt to override must already exist"); - attempt.owner = owner; - let updated = attempt.clone(); - write_state_durably(git_dir, &state).expect("test owner-override write"); - updated -} - -#[cfg(test)] -mod state_conformance { - use super::*; - use crate::services::hooks::mutation_scope_state_conformance::{ - mutation_scope_state_conformance_tests, CompletionView, FlushClaimView, RecoveryView, - StateConformance, - }; - - const FIXTURE_SESSION: &str = "ses-conformance"; - const FIXTURE_PHASES: [AttemptPhase; 3] = [ - AttemptPhase::Executed, - AttemptPhase::PendingAbandon, - AttemptPhase::PendingStart, - ]; - - struct PiStateConformance; - - fn fixture_key(index: usize) -> AttemptKey { - AttemptKey { - session_id: FIXTURE_SESSION.to_string(), - tool_call_id: format!("call-{index}"), - } - } - - impl StateConformance for PiStateConformance { - type State = AdapterState; - - const ADAPTER: &'static str = "pi"; - const SUPPORTED_VERSION: u32 = ADAPTER_STATE_VERSION; - - fn state_path(git_dir: &Path) -> PathBuf { - state_path(git_dir) - } - - fn fixture_state( - recovery: RecoveryView, - next_recovery_generation: u64, - attempt_count: usize, - ) -> AdapterState { - let mut state = AdapterState { - next_recovery_generation, - recovery: match recovery { - RecoveryView::Clear => RecoveryState::Clear, - RecoveryView::Pending(generation) => RecoveryState::Pending { generation }, - RecoveryView::Flushing(generation) => RecoveryState::Flushing { generation }, - }, - ..AdapterState::default() - }; - for index in 0..attempt_count { - allocate_pending_start(&mut state, &fixture_key(index), "write"); - state.attempts[index].phase = FIXTURE_PHASES[index % FIXTURE_PHASES.len()]; + if existing.phase == AttemptPhase::PendingAbandon { + return Ok(StateTransaction::Unchanged( + AdmitDecision::TerminalAttemptBlocked, + )); } - state - } - - fn read_state(git_dir: &Path) -> Result { - read_state(git_dir) + return Ok(StateTransaction::Unchanged(AdmitDecision::Admitted( + AllocatedAttempt { + attempt: existing.clone(), + reused: true, + }, + ))); } - fn persist(git_dir: &Path, state: &AdapterState) -> Result<()> { - write_state_durably(git_dir, state) + match state.recovery { + RecoveryState::Flushing { .. } => { + return Ok(StateTransaction::Unchanged(AdmitDecision::RecoveryBlocked)) + } + RecoveryState::Pending { generation } => { + state.recovery = RecoveryState::Flushing { generation }; + return Ok(StateTransaction::Persist(AdmitDecision::FlushClaimed { + generation, + })); + } + RecoveryState::Clear => {} } - fn persist_with_before_rename_hook( - git_dir: &Path, - state: &AdapterState, - before_rename: F, - ) -> Result<()> - where - F: FnOnce(&Path, &Path) -> Result<()>, + if state + .attempts + .iter() + .any(|attempt| attempt.phase == AttemptPhase::PendingAbandon) { - write_state_durably_inner(git_dir, state, before_rename) - } - - fn recovery_view(state: &AdapterState) -> RecoveryView { - match state.recovery { - RecoveryState::Clear => RecoveryView::Clear, - RecoveryState::Pending { generation } => RecoveryView::Pending(generation), - RecoveryState::Flushing { generation } => RecoveryView::Flushing(generation), - } + return Ok(StateTransaction::Unchanged( + AdmitDecision::UncertainAttemptBlocked, + )); } - fn next_recovery_generation(state: &AdapterState) -> u64 { - state.next_recovery_generation - } + let attempt = allocate_pending_start(state, &key, &tool_name); + Ok(StateTransaction::Persist(AdmitDecision::Admitted( + AllocatedAttempt { + attempt, + reused: false, + }, + ))) + }) + .await +} - fn scope_ids(state: &AdapterState) -> Vec { +pub(crate) async fn find_definitely_dead_attempts(git_dir: &Path) -> Result> { + with_locked_state(git_dir, |state| { + Ok(StateTransaction::Unchanged( state .attempts .iter() + .filter(|attempt| { + matches!( + attempt.phase, + AttemptPhase::PendingStart | AttemptPhase::Executed + ) && is_definitely_dead(&attempt.owner) + }) .map(|attempt| attempt.scope_id.clone()) - .collect() - } - - fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { - remove_attempt(git_dir, scope_id) - } + .collect(), + )) + }) + .await +} - fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { - normalize_recovery_after_boundary_lock_acquired(git_dir) - } +pub(crate) async fn mark_executed(git_dir: &Path, key: &AttemptKey) -> Result<()> { + let key = key.clone(); + with_locked_state(git_dir, move |state| { + let Some(attempt) = state + .attempts + .iter_mut() + .find(|attempt| attempt.matches_key(&key)) + else { + return Ok(StateTransaction::Unchanged(())); + }; - fn complete_recovery_flush(git_dir: &Path, generation: u64) -> Result { - Ok(match complete_recovery_flush(git_dir, generation)? { - RecoveryFlushCompletion::Cleared => CompletionView::Cleared, - RecoveryFlushCompletion::Superseded => CompletionView::Superseded, - }) + if attempt.phase == AttemptPhase::PendingStart { + attempt.phase = AttemptPhase::Executed; + return Ok(StateTransaction::Persist(())); } + Ok(StateTransaction::Unchanged(())) + }) + .await +} - fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { - relinquish_recovery_flush(git_dir, generation) +pub(crate) async fn remove_attempt(git_dir: &Path, scope_id: &str) -> Result<()> { + let scope_id = scope_id.to_string(); + with_locked_state(git_dir, move |state| { + let before = state.attempts.len(); + state + .attempts + .retain(|attempt| attempt.scope_id != scope_id); + if state.attempts.len() == before { + return Ok(StateTransaction::Unchanged(())); } + Ok(StateTransaction::Persist(())) + }) + .await +} - fn claim_flush_by_admitting_a_new_attempt( - git_dir: &Path, - contender: usize, - ) -> Result { - let decision = admit_tracked_attempt(git_dir, &fixture_key(contender), "bash")?; - Ok(match decision { - AdmitDecision::FlushClaimed { generation } => FlushClaimView::Claimed(generation), - AdmitDecision::RecoveryBlocked => FlushClaimView::Blocked, - other => FlushClaimView::Other(format!("{other:?}")), - }) +pub(crate) async fn normalize_recovery_after_boundary_lock_acquired(git_dir: &Path) -> Result<()> { + with_locked_state(git_dir, |state| { + if let RecoveryState::Flushing { generation } = state.recovery { + state.recovery = RecoveryState::Pending { generation }; + return Ok(StateTransaction::Persist(())); } - } - - mutation_scope_state_conformance_tests!(PiStateConformance); + Ok(StateTransaction::Unchanged(())) + }) + .await } -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - use std::thread; - - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-pi-mutation-scope-state-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - fn key(session_id: &str, tool_call_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - tool_call_id: tool_call_id.to_string(), +pub(crate) async fn begin_terminal_cleanup(git_dir: &Path, scope_ids: &[String]) -> Result { + let scope_ids = scope_ids.to_vec(); + with_locked_state(git_dir, move |state| { + for attempt in &mut state.attempts { + if scope_ids + .iter() + .any(|scope_id| scope_id == &attempt.scope_id) + { + attempt.phase = AttemptPhase::PendingAbandon; + } } - } - - fn admit(git_dir: &Path, key: &AttemptKey, tool_name: &str) -> AdmitDecision { - admit_tracked_attempt(git_dir, key, tool_name).expect("admit should not error") - } - - #[test] - fn admit_persists_the_pending_start_attempt_with_attempt_seq_one() { - let git_dir = unique_test_git_dir("admit-persists-pending-start"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let AdmitDecision::Admitted(allocated) = admit(&git_dir, &key("ses-1", "call-1"), "write") - else { - panic!("expected Admitted"); - }; - assert_eq!(allocated.attempt.attempt_seq, 1); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts.len(), 1); - assert_eq!(state.attempts[0], allocated.attempt); - assert_eq!(state.attempts[0].phase, AttemptPhase::PendingStart); - assert_eq!(state.attempts[0].session_id, "ses-1"); - assert_eq!(state.attempts[0].tool_call_id, "call-1"); - assert_eq!(state.attempts[0].tool_name, "write"); - assert_eq!(state.next_attempt_seq, 2); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn duplicate_live_delivery_reuses_the_same_attempt_and_scope_id() { - let git_dir = unique_test_git_dir("duplicate-reuse"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("ses-1", "call-1"); - - let AdmitDecision::Admitted(first) = admit(&git_dir, &attempt_key, "bash") else { - panic!("first admission should be Admitted"); - }; - assert!(!first.reused); - - let AdmitDecision::Admitted(second) = admit(&git_dir, &attempt_key, "bash") else { - panic!("duplicate delivery should still be Admitted"); - }; - assert!(second.reused); - assert_eq!(first.attempt.scope_id, second.attempt.scope_id); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts.len(), 1); - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_pending_start_attempt_never_blocks_a_concurrent_new_admission() { - let git_dir = unique_test_git_dir("pending-start-does-not-block"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - seed_attempt_for_tests( - &git_dir, - &key("ses-1", "call-a"), - "bash", - AttemptPhase::PendingStart, - ); - - let AdmitDecision::Admitted(second) = admit(&git_dir, &key("ses-1", "call-b"), "bash") - else { - panic!( - "D12: a lingering PendingStart from a still-executing tool call must not block \ - a genuinely concurrent tool call" - ); - }; - assert!(!second.reused); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts.len(), - 2 - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_blocks_a_new_admission_while_a_pending_abandon_exists() { - let git_dir = unique_test_git_dir("admit-blocks-on-pending-abandon"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - seed_attempt_for_tests( - &git_dir, - &key("ses-1", "call-a"), - "bash", - AttemptPhase::PendingAbandon, - ); - - assert_eq!( - admit(&git_dir, &key("ses-1", "call-b"), "bash"), - AdmitDecision::UncertainAttemptBlocked, - "an unresolved PendingAbandon must fail closed for a new admission" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_rejects_duplicate_delivery_of_a_pending_abandon_key() { - let git_dir = unique_test_git_dir("admit-duplicate-pending-abandon"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("ses-1", "call-1"); - - seed_attempt_for_tests(&git_dir, &attempt_key, "bash", AttemptPhase::PendingAbandon); - - assert_eq!( - admit(&git_dir, &attempt_key, "bash"), - AdmitDecision::TerminalAttemptBlocked, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn admit_claims_the_flush_when_recovery_is_pending() { - let git_dir = unique_test_git_dir("admit-claims-flush"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - seed_attempt_for_tests( - &git_dir, - &key("ses-1", "call-doomed"), - "bash", - AttemptPhase::PendingAbandon, - ); - let generation = begin_terminal_cleanup(&git_dir, &[]).expect("arming recovery succeeds"); - relinquish_recovery_flush(&git_dir, generation).expect("relinquish to Pending"); - - assert_eq!( - admit(&git_dir, &key("ses-1", "call-new"), "bash"), - AdmitDecision::FlushClaimed { generation }, - ); - assert_eq!( - read_state(&git_dir).expect("state readable").recovery, - RecoveryState::Flushing { generation }, - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_executed_transitions_pending_start_only() { - let git_dir = unique_test_git_dir("mark-executed"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - admit(&git_dir, &key("ses-1", "call-1"), "bash"); - - mark_executed(&git_dir, &key("ses-1", "call-1")).expect("mark_executed succeeds"); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::Executed, - ); - - mark_executed(&git_dir, &key("ses-unknown", "call-unknown")) - .expect("marking an unknown key executed is a safe no-op"); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn mark_executed_does_not_resurrect_a_pending_abandon_attempt() { - let git_dir = unique_test_git_dir("mark-executed-pending-abandon"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - seed_attempt_for_tests( - &git_dir, - &key("ses-1", "call-1"), - "bash", - AttemptPhase::PendingAbandon, - ); - - mark_executed(&git_dir, &key("ses-1", "call-1")).expect("mark_executed should not error"); - assert_eq!( - read_state(&git_dir).expect("state readable").attempts[0].phase, - AttemptPhase::PendingAbandon, - "a late tool_result must never move a PendingAbandon attempt back to Executed" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_new_attempt_after_terminal_cleanup_gets_a_fresh_attempt_seq_and_scope_id() { - let git_dir = unique_test_git_dir("terminal-scope-id-non-reuse"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let attempt_key = key("ses-1", "call-1"); - - let AdmitDecision::Admitted(first) = admit(&git_dir, &attempt_key, "bash") else { - panic!("expected Admitted"); - }; - assert_eq!(first.attempt.attempt_seq, 1); - remove_attempt(&git_dir, &first.attempt.scope_id).expect("terminal cleanup"); - - let AdmitDecision::Admitted(second) = admit(&git_dir, &attempt_key, "bash") else { - panic!("expected Admitted for the reused toolCallId"); + let generation = match state.recovery { + RecoveryState::Pending { generation } | RecoveryState::Flushing { generation } => { + generation + } + RecoveryState::Clear => { + let generation = state.next_recovery_generation; + state.next_recovery_generation += 1; + generation + } }; - assert!(!second.reused); - assert_eq!(second.attempt.attempt_seq, 2); - assert_ne!( - first.attempt.scope_id, second.attempt.scope_id, - "a reused toolCallId after terminal cleanup must never reactivate the old ScopeId" - ); - - remove_test_git_dir(&git_dir); - } - - const PARALLEL_ADMISSION_COUNT: usize = 6; - - #[test] - fn parallel_admissions_serialize_and_converge_without_lost_updates() { - let git_dir = unique_test_git_dir("parallel-admissions"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let handles: Vec<_> = (0..PARALLEL_ADMISSION_COUNT) - .map(|index| { - let git_dir = git_dir.clone(); - thread::spawn(move || { - let attempt_key = key("ses-1", &format!("call-{index}")); - admit_tracked_attempt(&git_dir, &attempt_key, "bash") - .expect("admit should not error") - }) - }) - .collect(); - - let mut scope_ids: Vec = handles - .into_iter() - .map(|handle| { - let AdmitDecision::Admitted(allocated) = - handle.join().expect("thread should not panic") - else { - panic!("D12: every genuinely distinct concurrent call must be admitted"); - }; - allocated.attempt.scope_id - }) - .collect(); - scope_ids.sort_unstable(); - scope_ids.dedup(); - assert_eq!(scope_ids.len(), PARALLEL_ADMISSION_COUNT); - - let state = read_state(&git_dir).expect("state should be readable"); - assert_eq!(state.attempts.len(), PARALLEL_ADMISSION_COUNT); - assert!(state - .attempts - .iter() - .all(|a| a.phase == AttemptPhase::PendingStart)); + state.recovery = RecoveryState::Flushing { generation }; + Ok(StateTransaction::Persist(generation)) + }) + .await +} - remove_test_git_dir(&git_dir); - } +pub(crate) async fn complete_recovery_flush( + git_dir: &Path, + generation: u64, +) -> Result { + with_locked_state(git_dir, move |state| match state.recovery { + RecoveryState::Flushing { generation: owned } if owned == generation => { + state.recovery = RecoveryState::Clear; + Ok(StateTransaction::Persist(RecoveryFlushCompletion::Cleared)) + } + _ => Ok(StateTransaction::Unchanged( + RecoveryFlushCompletion::Superseded, + )), + }) + .await +} + +pub(crate) async fn relinquish_recovery_flush(git_dir: &Path, generation: u64) -> Result<()> { + with_locked_state(git_dir, move |state| { + if let RecoveryState::Flushing { generation: owned } = state.recovery { + if owned == generation { + state.recovery = RecoveryState::Pending { generation: owned }; + return Ok(StateTransaction::Persist(())); + } + } + Ok(StateTransaction::Unchanged(())) + }) + .await } diff --git a/cli/src/services/hooks/pi_mutation_scope/tests.rs b/cli/src/services/hooks/pi_mutation_scope/tests.rs deleted file mode 100644 index a307be4c3..000000000 --- a/cli/src/services/hooks/pi_mutation_scope/tests.rs +++ /dev/null @@ -1,261 +0,0 @@ -use super::*; - -fn tool_event_json(hook_event_name: &str, overrides: &[(&str, Value)]) -> String { - let mut object = Map::new(); - object.insert( - HOOK_EVENT_NAME_FIELD.to_string(), - Value::String(hook_event_name.to_string()), - ); - object.insert( - SESSION_ID_FIELD.to_string(), - Value::String("01a091f4-session".to_string()), - ); - object.insert( - TOOL_CALL_ID_FIELD.to_string(), - Value::String("call_1|fc_1".to_string()), - ); - object.insert( - CWD_FIELD.to_string(), - Value::String("/repo/checkout".to_string()), - ); - object.insert( - TOOL_NAME_FIELD.to_string(), - Value::String("write".to_string()), - ); - for (field, value) in overrides { - object.insert((*field).to_string(), value.clone()); - } - Value::Object(object).to_string() -} - -fn key(session_id: &str, tool_call_id: &str) -> AttemptKey { - AttemptKey { - session_id: session_id.to_string(), - tool_call_id: tool_call_id.to_string(), - } -} - -fn tool_call(payload: &str) -> PiToolCall { - match parse_pi_hook_event(payload).expect("valid ToolCall parses") { - PiHookEvent::Call(call) => call, - other => panic!("expected ToolCall, got {other:?}"), - } -} - -mod ingress_conformance { - use super::*; - use crate::services::hooks::mutation_scope_ingress_conformance::{ - self as conformance, mutation_scope_ingress_conformance_tests, - CheckoutResolutionConformance, IngressConformance, - }; - - struct PiIngressConformance; - - impl IngressConformance for PiIngressConformance { - const ADAPTER: &'static str = "Pi"; - const EVENT_NAME_FIELD: &'static str = HOOK_EVENT_NAME_FIELD; - const REQUIRED_START_FIELDS: &'static [&'static str] = &[ - SESSION_ID_FIELD, - TOOL_CALL_ID_FIELD, - CWD_FIELD, - TOOL_NAME_FIELD, - ]; - const OPTIONAL_START_FIELDS: &'static [(&'static str, &'static str)] = - &[(MODEL_FIELD, "openai-codex/gpt-5.5")]; - const UNSUPPORTED_EVENT_NAMES: &'static [&'static str] = - &["PreToolUse", "tool_call", "chat.params"]; - - fn tracked_start() -> Map { - serde_json::from_str(&tool_event_json(HOOK_EVENT_TOOL_CALL, &[])) - .expect("the tracked ToolCall fixture is a JSON object") - } - - fn parse(payload: &str) -> Result<()> { - parse_pi_hook_event(payload).map(|_| ()) - } - - fn run(payload: &str) -> Result { - run_pi_mutation_scope_from_payload(payload, None) - } - - fn run_with_seams( - payload: &str, - resolve_git_dir: conformance::GitDirResolver, - seam: conformance::IngressSeam, - ) -> Result { - run_pi_mutation_scope_from_payload_with_seams(payload, None, resolve_git_dir, seam) - } - } - - impl CheckoutResolutionConformance for PiIngressConformance { - const CWD_FIELD: &'static str = CWD_FIELD; - const FAIL_CLOSED_MESSAGE: &'static str = FAIL_CLOSED_MESSAGE; - } - - mutation_scope_ingress_conformance_tests!(PiIngressConformance); - mutation_scope_ingress_conformance_tests!( - PiIngressConformance => tracked_start_fails_closed_when_its_checkout_cannot_be_resolved - ); -} - -#[test] -fn tool_call_parses_identity_and_model() { - let call = tool_call(&tool_event_json( - HOOK_EVENT_TOOL_CALL, - &[ - (TOOL_NAME_FIELD, Value::String("edit".to_string())), - ( - MODEL_FIELD, - Value::String("openai-codex/gpt-5.5".to_string()), - ), - ], - )); - assert_eq!(call.identity.session_id, "01a091f4-session"); - assert_eq!(call.identity.tool_call_id, "call_1|fc_1"); - assert_eq!(call.identity.tool_name, "edit"); - assert_eq!(call.model.as_deref(), Some("openai-codex/gpt-5.5")); - assert_eq!( - call.identity.classification(), - ToolClassification::TrackedMutation - ); -} - -#[test] -fn tool_call_model_is_optional() { - let call = tool_call(&tool_event_json(HOOK_EVENT_TOOL_CALL, &[])); - assert_eq!(call.model, None); -} - -#[test] -fn tool_result_and_tool_execution_end_parse_minimal_identity() { - for name in [ - HOOK_EVENT_TOOL_RESULT, - HOOK_EVENT_TOOL_EXECUTION_END, - HOOK_EVENT_TOOL_EXECUTION_ABANDON, - ] { - let event = parse_pi_hook_event(&tool_event_json(name, &[])).unwrap(); - let identity = match event { - PiHookEvent::Executed(identity) - | PiHookEvent::ExecutionEnd(identity) - | PiHookEvent::ExecutionAbandon(identity) => identity, - other => panic!("expected a minimal-identity event, got {other:?}"), - }; - assert_eq!( - identity.attempt_key(), - key("01a091f4-session", "call_1|fc_1") - ); - } -} - -#[test] -fn tool_execution_start_parses_and_is_never_evidence() { - let event = - parse_pi_hook_event(&tool_event_json(HOOK_EVENT_TOOL_EXECUTION_START, &[])).unwrap(); - let PiHookEvent::ExecutionStart(identity) = event else { - panic!("expected ToolExecutionStart"); - }; - assert_eq!(identity.tool_call_id, "call_1|fc_1"); -} - -#[test] -fn classification_table() { - let cases: &[(&str, ToolClassification)] = &[ - ("bash", ToolClassification::TrackedMutation), - ("edit", ToolClassification::TrackedMutation), - ("write", ToolClassification::TrackedMutation), - ("read", ToolClassification::Untracked), - ("grep", ToolClassification::Untracked), - ("find", ToolClassification::Untracked), - ("ls", ToolClassification::Untracked), - ("probe_mutate", ToolClassification::Untracked), - ("Bash", ToolClassification::Untracked), - ("some_future_pi_builtin", ToolClassification::Untracked), - ("", ToolClassification::Untracked), - ]; - for (tool_name, expected) in cases { - assert_eq!( - classify_tool(tool_name), - *expected, - "classify_tool({tool_name:?})" - ); - } -} - -#[test] -fn scope_id_embeds_attempt_seq_and_is_length_prefixed() { - let k = key("01a091f4-session", "call_1|fc_1"); - let scope_id = format_pi_scope_id(&k, 1); - assert_eq!( - scope_id, - "pi-tool-v1|n=1|s=16:01a091f4-session|c=11:call_1|fc_1" - ); - assert_ne!(format_pi_scope_id(&k, 1), format_pi_scope_id(&k, 2)); - assert_eq!( - pi_scope_start_event_id(&scope_id), - format!("{scope_id}|start") - ); - assert_eq!( - pi_scope_close_event_id(&scope_id), - format!("{scope_id}|close") - ); - assert_ne!( - pi_scope_start_event_id(&scope_id), - pi_scope_close_event_id(&scope_id) - ); -} - -#[test] -fn length_prefix_disambiguates_delimiter_collisions() { - let a = key("s|c=1:x", "y"); - let b = key("s", "1:x|y"); - assert_ne!(format_pi_scope_id(&a, 1), format_pi_scope_id(&b, 1)); -} - -#[test] -fn provenance_canonicalizes_the_session_and_normalizes_the_model() { - let provenance = pi_scope_provenance("01a091f4-session", Some("openai-codex/gpt-5.5")); - assert_eq!(provenance.session_id, "pi_01a091f4-session"); - assert_eq!(provenance.model_id.as_deref(), Some("openai-codex/gpt-5.5")); -} - -#[test] -fn provenance_keeps_an_already_prefixed_session_id() { - let provenance = pi_scope_provenance("pi_01a091f4-session", None); - assert_eq!(provenance.session_id, "pi_01a091f4-session"); -} - -#[test] -fn provenance_without_model_evidence_is_null() { - for model in [None, Some(""), Some(" ")] { - let provenance = pi_scope_provenance("01a091f4-session", model); - assert_eq!(provenance.model_id, None, "model {model:?}"); - } -} - -#[test] -fn run_from_payload_is_neutral_for_untracked_events() { - for tool_name in ["read", "grep", "find", "ls", "probe_mutate"] { - let payload = tool_event_json( - HOOK_EVENT_TOOL_CALL, - &[(TOOL_NAME_FIELD, Value::String(tool_name.to_string()))], - ); - assert_eq!( - run_pi_mutation_scope_from_payload(&payload, None).unwrap(), - String::new() - ); - } -} - -#[test] -fn tool_execution_start_is_always_a_no_op_regardless_of_classification() { - for tool_name in ["bash", "read"] { - let payload = tool_event_json( - HOOK_EVENT_TOOL_EXECUTION_START, - &[(TOOL_NAME_FIELD, Value::String(tool_name.to_string()))], - ); - assert_eq!( - run_pi_mutation_scope_from_payload(&payload, None).unwrap(), - String::new() - ); - } -} diff --git a/cli/src/services/hooks/runtime.rs b/cli/src/services/hooks/runtime.rs index 6fcd8d8ea..b86f9f5bf 100644 --- a/cli/src/services/hooks/runtime.rs +++ b/cli/src/services/hooks/runtime.rs @@ -4,8 +4,7 @@ use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; -#[cfg(test)] -use crate::services::agent_trace_storage::resolve_agent_trace_storage_for_hook_runtime_at_state_root; + use crate::services::agent_trace_storage::{ resolve_agent_trace_storage_for_hook_runtime, AgentTraceStorageContext, }; @@ -51,7 +50,7 @@ pub(crate) fn prefixed_session_id(tool_name: &str, raw_session_id: &str) -> Stri format!("{prefix}{raw_session_id}") } } -pub(crate) fn open_agent_trace_db_for_hook_runtime( +pub(crate) async fn open_agent_trace_db_for_hook_runtime( repository_root: &Path, context_message: &'static str, ) -> Result { @@ -64,25 +63,7 @@ pub(crate) fn open_agent_trace_db_for_hook_runtime( }; resolve_agent_trace_storage_for_hook_runtime(&storage_context) - .map(|storage| storage.db) - .context(context_message) -} - -#[cfg(test)] -pub(crate) fn open_agent_trace_db_for_hook_runtime_at_state_root( - repository_root: &Path, - state_root: &Path, - context_message: &'static str, -) -> Result { - let storage_config = config::resolve_agent_trace_storage_runtime_config(repository_root) - .context("Failed to resolve Agent Trace repository storage config.")?; - let storage_context = AgentTraceStorageContext { - repository_root, - explicit_repository_id: storage_config.repository_id.as_deref(), - repository_remote: &storage_config.repository_remote, - }; - - resolve_agent_trace_storage_for_hook_runtime_at_state_root(&storage_context, state_root) + .await .map(|storage| storage.db) .context(context_message) } diff --git a/cli/src/services/hooks/tests.rs b/cli/src/services/hooks/tests.rs deleted file mode 100644 index 8abfc31d9..000000000 --- a/cli/src/services/hooks/tests.rs +++ /dev/null @@ -1,4161 +0,0 @@ -use std::{ - cell::RefCell, - fs, - path::{Path, PathBuf}, - process::Command, - thread, - time::{Duration, SystemTime, UNIX_EPOCH}, -}; - -use super::*; -use crate::services::agent_trace_db::{ - ClaudeModelStateObservation, ObservationKind, ParsedDiffTracePatch, SkippedDiffTracePatch, -}; - -#[derive(Debug, Eq, PartialEq)] -struct CapturedPostCommitIntersectionInsert { - commit_id: String, - post_commit_time_ms: i64, - recent_window_cutoff_ms: i64, - recent_window_end_ms: i64, - loaded_diff_trace_count: i64, - skipped_diff_trace_count: i64, - intersection_patch: String, -} - -fn valid_patch_text(path: &str, content: &str) -> String { - format!( - "Index: {path}\n===================================================================\n--- {path}\n+++ {path}\n@@ -0,0 +1,1 @@\n+{content}\n" - ) -} - -fn valid_patch(path: &str, content: &str) -> ParsedPatch { - let patch_text = valid_patch_text(path, content); - - parse_patch_from_text(&patch_text, None).expect("test patch should parse") -} - -#[test] -fn conversation_trace_mixed_payload_maps_to_message_and_part_insert_inputs() { - let patch_text = valid_patch_text("src/lib.rs", "let answer = 42;"); - let question_text = serde_json::json!([ - { - "question": "Proceed?", - "answer": "Yes" - } - ]) - .to_string(); - let payload = serde_json::json!({ - "tool_name": "opencode", - "payloads": [ - { - "type": "message", - "session_id": "session-1", - "message_id": "message-1", - "role": "assistant", - "generated_at_unix_ms": 1_800_000_000_000_i64 - }, - { - "type": "message.part", - "session_id": "session-1", - "message_id": "message-1", - "part_type": "reasoning", - "text": "thinking through validation", - "generated_at_unix_ms": 1_800_000_000_001_i64 - }, - { - "type": "message.part", - "session_id": "session-1", - "message_id": "message-1", - "part_type": "patch", - "text": patch_text, - "generated_at_unix_ms": 1_800_000_000_002_i64 - }, - { - "type": "message.part", - "session_id": "session-1", - "message_id": "message-1", - "part_type": "question", - "text": question_text, - "generated_at_unix_ms": 1_800_000_000_003_i64 - } - ] - }); - - let parsed = parse_conversation_trace_payload(&payload.to_string()) - .expect("conversation-trace mixed payload should parse"); - - assert_eq!(parsed.attempted_count, 4); - assert!(parsed.skipped.is_empty()); - assert!(parsed.message_updated.skipped.is_empty()); - assert!(parsed.message_part_updated.skipped.is_empty()); - - assert_eq!(parsed.message_updated.inserts.len(), 1); - let message = &parsed.message_updated.inserts[0]; - assert_eq!(message.session_id, "oc_session-1"); - assert_eq!(message.message_id, "message-1"); - assert_eq!(message.role, MessageRole::Assistant); - assert_eq!(message.generated_at_unix_ms, 1_800_000_000_000_i64); - - assert_eq!(parsed.message_part_updated.inserts.len(), 3); - let reasoning_part = &parsed.message_part_updated.inserts[0]; - assert_eq!(reasoning_part.session_id, "oc_session-1"); - assert_eq!(reasoning_part.message_id, "message-1"); - assert_eq!(reasoning_part.part_type, PartType::Reasoning); - assert_eq!(reasoning_part.text, "thinking through validation"); - assert_eq!(reasoning_part.generated_at_unix_ms, 1_800_000_000_001_i64); - - let patch_part = &parsed.message_part_updated.inserts[1]; - assert_eq!(patch_part.session_id, "oc_session-1"); - assert_eq!(patch_part.message_id, "message-1"); - assert_eq!(patch_part.part_type, PartType::Patch); - assert_eq!( - patch_part.text, - serialize_to_json(&valid_patch("src/lib.rs", "let answer = 42;")) - .expect("test patch should serialize") - ); - assert_eq!(patch_part.generated_at_unix_ms, 1_800_000_000_002_i64); - - let question_part = &parsed.message_part_updated.inserts[2]; - assert_eq!(question_part.session_id, "oc_session-1"); - assert_eq!(question_part.message_id, "message-1"); - assert_eq!(question_part.part_type, PartType::Question); - assert_eq!(question_part.text, question_text); - assert_eq!(question_part.generated_at_unix_ms, 1_800_000_000_003_i64); -} - -#[test] -fn conversation_trace_mixed_payload_skips_malformed_sibling_items() { - let invalid_question_text = serde_json::json!({ - "question": "Proceed?", - "answer": "Yes" - }) - .to_string(); - let payload = serde_json::json!({ - "tool_name": "opencode", - "payloads": [ - { - "type": "message", - "session_id": "session-1", - "message_id": "message-1", - "role": "assistant", - "generated_at_unix_ms": 1_800_000_000_000_i64 - }, - { - "type": "message", - "session_id": "session-2", - "message_id": "message-2", - "role": "system", - "generated_at_unix_ms": 1_800_000_000_002_i64 - }, - { - "type": "message.part", - "session_id": "session-3", - "message_id": "message-3", - "part_type": "text", - "generated_at_unix_ms": 1_800_000_000_003_i64 - }, - { - "type": "message.part", - "session_id": "session-4", - "message_id": "message-4", - "part_type": "patch", - "text": "--- src/main.rs", - "generated_at_unix_ms": 1_800_000_000_004_i64 - }, - { - "type": "message.part", - "session_id": "session-5", - "message_id": "message-5", - "part_type": "question", - "text": invalid_question_text, - "generated_at_unix_ms": 1_800_000_000_005_i64 - }, - { - "type": "session.started", - "session_id": "session-6" - }, - 42, - { - "type": null, - "session_id": "session-7" - } - ] - }); - - let parsed = parse_conversation_trace_payload(&payload.to_string()) - .expect("conversation-trace mixed payload should parse with skipped items"); - - assert_eq!(parsed.attempted_count, 8); - assert_eq!(parsed.message_updated.inserts.len(), 1); - assert_eq!(parsed.message_updated.skipped.len(), 1); - assert_eq!(parsed.message_updated.skipped[0].index, 1); - assert!(parsed.message_updated.skipped[0] - .reason - .contains("field 'role'")); - assert_eq!(parsed.message_part_updated.inserts.len(), 0); - assert_eq!(parsed.message_part_updated.skipped.len(), 3); - assert_eq!(parsed.message_part_updated.skipped[0].index, 2); - assert!(parsed.message_part_updated.skipped[0] - .reason - .contains("missing required field 'text'")); - assert_eq!(parsed.message_part_updated.skipped[1].index, 3); - assert!(parsed.message_part_updated.skipped[1] - .reason - .contains("neither valid patch-JSON nor a valid patch")); - assert_eq!(parsed.message_part_updated.skipped[2].index, 4); - assert!(parsed.message_part_updated.skipped[2] - .reason - .contains("question part must be a JSON array")); - assert_eq!(parsed.skipped.len(), 3); - assert_eq!(parsed.skipped[0].index, 5); - assert!(parsed.skipped[0].reason.contains("field 'type'")); - assert_eq!(parsed.skipped[1].index, 6); - assert!(parsed.skipped[1] - .reason - .contains("payloads[6] must be an object")); - assert_eq!(parsed.skipped[2].index, 7); - assert!(parsed.skipped[2] - .reason - .contains("field 'type' must be a string")); -} - -fn normalized_conversation_trace_message_payload(tool_name: &str, session_id: &str) -> String { - serde_json::json!({ - "tool_name": tool_name, - "payloads": [ - { - "type": "message", - "session_id": session_id, - "message_id": "message-1", - "role": "assistant", - "generated_at_unix_ms": 1_800_000_000_000_i64 - } - ] - }) - .to_string() -} - -#[test] -fn conversation_trace_normalized_payload_accepts_pi_tool_name_with_prefixed_session_id() { - let stdin_payload = normalized_conversation_trace_message_payload("pi", "session-1"); - - let parsed = parse_conversation_trace_payload(&stdin_payload) - .expect("Pi normalized conversation-trace payload should parse"); - - assert_eq!(parsed.message_updated.inserts.len(), 1); - assert_eq!(parsed.message_updated.inserts[0].session_id, "pi_session-1"); -} - -#[test] -fn conversation_trace_normalized_payload_rejects_unsupported_tool_name() { - let stdin_payload = normalized_conversation_trace_message_payload("cursor", "session-1"); - - let error = parse_conversation_trace_payload(&stdin_payload) - .expect_err("unsupported tool_name should be rejected"); - - assert!(error.to_string().contains("unsupported tool_name 'cursor'")); - assert!(error.to_string().contains("'opencode'")); - assert!(error.to_string().contains("'pi'")); -} - -#[test] -fn conversation_trace_normalized_payload_rejects_empty_tool_name() { - let stdin_payload = normalized_conversation_trace_message_payload("", "session-1"); - - let error = parse_conversation_trace_payload(&stdin_payload) - .expect_err("empty tool_name should be rejected"); - - assert!(error - .to_string() - .contains("field 'tool_name' must be a non-empty string")); -} - -#[test] -fn conversation_trace_normalized_payload_rejects_missing_tool_name() { - let stdin_payload = serde_json::json!({ - "payloads": [ - { - "type": "message", - "session_id": "session-1", - "message_id": "message-1", - "role": "assistant", - "generated_at_unix_ms": 1_800_000_000_000_i64 - } - ] - }) - .to_string(); - - let error = parse_conversation_trace_payload(&stdin_payload) - .expect_err("missing tool_name should be rejected"); - - assert!(error - .to_string() - .contains("missing required field 'tool_name'")); -} - -#[test] -fn conversation_trace_normalized_payload_keeps_already_prefixed_session_id() { - let stdin_payload = normalized_conversation_trace_message_payload("opencode", "oc_session-1"); - - let parsed = parse_conversation_trace_payload(&stdin_payload) - .expect("already-prefixed OpenCode session ID should parse"); - - assert_eq!(parsed.message_updated.inserts[0].session_id, "oc_session-1"); -} - -#[test] -fn conversation_trace_raw_claude_event_uses_claude_identity_with_cc_prefixed_session_id() { - let stdin_payload = serde_json::json!({ - "hook_event_name": "UserPromptSubmit", - "session_id": "session-1", - "prompt": "hello" - }) - .to_string(); - - let parsed = parse_conversation_trace_payload(&stdin_payload) - .expect("raw Claude UserPromptSubmit event should parse"); - - assert_eq!(parsed.message_updated.inserts.len(), 1); - assert_eq!(parsed.message_updated.inserts[0].session_id, "cc_session-1"); -} - -fn diff_trace_payload(model_id: Option<&str>, tool_version: Option<&str>) -> DiffTracePayload { - diff_trace_payload_with( - "claude", - "session-123", - PAYLOAD_TYPE_STRUCTURED, - model_id, - tool_version, - ) -} - -fn diff_trace_payload_with( - tool_name: &str, - session_id: &str, - payload_type: &str, - model_id: Option<&str>, - tool_version: Option<&str>, -) -> DiffTracePayload { - DiffTracePayload { - session_id: String::from(session_id), - diff: String::from("diff text"), - time: 1_800_000_000_000_u64, - model_id: model_id.map(String::from), - agent_id: None, - transcript_path: None, - tool_name: String::from(tool_name), - tool_version: tool_version.map(String::from), - payload_type: String::from(payload_type), - } -} - -fn claude_model_test_event(transcript_path: &Path, tool_use_id: &str) -> Value { - json!({ - "hook_event_name": "PostToolUse", - "session_id": "session-123", - "tool_name": "Write", - "tool_use_id": tool_use_id, - "transcript_path": transcript_path, - "tool_input": { - "file_path": "docs/status.md", - "content": "# Status\n\nThe new state is complete.\n" - }, - "tool_response": { - "originalFile": "# Status\n\nThe old state is pending.\n", - "structuredPatch": { - "hunks": [{ - "oldStart": 1, - "oldCount": 3, - "newStart": 1, - "newCount": 3, - "lines": [ - " # Status", - " ", - "-The old state is pending.", - "+The new state is complete." - ] - }] - } - } - }) -} - -fn parsed_claude_model_id(event: &Value) -> Option { - match parse_diff_trace_payload(&event.to_string()) - .expect("Claude PostToolUse diff-trace payload should parse") - { - DiffTraceParseResult::Persist(payload) => payload.model_id, - DiffTraceParseResult::NoOp(message) => { - panic!("Claude Write payload should persist, got no-op: {message}") - } - } -} - -fn parsed_claude_diff_trace(event: &Value) -> DiffTracePayload { - match parse_diff_trace_payload(&event.to_string()) - .expect("Claude PostToolUse diff-trace payload should parse") - { - DiffTraceParseResult::Persist(payload) => payload, - DiffTraceParseResult::NoOp(message) => { - panic!("Claude Write payload should persist, got no-op: {message}") - } - } -} - -fn unique_attribution_db_path(label: &str) -> PathBuf { - let suffix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!("sce-claude-model-attribution-{label}-{suffix}")) - .join("agent-trace.db") -} - -fn resolved_claude_model_id_with(event: &Value, transcript_lookup: F) -> Option -where - F: FnOnce(&Path, &str) -> Option, -{ - resolve_claude_model_id_with( - event.as_object().expect("test event should be an object"), - transcript_lookup, - ) -} - -fn run_attribution_git(repo_root: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(repo_root) - .output() - .expect("git should start"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); -} - -fn init_attribution_git_repo(label: &str) -> PathBuf { - let repo_root = unique_attribution_db_path(label) - .parent() - .expect("test repository should have a parent") - .to_path_buf(); - fs::create_dir_all(&repo_root).expect("test repository directory should be created"); - run_attribution_git(&repo_root, &["init", "-q"]); - run_attribution_git( - &repo_root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - repo_root -} - -fn model_less_claude_diff_event( - session_id: &str, - tool_use_id: &str, - agent_id: Option<&str>, -) -> Value { - let mut event = claude_model_test_event(Path::new("/virtual/missing.jsonl"), tool_use_id); - let object = event - .as_object_mut() - .expect("Claude test event should be an object"); - object.insert("session_id".to_string(), json!(session_id)); - object.remove("transcript_path"); - object.remove("tool_use_id"); - if let Some(agent_id) = agent_id { - object.insert("agent_id".to_string(), json!(agent_id)); - } - event -} - -fn persisted_model_ids(db: &RepositoryAgentTraceDb) -> Vec> { - db.query_map( - "SELECT model_id FROM diff_traces ORDER BY id ASC", - (), - |row| row.get::>(0).map_err(Into::into), - ) - .expect("persisted model IDs should be readable") -} - -#[test] -fn claude_model_direct_nested_metadata_wins_over_transcript_without_double_prefixing() { - let transcript_path = Path::new("/unused/direct-precedence.jsonl"); - let mut event = claude_model_test_event(transcript_path, "tool-123"); - event - .as_object_mut() - .expect("test event should be an object") - .insert("model".to_string(), json!({ "id": "claude/direct-model" })); - - let model_id = resolved_claude_model_id_with(&event, |_, _| { - panic!("transcript lookup must not run when direct metadata is present") - }); - - assert_eq!(model_id.as_deref(), Some("claude/direct-model")); - assert_eq!(parsed_claude_model_id(&event), model_id); -} - -#[test] -fn claude_model_falls_back_to_matching_transcript_and_normalizes_model() { - let transcript_path = Path::new("/virtual/transcript-fallback.jsonl"); - let event = claude_model_test_event(transcript_path, "tool-123"); - - let model_id = resolved_claude_model_id_with(&event, |path, tool_use_id| { - assert_eq!(path, transcript_path); - assert_eq!(tool_use_id, "tool-123"); - Some(String::from("claude/claude-opus-4-1")) - }); - - assert_eq!(model_id.as_deref(), Some("claude/claude-opus-4-1")); -} - -#[test] -fn claude_model_remains_none_when_transcript_lookup_cannot_succeed() { - let event = claude_model_test_event(Path::new("/virtual/missing.jsonl"), "tool-123"); - assert_eq!(resolved_claude_model_id_with(&event, |_, _| None), None); - - let mut event_without_lookup_fields = event; - let payload = event_without_lookup_fields - .as_object_mut() - .expect("test event should be an object"); - payload.remove("transcript_path"); - payload.remove("tool_use_id"); - assert_eq!( - resolved_claude_model_id_with(&event_without_lookup_fields, |_, _| { - panic!("lookup must not run without transcript event metadata") - }), - None - ); -} - -#[test] -fn claude_diff_trace_parser_keeps_agent_id_ephemeral_and_storage_free() { - let mut event = claude_model_test_event(Path::new("/virtual/missing.jsonl"), "tool-123"); - event - .as_object_mut() - .expect("test event should be an object") - .insert("agent_id".to_string(), json!(" agent-1 ")); - - let payload = parsed_claude_diff_trace(&event); - - assert_eq!(payload.agent_id.as_deref(), Some("agent-1")); - assert!(serde_json::to_value(&payload) - .expect("internal payload should serialize") - .get("agent_id") - .is_none()); -} - -#[test] -fn claude_diff_trace_parser_keeps_transcript_path_ephemeral_and_storage_free() { - let transcript_path = Path::new("/virtual/session-123.jsonl"); - let event = claude_model_test_event(transcript_path, "tool-123"); - - let payload = parsed_claude_diff_trace(&event); - - assert_eq!( - payload.transcript_path.as_deref(), - Some("/virtual/session-123.jsonl") - ); - assert!(serde_json::to_value(&payload) - .expect("internal payload should serialize") - .get("transcript_path") - .is_none()); -} - -#[test] -fn claude_diff_trace_parser_leaves_transcript_path_none_without_the_field() { - let mut event = claude_model_test_event(Path::new("/virtual/missing.jsonl"), "tool-123"); - event - .as_object_mut() - .expect("test event should be an object") - .remove("transcript_path"); - - assert_eq!(parsed_claude_diff_trace(&event).transcript_path, None); -} - -#[test] -fn claude_diff_trace_normalized_opencode_payload_carries_no_transcript_path() { - let stdin_payload = serde_json::json!({ - "sessionID": "session-123", - "diff": "diff text", - "time": 1_800_000_000_000_u64, - "model_id": "anthropic/claude-opus-4", - "tool_name": "opencode", - "tool_version": null - }) - .to_string(); - - let parsed = parse_diff_trace_payload(&stdin_payload) - .expect("normalized OpenCode diff-trace payload should parse"); - let payload = match parsed { - DiffTraceParseResult::Persist(payload) => payload, - DiffTraceParseResult::NoOp(message) => { - panic!("normalized OpenCode payload should persist, got no-op: {message}") - } - }; - - assert_eq!(payload.transcript_path, None); -} - -#[test] -#[allow(clippy::too_many_lines)] -fn claude_model_attribution_end_to_end_persists_lifecycle_fallback_precedence_and_scope() { - let repo_root = init_attribution_git_repo("end-to-end"); - let state_root = unique_attribution_db_path("end-to-end-state") - .parent() - .expect("test state should have a parent") - .to_path_buf(); - let storage = resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &repo_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("setup path should initialize the test repository DB"); - drop(storage); - - let session_start = json!({ - "hook_event_name": "SessionStart", - "session_id": "session-123", - "model": "model-a", - "source": "startup" - }); - assert_eq!( - claude_model_state::run_claude_model_state_from_payload_at_state_root( - &repo_root, - &state_root, - &session_start.to_string(), - None, - || Ok(10), - ), - "" - ); - - let db = open_agent_trace_db_for_hook_runtime_at_state_root( - &repo_root, - &state_root, - "test DB should open after SessionStart", - ) - .expect("test DB should open after SessionStart"); - assert_eq!( - db.claude_model_state_by_session_and_agent("cc_session-123", "") - .expect("SessionStart state should be readable") - .expect("SessionStart should seed state") - .model_id, - "claude/model-a" - ); - let session_start_event = model_less_claude_diff_event("session-123", "tool-a", None); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&session_start_event), - ) - .expect("SessionStart state should attribute the next diff trace"); - drop(db); - - let post_model_switch = json!({ - "hook_event_name": "PostModelSwitch", - "session_id": "session-123", - "from_model": "model-a", - "to_model": "model-b", - "source": "picker" - }); - assert_eq!( - claude_model_state::run_claude_model_state_from_payload_at_state_root( - &repo_root, - &state_root, - &post_model_switch.to_string(), - None, - || Ok(20), - ), - "" - ); - - let db = open_agent_trace_db_for_hook_runtime_at_state_root( - &repo_root, - &state_root, - "test DB should open after PostModelSwitch", - ) - .expect("test DB should open after PostModelSwitch"); - assert_eq!( - db.claude_model_state_by_session_and_agent("cc_session-123", "") - .expect("PostModelSwitch state should be readable") - .expect("PostModelSwitch should update state") - .model_id, - "claude/model-b" - ); - let switched_event = model_less_claude_diff_event("session-123", "tool-b", None); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&switched_event), - ) - .expect("PostModelSwitch state should attribute the next diff trace"); - - let mut direct_event = model_less_claude_diff_event("session-123", "tool-direct", None); - direct_event - .as_object_mut() - .expect("Claude test event should be an object") - .insert("model".to_string(), json!("model-c")); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&direct_event), - ) - .expect("direct model attribution should persist"); - - let transcript_path = state_root.join("transcript.jsonl"); - fs::write( - &transcript_path, - concat!( - r#"{"type":"assistant","message":{"role":"assistant","model":"model-c","content":[{"type":"tool_use","id":"tool-transcript"}]}}"#, - "\n" - ), - ) - .expect("transcript fixture should be written"); - let transcript_event = claude_model_test_event(&transcript_path, "tool-transcript"); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&transcript_event), - ) - .expect("transcript model attribution should persist"); - - let no_state_event = model_less_claude_diff_event("session-without-state", "tool-none", None); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&no_state_event), - ) - .expect("an attribution-less diff trace should still persist"); - - let subagent_event = - model_less_claude_diff_event("session-123", "tool-subagent", Some("subagent-1")); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&subagent_event), - ) - .expect("a subagent diff trace should persist"); - - assert_eq!( - persisted_model_ids(&db), - vec![ - Some(String::from("claude/model-a")), - Some(String::from("claude/model-b")), - Some(String::from("claude/model-c")), - Some(String::from("claude/model-c")), - None, - None, - ] - ); - - drop(db); - fs::remove_file(transcript_path).expect("transcript fixture should be removed"); - fs::remove_dir_all(repo_root).expect("test repository should be removed"); - fs::remove_dir_all(state_root).expect("test state should be removed"); -} - -#[test] -fn claude_model_attribution_bridge_inheritance_seeds_state_and_diff_trace() { - let repo_root = init_attribution_git_repo("bridge-inheritance"); - let state_root = unique_attribution_db_path("bridge-inheritance-state") - .parent() - .expect("test state should have a parent") - .to_path_buf(); - let storage = resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &repo_root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("setup path should initialize the test repository DB"); - drop(storage); - - let sibling_transcript = state_root.join("session-old.jsonl"); - let current_transcript = state_root.join("session-current.jsonl"); - fs::write( - &sibling_transcript, - concat!( - r#"{"type":"file-history-snapshot"}"#, - "\n", - r#"{"type":"bridge-session","sessionId":"session-old","bridgeSessionId":"cse_shared"}"#, - "\n", - ), - ) - .expect("sibling transcript fixture should be written"); - fs::write( - ¤t_transcript, - concat!( - r#"{"type":"file-history-snapshot"}"#, - "\n", - r#"{"type":"bridge-session","sessionId":"session-current","bridgeSessionId":"cse_shared"}"#, - "\n", - ), - ) - .expect("current transcript fixture should be written"); - - let db = open_agent_trace_db_for_hook_runtime_at_state_root( - &repo_root, - &state_root, - "test DB should open before bridge inheritance", - ) - .expect("test DB should open before bridge inheritance"); - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-old"), - agent_id: String::new(), - model_id: String::from("claude/inherited-model"), - observation_kind: ObservationKind::SessionStart, - source: String::from("startup"), - observed_at_ms: 5, - }) - .expect("sibling state should be seeded"); - drop(db); - - let session_start = json!({ - "hook_event_name": "SessionStart", - "session_id": "session-current", - "source": "clear", - "transcript_path": current_transcript, - }); - assert_eq!( - claude_model_state::run_claude_model_state_from_payload_at_state_root( - &repo_root, - &state_root, - &session_start.to_string(), - None, - || Ok(10), - ), - "" - ); - - let db = open_agent_trace_db_for_hook_runtime_at_state_root( - &repo_root, - &state_root, - "test DB should open after bridge inheritance", - ) - .expect("test DB should open after bridge inheritance"); - let inherited = db - .claude_model_state_by_session_and_agent("cc_session-current", "") - .expect("inherited state lookup should succeed") - .expect("current session should inherit sibling state"); - assert_eq!(inherited.model_id, "claude/inherited-model"); - assert_eq!(inherited.source, "bridge_inherited"); - assert_eq!(inherited.observation_kind, ObservationKind::SessionStart); - assert_eq!(inherited.observed_at_ms, 10); - - let diff_event = model_less_claude_diff_event("session-current", "tool-inherited", None); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&diff_event), - ) - .expect("inherited state should attribute the diff trace"); - assert_eq!( - persisted_model_ids(&db), - vec![Some(String::from("claude/inherited-model"))] - ); - - drop(db); - fs::remove_file(sibling_transcript).expect("sibling transcript should be removed"); - fs::remove_file(current_transcript).expect("current transcript should be removed"); - fs::remove_dir_all(repo_root).expect("test repository should be removed"); - fs::remove_dir_all(state_root).expect("test state should be removed"); -} - -#[test] -fn claude_diff_trace_persistence_uses_state_only_after_direct_and_transcript() { - let db_path = unique_attribution_db_path("precedence"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-123"), - agent_id: String::new(), - model_id: String::from("claude/state-model"), - observation_kind: ObservationKind::SessionStart, - source: String::from("startup"), - observed_at_ms: 1, - }) - .expect("state should be seeded"); - - let mut state_event = claude_model_test_event(Path::new("/virtual/missing.jsonl"), "state"); - let state_object = state_event - .as_object_mut() - .expect("test event should be an object"); - state_object.remove("transcript_path"); - state_object.remove("tool_use_id"); - let state_payload = parsed_claude_diff_trace(&state_event); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &state_payload) - .expect("state fallback should persist"); - - let mut direct_event = state_event.clone(); - direct_event - .as_object_mut() - .expect("test event should be an object") - .insert("model".to_string(), json!("direct-model")); - let direct_payload = parsed_claude_diff_trace(&direct_event); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &direct_payload) - .expect("direct attribution should persist"); - - let transcript_path = db_path.with_extension("jsonl"); - fs::write( - &transcript_path, - concat!( - r#"{"type":"assistant","message":{"role":"assistant","model":"transcript-model","content":[{"type":"tool_use","id":"transcript"}]}}"#, - "\n" - ), - ) - .expect("transcript fixture should be written"); - let transcript_event = claude_model_test_event(&transcript_path, "transcript"); - let transcript_payload = parsed_claude_diff_trace(&transcript_event); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &transcript_payload) - .expect("transcript attribution should persist"); - - let models = db - .query_map( - "SELECT model_id FROM diff_traces ORDER BY id ASC", - (), - |row| row.get::>(0).map_err(Into::into), - ) - .expect("persisted models should be readable"); - assert_eq!( - models, - vec![ - Some(String::from("claude/state-model")), - Some(String::from("claude/direct-model")), - Some(String::from("claude/transcript-model")), - ] - ); - - drop(db); - fs::remove_file(transcript_path).expect("transcript fixture should be removed"); - fs::remove_dir_all(db_path.parent().expect("test DB should have a parent")) - .expect("test DB directory should be removed"); -} - -#[test] -fn normalized_claude_tool_name_does_not_use_claude_state_fallback() { - let db_path = unique_attribution_db_path("normalized-claude"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-123"), - agent_id: String::new(), - model_id: String::from("claude/parent-model"), - observation_kind: ObservationKind::SessionStart, - source: String::from("startup"), - observed_at_ms: 1, - }) - .expect("parent state should be seeded"); - - let payload = diff_trace_payload_with( - CLAUDE_TOOL_NAME, - "session-123", - PAYLOAD_TYPE_PATCH, - None, - None, - ); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &payload) - .expect("normalized Claude payload should persist"); - - let model = db - .query_map("SELECT model_id FROM diff_traces LIMIT 1", (), |row| { - row.get::>(0).map_err(Into::into) - }) - .expect("persisted model should be readable") - .into_iter() - .next() - .expect("diff trace row should exist"); - assert_eq!(model, None); - - drop(db); - fs::remove_dir_all(db_path.parent().expect("test DB should have a parent")) - .expect("test DB directory should be removed"); -} - -#[test] -fn claude_diff_trace_state_lookup_isolated_to_exact_subagent_scope() { - let db_path = unique_attribution_db_path("subagent"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-123"), - agent_id: String::new(), - model_id: String::from("claude/parent-model"), - observation_kind: ObservationKind::SessionStart, - source: String::from("startup"), - observed_at_ms: 1, - }) - .expect("parent state should be seeded"); - - let mut event = claude_model_test_event(Path::new("/virtual/missing.jsonl"), "subagent"); - let event_object = event - .as_object_mut() - .expect("test event should be an object"); - event_object.remove("transcript_path"); - event_object.remove("tool_use_id"); - event_object.insert("agent_id".to_string(), json!("subagent-1")); - let payload = parsed_claude_diff_trace(&event); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &payload) - .expect("subagent diff trace should persist"); - - let model = db - .query_map("SELECT model_id FROM diff_traces LIMIT 1", (), |row| { - row.get::>(0).map_err(Into::into) - }) - .expect("persisted model should be readable") - .into_iter() - .next() - .expect("diff trace row should exist"); - assert_eq!(model, None); - - drop(db); - fs::remove_dir_all(db_path.parent().expect("test DB should have a parent")) - .expect("test DB directory should be removed"); -} - -fn write_bridge_transcript(path: &Path, bridge_session_id: &str) { - fs::write( - path, - format!( - concat!( - "{{\"type\":\"file-history-snapshot\"}}\n", - "{{\"type\":\"bridge-session\",\"sessionId\":\"s\",", - "\"bridgeSessionId\":\"{bridge_session_id}\"}}\n" - ), - bridge_session_id = bridge_session_id, - ), - ) - .expect("bridge transcript fixture should be written"); -} - -#[test] -fn claude_diff_trace_seeds_bridge_chain_state_on_state_miss_and_reuses_it() { - let db_path = unique_attribution_db_path("bridge-chain-seed"); - let dir = db_path - .parent() - .expect("test DB should have a parent") - .to_path_buf(); - fs::create_dir_all(&dir).expect("test DB directory should be created"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-member"), - agent_id: String::new(), - model_id: String::from("claude/chain-model"), - observation_kind: ObservationKind::SessionStart, - source: String::from("startup"), - observed_at_ms: 100, - }) - .expect("chain member state should seed"); - - let current_transcript = dir.join("session-current.jsonl"); - let member_transcript = dir.join("session-member.jsonl"); - write_bridge_transcript(¤t_transcript, "cse_chain"); - write_bridge_transcript(&member_transcript, "cse_chain"); - - let payload = parsed_claude_diff_trace(&claude_model_test_event(¤t_transcript, "a")); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &payload) - .expect("bridge chain seeding should persist"); - - assert_eq!( - persisted_model_ids(&db), - vec![Some(String::from("claude/chain-model"))] - ); - let seeded = db - .claude_model_state_by_session_and_agent("cc_session-123", "") - .expect("seeded lookup should succeed") - .expect("current session should be seeded"); - assert_eq!(seeded.model_id, "claude/chain-model"); - assert_eq!(seeded.source, "bridge_inherited"); - - fs::remove_file(&member_transcript).expect("member transcript should be removed"); - let payload_two = parsed_claude_diff_trace(&claude_model_test_event(¤t_transcript, "b")); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &payload_two) - .expect("second diff trace should persist"); - assert_eq!( - persisted_model_ids(&db), - vec![ - Some(String::from("claude/chain-model")), - Some(String::from("claude/chain-model")), - ] - ); - let after = db - .claude_model_state_by_session_and_agent("cc_session-123", "") - .expect("lookup should succeed") - .expect("row should still exist"); - assert_eq!(after.observed_at_ms, seeded.observed_at_ms); - - drop(db); - fs::remove_dir_all(&dir).expect("test DB directory should be removed"); -} - -#[test] -fn claude_diff_trace_bridge_chain_selects_newest_observation_across_members() { - let db_path = unique_attribution_db_path("bridge-chain-newest"); - let dir = db_path - .parent() - .expect("test DB should have a parent") - .to_path_buf(); - fs::create_dir_all(&dir).expect("test DB directory should be created"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-root"), - agent_id: String::new(), - model_id: String::from("claude/sonnet-5"), - observation_kind: ObservationKind::SessionStart, - source: String::from("startup"), - observed_at_ms: 10, - }) - .expect("root state should seed"); - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-mid"), - agent_id: String::new(), - model_id: String::from("claude/opus-5"), - observation_kind: ObservationKind::PostModelSwitch, - source: String::from("picker"), - observed_at_ms: 20, - }) - .expect("mid state should seed"); - - let current_transcript = dir.join("session-current.jsonl"); - let root_transcript = dir.join("session-root.jsonl"); - let mid_transcript = dir.join("session-mid.jsonl"); - write_bridge_transcript(&mid_transcript, "cse_chain"); - write_bridge_transcript(¤t_transcript, "cse_chain"); - thread::sleep(Duration::from_millis(15)); - write_bridge_transcript(&root_transcript, "cse_chain"); - - let payload = parsed_claude_diff_trace(&claude_model_test_event(¤t_transcript, "x")); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &payload) - .expect("newest-observation resolution should persist"); - - assert_eq!( - persisted_model_ids(&db), - vec![Some(String::from("claude/opus-5"))] - ); - - drop(db); - fs::remove_dir_all(&dir).expect("test DB directory should be removed"); -} - -#[test] -fn claude_diff_trace_bridge_chain_fails_open_without_write_or_attribution() { - let db_path = unique_attribution_db_path("bridge-chain-fail-open"); - let dir = db_path - .parent() - .expect("test DB should have a parent") - .to_path_buf(); - fs::create_dir_all(&dir).expect("test DB directory should be created"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - - let mut event = claude_model_test_event(Path::new("/virtual/missing.jsonl"), "no-transcript"); - event - .as_object_mut() - .expect("event should be an object") - .remove("transcript_path"); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &parsed_claude_diff_trace(&event)) - .expect("missing transcript should fail open"); - - let current_transcript = dir.join("session-current.jsonl"); - let member_transcript = dir.join("session-member.jsonl"); - write_bridge_transcript(¤t_transcript, "cse_chain"); - write_bridge_transcript(&member_transcript, "cse_chain"); - persist_diff_trace_payload_to_agent_trace_db_with_db( - &db, - &parsed_claude_diff_trace(&claude_model_test_event(¤t_transcript, "no-state")), - ) - .expect("stateless chain should fail open"); - - assert_eq!(persisted_model_ids(&db), vec![None, None]); - assert!( - db.claude_model_state_by_session_and_agent("cc_session-123", "") - .expect("lookup should succeed") - .is_none(), - "no state row should be written on a fail-open branch" - ); - - drop(db); - fs::remove_dir_all(&dir).expect("test DB directory should be removed"); -} - -#[test] -fn claude_diff_trace_bridge_chain_does_not_seed_subagent_scope() { - let db_path = unique_attribution_db_path("bridge-chain-subagent"); - let dir = db_path - .parent() - .expect("test DB should have a parent") - .to_path_buf(); - fs::create_dir_all(&dir).expect("test DB directory should be created"); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test DB should open"); - - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: String::from("cc_session-member"), - agent_id: String::new(), - model_id: String::from("claude/chain-model"), - observation_kind: ObservationKind::SessionStart, - source: String::from("startup"), - observed_at_ms: 100, - }) - .expect("chain member state should seed"); - - let current_transcript = dir.join("session-current.jsonl"); - let member_transcript = dir.join("session-member.jsonl"); - write_bridge_transcript(¤t_transcript, "cse_chain"); - write_bridge_transcript(&member_transcript, "cse_chain"); - - let mut event = claude_model_test_event(¤t_transcript, "subagent"); - event - .as_object_mut() - .expect("event should be an object") - .insert("agent_id".to_string(), json!("subagent-1")); - persist_diff_trace_payload_to_agent_trace_db_with_db(&db, &parsed_claude_diff_trace(&event)) - .expect("subagent diff trace should persist"); - - assert_eq!(persisted_model_ids(&db), vec![None]); - assert!( - db.claude_model_state_by_session_and_agent("cc_session-123", "subagent-1") - .expect("lookup should succeed") - .is_none(), - "subagent scope must not inherit main-session chain state" - ); - - drop(db); - fs::remove_dir_all(&dir).expect("test DB directory should be removed"); -} - -#[test] -fn prefixed_diff_trace_session_id_prefixes_fresh_pi_session_id() { - assert_eq!( - prefixed_diff_trace_session_id("pi", "session-123"), - "pi_session-123" - ); -} - -#[test] -fn prefixed_diff_trace_session_id_keeps_already_prefixed_pi_session_id() { - assert_eq!( - prefixed_diff_trace_session_id("pi", "pi_session-123"), - "pi_session-123" - ); -} - -#[test] -fn prefixed_diff_trace_session_id_prefixes_fresh_codex_session_id() { - assert_eq!( - prefixed_diff_trace_session_id("codex", "session-123"), - "cx_session-123" - ); -} - -#[test] -fn prefixed_diff_trace_session_id_keeps_already_prefixed_codex_session_id() { - assert_eq!( - prefixed_diff_trace_session_id("codex", "cx_session-123"), - "cx_session-123" - ); -} - -#[test] -fn prefixed_diff_trace_session_id_adding_codex_does_not_affect_other_tool_prefixes() { - assert_eq!( - prefixed_diff_trace_session_id("opencode", "session-123"), - "oc_session-123" - ); - assert_eq!( - prefixed_diff_trace_session_id("claude", "session-123"), - "cc_session-123" - ); - assert_eq!( - prefixed_diff_trace_session_id("pi", "session-123"), - "pi_session-123" - ); -} - -#[test] -fn normalize_codex_model_id_preserves_fresh_model_id() { - assert_eq!( - normalize_codex_model_id("gpt-5.6-codex").as_deref(), - Some("gpt-5.6-codex") - ); -} - -#[test] -fn normalize_codex_model_id_preserves_qualified_model_ids() { - for model in ["openai/gpt-x", "qualified/custom-provider/model"] { - assert_eq!(normalize_codex_model_id(model).as_deref(), Some(model)); - } -} - -#[test] -fn normalize_codex_model_id_preserves_unqualified_model_ids() { - assert_eq!( - normalize_codex_model_id("custom-codex-model").as_deref(), - Some("custom-codex-model") - ); -} - -#[test] -fn normalize_codex_model_id_returns_none_for_blank_model_ids() { - assert_eq!(normalize_codex_model_id(" "), None); -} - -#[test] -fn normalize_opencode_model_id_preserves_qualified_and_unqualified_ids() { - for model in [ - "opencode/big-pickle", - "anthropic/claude-sonnet-4", - "custom-model", - ] { - assert_eq!(normalize_opencode_model_id(model).as_deref(), Some(model)); - } - assert_eq!( - normalize_opencode_model_id(" opencode/big-pickle ").as_deref(), - Some("opencode/big-pickle") - ); -} - -#[test] -fn normalize_opencode_model_id_returns_none_for_blank_model_ids() { - assert_eq!(normalize_opencode_model_id(""), None); - assert_eq!(normalize_opencode_model_id(" "), None); -} - -#[test] -fn pi_normalized_diff_trace_payload_persists_with_pi_prefixed_session_id() { - let stdin_payload = serde_json::json!({ - "sessionID": "session-123", - "diff": "diff text", - "time": 1_800_000_000_000_u64, - "model_id": "anthropic/claude-opus-4", - "tool_name": "pi", - "tool_version": null - }) - .to_string(); - - let parsed = parse_diff_trace_payload(&stdin_payload) - .expect("normalized Pi diff-trace payload should parse"); - let payload = match parsed { - DiffTraceParseResult::Persist(payload) => payload, - DiffTraceParseResult::NoOp(message) => { - panic!("Pi payload should persist, got no-op: {message}") - } - }; - - assert_eq!(payload.tool_name, "pi"); - assert_eq!(payload.model_id.as_deref(), Some("anthropic/claude-opus-4")); - assert_eq!(payload.tool_version, None); - - persist_diff_trace_payload_to_agent_trace_db_with( - &payload, - payload.model_id.as_deref(), - payload.tool_version.as_deref(), - |input| { - assert_eq!(input.time_ms, 1_800_000_000_000_i64); - assert_eq!(input.session_id, "pi_session-123"); - assert_eq!(input.model_id, Some("anthropic/claude-opus-4")); - assert_eq!(input.tool_name, "pi"); - assert_eq!(input.tool_version, None); - assert_eq!(input.payload_type, PAYLOAD_TYPE_PATCH); - - Ok(()) - }, - ) - .expect("Pi diff-trace payload should be persisted"); -} - -#[test] -fn post_commit_intersection_flow_preserves_pi_provenance() { - let now_ms = 1_800_000_000_000_i64; - let commit_time_ms = now_ms - 1_000; - - let output = run_post_commit_intersection_flow_with( - Path::new("/repo"), - |_| { - Ok(PostCommitPatchData { - commit_oid: String::from("def456"), - commit_time_ms, - parsed_patch: valid_patch("src/lib.rs", "shared line"), - }) - }, - || Ok(now_ms), - |_, _| { - Ok(RecentDiffTracePatches { - patches: vec![ParsedDiffTracePatch { - id: 9, - time_ms: now_ms - 500, - session_id: String::from("pi_valid-session"), - patch: valid_patch("src/lib.rs", "shared line"), - tool_name: Some(String::from("pi")), - tool_version: None, - payload_type: String::from(PAYLOAD_TYPE_PATCH), - }], - skipped: vec![], - }) - }, - |_| Ok(()), - ) - .expect("post-commit intersection flow should succeed"); - - assert_eq!(output.combined_recent_patch.files.len(), 1); - assert_eq!(output.tool_name, Some(String::from("pi"))); - assert_eq!(output.tool_version, None); -} - -#[test] -fn diff_trace_db_persistence_uses_direct_payload_model_and_tool_version() { - let payload = diff_trace_payload(Some("direct-model"), None); - - persist_diff_trace_payload_to_agent_trace_db_with( - &payload, - Some("direct-model"), - Some("Claude Code 1.2.3"), - |input| { - assert_eq!(input.time_ms, 1_800_000_000_000_i64); - assert_eq!(input.session_id, "cc_session-123"); - assert_eq!(input.model_id, Some("direct-model")); - assert_eq!(input.tool_name, "claude"); - assert_eq!(input.tool_version, Some("Claude Code 1.2.3")); - assert_eq!(input.payload_type, PAYLOAD_TYPE_STRUCTURED); - - Ok(()) - }, - ) - .expect("direct diff-trace attribution should be persisted"); -} - -#[test] -fn post_commit_intersection_flow_uses_same_window_end_for_query_and_persistence() { - let now_ms = 1_800_000_000_000_i64; - let commit_time_ms = now_ms - 1_000; - let expected_cutoff_ms = now_ms - RECENT_DAYS_MILLIS; - let query_window = RefCell::new(None); - let persisted = RefCell::new(None); - - let output = run_post_commit_intersection_flow_with( - Path::new("/repo"), - |_| { - Ok(PostCommitPatchData { - commit_oid: String::from("abc123"), - commit_time_ms, - parsed_patch: valid_patch("src/lib.rs", "shared line"), - }) - }, - || Ok(now_ms), - |cutoff_ms, end_ms| { - *query_window.borrow_mut() = Some((cutoff_ms, end_ms)); - - Ok(RecentDiffTracePatches { - patches: vec![ParsedDiffTracePatch { - id: 7, - time_ms: now_ms - 500, - session_id: String::from("oc_valid-session"), - patch: valid_patch("src/lib.rs", "shared line"), - tool_name: Some(String::from("opencode")), - tool_version: Some(String::from("1.2.3")), - payload_type: String::from(PAYLOAD_TYPE_PATCH), - }], - skipped: vec![SkippedDiffTracePatch { - id: 8, - time_ms: now_ms - 250, - session_id: String::from("oc_malformed-session"), - reason: String::from("invalid hunk header"), - }], - }) - }, - |insert_input| { - *persisted.borrow_mut() = Some(CapturedPostCommitIntersectionInsert { - commit_id: insert_input.commit_id.to_string(), - post_commit_time_ms: insert_input.post_commit_time_ms, - recent_window_cutoff_ms: insert_input.recent_window_cutoff_ms, - recent_window_end_ms: insert_input.recent_window_end_ms, - loaded_diff_trace_count: insert_input.loaded_diff_trace_count, - skipped_diff_trace_count: insert_input.skipped_diff_trace_count, - intersection_patch: insert_input.intersection_patch.to_string(), - }); - - Ok(()) - }, - ) - .expect("post-commit intersection flow should succeed"); - - assert_eq!( - query_window.into_inner(), - Some((expected_cutoff_ms, now_ms)) - ); - - let persisted = persisted - .into_inner() - .expect("intersection row should be persisted"); - assert_eq!(persisted.commit_id, "abc123"); - assert_eq!(persisted.post_commit_time_ms, commit_time_ms); - assert_eq!(persisted.recent_window_cutoff_ms, expected_cutoff_ms); - assert_eq!(persisted.recent_window_end_ms, now_ms); - assert_eq!(persisted.loaded_diff_trace_count, 1); - assert_eq!(persisted.skipped_diff_trace_count, 1); - - let intersection: ParsedPatch = serde_json::from_str(&persisted.intersection_patch) - .expect("persisted intersection patch should deserialize"); - assert_eq!(intersection.files.len(), 1); - assert_eq!(intersection.files[0].new_path, "src/lib.rs"); - assert_eq!(intersection.files[0].hunks[0].lines.len(), 1); - assert_eq!( - intersection.files[0].hunks[0].lines[0].content, - "shared line" - ); - - assert_eq!(output.post_commit_data.commit_oid, "abc123"); - assert_eq!(output.post_commit_data.commit_time_ms, commit_time_ms); - assert_eq!(output.combined_recent_patch.files.len(), 1); - assert_eq!(output.combined_recent_patch.files[0].new_path, "src/lib.rs"); - assert_eq!(output.tool_name, Some(String::from("opencode"))); - assert_eq!(output.tool_version, Some(String::from("1.2.3"))); -} - -fn post_commit_flow_result() -> PostCommitIntersectionFlowResult { - PostCommitIntersectionFlowResult { - combined_recent_patch: valid_patch("src/lib.rs", "shared line"), - post_commit_data: PostCommitPatchData { - commit_oid: String::from("abc123"), - commit_time_ms: 1_800_000_000_000, - parsed_patch: valid_patch("src/lib.rs", "shared line"), - }, - tool_name: None, - tool_version: None, - } -} - -fn minimal_agent_trace() -> AgentTrace { - serde_json::from_value(json!({ "files": [] })).expect("minimal Agent Trace should deserialize") -} - -#[test] -fn post_commit_auto_sync_launches_after_successful_persistence_when_enabled() { - let events = RefCell::new(Vec::new()); - - let output = run_post_commit_subcommand_with( - Path::new("/repo"), - None, - "", - |_| { - events.borrow_mut().push("intersection"); - Ok(post_commit_flow_result()) - }, - |_, _, _, _| { - events.borrow_mut().push("persistence"); - Ok(minimal_agent_trace()) - }, - |_| { - events.borrow_mut().push("config"); - Ok(true) - }, - |_| { - events.borrow_mut().push("launch"); - Ok(()) - }, - |_| { - events.borrow_mut().push("checkpoint"); - Ok(()) - }, - None, - ) - .expect("successful post-commit should remain successful"); - - assert!(output.contains("post-commit hook processed intersection")); - assert_eq!( - events.into_inner(), - vec![ - "intersection", - "persistence", - "checkpoint", - "config", - "launch" - ] - ); -} - -#[test] -fn post_commit_validation_failure_does_not_resolve_or_launch_auto_sync() { - let validation_called = RefCell::new(false); - let config_called = RefCell::new(false); - let launch_called = RefCell::new(false); - - let error = run_post_commit_subcommand_with( - Path::new("/repo"), - None, - "", - |_| Ok(post_commit_flow_result()), - |_, flow_result, vcs_type, remote_url| { - run_post_commit_agent_trace_flow_with( - flow_result, - vcs_type, - remote_url, - &ParsedPatch { files: Vec::new() }, - |_| { - *validation_called.borrow_mut() = true; - Err(anyhow!("Agent Trace validation failed")) - }, - |_| panic!("Agent Trace persistence must not run after validation failure"), - ) - }, - |_| { - *config_called.borrow_mut() = true; - Ok(true) - }, - |_| { - *launch_called.borrow_mut() = true; - Ok(()) - }, - |_| panic!("checkpoint must not run after persistence failure"), - None, - ) - .expect_err("validation failure should be returned"); - - assert!(*validation_called.borrow()); - assert!(!error.to_string().is_empty()); - assert!(!*config_called.borrow()); - assert!(!*launch_called.borrow()); -} - -fn post_commit_flow_result_for( - direct: ParsedPatch, - committed: ParsedPatch, -) -> PostCommitIntersectionFlowResult { - PostCommitIntersectionFlowResult { - combined_recent_patch: direct, - post_commit_data: PostCommitPatchData { - commit_oid: String::from("abc123"), - commit_time_ms: 1_800_000_000_000, - parsed_patch: committed, - }, - tool_name: Some(String::from("claude")), - tool_version: Some(String::from("9.9.9")), - } -} - -fn persisted_post_commit_trace( - flow_result: &PostCommitIntersectionFlowResult, - mutation_ai_patch: &ParsedPatch, -) -> Value { - let persisted = RefCell::new(None); - - run_post_commit_agent_trace_flow_with( - flow_result, - Some(AgentTraceVcsType::Git), - "", - mutation_ai_patch, - |_| Ok(()), - |insert| { - *persisted.borrow_mut() = Some(insert.trace_json.to_string()); - Ok(()) - }, - ) - .expect("post-commit Agent Trace flow should build and persist"); - - serde_json::from_str( - persisted - .into_inner() - .expect("trace should have been persisted") - .as_str(), - ) - .expect("persisted trace JSON should parse") -} - -#[test] -fn post_commit_agent_trace_flow_attributes_mutation_only_lines_as_ai_without_provenance() { - let flow_result = post_commit_flow_result_for( - ParsedPatch { files: Vec::new() }, - valid_patch("src/lib.rs", "mutated line"), - ); - let mutation_ai_patch = valid_patch("src/lib.rs", "mutated line"); - - let trace = persisted_post_commit_trace(&flow_result, &mutation_ai_patch); - - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(1) - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["unknown"]["added"], - json!(0) - ); - assert!( - trace.get("tool").is_none(), - "mutation-only coverage fabricates no tool provenance" - ); - let contributor = &trace["files"][0]["conversations"][0]["contributor"]; - assert_eq!(contributor["type"], json!("ai")); - assert!( - contributor.get("model_id").is_none(), - "mutation-only coverage carries no model provenance" - ); - assert!( - trace["files"][0]["conversations"][0] - .get("related") - .is_none(), - "mutation-only coverage carries no session provenance" - ); -} - -#[test] -fn post_commit_agent_trace_flow_keeps_direct_provenance_when_direct_covers_the_line() { - let flow_result = post_commit_flow_result_for( - valid_patch("src/lib.rs", "shared line"), - valid_patch("src/lib.rs", "shared line"), - ); - - let trace = persisted_post_commit_trace(&flow_result, &ParsedPatch { files: Vec::new() }); - - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(1) - ); - assert_eq!( - trace["tool"], - json!({ "name": "claude", "version": "9.9.9" }) - ); - assert_eq!( - trace["files"][0]["conversations"][0]["contributor"]["type"], - json!("ai") - ); -} - -#[test] -fn post_commit_agent_trace_flow_with_empty_mutation_patch_leaves_uncovered_lines_unknown() { - let flow_result = post_commit_flow_result_for( - ParsedPatch { files: Vec::new() }, - valid_patch("src/lib.rs", "human line"), - ); - - let trace = persisted_post_commit_trace(&flow_result, &ParsedPatch { files: Vec::new() }); - - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["unknown"]["added"], - json!(1) - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(0) - ); - assert!(trace.get("tool").is_none()); - assert_eq!( - trace["files"][0]["conversations"][0]["contributor"]["type"], - json!("unknown") - ); -} - -mod mutation_attribution_e2e { - use super::*; - use crate::services::mutation_trace::runtime::resolve_post_commit_mutation_ai_patch; - use crate::services::mutation_trace::runtime::resolve_worktree_id; - use crate::services::mutation_trace::store::encode_revision; - - fn git(repo: &Path, args: &[&str]) -> String { - let output = Command::new("git") - .args(args) - .current_dir(repo) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).expect("git output should be UTF-8") - } - - fn commit_all(repo: &Path, message: &str) { - git(repo, &["add", "-A"]); - git( - repo, - &[ - "-c", - "user.name=SCE Test", - "-c", - "user.email=sce@example.invalid", - "commit", - "-qm", - message, - ], - ); - } - - struct E2eRepo { - _temp: tempfile::TempDir, - root: PathBuf, - db_path: PathBuf, - } - - impl E2eRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-mutation-attr-e2e-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.rs"), "one\n").expect("seed file should write"); - commit_all(&root, "base"); - let db_path = temp.path().join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path) - .expect("repository DB should open with schema"); - Self { - _temp: temp, - root, - db_path, - } - } - - fn db(&self) -> RepositoryAgentTraceDb { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) - .expect("repository DB should reopen") - } - - fn head_tree(&self) -> String { - git(&self.root, &["rev-parse", "HEAD^{tree}"]) - .trim() - .to_owned() - } - - fn parent_tree(&self) -> String { - git(&self.root, &["rev-parse", "HEAD~1^{tree}"]) - .trim() - .to_owned() - } - - fn checkout_id(&self) -> String { - resolve_worktree_id(&self.root) - .expect("worktree identity should resolve") - .0 - } - } - - fn seed_event( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - before_tree: &str, - after_tree: &str, - attribution_kind: &str, - attribution_scope_id: Option<&str>, - ) { - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, - boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', ?5, ?6, 'flush', NULL, NULL)", - ( - worktree_id, - encode_revision(revision).as_slice(), - before_tree, - after_tree, - attribution_kind, - attribution_scope_id, - ), - ) - .expect("mutation event insert should succeed"); - } - - fn row_count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("count row should exist") - } - - fn touched_line_count(patch: &ParsedPatch) -> usize { - patch - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .map(|hunk| hunk.lines.len()) - .sum() - } - - fn flow_result_for(repo: &E2eRepo, direct: ParsedPatch) -> PostCommitIntersectionFlowResult { - let post_commit_data = capture_post_commit_patch_from_git(&repo.root) - .expect("capturing the post-commit patch should succeed"); - PostCommitIntersectionFlowResult { - combined_recent_patch: direct, - post_commit_data, - tool_name: None, - tool_version: None, - } - } - - fn resolve_mutation_ai( - repo: &E2eRepo, - db: &RepositoryAgentTraceDb, - flow_result: &PostCommitIntersectionFlowResult, - ) -> ParsedPatch { - let direct_intersection = intersect_patches_fn( - &flow_result.combined_recent_patch, - &flow_result.post_commit_data.parsed_patch, - ); - resolve_post_commit_mutation_ai_patch( - &repo.root, - db, - &direct_intersection, - &flow_result.post_commit_data.parsed_patch, - ) - } - - fn persist_trace( - flow_result: &PostCommitIntersectionFlowResult, - db: &RepositoryAgentTraceDb, - mutation_ai_patch: &ParsedPatch, - ) -> Value { - let persisted = RefCell::new(None); - run_post_commit_agent_trace_flow_with( - flow_result, - Some(AgentTraceVcsType::Git), - "git@github.com:acme/widgets.git", - mutation_ai_patch, - |value| validate_agent_trace_value(value).map_err(|error| anyhow!(error.to_string())), - |insert| { - *persisted.borrow_mut() = Some(insert.trace_json.to_string()); - db.insert_agent_trace(insert).map(|_| ()) - }, - ) - .expect("the post-commit Agent Trace flow should build, validate, and persist"); - - serde_json::from_str( - persisted - .into_inner() - .expect("a trace should have been persisted") - .as_str(), - ) - .expect("the persisted trace JSON should parse") - } - - #[test] - fn a_mutation_only_line_persists_as_ai_without_fabricated_provenance() { - let repo = E2eRepo::new("mutation-only"); - fs::write(repo.root.join("file.rs"), "one\ntwo\n").expect("the edit should write"); - commit_all(&repo.root, "add two"); - - let db = repo.db(); - seed_event( - &db, - &repo.checkout_id(), - 1, - &repo.parent_tree(), - &repo.head_tree(), - "ai_exclusive", - Some("scope-x"), - ); - - let flow_result = flow_result_for(&repo, ParsedPatch { files: Vec::new() }); - let mutation_ai_patch = resolve_mutation_ai(&repo, &db, &flow_result); - assert_eq!( - touched_line_count(&mutation_ai_patch), - 1, - "a healthy untainted exclusive event covers the committed line" - ); - - let trace = persist_trace(&flow_result, &db, &mutation_ai_patch); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(1) - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["unknown"]["added"], - json!(0) - ); - assert!( - trace.get("tool").is_none(), - "mutation-only coverage fabricates no tool provenance" - ); - let contributor = &trace["files"][0]["conversations"][0]["contributor"]; - assert_eq!(contributor["type"], json!("ai")); - assert!( - contributor.get("model_id").is_none(), - "mutation-only coverage carries no model provenance" - ); - - assert_eq!( - row_count(&db, "diff_traces"), - 0, - "mutation evidence is never inserted into diff_traces" - ); - assert_eq!( - row_count(&db, "post_commit_patch_intersections"), - 0, - "the direct-only intersection table is untouched by this flow" - ); - assert_eq!(row_count(&db, "agent_traces"), 1); - } - - #[test] - fn direct_plus_mutation_evidence_completes_hunk_coverage_and_keeps_direct_provenance() { - let repo = E2eRepo::new("direct-plus-mutation"); - fs::write(repo.root.join("file.rs"), "one\ntwo\nthree\n").expect("the edit should write"); - commit_all(&repo.root, "add two and three"); - - let db = repo.db(); - seed_event( - &db, - &repo.checkout_id(), - 1, - &repo.parent_tree(), - &repo.head_tree(), - "ai_exclusive", - Some("scope-x"), - ); - - let direct = parse_patch_from_text( - "diff --git a/file.rs b/file.rs\n--- a/file.rs\n+++ b/file.rs\n@@ -1,1 +1,2 @@\n one\n+two\n", - None, - ) - .expect("the direct patch should parse"); - let mut flow_result = flow_result_for(&repo, direct); - flow_result.tool_name = Some(String::from("claude")); - flow_result.tool_version = Some(String::from("9.9.9")); - - let mutation_ai_patch = resolve_mutation_ai(&repo, &db, &flow_result); - assert_eq!( - touched_line_count(&mutation_ai_patch), - 1, - "only the line direct evidence did not cover is resolved from mutation history" - ); - - let trace = persist_trace(&flow_result, &db, &mutation_ai_patch); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(2), - "the union of direct and mutation coverage classifies the hunk ai" - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["unknown"]["added"], - json!(0) - ); - assert_eq!( - trace["tool"], - json!({ "name": "claude", "version": "9.9.9" }) - ); - } - - #[test] - fn a_newer_nonexclusive_event_keeps_the_line_non_ai() { - let repo = E2eRepo::new("newer-nonexclusive"); - fs::write(repo.root.join("file.rs"), "one\ntwo\n").expect("the edit should write"); - commit_all(&repo.root, "add two"); - - let db = repo.db(); - let worktree = repo.checkout_id(); - seed_event( - &db, - &worktree, - 1, - &repo.parent_tree(), - &repo.head_tree(), - "ai_exclusive", - Some("scope-old"), - ); - seed_event( - &db, - &worktree, - 2, - &repo.parent_tree(), - &repo.head_tree(), - "ai_contended", - None, - ); - - let flow_result = flow_result_for(&repo, ParsedPatch { files: Vec::new() }); - let mutation_ai_patch = resolve_mutation_ai(&repo, &db, &flow_result); - assert_eq!( - touched_line_count(&mutation_ai_patch), - 0, - "the newer contended match resolves the line and blocks the older exclusive event" - ); - - let trace = persist_trace(&flow_result, &db, &mutation_ai_patch); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["unknown"]["added"], - json!(1) - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(0) - ); - assert_eq!( - trace["files"][0]["conversations"][0]["contributor"]["type"], - json!("unknown") - ); - } - - #[test] - fn an_adversarial_foreign_worktree_event_cannot_block_the_current_worktrees_exclusive_event() { - let repo = E2eRepo::new("adversarial-linked"); - - let linked_root = repo - .root - .parent() - .expect("the repo should have a parent directory") - .join("linked"); - git( - &repo.root, - &[ - "worktree", - "add", - "-q", - linked_root.to_str().expect("worktree path should be UTF-8"), - ], - ); - - fs::write(repo.root.join("file.rs"), "one\ntwo\n").expect("the edit should write"); - commit_all(&repo.root, "add two"); - - let db = repo.db(); - let current_worktree = repo.checkout_id(); - let foreign_worktree = resolve_worktree_id(&linked_root) - .expect("the linked worktree's identity should resolve") - .0; - assert_ne!( - current_worktree, foreign_worktree, - "the linked worktree must derive its own distinct identity" - ); - - seed_event( - &db, - ¤t_worktree, - 1, - &repo.parent_tree(), - &repo.head_tree(), - "ai_exclusive", - Some("scope-current"), - ); - seed_event( - &db, - &foreign_worktree, - 2, - &repo.parent_tree(), - &repo.head_tree(), - "ai_contended", - None, - ); - - let flow_result = flow_result_for(&repo, ParsedPatch { files: Vec::new() }); - let mutation_ai_patch = resolve_mutation_ai(&repo, &db, &flow_result); - assert_eq!( - touched_line_count(&mutation_ai_patch), - 1, - "only the current worktree's history is eligible, so the older exclusive event contributes" - ); - - let trace = persist_trace(&flow_result, &db, &mutation_ai_patch); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(1), - "worktree isolation lets the current worktree's exclusive event classify the target ai" - ); - assert_eq!( - trace["files"][0]["conversations"][0]["contributor"]["type"], - json!("ai") - ); - assert!(trace.get("tool").is_none()); - } - - fn touched_contents(patch: &ParsedPatch) -> Vec { - patch - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .flat_map(|hunk| hunk.lines.iter()) - .map(|line| line.content.clone()) - .collect() - } - - #[test] - #[allow(clippy::too_many_lines)] - fn persistence_boundaries_stay_separated_across_diff_traces_intersection_and_agent_trace() { - let repo = E2eRepo::new("persistence-boundary"); - - fs::write(repo.root.join("file.rs"), "one\ntwo\n").expect("the direct edit should write"); - git(&repo.root, &["add", "-A"]); - let intermediate_tree = git(&repo.root, &["write-tree"]).trim().to_owned(); - - fs::write(repo.root.join("file.rs"), "one\ntwo\nthree\n") - .expect("the mutation edit should write"); - commit_all(&repo.root, "add two and three"); - - let base_tree = repo.parent_tree(); - let final_tree = repo.head_tree(); - assert_ne!( - base_tree, intermediate_tree, - "the direct edit must move the tree" - ); - assert_ne!( - intermediate_tree, final_tree, - "the mutation edit must move the tree again" - ); - - let db = repo.db(); - - let now_ms = current_unix_time_ms().expect("the clock should resolve"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: now_ms - 60_000, - session_id: "cc_session-direct", - patch: "diff --git a/file.rs b/file.rs\n--- a/file.rs\n+++ b/file.rs\n@@ -1,1 +1,2 @@\n one\n+two\n", - model_id: Some("claude/model-direct"), - tool_name: "claude", - tool_version: Some("9.9.9"), - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("the direct diff_traces row should insert"); - - seed_event( - &db, - &repo.checkout_id(), - 1, - &intermediate_tree, - &final_tree, - "ai_exclusive", - Some("scope-mutation"), - ); - - let flow_result = run_post_commit_intersection_flow_with( - &repo.root, - capture_post_commit_patch_from_git, - current_unix_time_ms, - |cutoff_ms, end_ms| db.recent_diff_trace_patches(cutoff_ms, end_ms), - |insert| db.insert_post_commit_patch_intersection(insert).map(|_| ()), - ) - .expect("the real post-commit intersection flow should run"); - assert_eq!( - touched_contents(&flow_result.combined_recent_patch), - vec!["two".to_owned()], - "the combined recent patch comes from the real diff_traces query, not an in-memory patch" - ); - - let mutation_ai_patch = resolve_mutation_ai(&repo, &db, &flow_result); - assert_eq!( - touched_contents(&mutation_ai_patch), - vec!["three".to_owned()], - "mutation history resolves only the committed line direct evidence missed" - ); - - persist_trace(&flow_result, &db, &mutation_ai_patch); - - assert_eq!( - row_count(&db, "diff_traces"), - 1, - "mutation attribution must not create another diff_traces row" - ); - let stored_direct_patch: String = db - .query_map("SELECT patch FROM diff_traces", (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("diff_traces query should succeed") - .into_iter() - .next() - .expect("one diff_traces row should exist"); - let stored_direct = parse_patch_from_text(&stored_direct_patch, None) - .expect("the stored direct patch should parse"); - assert_eq!( - touched_contents(&stored_direct), - vec!["two".to_owned()], - "the direct diff_traces row contains 'two' and never 'three'" - ); - - assert_eq!( - row_count(&db, "post_commit_patch_intersections"), - 1, - "the intersection flow persists exactly one direct-only row" - ); - let stored_intersection_json: String = db - .query_map( - "SELECT intersection_patch FROM post_commit_patch_intersections", - (), - |row| row.get::(0).map_err(anyhow::Error::from), - ) - .expect("intersection query should succeed") - .into_iter() - .next() - .expect("one intersection row should exist"); - let stored_intersection = load_patch_from_json(&stored_intersection_json) - .expect("the persisted intersection patch should reconstruct"); - assert_eq!( - touched_contents(&stored_intersection), - vec!["two".to_owned()], - "post_commit_patch_intersections stays direct-only; the mutation line 'three' \ - must never contaminate this table" - ); - - assert_eq!(row_count(&db, "agent_traces"), 1); - let stored_trace_json: String = db - .query_map("SELECT trace_json FROM agent_traces", (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("agent_traces query should succeed") - .into_iter() - .next() - .expect("one Agent Trace row should exist"); - let trace: Value = serde_json::from_str(&stored_trace_json) - .expect("the persisted Agent Trace JSON should parse"); - validate_agent_trace_value(&trace).expect( - "the persisted agent_traces.trace_json validates against the embedded Agent Trace schema", - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(2), - "direct + mutation coverage classifies both committed added lines as ai" - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["unknown"]["added"], - json!(0) - ); - assert_eq!( - trace["files"][0]["conversations"][0]["contributor"]["type"], - json!("ai") - ); - - assert_eq!( - trace["tool"], - json!({ "name": "claude", "version": "9.9.9" }) - ); - - assert_eq!( - row_count(&db, "mutation_trace_events"), - 1, - "attribution performs no mutation-cursor write" - ); - } -} - -mod mutation_provenance_e2e { - use super::*; - use crate::services::agent_trace_db::{ClaudeModelStateObservation, ObservationKind}; - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - use crate::services::hooks::claude_mutation_scope; - use crate::services::hooks::codex_mutation_scope; - use crate::services::hooks::opencode_mutation_scope; - use crate::services::hooks::pi_mutation_scope; - use crate::services::mutation_trace::runtime::resolve_git_dir; - use crate::services::mutation_trace::runtime::resolve_post_commit_mutation_ai_patch; - - fn git(repo: &Path, args: &[&str]) -> String { - let output = Command::new("git") - .args(args) - .current_dir(repo) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).expect("git output should be UTF-8") - } - - fn row_count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("count row should exist") - } - - struct ProvenanceE2eRepo { - _temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, - db_path: PathBuf, - } - - impl ProvenanceE2eRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-mutation-provenance-e2e-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); - git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], - ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - let storage = resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); - - Self { - _temp: temp, - root, - state_root, - db_path: storage.db_path, - } - } - - fn db(&self) -> RepositoryAgentTraceDb { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) - .expect("repository DB should reopen") - } - - fn cwd(&self) -> String { - self.root.to_string_lossy().into_owned() - } - - fn write_change(&self, content: &str) { - fs::write(self.root.join("file.txt"), content).expect("mutation should write"); - } - - fn commit_change(&self) { - git(&self.root, &["add", "-A"]); - git(&self.root, &["commit", "-qm", "AI mutation"]); - } - - fn mutation_events(&self) -> Vec<(String, Option)> { - self.db() - .query_map( - "SELECT attribution_kind, attribution_scope_id \ - FROM mutation_trace_events ORDER BY revision", - (), - |row| { - let attribution_kind = row.get::(0).map_err(anyhow::Error::from)?; - let attribution_scope_id = - row.get::>(1).map_err(anyhow::Error::from)?; - Ok((attribution_kind, attribution_scope_id)) - }, - ) - .expect("mutation-events query should succeed") - } - - fn run_post_commit(&self) -> Value { - let db = self.db(); - run_post_commit_subcommand_with( - &self.root, - Some(AgentTraceVcsType::Git), - "git@github.com:acme/widgets.git", - |root| { - run_post_commit_intersection_flow_with( - root, - capture_post_commit_patch_from_git, - current_unix_time_ms, - |cutoff_ms, end_ms| db.recent_diff_trace_patches(cutoff_ms, end_ms), - |insert| db.insert_post_commit_patch_intersection(insert).map(|_| ()), - ) - }, - |root, flow_result, vcs_type, remote_url| { - let direct_intersection = intersect_patches_fn( - &flow_result.combined_recent_patch, - &flow_result.post_commit_data.parsed_patch, - ); - let mutation_ai_patch = resolve_post_commit_mutation_ai_patch( - root, - &db, - &direct_intersection, - &flow_result.post_commit_data.parsed_patch, - ); - - run_post_commit_agent_trace_flow_with( - flow_result, - vcs_type, - remote_url, - &mutation_ai_patch, - |value| { - validate_agent_trace_value(value) - .map_err(|error| anyhow!(error.to_string())) - }, - |insert| db.insert_agent_trace(insert).map(|_| ()), - ) - }, - |_| Ok(false), - |_| Ok(()), - |_| db.passive_checkpoint(), - None, - ) - .expect("the real post-commit hook flow should persist Agent Trace"); - - db.query_map("SELECT trace_json FROM agent_traces", (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("persisted Agent Trace should be readable") - .into_iter() - .next() - .map(|trace| serde_json::from_str(&trace).expect("trace JSON should parse")) - .expect("one Agent Trace row should exist") - } - } - - fn assert_mutation_trace_provenance(trace: &Value, model_id: &str, session_id: &str) { - assert_eq!(trace["files"][0]["path"], json!("file.txt")); - assert_eq!( - trace["files"][0]["conversations"][0]["contributor"], - json!({"type": "ai", "model_id": model_id}) - ); - assert_eq!( - trace["files"][0]["conversations"][0]["related"], - json!([{ - "type": "session", - "url": format!("https://sce.crocoder.dev/sessions/{session_id}"), - }]) - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["ai"]["added"], - json!(1) - ); - assert_eq!( - trace["metadata"]["sce"]["line_changes"]["unknown"]["added"], - json!(0) - ); - } - - fn opencode_before( - cwd: &str, - session_id: &str, - call_id: &str, - tool_name: &str, - model: Option<&str>, - ) -> String { - let mut payload = json!({ - "hook_event_name": "ToolExecuteBefore", - "session_id": session_id, - "call_id": call_id, - "cwd": cwd, - "tool_name": tool_name, - }); - if let Some(model) = model { - payload["model"] = json!(model); - } - payload.to_string() - } - - fn opencode_shell_env( - cwd: &str, - session_id: &str, - call_id: &str, - model: Option<&str>, - ) -> String { - let mut payload = json!({ - "hook_event_name": "ShellEnv", - "session_id": session_id, - "call_id": call_id, - "cwd": cwd, - }); - if let Some(model) = model { - payload["model"] = json!(model); - } - payload.to_string() - } - - fn opencode_after(cwd: &str, session_id: &str, call_id: &str, tool_name: &str) -> String { - json!({ - "hook_event_name": "ToolExecuteAfter", - "session_id": session_id, - "call_id": call_id, - "cwd": cwd, - "tool_name": tool_name, - }) - .to_string() - } - - fn opencode_tool_error(cwd: &str, session_id: &str, call_id: &str, tool_name: &str) -> String { - json!({ - "hook_event_name": "ToolError", - "session_id": session_id, - "call_id": call_id, - "cwd": cwd, - "tool_name": tool_name, - }) - .to_string() - } - - fn drive_opencode(repo: &ProvenanceE2eRepo, payload: &str) -> Result { - opencode_mutation_scope::run_opencode_mutation_scope_from_payload_at_state_root( - &repo.state_root, - payload, - None, - ) - } - - #[test] - fn opencode_bash_mutation_persists_model_and_session_in_agent_trace() { - let repo = ProvenanceE2eRepo::new("opencode-bash"); - let session_id = "ses_opencode_bash"; - let cwd = repo.cwd(); - - drive_opencode( - &repo, - &opencode_shell_env(&cwd, session_id, "call_bash", Some("opencode/big-pickle")), - ) - .expect("OpenCode shell.env should establish the bash scope"); - - repo.write_change("one\nopencode bash mutation\n"); - - drive_opencode( - &repo, - &opencode_after(&cwd, session_id, "call_bash", "bash"), - ) - .expect("OpenCode ToolExecuteAfter should close the bash scope"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_mutation_trace_provenance(&trace, "opencode/big-pickle", "oc_ses_opencode_bash"); - assert_eq!(row_count(&repo.db(), "diff_traces"), 0); - assert_eq!(row_count(&repo.db(), "post_commit_patch_intersections"), 1); - assert_eq!(row_count(&repo.db(), "mutation_trace_events"), 1); - assert_eq!(row_count(&repo.db(), "agent_traces"), 1); - } - - #[test] - fn opencode_apply_patch_mutation_with_missing_model_persists_no_model_in_agent_trace() { - let repo = ProvenanceE2eRepo::new("opencode-apply-patch"); - let session_id = "ses_opencode_no_model"; - let cwd = repo.cwd(); - - drive_opencode( - &repo, - &opencode_before(&cwd, session_id, "call_patch", "apply_patch", None), - ) - .expect("OpenCode ToolExecuteBefore should establish the apply_patch scope"); - - repo.write_change("one\npatched without model evidence\n"); - - drive_opencode( - &repo, - &opencode_after(&cwd, session_id, "call_patch", "apply_patch"), - ) - .expect("OpenCode ToolExecuteAfter should close the apply_patch scope"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_eq!(trace["files"][0]["path"], json!("file.txt")); - let contributor = &trace["files"][0]["conversations"][0]["contributor"]; - assert_eq!(contributor["type"], json!("ai")); - assert!( - contributor.get("model_id").is_none(), - "absent model evidence must never be guessed or fabricated" - ); - assert_eq!( - trace["files"][0]["conversations"][0]["related"], - json!([{ - "type": "session", - "url": "https://sce.crocoder.dev/sessions/oc_ses_opencode_no_model", - }]) - ); - } - - #[test] - fn opencode_write_mutation_persists_model_while_task_delegation_stays_zero_footprint() { - let repo = ProvenanceE2eRepo::new("opencode-write-task"); - let session_id = "ses_opencode_write"; - let cwd = repo.cwd(); - - drive_opencode( - &repo, - &opencode_before(&cwd, session_id, "call_task", "task", None), - ) - .expect("task delegation ToolExecuteBefore is neutral"); - drive_opencode( - &repo, - &opencode_after(&cwd, session_id, "call_task", "task"), - ) - .expect("task delegation ToolExecuteAfter is neutral"); - - assert_eq!( - row_count(&repo.db(), "mutation_trace_scopes"), - 0, - "a delegation event must create no mutation scope" - ); - - drive_opencode( - &repo, - &opencode_before( - &cwd, - session_id, - "call_write", - "write", - Some("opencode/big-pickle"), - ), - ) - .expect("OpenCode write ToolExecuteBefore should establish the scope"); - repo.write_change("one\nwrite mutation\n"); - drive_opencode( - &repo, - &opencode_after(&cwd, session_id, "call_write", "write"), - ) - .expect("OpenCode write ToolExecuteAfter should close the scope"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_mutation_trace_provenance(&trace, "opencode/big-pickle", "oc_ses_opencode_write"); - assert_eq!( - row_count(&repo.db(), "mutation_trace_scopes"), - 1, - "only the tracked write call created a scope" - ); - } - - #[test] - fn opencode_unknown_tool_events_create_no_scope_or_mutation_state() { - let repo = ProvenanceE2eRepo::new("opencode-untracked"); - let session_id = "ses_opencode_untracked"; - let cwd = repo.cwd(); - - for tool_name in ["read", "custom_mcp_tool", "totally_unknown_future_tool"] { - let call_id = format!("call_{tool_name}"); - drive_opencode( - &repo, - &opencode_before(&cwd, session_id, &call_id, tool_name, None), - ) - .unwrap_or_else(|_| panic!("{tool_name} ToolExecuteBefore should be neutral")); - drive_opencode( - &repo, - &opencode_after(&cwd, session_id, &call_id, tool_name), - ) - .unwrap_or_else(|_| panic!("{tool_name} ToolExecuteAfter should be neutral")); - drive_opencode( - &repo, - &opencode_tool_error(&cwd, session_id, &call_id, tool_name), - ) - .unwrap_or_else(|_| panic!("{tool_name} ToolError should be neutral")); - } - - assert_eq!(row_count(&repo.db(), "mutation_trace_scopes"), 0); - assert_eq!(row_count(&repo.db(), "mutation_trace_events"), 0); - } - - #[test] - fn opencode_child_task_session_gets_its_own_independent_scope_and_provenance() { - let repo = ProvenanceE2eRepo::new("opencode-child-session"); - let parent_session = "ses_opencode_parent"; - let child_session = "ses_opencode_child"; - let cwd = repo.cwd(); - - drive_opencode( - &repo, - &opencode_before(&cwd, parent_session, "call_task", "task", None), - ) - .expect("the parent's task delegation is neutral"); - - drive_opencode( - &repo, - &opencode_before( - &cwd, - child_session, - "call_child_write", - "write", - Some("opencode/child-model"), - ), - ) - .expect("the child session's write ToolExecuteBefore should establish its own scope"); - repo.write_change("one\nchild session mutation\n"); - drive_opencode( - &repo, - &opencode_after(&cwd, child_session, "call_child_write", "write"), - ) - .expect("the child session's write ToolExecuteAfter should close its own scope"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_mutation_trace_provenance(&trace, "opencode/child-model", "oc_ses_opencode_child"); - assert_eq!( - row_count(&repo.db(), "mutation_trace_scopes"), - 1, - "the parent's task delegation created no scope; only the child session's write did" - ); - } - - #[test] - fn opencode_concurrent_reject_and_confirm_keeps_only_the_confirmed_mutation_ai() { - let repo = ProvenanceE2eRepo::new("opencode-concurrent-reject"); - let session_id = "ses_opencode_concurrent"; - let cwd = repo.cwd(); - - drive_opencode( - &repo, - &opencode_before( - &cwd, - session_id, - "call_a_edit", - "edit", - Some("opencode/big-pickle"), - ), - ) - .expect("A's edit ToolExecuteBefore should establish a scope"); - drive_opencode( - &repo, - &opencode_before( - &cwd, - session_id, - "call_b_write", - "write", - Some("opencode/big-pickle"), - ), - ) - .expect("B's write ToolExecuteBefore should establish a distinct concurrent scope"); - - fs::write(repo.root.join("rejected.txt"), "rejected mutation\n") - .expect("A's mutation should write"); - fs::write( - repo.root.join("ambiguous.txt"), - "B's mutation before recovery\n", - ) - .expect("B's pre-recovery mutation should write"); - - drive_opencode( - &repo, - &opencode_tool_error(&cwd, session_id, "call_a_edit", "edit"), - ) - .expect("A's ToolError should abandon A's scope and consume the shared ambiguous interval"); - - fs::write( - repo.root.join("confirmed.txt"), - "B's mutation after recovery\n", - ) - .expect("B's post-recovery mutation should write"); - drive_opencode( - &repo, - &opencode_after(&cwd, session_id, "call_b_write", "write"), - ) - .expect("B's ToolExecuteAfter should confirm exactly B's own surviving scope"); - - git(&repo.root, &["add", "-A"]); - git(&repo.root, &["commit", "-qm", "concurrent mutation"]); - - let db = repo.db(); - let post_commit_data = capture_post_commit_patch_from_git(&repo.root) - .expect("capturing the post-commit patch should succeed"); - let mutation_ai_patch = resolve_post_commit_mutation_ai_patch( - &repo.root, - &db, - &ParsedPatch { files: Vec::new() }, - &post_commit_data.parsed_patch, - ); - - let ai_paths: Vec<&str> = mutation_ai_patch - .files - .iter() - .map(|file| file.new_path.as_str()) - .collect(); - assert!( - !ai_paths.contains(&"rejected.txt"), - "the abandoned scope's own mutation must never enter mutation_ai_patch" - ); - assert!( - !ai_paths.contains(&"ambiguous.txt"), - "B's mutation made before the ambiguity-consuming flush is genuinely \ - indistinguishable from A's and must stay non-AI, not merely non-A" - ); - assert!( - ai_paths.contains(&"confirmed.txt"), - "B's own later mutation, made after A's interval was consumed and confirmed \ - by B's own Close, must be attributed AI" - ); - } - - #[test] - fn opencode_and_codex_unconfirmed_overlap_stays_ineligible_until_codex_confirms() { - let repo = ProvenanceE2eRepo::new("opencode-codex-overlap"); - let oc_session = "ses_opencode_overlap"; - let codex_session = "codex-overlap-session"; - let cwd = repo.cwd(); - - drive_opencode( - &repo, - &opencode_before( - &cwd, - oc_session, - "call_oc", - "write", - Some("opencode/big-pickle"), - ), - ) - .expect("OpenCode write should establish a scope"); - - let codex_pre = json!({ - "hook_event_name": "PreToolUse", - "session_id": codex_session, - "turn_id": "codex-overlap-turn", - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "codex-overlap-bash", - "model": "gpt-5.6-sol", - "tool_input": {"command": "true"}, - }); - codex_mutation_scope::run_codex_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &codex_pre.to_string(), - None, - ) - .expect("Codex Bash PreToolUse should establish a concurrent scope"); - - repo.write_change("one\nopencode overlap mutation\n"); - - drive_opencode(&repo, &opencode_after(&cwd, oc_session, "call_oc", "write")) - .expect("OpenCode ToolExecuteAfter should close its own scope"); - - let attribution_after_first_close = repo.mutation_events(); - assert_eq!( - attribution_after_first_close - .last() - .map(|(kind, _)| kind.as_str()), - Some("ineligible_unscoped"), - "an unconfirmed live Codex scope must suppress OpenCode's own confirming close" - ); - - let codex_post = json!({ - "hook_event_name": "PostToolUse", - "session_id": codex_session, - "turn_id": "codex-overlap-turn", - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "codex-overlap-bash", - }); - codex_mutation_scope::run_codex_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &codex_post.to_string(), - None, - ) - .expect("Codex PostToolUse should close its own scope"); - - drive_opencode( - &repo, - &opencode_before( - &cwd, - oc_session, - "call_oc_2", - "write", - Some("opencode/big-pickle"), - ), - ) - .expect("a fresh OpenCode write should establish a new scope"); - repo.write_change("one\nopencode overlap mutation\nsecond change\n"); - drive_opencode( - &repo, - &opencode_after(&cwd, oc_session, "call_oc_2", "write"), - ) - .expect("the fresh OpenCode scope should close cleanly once Codex is confirmed"); - - let attribution_after_second_close = repo.mutation_events(); - assert_eq!( - attribution_after_second_close - .last() - .map(|(kind, _)| kind.as_str()), - Some("ai_exclusive"), - "once every other live scope is confirmation-safe, a solo confirming close is AiExclusive" - ); - } - - #[test] - fn opencode_and_claude_overlap_produces_ai_contended() { - let repo = ProvenanceE2eRepo::new("opencode-claude-overlap"); - let oc_session = "ses_opencode_contended"; - let claude_session = "claude-overlap-session"; - let cwd = repo.cwd(); - - drive_opencode( - &repo, - &opencode_before( - &cwd, - oc_session, - "call_oc_contended", - "write", - Some("opencode/big-pickle"), - ), - ) - .expect("OpenCode write should establish a scope"); - - let claude_pre = json!({ - "hook_event_name": "PreToolUse", - "session_id": claude_session, - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "claude-overlap-bash", - "tool_input": {"command": "printf mutation"}, - }); - claude_mutation_scope::run_claude_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &claude_pre.to_string(), - None, - ) - .expect( - "Claude Bash PreToolUse should establish a concurrent, non-confirmation-required scope", - ); - - repo.write_change("one\ncontended mutation\n"); - - drive_opencode( - &repo, - &opencode_after(&cwd, oc_session, "call_oc_contended", "write"), - ) - .expect("OpenCode ToolExecuteAfter should confirm its own scope"); - - let attribution = repo.mutation_events(); - assert_eq!( - attribution.last().map(|(kind, _)| kind.as_str()), - Some("ai_contended"), - "a confirmed OpenCode close alongside a live non-confirmation-required Claude scope is contended, not suppressed" - ); - - let claude_post = json!({ - "hook_event_name": "PostToolUse", - "session_id": claude_session, - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "claude-overlap-bash", - }); - claude_mutation_scope::run_claude_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &claude_post.to_string(), - None, - ) - .expect("Claude PostToolUse should close its own scope"); - } - - #[test] - fn claude_bash_mutation_persists_model_and_session_in_agent_trace() { - let repo = ProvenanceE2eRepo::new("claude"); - let session_id = "claude-session-e2e"; - let db = repo.db(); - db.upsert_claude_model_state(ClaudeModelStateObservation { - session_id: format!("cc_{session_id}"), - agent_id: String::new(), - model_id: String::from("claude/opus-4-1"), - observation_kind: ObservationKind::SessionStart, - source: String::from("test"), - observed_at_ms: 1, - }) - .expect("Claude model state should be persisted"); - - let cwd = repo.cwd(); - let pre = json!({ - "hook_event_name": "PreToolUse", - "session_id": session_id, - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "claude-bash-e2e", - "tool_input": {"command": "printf mutation"}, - }); - claude_mutation_scope::run_claude_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &pre.to_string(), - None, - ) - .expect("Claude Bash PreToolUse should establish a scope"); - - let post = json!({ - "hook_event_name": "PostToolUse", - "session_id": session_id, - "cwd": repo.cwd(), - "tool_name": "Bash", - "tool_use_id": "claude-bash-e2e", - }); - repo.write_change("one\nclaude mutation\n"); - claude_mutation_scope::run_claude_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &post.to_string(), - None, - ) - .expect("Claude Bash PostToolUse should close the scope"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_mutation_trace_provenance(&trace, "claude/opus-4-1", "cc_claude-session-e2e"); - assert_eq!(row_count(&repo.db(), "diff_traces"), 0); - assert_eq!(row_count(&repo.db(), "post_commit_patch_intersections"), 1); - assert_eq!(row_count(&repo.db(), "mutation_trace_events"), 1); - assert_eq!(row_count(&repo.db(), "agent_traces"), 1); - } - - #[test] - fn codex_bash_mutation_persists_model_and_session_in_agent_trace() { - let repo = ProvenanceE2eRepo::new("codex"); - let session_id = "codex-session-e2e"; - let cwd = repo.cwd(); - let pre = json!({ - "hook_event_name": "PreToolUse", - "session_id": session_id, - "turn_id": "codex-turn-e2e", - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "codex-bash-e2e", - "model": "gpt-5.6-sol", - "tool_input": {"command": "true"}, - }); - codex_mutation_scope::run_codex_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &pre.to_string(), - None, - ) - .expect("Codex Bash PreToolUse should establish a scope"); - - let post = json!({ - "hook_event_name": "PostToolUse", - "session_id": session_id, - "turn_id": "codex-turn-e2e", - "cwd": repo.cwd(), - "tool_name": "Bash", - "tool_use_id": "codex-bash-e2e", - }); - repo.write_change("one\ncodex mutation\n"); - codex_mutation_scope::run_codex_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &post.to_string(), - None, - ) - .expect("Codex Bash PostToolUse should close the scope"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_mutation_trace_provenance(&trace, "gpt-5.6-sol", "cx_codex-session-e2e"); - assert_eq!(row_count(&repo.db(), "diff_traces"), 0); - assert_eq!(row_count(&repo.db(), "post_commit_patch_intersections"), 1); - assert_eq!(row_count(&repo.db(), "mutation_trace_events"), 1); - assert_eq!(row_count(&repo.db(), "agent_traces"), 1); - } - - fn pi_tool_call( - cwd: &str, - session_id: &str, - tool_call_id: &str, - tool_name: &str, - model: Option<&str>, - ) -> String { - let mut payload = json!({ - "hook_event_name": "ToolCall", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": cwd, - "tool_name": tool_name, - }); - if let Some(model) = model { - payload["model"] = json!(model); - } - payload.to_string() - } - - fn pi_tool_result(cwd: &str, session_id: &str, tool_call_id: &str, tool_name: &str) -> String { - json!({ - "hook_event_name": "ToolResult", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": cwd, - "tool_name": tool_name, - }) - .to_string() - } - - fn pi_tool_execution_end( - cwd: &str, - session_id: &str, - tool_call_id: &str, - tool_name: &str, - ) -> String { - json!({ - "hook_event_name": "ToolExecutionEnd", - "session_id": session_id, - "tool_call_id": tool_call_id, - "cwd": cwd, - "tool_name": tool_name, - }) - .to_string() - } - - fn drive_pi(repo: &ProvenanceE2eRepo, payload: &str) -> Result { - pi_mutation_scope::run_pi_mutation_scope_from_payload_at_state_root( - &repo.state_root, - payload, - None, - ) - } - - fn pi_confirmed_tool_case(tool_name: &str, label: &str) { - let repo = ProvenanceE2eRepo::new(label); - let session_id = format!("ses-{label}"); - let call_id = format!("call-{label}"); - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - &session_id, - &call_id, - tool_name, - Some("anthropic/opus-5"), - ), - ) - .expect("Pi ToolCall should establish the tracked scope before execution"); - - repo.write_change(&format!("one\npi {tool_name} mutation\n")); - - drive_pi( - &repo, - &pi_tool_result(&cwd, &session_id, &call_id, tool_name), - ) - .expect("Pi ToolResult should mark the attempt executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, &session_id, &call_id, tool_name), - ) - .expect("Pi ToolExecutionEnd paired with an observed ToolResult should Close"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_mutation_trace_provenance(&trace, "anthropic/opus-5", &format!("pi_{session_id}")); - assert_eq!(row_count(&repo.db(), "diff_traces"), 0); - assert_eq!(row_count(&repo.db(), "post_commit_patch_intersections"), 1); - assert_eq!(row_count(&repo.db(), "mutation_trace_events"), 1); - assert_eq!(row_count(&repo.db(), "agent_traces"), 1); - } - - #[test] - fn pi_bash_mutation_persists_model_and_session_in_agent_trace() { - pi_confirmed_tool_case("bash", "pi-bash"); - } - - #[test] - fn pi_write_mutation_persists_model_and_session_in_agent_trace() { - pi_confirmed_tool_case("write", "pi-write"); - } - - #[test] - fn pi_edit_mutation_persists_model_and_session_in_agent_trace() { - pi_confirmed_tool_case("edit", "pi-edit"); - } - - #[test] - fn pi_missing_model_preserves_session_with_null_model_in_agent_trace() { - let repo = ProvenanceE2eRepo::new("pi-no-model"); - let session_id = "ses-pi-no-model"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call(&cwd, session_id, "call-1", "bash", None), - ) - .expect("Pi ToolCall should establish the scope without model evidence"); - - repo.write_change("one\npi mutation without model\n"); - - drive_pi(&repo, &pi_tool_result(&cwd, session_id, "call-1", "bash")) - .expect("Pi ToolResult should mark the attempt executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, session_id, "call-1", "bash"), - ) - .expect("Pi ToolExecutionEnd should close the scope"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_eq!(trace["files"][0]["path"], json!("file.txt")); - let contributor = &trace["files"][0]["conversations"][0]["contributor"]; - assert_eq!(contributor["type"], json!("ai")); - assert!( - contributor.get("model_id").is_none(), - "absent model evidence must never be guessed or fabricated" - ); - assert_eq!( - trace["files"][0]["conversations"][0]["related"], - json!([{ - "type": "session", - "url": "https://sce.crocoder.dev/sessions/pi_ses-pi-no-model", - }]) - ); - } - - #[test] - fn pi_read_only_and_unknown_tools_create_no_scope_or_mutation_state() { - let repo = ProvenanceE2eRepo::new("pi-untracked"); - let session_id = "ses-pi-untracked"; - let cwd = repo.cwd(); - - for tool_name in [ - "read", - "grep", - "find", - "ls", - "custom_mcp_tool", - "totally_unknown_future_tool", - "user_bash", - ] { - let call_id = format!("call-{tool_name}"); - drive_pi( - &repo, - &pi_tool_call(&cwd, session_id, &call_id, tool_name, None), - ) - .unwrap_or_else(|_| panic!("{tool_name} ToolCall should be neutral")); - drive_pi( - &repo, - &pi_tool_result(&cwd, session_id, &call_id, tool_name), - ) - .unwrap_or_else(|_| panic!("{tool_name} ToolResult should be neutral")); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, session_id, &call_id, tool_name), - ) - .unwrap_or_else(|_| panic!("{tool_name} ToolExecutionEnd should be neutral")); - } - - assert_eq!(row_count(&repo.db(), "mutation_trace_scopes"), 0); - assert_eq!(row_count(&repo.db(), "mutation_trace_events"), 0); - } - - #[test] - fn pi_later_extension_rejection_after_start_produces_no_mutation_ai_patch() { - let repo = ProvenanceE2eRepo::new("pi-later-rejection"); - let session_id = "ses-pi-rejected"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call(&cwd, session_id, "call-1", "bash", Some("anthropic/opus-5")), - ) - .expect("Pi ToolCall should establish the scope before a later extension can reject it"); - - fs::write( - repo.root.join("rejected.txt"), - "should never be attributed AI\n", - ) - .expect("the blocked attempt's incidental write should still land on disk"); - - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, session_id, "call-1", "bash"), - ) - .expect("ToolExecutionEnd with no preceding ToolResult must abandon, not error"); - - let scope_status = repo - .db() - .query_map("SELECT status FROM mutation_trace_scopes", (), |row| { - row.get::(0).map_err(anyhow::Error::from) - }) - .expect("scope-status query should succeed"); - assert_eq!( - scope_status, - vec!["abandoned".to_string()], - "the D7 abandon path must leave the scope durably abandoned, never closed or active" - ); - - git(&repo.root, &["add", "-A"]); - git(&repo.root, &["commit", "-qm", "rejected mutation"]); - - let db = repo.db(); - let post_commit_data = capture_post_commit_patch_from_git(&repo.root) - .expect("capturing the post-commit patch should succeed"); - let mutation_ai_patch = resolve_post_commit_mutation_ai_patch( - &repo.root, - &db, - &ParsedPatch { files: Vec::new() }, - &post_commit_data.parsed_patch, - ); - - assert!( - mutation_ai_patch.files.is_empty(), - "a Start that never reached a confirmed Close must never produce mutation_ai_patch entries" - ); - } - - #[test] - fn pi_mutate_then_error_still_persists_confirmed_mutation_through_close() { - let repo = ProvenanceE2eRepo::new("pi-error-executed"); - let session_id = "ses-pi-error"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call(&cwd, session_id, "call-1", "bash", Some("anthropic/opus-5")), - ) - .expect("Pi ToolCall should establish the scope"); - - repo.write_change("one\npartial mutation before failure\n"); - - let mut result_payload: Value = - serde_json::from_str(&pi_tool_result(&cwd, session_id, "call-1", "bash")) - .expect("tool_result payload should parse as JSON"); - result_payload["isError"] = json!(true); - drive_pi(&repo, &result_payload.to_string()) - .expect("a failed-but-executed ToolResult is still positive execution evidence"); - - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, session_id, "call-1", "bash"), - ) - .expect("ToolExecutionEnd paired with an observed ToolResult must Close, not abandon"); - repo.commit_change(); - - let trace = repo.run_post_commit(); - assert_mutation_trace_provenance(&trace, "anthropic/opus-5", "pi_ses-pi-error"); - } - - #[test] - fn pi_concurrent_reject_and_confirm_keeps_only_the_confirmed_mutation_ai() { - let repo = ProvenanceE2eRepo::new("pi-concurrent-reject"); - let session_id = "ses-pi-concurrent"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - session_id, - "call-a-edit", - "edit", - Some("anthropic/opus-5"), - ), - ) - .expect("A's edit ToolCall should establish a scope"); - drive_pi( - &repo, - &pi_tool_call( - &cwd, - session_id, - "call-b-write", - "write", - Some("anthropic/opus-5"), - ), - ) - .expect("B's write ToolCall should establish a distinct concurrent scope"); - - fs::write(repo.root.join("rejected.txt"), "rejected mutation\n") - .expect("A's mutation should write"); - fs::write( - repo.root.join("ambiguous.txt"), - "B's mutation before recovery\n", - ) - .expect("B's pre-recovery mutation should write"); - - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, session_id, "call-a-edit", "edit"), - ) - .expect( - "A's ToolExecutionEnd with no ToolResult should abandon A's scope and consume \ - the shared ambiguous interval", - ); - - fs::write( - repo.root.join("confirmed.txt"), - "B's mutation after recovery\n", - ) - .expect("B's post-recovery mutation should write"); - - drive_pi( - &repo, - &pi_tool_result(&cwd, session_id, "call-b-write", "write"), - ) - .expect("B's ToolResult should mark it executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, session_id, "call-b-write", "write"), - ) - .expect("B's ToolExecutionEnd should confirm exactly B's own surviving scope"); - - git(&repo.root, &["add", "-A"]); - git(&repo.root, &["commit", "-qm", "concurrent mutation"]); - - let db = repo.db(); - let post_commit_data = capture_post_commit_patch_from_git(&repo.root) - .expect("capturing the post-commit patch should succeed"); - let mutation_ai_patch = resolve_post_commit_mutation_ai_patch( - &repo.root, - &db, - &ParsedPatch { files: Vec::new() }, - &post_commit_data.parsed_patch, - ); - - let ai_paths: Vec<&str> = mutation_ai_patch - .files - .iter() - .map(|file| file.new_path.as_str()) - .collect(); - assert!( - !ai_paths.contains(&"rejected.txt"), - "the abandoned scope's own mutation must never enter mutation_ai_patch" - ); - assert!( - !ai_paths.contains(&"ambiguous.txt"), - "B's mutation made before the ambiguity-consuming flush is genuinely \ - indistinguishable from A's and must stay non-AI, not merely non-A" - ); - assert!( - ai_paths.contains(&"confirmed.txt"), - "B's own later mutation, made after A's interval was consumed and confirmed \ - by B's own Close, must be attributed AI" - ); - } - - #[test] - fn pi_and_claude_overlap_produces_ai_contended() { - let repo = ProvenanceE2eRepo::new("pi-claude-overlap"); - let pi_session = "ses-pi-contended"; - let claude_session = "claude-pi-overlap-session"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - pi_session, - "call-pi-contended", - "write", - Some("anthropic/opus-5"), - ), - ) - .expect("Pi write should establish a scope"); - - let claude_pre = json!({ - "hook_event_name": "PreToolUse", - "session_id": claude_session, - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "claude-pi-overlap-bash", - "tool_input": {"command": "printf mutation"}, - }); - claude_mutation_scope::run_claude_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &claude_pre.to_string(), - None, - ) - .expect( - "Claude Bash PreToolUse should establish a concurrent, non-confirmation-required scope", - ); - - repo.write_change("one\npi+claude contended mutation\n"); - - drive_pi( - &repo, - &pi_tool_result(&cwd, pi_session, "call-pi-contended", "write"), - ) - .expect("Pi ToolResult should mark the attempt executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, pi_session, "call-pi-contended", "write"), - ) - .expect("Pi ToolExecutionEnd should confirm its own scope"); - - let attribution = repo.mutation_events(); - assert_eq!( - attribution.last().map(|(kind, _)| kind.as_str()), - Some("ai_contended"), - "a confirmed Pi close alongside a live non-confirmation-required Claude scope is contended, not suppressed" - ); - - let claude_post = json!({ - "hook_event_name": "PostToolUse", - "session_id": claude_session, - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "claude-pi-overlap-bash", - }); - claude_mutation_scope::run_claude_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &claude_post.to_string(), - None, - ) - .expect("Claude PostToolUse should close its own scope"); - } - - #[test] - fn pi_and_codex_overlap_stays_ineligible_until_codex_confirms() { - let repo = ProvenanceE2eRepo::new("pi-codex-overlap"); - let pi_session = "ses-pi-overlap"; - let codex_session = "codex-pi-overlap-session"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - pi_session, - "call-pi", - "write", - Some("anthropic/opus-5"), - ), - ) - .expect("Pi write should establish a scope"); - - let codex_pre = json!({ - "hook_event_name": "PreToolUse", - "session_id": codex_session, - "turn_id": "codex-pi-overlap-turn", - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "codex-pi-overlap-bash", - "model": "gpt-5.6-sol", - "tool_input": {"command": "true"}, - }); - codex_mutation_scope::run_codex_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &codex_pre.to_string(), - None, - ) - .expect("Codex Bash PreToolUse should establish a concurrent scope"); - - repo.write_change("one\npi codex overlap mutation\n"); - - drive_pi(&repo, &pi_tool_result(&cwd, pi_session, "call-pi", "write")) - .expect("Pi ToolResult should mark the attempt executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, pi_session, "call-pi", "write"), - ) - .expect("Pi ToolExecutionEnd should attempt to confirm its own scope"); - - let attribution_after_first_close = repo.mutation_events(); - assert_eq!( - attribution_after_first_close - .last() - .map(|(kind, _)| kind.as_str()), - Some("ineligible_unscoped"), - "an unconfirmed live Codex scope must suppress Pi's own confirming close" - ); - - let codex_post = json!({ - "hook_event_name": "PostToolUse", - "session_id": codex_session, - "turn_id": "codex-pi-overlap-turn", - "cwd": cwd, - "tool_name": "Bash", - "tool_use_id": "codex-pi-overlap-bash", - }); - codex_mutation_scope::run_codex_mutation_scope_from_payload_at_state_root( - &repo.state_root, - &codex_post.to_string(), - None, - ) - .expect("Codex PostToolUse should close its own scope"); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - pi_session, - "call-pi-2", - "write", - Some("anthropic/opus-5"), - ), - ) - .expect("a fresh Pi write should establish a new scope"); - repo.write_change("one\npi codex overlap mutation\nsecond change\n"); - drive_pi( - &repo, - &pi_tool_result(&cwd, pi_session, "call-pi-2", "write"), - ) - .expect("Pi ToolResult should mark the fresh attempt executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, pi_session, "call-pi-2", "write"), - ) - .expect("the fresh Pi scope should close cleanly once Codex is confirmed"); - - let attribution_after_second_close = repo.mutation_events(); - assert_eq!( - attribution_after_second_close - .last() - .map(|(kind, _)| kind.as_str()), - Some("ai_exclusive"), - "once every other live scope is confirmation-safe, a solo confirming close is AiExclusive" - ); - } - - #[test] - fn pi_and_opencode_overlap_stays_ineligible_until_opencode_confirms() { - let repo = ProvenanceE2eRepo::new("pi-opencode-overlap"); - let pi_session = "ses-pi-oc-overlap"; - let oc_session = "ses_opencode_pi_overlap"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - pi_session, - "call-pi", - "write", - Some("anthropic/opus-5"), - ), - ) - .expect("Pi write should establish a scope"); - - drive_opencode( - &repo, - &opencode_before( - &cwd, - oc_session, - "call_oc", - "write", - Some("opencode/big-pickle"), - ), - ) - .expect("OpenCode write ToolExecuteBefore should establish a concurrent scope"); - - repo.write_change("one\npi opencode overlap mutation\n"); - - drive_pi(&repo, &pi_tool_result(&cwd, pi_session, "call-pi", "write")) - .expect("Pi ToolResult should mark the attempt executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, pi_session, "call-pi", "write"), - ) - .expect("Pi ToolExecutionEnd should attempt to confirm its own scope"); - - let attribution_after_first_close = repo.mutation_events(); - assert_eq!( - attribution_after_first_close - .last() - .map(|(kind, _)| kind.as_str()), - Some("ineligible_unscoped"), - "an unconfirmed live OpenCode scope must suppress Pi's own confirming close" - ); - - drive_opencode(&repo, &opencode_after(&cwd, oc_session, "call_oc", "write")) - .expect("OpenCode ToolExecuteAfter should close its own scope"); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - pi_session, - "call-pi-2", - "write", - Some("anthropic/opus-5"), - ), - ) - .expect("a fresh Pi write should establish a new scope"); - repo.write_change("one\npi opencode overlap mutation\nsecond change\n"); - drive_pi( - &repo, - &pi_tool_result(&cwd, pi_session, "call-pi-2", "write"), - ) - .expect("Pi ToolResult should mark the fresh attempt executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, pi_session, "call-pi-2", "write"), - ) - .expect("the fresh Pi scope should close cleanly once OpenCode is confirmed"); - - let attribution_after_second_close = repo.mutation_events(); - assert_eq!( - attribution_after_second_close - .last() - .map(|(kind, _)| kind.as_str()), - Some("ai_exclusive"), - "once every other live scope is confirmation-safe, a solo confirming close is AiExclusive" - ); - } - - #[test] - fn pi_stale_process_recovery_discards_ambiguous_interval_while_fresh_pi_work_remains_usable() { - let repo = ProvenanceE2eRepo::new("pi-stale-recovery"); - let stale_session = "ses-pi-stale"; - let fresh_session = "ses-pi-fresh"; - let cwd = repo.cwd(); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - stale_session, - "call-stale", - "bash", - Some("anthropic/opus-5"), - ), - ) - .expect("the stale attempt's Pi ToolCall should establish a scope"); - - let git_dir = resolve_git_dir(&repo.root).expect("git dir should resolve"); - let scope_id = pi_mutation_scope::state::read_state(&git_dir) - .expect("state should be readable") - .attempts - .iter() - .find(|attempt| attempt.session_id == stale_session) - .expect("the stale attempt should exist") - .scope_id - .clone(); - pi_mutation_scope::force_attempt_owner_dead_for_tests(&git_dir, &scope_id); - - fs::write( - repo.root.join("ambiguous.txt"), - "left behind by the dead Pi process\n", - ) - .expect("the stale attempt's own mutation should still land on disk"); - - drive_pi( - &repo, - &pi_tool_call( - &cwd, - fresh_session, - "call-fresh", - "write", - Some("anthropic/opus-5"), - ), - ) - .expect("a fresh Pi ToolCall should trigger dead-owner recovery and then establish its own scope"); - - fs::write(repo.root.join("confirmed.txt"), "the fresh Pi work\n") - .expect("the fresh attempt's mutation should write"); - - drive_pi( - &repo, - &pi_tool_result(&cwd, fresh_session, "call-fresh", "write"), - ) - .expect("the fresh attempt's ToolResult should mark it executed"); - drive_pi( - &repo, - &pi_tool_execution_end(&cwd, fresh_session, "call-fresh", "write"), - ) - .expect("the fresh attempt should close and reach AiExclusive"); - - git(&repo.root, &["add", "-A"]); - git(&repo.root, &["commit", "-qm", "stale recovery"]); - - let db = repo.db(); - let post_commit_data = capture_post_commit_patch_from_git(&repo.root) - .expect("capturing the post-commit patch should succeed"); - let mutation_ai_patch = resolve_post_commit_mutation_ai_patch( - &repo.root, - &db, - &ParsedPatch { files: Vec::new() }, - &post_commit_data.parsed_patch, - ); - - let ai_paths: Vec<&str> = mutation_ai_patch - .files - .iter() - .map(|file| file.new_path.as_str()) - .collect(); - assert!( - !ai_paths.contains(&"ambiguous.txt"), - "the dead process's ambiguous interval must never be attributed AI" - ); - assert!( - ai_paths.contains(&"confirmed.txt"), - "later fresh Pi work must remain usable and reach AiExclusive" - ); - - let attribution = repo.mutation_events(); - assert_eq!( - attribution.last().map(|(kind, _)| kind.as_str()), - Some("ai_exclusive"), - "the fresh attempt, unencumbered by the recovered stale scope, should reach AiExclusive" - ); - } -} - -#[test] -fn post_commit_auto_sync_does_not_launch_when_disabled() { - let launch_called = RefCell::new(false); - - run_post_commit_subcommand_with( - Path::new("/repo"), - None, - "", - |_| Ok(post_commit_flow_result()), - |_, _, _, _| Ok(minimal_agent_trace()), - |_| Ok(false), - |_| { - *launch_called.borrow_mut() = true; - Ok(()) - }, - |_| Ok(()), - None, - ) - .expect("disabled auto-sync should not affect post-commit success"); - - assert!(!*launch_called.borrow()); -} - -#[test] -fn post_commit_persistence_failure_does_not_launch_auto_sync() { - let launch_called = RefCell::new(false); - - let error = run_post_commit_subcommand_with( - Path::new("/repo"), - None, - "", - |_| Ok(post_commit_flow_result()), - |_, _, _, _| Err(anyhow!("Agent Trace persistence failed")), - |_| panic!("auto-sync config must not be resolved after persistence failure"), - |_| { - *launch_called.borrow_mut() = true; - Ok(()) - }, - |_| panic!("checkpoint must not run after persistence failure"), - None, - ) - .expect_err("persistence failure should be returned"); - - assert!(error.to_string().contains("persistence failed")); - assert!(!*launch_called.borrow()); -} - -#[test] -fn post_commit_auto_sync_launcher_failure_is_fail_open() { - let output = run_post_commit_subcommand_with( - Path::new("/repo"), - None, - "", - |_| Ok(post_commit_flow_result()), - |_, _, _, _| Ok(minimal_agent_trace()), - |_| Ok(true), - |_| Err(anyhow!("spawn unavailable")), - |_| Ok(()), - None, - ) - .expect("launcher failure must not affect post-commit success"); - - assert!(output.contains("post-commit hook processed intersection")); -} - -#[derive(Default)] -struct RecordingLogger { - warnings: std::sync::Mutex>, -} - -impl Logger for RecordingLogger { - fn info( - &self, - _event_id: &str, - _message: &str, - _fields: &[(&str, &str)], - _session_id: Option<&str>, - ) { - } - - fn debug( - &self, - _event_id: &str, - _message: &str, - _fields: &[(&str, &str)], - _session_id: Option<&str>, - ) { - } - - fn warn( - &self, - event_id: &str, - message: &str, - _fields: &[(&str, &str)], - _session_id: Option<&str>, - ) { - self.warnings - .lock() - .expect("warnings mutex should not be poisoned") - .push((event_id.to_string(), message.to_string())); - } - - fn error( - &self, - _event_id: &str, - _message: &str, - _fields: &[(&str, &str)], - _session_id: Option<&str>, - ) { - } - - fn log_cli_error(&self, _error: &crate::services::error::CliError, _session_id: Option<&str>) {} -} - -#[test] -fn post_commit_checkpoint_runs_once_after_successful_persistence() { - let events = RefCell::new(Vec::new()); - - let output = run_post_commit_subcommand_with( - Path::new("/repo"), - None, - "", - |_| Ok(post_commit_flow_result()), - |_, _, _, _| { - events.borrow_mut().push("persistence"); - Ok(minimal_agent_trace()) - }, - |_| Ok(false), - |_| Ok(()), - |_| { - events.borrow_mut().push("checkpoint"); - Ok(()) - }, - None, - ) - .expect("successful checkpoint should not affect post-commit success"); - - assert!(output.contains("post-commit hook processed intersection")); - assert_eq!(events.into_inner(), vec!["persistence", "checkpoint"]); -} - -#[test] -fn post_commit_checkpoint_failure_is_fail_open_and_logs_warning() { - let logger = RecordingLogger::default(); - let persisted = RefCell::new(false); - - let output = run_post_commit_subcommand_with( - Path::new("/repo"), - None, - "", - |_| Ok(post_commit_flow_result()), - |_, _, _, _| { - *persisted.borrow_mut() = true; - Ok(minimal_agent_trace()) - }, - |_| Ok(false), - |_| Ok(()), - |_| Err(anyhow!("checkpoint failed")), - Some(&logger), - ) - .expect("checkpoint failure must not affect post-commit success"); - - assert!(output.contains("post-commit hook processed intersection")); - assert!(*persisted.borrow()); - assert_eq!( - logger - .warnings - .into_inner() - .expect("warnings mutex should not be poisoned"), - vec![( - String::from("sce.agent_trace_db.passive_checkpoint_failed"), - String::from("checkpoint failed") - )] - ); -} - -#[test] -fn async_dispatch_doctor_and_hook_persist_before_completion() { - use std::io::Write; - use std::process::Stdio; - - let repo = init_attribution_git_repo("async-dispatch"); - let sandbox = tempfile::tempdir().unwrap(); - let payload = json!({ - "sessionID": "async-boundary-session", - "diff": valid_patch_text("src/boundary.rs", "completed"), - "time": 1_800_000_000_000_u64, - "model_id": "test/model", - "tool_name": "opencode", - "tool_version": null, - }); - let mut child = Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "services::hooks::tests::isolated_async_dispatch_boundary", - "--nocapture", - ]) - .current_dir(&repo) - .env("SCE_HOOK_BOUNDARY_CHILD", "1") - .env("HOME", sandbox.path()) - .env("XDG_CONFIG_HOME", sandbox.path().join("config")) - .env("XDG_STATE_HOME", sandbox.path().join("state")) - .env("XDG_DATA_HOME", sandbox.path().join("data")) - .env("XDG_CACHE_HOME", sandbox.path().join("cache")) - .env("NO_COLOR", "1") - .env_remove("SCE_DISABLED") - .env_remove("SCE_CONFIG_FILE") - .env_remove("SCE_ATTRIBUTION_HOOKS_DISABLED") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .unwrap(); - child - .stdin - .take() - .unwrap() - .write_all(payload.to_string().as_bytes()) - .unwrap(); - let output = child.wait_with_output().unwrap(); - fs::remove_dir_all(&repo).unwrap(); - assert!( - output.status.success(), - "{}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); -} - -#[tokio::test(flavor = "multi_thread")] -async fn isolated_async_dispatch_boundary() { - use crate::app::AppContext; - use crate::services::capabilities::{ProcessGitOps, StdFsOps}; - use crate::services::command_registry::{default_runtime_command, RuntimeCommand}; - use crate::services::observability::traits::{NoopLogger, NoopTelemetry}; - - if std::env::var_os("SCE_HOOK_BOUNDARY_CHILD").is_none() { - return; - } - assert_eq!( - tokio::runtime::Handle::current().runtime_flavor(), - tokio::runtime::RuntimeFlavor::MultiThread - ); - let repo = std::env::current_dir().unwrap(); - let storage_context = AgentTraceStorageContext { - repository_root: &repo, - explicit_repository_id: None, - repository_remote: "origin", - }; - let state_root = crate::services::default_paths::resolve_state_data_root().unwrap(); - let db_path = tokio::task::block_in_place(|| { - let storage = - resolve_agent_trace_storage_at_state_root(&storage_context, &state_root).unwrap(); - storage.db_path.clone() - }); - let context = AppContext::new( - &NoopLogger, - &NoopTelemetry, - &StdFsOps, - &ProcessGitOps, - Some(repo.clone()), - ); - let doctor = default_runtime_command("doctor").unwrap(); - let doctor_output = doctor.execute(&context).await.unwrap(); - assert!(doctor_output.contains("Agent Trace"), "{doctor_output}"); - assert!( - !doctor_output.contains("schema is not ready"), - "{doctor_output}" - ); - let hook = RuntimeCommand::Hooks(crate::services::hooks::command::HooksCommand { - subcommand: HookSubcommand::DiffTrace, - }); - assert_eq!( - hook.execute(&context).await.unwrap(), - "diff-trace hook intake persisted payload to AgentTraceDb." - ); - tokio::task::block_in_place(|| { - let db = RepositoryAgentTraceDb::open_without_migrations_at(&db_path).unwrap(); - let rows = db - .query_map( - "SELECT session_id, model_id, tool_name FROM diff_traces ORDER BY id", - (), - |row| { - Ok(( - row.get::(0)?, - row.get::(1)?, - row.get::(2)?, - )) - }, - ) - .unwrap(); - assert_eq!( - rows, - vec![( - "oc_async-boundary-session".to_string(), - "test/model".to_string(), - "opencode".to_string() - )] - ); - }); -} diff --git a/cli/src/services/lifecycle.rs b/cli/src/services/lifecycle.rs index 7161d7394..a343f1b2c 100644 --- a/cli/src/services/lifecycle.rs +++ b/cli/src/services/lifecycle.rs @@ -131,19 +131,20 @@ pub struct SetupOutcome { pub required_hooks_install: Option, } -pub trait ServiceLifecycle: Send + Sync { - #[allow(dead_code)] - fn id(&self) -> LifecycleProviderId; - - fn diagnose(&self, _ctx: &C) -> Vec { +pub(crate) trait ServiceLifecycle: Send + Sync { + async fn diagnose(&self, _ctx: &C) -> Vec { Vec::new() } - fn fix(&self, _ctx: &C, _problems: &[HealthProblem]) -> Vec { + async fn fix( + &self, + _ctx: &C, + _problems: &[HealthProblem], + ) -> Vec { Vec::new() } - fn setup(&self, _ctx: &C) -> Result { + async fn setup(&self, _ctx: &C) -> Result { Ok(SetupOutcome::default()) } } @@ -168,41 +169,97 @@ impl LifecycleProvider { } } - pub fn diagnose(self, ctx: &C) -> Vec { + pub async fn diagnose(self, ctx: &C) -> Vec { match self { - Self::Config => crate::services::config::lifecycle::ConfigLifecycle.diagnose(ctx), - Self::LocalDb => crate::services::local_db::lifecycle::LocalDbLifecycle.diagnose(ctx), - Self::AuthDb => crate::services::auth_db::lifecycle::AuthDbLifecycle.diagnose(ctx), + Self::Config => { + crate::services::config::lifecycle::ConfigLifecycle + .diagnose(ctx) + .await + } + Self::LocalDb => { + crate::services::local_db::lifecycle::LocalDbLifecycle + .diagnose(ctx) + .await + } + Self::AuthDb => { + crate::services::auth_db::lifecycle::AuthDbLifecycle + .diagnose(ctx) + .await + } Self::AgentTraceDb => { - crate::services::agent_trace_db::lifecycle::AgentTraceDbLifecycle.diagnose(ctx) + crate::services::agent_trace_db::lifecycle::AgentTraceDbLifecycle + .diagnose(ctx) + .await + } + Self::Hooks => { + crate::services::hooks::lifecycle::HooksLifecycle + .diagnose(ctx) + .await } - Self::Hooks => crate::services::hooks::lifecycle::HooksLifecycle.diagnose(ctx), } } - pub fn fix(self, ctx: &C, problems: &[HealthProblem]) -> Vec { + pub async fn fix( + self, + ctx: &C, + problems: &[HealthProblem], + ) -> Vec { match self { - Self::Config => crate::services::config::lifecycle::ConfigLifecycle.fix(ctx, problems), + Self::Config => { + crate::services::config::lifecycle::ConfigLifecycle + .fix(ctx, problems) + .await + } Self::LocalDb => { - crate::services::local_db::lifecycle::LocalDbLifecycle.fix(ctx, problems) + crate::services::local_db::lifecycle::LocalDbLifecycle + .fix(ctx, problems) + .await + } + Self::AuthDb => { + crate::services::auth_db::lifecycle::AuthDbLifecycle + .fix(ctx, problems) + .await } - Self::AuthDb => crate::services::auth_db::lifecycle::AuthDbLifecycle.fix(ctx, problems), Self::AgentTraceDb => { - crate::services::agent_trace_db::lifecycle::AgentTraceDbLifecycle.fix(ctx, problems) + crate::services::agent_trace_db::lifecycle::AgentTraceDbLifecycle + .fix(ctx, problems) + .await + } + Self::Hooks => { + crate::services::hooks::lifecycle::HooksLifecycle + .fix(ctx, problems) + .await } - Self::Hooks => crate::services::hooks::lifecycle::HooksLifecycle.fix(ctx, problems), } } - pub fn setup(self, ctx: &C) -> Result { + pub async fn setup(self, ctx: &C) -> Result { match self { - Self::Config => crate::services::config::lifecycle::ConfigLifecycle.setup(ctx), - Self::LocalDb => crate::services::local_db::lifecycle::LocalDbLifecycle.setup(ctx), - Self::AuthDb => crate::services::auth_db::lifecycle::AuthDbLifecycle.setup(ctx), + Self::Config => { + crate::services::config::lifecycle::ConfigLifecycle + .setup(ctx) + .await + } + Self::LocalDb => { + crate::services::local_db::lifecycle::LocalDbLifecycle + .setup(ctx) + .await + } + Self::AuthDb => { + crate::services::auth_db::lifecycle::AuthDbLifecycle + .setup(ctx) + .await + } Self::AgentTraceDb => { - crate::services::agent_trace_db::lifecycle::AgentTraceDbLifecycle.setup(ctx) + crate::services::agent_trace_db::lifecycle::AgentTraceDbLifecycle + .setup(ctx) + .await + } + Self::Hooks => { + crate::services::hooks::lifecycle::HooksLifecycle + .setup(ctx) + .await } - Self::Hooks => crate::services::hooks::lifecycle::HooksLifecycle.setup(ctx), } } } diff --git a/cli/src/services/local_db/lifecycle.rs b/cli/src/services/local_db/lifecycle.rs index edb7e33b2..329e661ef 100644 --- a/cli/src/services/local_db/lifecycle.rs +++ b/cli/src/services/local_db/lifecycle.rs @@ -5,7 +5,7 @@ use crate::services::db::{bootstrap_db_parent, collect_db_path_health, DbSpec}; use crate::services::default_paths::local_db_path; use crate::services::lifecycle::{ FixOutcome, FixResultRecord, HealthCategory, HealthFixability, HealthProblem, - HealthProblemKind, HealthSeverity, LifecycleProviderId, ServiceLifecycle, SetupOutcome, + HealthProblemKind, HealthSeverity, ServiceLifecycle, SetupOutcome, }; use super::{LocalDb, LocalDbSpec}; @@ -14,15 +14,15 @@ use super::{LocalDb, LocalDbSpec}; pub struct LocalDbLifecycle; impl ServiceLifecycle for LocalDbLifecycle { - fn id(&self) -> LifecycleProviderId { - LifecycleProviderId::LocalDb - } - - fn diagnose(&self, _ctx: &C) -> Vec { + async fn diagnose(&self, _ctx: &C) -> Vec { diagnose_local_db_health() } - fn fix(&self, _ctx: &C, problems: &[HealthProblem]) -> Vec { + async fn fix( + &self, + _ctx: &C, + problems: &[HealthProblem], + ) -> Vec { let should_bootstrap_parent = problems.iter().any(|problem| { problem.category == HealthCategory::GlobalState && problem.fixability == HealthFixability::AutoFixable @@ -48,8 +48,10 @@ impl ServiceLifecycle for LocalDbLifecycle { } } - fn setup(&self, _ctx: &C) -> Result { - LocalDb::new().context("Local DB lifecycle setup failed while initializing local DB")?; + async fn setup(&self, _ctx: &C) -> Result { + LocalDb::new() + .await + .context("Local DB lifecycle setup failed while initializing local DB")?; Ok(SetupOutcome::default()) } } diff --git a/cli/src/services/local_db/mod.rs b/cli/src/services/local_db/mod.rs index 0f7ad14d9..bf6987392 100644 --- a/cli/src/services/local_db/mod.rs +++ b/cli/src/services/local_db/mod.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use anyhow::Result; use crate::services::{ - db::{DbSpec, TursoDb}, + db::{DatabaseKind, DbSpec, TursoDb}, default_paths::local_db_path, }; @@ -27,9 +27,7 @@ impl DbSpec for LocalDbSpec { &[] } - fn db_config_key() -> &'static str { - "local_db" - } + const KIND: DatabaseKind = DatabaseKind::Local; } /// Local Turso database adapter. diff --git a/cli/src/services/mod.rs b/cli/src/services/mod.rs index 1f7690f8e..899ef6907 100644 --- a/cli/src/services/mod.rs +++ b/cli/src/services/mod.rs @@ -1,16 +1,12 @@ pub mod agent_trace; pub mod agent_trace_db; -#[allow(dead_code)] pub mod agent_trace_export; -#[allow(dead_code)] pub mod agent_trace_storage; -#[allow(dead_code)] pub mod agent_trace_sync; pub mod app_support; pub mod auth; pub mod auth_command; pub mod auth_db; -#[allow(dead_code)] pub mod bash_policy; pub mod capabilities; pub(crate) mod codex_hook_config; @@ -27,13 +23,11 @@ pub mod help; pub mod hooks; pub mod lifecycle; pub mod local_db; -#[allow(dead_code)] pub mod mutation_trace; pub mod observability; pub mod output_format; pub mod parse; pub mod patch; -#[allow(dead_code)] pub mod repository_identity; pub mod resilience; pub mod security; diff --git a/cli/src/services/mutation_trace/lineage.rs b/cli/src/services/mutation_trace/lineage.rs index 74e69527a..4f28203ed 100644 --- a/cli/src/services/mutation_trace/lineage.rs +++ b/cli/src/services/mutation_trace/lineage.rs @@ -72,10 +72,6 @@ impl MutationLineage { } } - pub fn reset_file(&mut self, path: &str, content: Option<&str>) { - self.files.insert(path.to_owned(), unknown_lines(content)); - } - pub fn reset_all(&mut self, files: &BTreeMap>) { self.files = files .iter() @@ -83,10 +79,6 @@ impl MutationLineage { .collect(); } - pub fn tracked_paths(&self) -> impl Iterator { - self.files.keys() - } - pub fn apply( &mut self, patch: &ParsedPatch, diff --git a/cli/src/services/mutation_trace/lineage/tests.rs b/cli/src/services/mutation_trace/lineage/tests.rs index d2f0f093a..4a49dd069 100644 --- a/cli/src/services/mutation_trace/lineage/tests.rs +++ b/cli/src/services/mutation_trace/lineage/tests.rs @@ -57,7 +57,6 @@ fn a_removed_line_loses_its_provenance_permanently() { lineage.provenance_at("f.rs", 2, "foo"), LineProvenance::Unknown ); - assert_eq!(lineage.tracked_paths().count(), 1); } #[test] @@ -161,31 +160,3 @@ fn duplicate_lines_do_not_let_provenance_jump_between_occurrences() { LineProvenance::Unknown ); } - -#[test] -fn a_deleted_file_drops_out_of_the_lineage() { - let mut lineage = baseline(&[("f.rs", Some("a\nb\n"))]); - lineage - .apply( - &diff("diff --git a/f.rs b/f.rs\ndeleted file mode 100644\n--- a/f.rs\n+++ /dev/null\n@@ -1,2 +0,0 @@\n-a\n-b\n"), - &TransitionOrigin::MutationNonAi, - ) - .expect("delete"); - assert_eq!(lineage.tracked_paths().count(), 0); -} - -#[test] -fn reset_file_returns_a_file_to_a_conservative_baseline() { - let mut lineage = baseline(&[("f.rs", Some("a\n"))]); - lineage - .apply( - &diff("diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n"), - &TransitionOrigin::MutationAi(scope("ai")), - ) - .expect("add"); - lineage.reset_file("f.rs", Some("a\nfoo\n")); - assert_eq!( - lineage.provenance_at("f.rs", 2, "foo"), - LineProvenance::Unknown - ); -} diff --git a/cli/src/services/mutation_trace/mbt/model.rs b/cli/src/services/mutation_trace/mbt/model.rs index c257e4088..4876c3451 100644 --- a/cli/src/services/mutation_trace/mbt/model.rs +++ b/cli/src/services/mutation_trace/mbt/model.rs @@ -443,6 +443,10 @@ pub struct ModelState { /// variables are still accepted, but only via an explicit [`IgnoredAny`] /// field — modeling a new Quint variable here as `IgnoredAny` is a /// deliberate "not semantic state" decision, not an oversight. +#[allow( + dead_code, + reason = "verification-only Quint variables are deserialized to satisfy deny_unknown_fields and never read" +)] #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct WireModelState { diff --git a/cli/src/services/mutation_trace/runtime/coordinator.rs b/cli/src/services/mutation_trace/runtime/coordinator.rs index e6f1e038f..1fa06e61d 100644 --- a/cli/src/services/mutation_trace/runtime/coordinator.rs +++ b/cli/src/services/mutation_trace/runtime/coordinator.rs @@ -1,4 +1,5 @@ use std::path::Path; +use std::time::Duration; use anyhow::Result; use uuid::Uuid; @@ -14,7 +15,8 @@ use crate::services::mutation_trace::types::{ }; use super::git_snapshot::GitSnapshotService; -use super::protected_worktree::{ProtectedWorktree, ProtectedWorktreeError}; +use super::protected_worktree::{ProtectedWorktree, ProtectedWorktreeError, WORKTREE_LOCK_TIMEOUT}; +use super::worktree_lock::WorktreeLockLease; pub use super::protected_worktree::ExternalTaintOperation; @@ -49,10 +51,30 @@ pub enum RuntimeBoundary { #[derive(Debug)] pub struct CoordinateOutcome { + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub worktree_id: WorktreeId, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub observed_tree: TreeId, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub revision: u64, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub evaluation: protocol::CommitEvaluation, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] pub mutation_event: Option, } @@ -89,6 +111,10 @@ pub enum CoordinateError { /// next invocation conservatively recovers. MarkerClearAfterCommit { source: anyhow::Error, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] committed: Box, }, /// The caller-supplied Agent Trace DB provider returned `Err` after the @@ -144,66 +170,83 @@ impl std::fmt::Display for CoordinateError { impl std::error::Error for CoordinateError {} pub trait SnapshotCapture { - fn capture(&self) -> Result; - fn pin(&self, worktree_id: &WorktreeId, tree: &TreeId) -> Result<()>; + async fn capture(&self) -> Result; + async fn pin( + &self, + lease: WorktreeLockLease, + worktree_id: &WorktreeId, + tree: &TreeId, + ) -> Result<()>; } impl SnapshotCapture for GitSnapshotService { - fn capture(&self) -> Result { - self.capture_tree() + async fn capture(&self) -> Result { + self.capture_tree().await } - fn pin(&self, worktree_id: &WorktreeId, tree: &TreeId) -> Result<()> { - self.pin_tree(worktree_id, tree) + async fn pin( + &self, + lease: WorktreeLockLease, + worktree_id: &WorktreeId, + tree: &TreeId, + ) -> Result<()> { + self.pin_tree(lease, worktree_id, tree).await } } -pub fn coordinate

( +pub async fn coordinate

( repository_root: &Path, boundary: &RuntimeBoundary, open_db: P, ) -> Result where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, { coordinate_inner( repository_root, boundary, open_db, + WORKTREE_LOCK_TIMEOUT, || {}, |_attempt| {}, |_attempt| Ok(()), ) + .await } -pub(super) fn coordinate_inner( +pub(super) async fn coordinate_inner( repository_root: &Path, boundary: &RuntimeBoundary, open_db: P, + lock_timeout: Duration, on_lock_contention: F, after_load: L, after_recovery: R, ) -> Result where - P: FnOnce() -> anyhow::Result, - F: FnOnce(), + P: std::ops::AsyncFnOnce() -> anyhow::Result, + F: FnOnce() + Send + 'static, L: FnMut(u32), R: FnMut(u32) -> Result<()>, { - let protected = ProtectedWorktree::acquire_inner(repository_root, on_lock_contention) - .map_err(protected_worktree_failure)?; + let protected = + ProtectedWorktree::acquire_with_timeout(repository_root, lock_timeout, on_lock_contention) + .await + .map_err(protected_worktree_failure)?; let outcome = coordinate_protected( repository_root, + protected.lock_lease(), protected.worktree_id(), boundary, open_db, protected.inherited_external_taint(), after_load, after_recovery, - )?; + ) + .await?; - match protected.complete() { + match protected.complete().await { Ok(()) => Ok(outcome), Err(source) => Err(CoordinateError::MarkerClearAfterCommit { source, @@ -212,8 +255,10 @@ where } } -fn coordinate_protected( +#[allow(clippy::too_many_arguments)] +async fn coordinate_protected( repository_root: &Path, + pin_lease: WorktreeLockLease, worktree_id: &WorktreeId, boundary: &RuntimeBoundary, open_db: P, @@ -222,37 +267,45 @@ fn coordinate_protected( after_recovery: R, ) -> Result where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, L: FnMut(u32), R: FnMut(u32) -> Result<()>, { - let db = open_db().map_err(CoordinateError::AgentTraceDbUnavailable)?; + let mut db = open_db() + .await + .map_err(CoordinateError::AgentTraceDbUnavailable)?; - let snapshot = GitSnapshotService::new(repository_root).map_err(CoordinateError::Other)?; + let snapshot = GitSnapshotService::new(repository_root) + .await + .map_err(CoordinateError::Other)?; coordinate_boundary_inner( - &db, + &mut db, &snapshot, + pin_lease, worktree_id, boundary, inherited_external_taint, after_load, after_recovery, ) + .await } -pub(super) fn coordinate_on_held_worktree

( +pub(super) async fn coordinate_on_held_worktree

( repository_root: &Path, + pin_lease: WorktreeLockLease, worktree_id: &WorktreeId, boundary: &RuntimeBoundary, open_db: P, force_recovery: bool, ) -> Result where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, { coordinate_protected( repository_root, + pin_lease, worktree_id, boundary, open_db, @@ -260,6 +313,7 @@ where |_attempt| {}, |_attempt| Ok(()), ) + .await } fn protected_worktree_failure(error: ProtectedWorktreeError) -> CoordinateError { @@ -272,31 +326,30 @@ fn protected_worktree_failure(error: ProtectedWorktreeError) -> CoordinateError ProtectedWorktreeError::ExternalTaintMarker { operation, source } => { CoordinateError::ExternalTaintMarker { operation, source } } + error @ ProtectedWorktreeError::MarkerWorkerFailed(_) => { + CoordinateError::Other(anyhow::Error::new(error)) + } } } -#[cfg(test)] -fn coordinate_boundary( - db: &RepositoryAgentTraceDb, +async fn capture_and_pin( capture: &C, + pin_lease: WorktreeLockLease, worktree_id: &WorktreeId, - boundary: &RuntimeBoundary, - inherited_external_taint: bool, -) -> Result { - coordinate_boundary_inner( - db, - capture, - worktree_id, - boundary, - inherited_external_taint, - |_attempt| {}, - |_attempt| Ok(()), - ) +) -> Result +where + C: SnapshotCapture, +{ + let tree = capture.capture().await?; + capture.pin(pin_lease, worktree_id, &tree).await?; + Ok(tree) } -fn coordinate_boundary_inner( - db: &RepositoryAgentTraceDb, +#[allow(clippy::too_many_arguments)] +async fn coordinate_boundary_inner( + db: &mut RepositoryAgentTraceDb, capture: &C, + pin_lease: WorktreeLockLease, worktree_id: &WorktreeId, boundary: &RuntimeBoundary, inherited_external_taint: bool, @@ -308,30 +361,29 @@ where AfterLoad: FnMut(u32), AfterRecovery: FnMut(u32) -> Result<()>, { - let store = MutationTraceStore::new(db); + let mut store = MutationTraceStore::new(db); - let observed_tree = match capture.capture().and_then(|tree| { - capture.pin(worktree_id, &tree)?; - Ok(tree) - }) { + let observed_tree = match capture_and_pin(capture, pin_lease, worktree_id).await { Ok(tree) => tree, - Err(source) => return Err(handle_snapshot_failure(&store, worktree_id, source)), + Err(source) => return Err(handle_snapshot_failure(&mut store, worktree_id, source).await), }; store .initialize_worktree(worktree_id, &observed_tree) + .await .map_err(CoordinateError::Other)?; let registered_scope = match hook_identity(boundary) { Some((scope, actor_kind)) => Some( store .register_scope(scope, worktree_id, actor_kind) + .await .map_err(CoordinateError::ScopeIdentityConflict)?, ), None => None, }; - register_start_provenance(&store, boundary, registered_scope.as_ref())?; + register_start_provenance(&store, boundary, registered_scope.as_ref()).await?; let type_boundary = into_protocol_boundary(boundary, worktree_id); let scope_ref = types::boundary_scope(&type_boundary); @@ -342,6 +394,7 @@ where for attempt_index in 0..MAX_CAS_RETRY_ATTEMPTS { let Some(projection) = store .load_worktree(worktree_id, scope_ref.as_ref(), event_key_ref.as_ref()) + .await .map_err(CoordinateError::Other)? else { return Err(CoordinateError::Other(anyhow::anyhow!( @@ -373,7 +426,11 @@ where }); }; - match store.commit(&transition).map_err(CoordinateError::Other)? { + match store + .commit(&transition) + .await + .map_err(CoordinateError::Other)? + { CasResult::Applied => { state = recovered; external_taint_pending = false; @@ -395,7 +452,11 @@ where match DurableTransition::between(&state, &outcome.state, worktree_id) .map_err(CoordinateError::Other)? { - Some(transition) => match store.commit(&transition).map_err(CoordinateError::Other)? { + Some(transition) => match store + .commit(&transition) + .await + .map_err(CoordinateError::Other)? + { CasResult::Applied => { return Ok(build_outcome(worktree_id, observed_tree, &outcome)) } @@ -421,7 +482,7 @@ where /// `Start`. Once the scope has crossed protocol admission, absent provenance /// stays absent permanently: provenance describes its scope as observed at /// admission, so it is never attached retroactively. -fn register_start_provenance( +async fn register_start_provenance( store: &MutationTraceStore<'_>, boundary: &RuntimeBoundary, registered_scope: Option<&types::ScopeState>, @@ -432,6 +493,7 @@ fn register_start_provenance( let stored = store .load_scope_provenance(scope) + .await .map_err(CoordinateError::ScopeProvenanceRegistration)?; let before_admission = registered_scope.is_some_and(|scope_state| scope_state.status == ScopeStatus::NeverSeen); @@ -446,6 +508,7 @@ fn register_start_provenance( session_id: provenance.session_id.clone(), model_id: provenance.model_id.clone(), }) + .await .map_err(CoordinateError::ScopeProvenanceRegistration)?; Ok(()) @@ -530,12 +593,12 @@ fn hook_identity(boundary: &RuntimeBoundary) -> Option<(&ScopeId, ActorKind)> { } } -fn handle_snapshot_failure( - store: &MutationTraceStore<'_>, +async fn handle_snapshot_failure( + store: &mut MutationTraceStore<'_>, worktree_id: &WorktreeId, source: anyhow::Error, ) -> CoordinateError { - match run_taint_retry_loop(store, worktree_id) { + match run_taint_retry_loop(store, worktree_id).await { Ok(persisted_taint) => CoordinateError::SnapshotFailure { persisted_taint, source, @@ -544,12 +607,15 @@ fn handle_snapshot_failure( } } -fn run_taint_retry_loop(store: &MutationTraceStore<'_>, worktree_id: &WorktreeId) -> Result { - run_taint_retry_loop_inner(store, worktree_id, |_attempt| {}) +async fn run_taint_retry_loop( + store: &mut MutationTraceStore<'_>, + worktree_id: &WorktreeId, +) -> Result { + run_taint_retry_loop_inner(store, worktree_id, |_attempt| {}).await } -fn run_taint_retry_loop_inner( - store: &MutationTraceStore<'_>, +async fn run_taint_retry_loop_inner( + store: &mut MutationTraceStore<'_>, worktree_id: &WorktreeId, mut after_load: F, ) -> Result @@ -557,7 +623,7 @@ where F: FnMut(u32), { for attempt in 0..MAX_CAS_RETRY_ATTEMPTS { - let Some(projection) = store.load_worktree(worktree_id, None, None)? else { + let Some(projection) = store.load_worktree(worktree_id, None, None).await? else { return Ok(false); }; after_load(attempt); @@ -575,7 +641,7 @@ where }); return Ok(currently_unhealthy); } - Some(transition) => match store.commit(&transition)? { + Some(transition) => match store.commit(&transition).await? { CasResult::Applied => return Ok(true), CasResult::Conflict => {} }, @@ -583,2346 +649,3 @@ where } Ok(false) } - -#[cfg(test)] -mod tests { - use std::cell::{Cell, RefCell}; - use std::collections::BTreeSet; - use std::collections::VecDeque; - use std::process::Command; - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::mpsc; - use std::thread; - use std::time::Duration; - - use super::*; - use crate::services::mutation_trace::runtime::external_taint::ExternalTaintMarker; - use crate::services::mutation_trace::runtime::git_snapshot::{ - resolve_git_dir, resolve_worktree_id, - }; - use crate::services::mutation_trace::runtime::worktree_lock::acquire_inner; - use crate::services::mutation_trace::store::encode_revision; - use crate::services::mutation_trace::types::{Attribution, EventKey, FailureKind, ScopeStatus}; - - static NEXT_TEST_DB_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_db_path(label: &str) -> std::path::PathBuf { - let id = NEXT_TEST_DB_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir() - .join(format!( - "sce-mutation-trace-coordinator-{label}-{}-{id}", - std::process::id() - )) - .join("agent-trace.db") - } - - fn test_db(label: &str) -> (RepositoryAgentTraceDb, std::path::PathBuf) { - let db_path = unique_test_db_path(label); - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test db should open"); - (db, db_path) - } - - fn remove_test_db(db_path: &std::path::Path) { - if let Some(parent) = db_path.parent() { - let _ = std::fs::remove_dir_all(parent); - } - } - - fn unique_test_repo(label: &str) -> std::path::PathBuf { - let id = NEXT_TEST_DB_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-mutation-trace-coordinator-repo-{label}-{}-{id}", - std::process::id() - )) - } - - fn init_repo(repo_root: &std::path::Path) { - std::fs::create_dir_all(repo_root).expect("repo root should be created"); - run_git(repo_root, &["init", "--quiet"]); - run_git(repo_root, &["config", "user.email", "test@example.com"]); - run_git(repo_root, &["config", "user.name", "Test"]); - } - - fn run_git(repo_root: &std::path::Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(repo_root) - .output() - .expect("git command should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - fn remove_test_repo(repo_root: &std::path::Path) { - let _ = std::fs::remove_dir_all(repo_root); - } - - enum FakeOutcome { - Succeed(TreeId), - Fail(String), - } - - struct FakeSnapshotCapture { - outcomes: RefCell>, - default_tree: TreeId, - capture_calls: Cell, - pin_calls: Cell, - } - - impl FakeSnapshotCapture { - fn new(default_tree: TreeId) -> Self { - Self { - outcomes: RefCell::new(VecDeque::new()), - default_tree, - capture_calls: Cell::new(0), - pin_calls: Cell::new(0), - } - } - - fn push_success(&self, tree: TreeId) { - self.outcomes - .borrow_mut() - .push_back(FakeOutcome::Succeed(tree)); - } - - fn push_failure(&self, message: &str) { - self.outcomes - .borrow_mut() - .push_back(FakeOutcome::Fail(message.to_string())); - } - - fn capture_call_count(&self) -> u32 { - self.capture_calls.get() - } - - fn pin_call_count(&self) -> u32 { - self.pin_calls.get() - } - } - - impl SnapshotCapture for FakeSnapshotCapture { - fn capture(&self) -> Result { - self.capture_calls.set(self.capture_calls.get() + 1); - match self.outcomes.borrow_mut().pop_front() { - Some(FakeOutcome::Succeed(tree)) => Ok(tree), - Some(FakeOutcome::Fail(message)) => Err(anyhow::anyhow!(message)), - None => Ok(self.default_tree.clone()), - } - } - - fn pin(&self, _worktree_id: &WorktreeId, _tree: &TreeId) -> Result<()> { - self.pin_calls.set(self.pin_calls.get() + 1); - Ok(()) - } - } - - struct HookedFailingCapture<'a> { - message: String, - hook: RefCell>>, - } - - impl<'a> HookedFailingCapture<'a> { - fn new(message: &str, hook: impl FnMut() + 'a) -> Self { - Self { - message: message.to_string(), - hook: RefCell::new(Some(Box::new(hook))), - } - } - } - - impl SnapshotCapture for HookedFailingCapture<'_> { - fn capture(&self) -> Result { - if let Some(hook) = self.hook.borrow_mut().as_mut() { - hook(); - } - Err(anyhow::anyhow!(self.message.clone())) - } - - fn pin(&self, _worktree_id: &WorktreeId, _tree: &TreeId) -> Result<()> { - Ok(()) - } - } - - fn commit_competing_advance( - store: &MutationTraceStore<'_>, - worktree: &WorktreeId, - scope: &ScopeId, - event_id: &str, - first: bool, - ) { - if first { - store - .register_scope(scope, worktree, ActorKind::ClaudeCode) - .expect("competing register_scope should succeed"); - } - - let loaded = store - .load_worktree(worktree, Some(scope), None) - .expect("competing load should succeed") - .expect("worktree should already be materialized") - .into_protocol_state(); - let cursor_tree = loaded.worktrees[worktree].cursor_tree.clone(); - let boundary = if first { - Boundary::Start { - scope: scope.clone(), - event: EventId(event_id.to_string()), - } - } else { - Boundary::Advance { - scope: scope.clone(), - event: EventId(event_id.to_string()), - } - }; - let attempt_id = AttemptId(format!("competing-{event_id}")); - let prepared = protocol::prepare(&loaded, attempt_id.clone(), boundary, cursor_tree); - let outcome = protocol::commit(&prepared, &attempt_id); - let transition = DurableTransition::between(&loaded, &outcome.state, worktree) - .expect("diff should succeed") - .expect("each competing advance should durably bump the revision"); - assert_eq!( - store - .commit(&transition) - .expect("competing commit should not error"), - CasResult::Applied, - "the competing commit should land before this iteration's own commit" - ); - } - - fn insert_worktree_at_revision( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - cursor_tree: &str, - revision: u64, - tainted: bool, - failure_kind: &str, - needs_rebaseline: bool, - ) { - db.execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - ( - worktree_id, - cursor_tree, - encode_revision(revision).as_slice(), - tainted, - failure_kind, - needs_rebaseline, - ), - ) - .expect("worktree insert should succeed"); - } - - #[test] - fn needs_recovery_reads_health_from_failure_kind_even_if_tainted_disagrees() { - let worktree_id = WorktreeId("wt0".to_string()); - let mut state = types::ProtocolState::default(); - state.worktrees.insert( - worktree_id.clone(), - types::WorktreeState { - cursor_tree: TreeId("tree0".to_string()), - revision: 1, - tainted: false, - failure_kind: FailureKind::SnapshotFailure, - needs_rebaseline: false, - }, - ); - - assert!(needs_recovery(&state, &worktree_id)); - } - - #[test] - fn first_observation_establishes_baseline_without_evidence() { - let (db, db_path) = test_db("ac1-first-observation"); - let worktree = WorktreeId("wt-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - let outcome = coordinate_boundary(&db, &capture, &worktree, &RuntimeBoundary::Flush, false) - .expect("first observation should succeed"); - - assert_eq!(outcome.observed_tree, TreeId("tree-a".to_string())); - assert_eq!( - outcome.revision, 0, - "a Flush that observes no change from the freshly established baseline must not advance the revision" - ); - assert!( - outcome.mutation_event.is_none(), - "no mutation evidence should be emitted for the worktree's first observation" - ); - assert_eq!(capture.capture_call_count(), 1); - assert_eq!(capture.pin_call_count(), 1); - - remove_test_db(&db_path); - } - - #[test] - fn exclusive_edit_between_start_and_advance_commits_one_event() { - let (db, db_path) = test_db("ac2-exclusive-edit"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("start should succeed"); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - false, - ) - .expect("advance should succeed"); - - let event = outcome - .mutation_event - .expect("an edit observed between Start and Advance should commit exactly one event"); - assert_eq!(event.before_tree, TreeId("tree-a".to_string())); - assert_eq!(event.after_tree, TreeId("tree-b".to_string())); - assert_eq!(event.attribution, Attribution::AiExclusive(scope)); - - remove_test_db(&db_path); - } - - #[test] - fn replaying_the_same_scope_event_key_does_not_duplicate_evidence() { - let (db, db_path) = test_db("ac3-replay"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("start should succeed"); - - let advance = RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }; - - capture.push_success(TreeId("tree-b".to_string())); - let first = coordinate_boundary(&db, &capture, &worktree, &advance, false) - .expect("first advance should commit"); - assert!(first.mutation_event.is_some()); - - capture.push_success(TreeId("tree-b".to_string())); - let replay = coordinate_boundary(&db, &capture, &worktree, &advance, false).expect( - "replaying the identical (scope, event) boundary must be a no-op, not an error", - ); - assert!( - replay.mutation_event.is_none(), - "replay must not duplicate mutation evidence" - ); - assert_eq!( - replay.revision, first.revision, - "replay must not advance the revision beyond the original commit" - ); - - remove_test_db(&db_path); - } - - #[test] - fn close_boundary_attributes_using_pre_close_scope_set() { - let (db, db_path) = test_db("ac4-close-attribution"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("start should succeed"); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: scope.clone(), - event: EventId("evt-close".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - false, - ) - .expect("close should succeed"); - - let event = outcome - .mutation_event - .expect("a real tree change observed at Close should still commit an event"); - assert_eq!( - event.attribution, - Attribution::AiExclusive(scope.clone()), - "the Close boundary's own emitted event must still attribute to the scope it is about to close" - ); - - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - projection.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::Closed) - ); - - remove_test_db(&db_path); - } - - fn assert_contended_attribution(label: &str, actor_a: ActorKind, actor_b: ActorKind) { - let (db, db_path) = test_db(label); - let worktree = WorktreeId("wt-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: ScopeId("scope-a".to_string()), - event: EventId("evt-start-a".to_string()), - actor_kind: actor_a, - provenance: None, - }, - false, - ) - .expect("starting scope a should succeed"); - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: ScopeId("scope-b".to_string()), - event: EventId("evt-start-b".to_string()), - actor_kind: actor_b, - provenance: None, - }, - false, - ) - .expect("starting scope b should succeed"); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: ScopeId("scope-b".to_string()), - event: EventId("evt-close-b".to_string()), - actor_kind: actor_b, - }, - false, - ) - .expect("close should succeed"); - - let event = outcome - .mutation_event - .expect("a real tree change with two live scopes should commit an event"); - assert_eq!(event.attribution, Attribution::AiContended); - - remove_test_db(&db_path); - } - - #[test] - fn contended_scopes_yield_ai_contended_same_and_different_actor() { - assert_contended_attribution( - "ac5-same-actor", - ActorKind::ClaudeCode, - ActorKind::ClaudeCode, - ); - assert_contended_attribution("ac5-different-actor", ActorKind::ClaudeCode, ActorKind::Pi); - } - - #[test] - fn an_unconfirmed_codex_scope_makes_another_harness_boundary_ineligible() { - let (db, db_path) = test_db("codex-unconfirmed-cross-harness"); - let worktree = WorktreeId("wt-1".to_string()); - let codex = ScopeId("codex-a".to_string()); - let claude = ScopeId("claude-c".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: codex.clone(), - event: EventId("evt-codex-start".to_string()), - actor_kind: ActorKind::Codex, - provenance: None, - }, - false, - ) - .expect("starting the Codex scope should succeed"); - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: claude.clone(), - event: EventId("evt-claude-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("starting the Claude scope should succeed"); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: claude.clone(), - event: EventId("evt-claude-close".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - false, - ) - .expect("closing the Claude scope should succeed"); - - let event = outcome - .mutation_event - .expect("a real tree change with two live scopes should still commit an event"); - assert_eq!( - event.active_scopes, - BTreeSet::from([codex.clone(), claude.clone()]) - ); - assert_eq!(event.attribution, Attribution::IneligibleUnscoped); - assert_ne!(event.attribution, Attribution::AiContended); - - capture.push_success(TreeId("tree-c".to_string())); - let confirmed = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: codex.clone(), - event: EventId("evt-codex-close".to_string()), - actor_kind: ActorKind::Codex, - }, - false, - ) - .expect("closing the Codex scope should succeed"); - - let confirmed_event = confirmed - .mutation_event - .expect("the Codex Close should commit its own observed change"); - assert_eq!(confirmed_event.active_scopes, BTreeSet::from([codex])); - assert_eq!( - confirmed_event.attribution, - Attribution::AiExclusive(ScopeId("codex-a".to_string())) - ); - - remove_test_db(&db_path); - } - - #[test] - fn a_confirming_codex_close_contends_with_a_live_non_codex_scope() { - let (db, db_path) = test_db("codex-confirmed-cross-harness"); - let worktree = WorktreeId("wt-1".to_string()); - let codex = ScopeId("codex-a".to_string()); - let claude = ScopeId("claude-c".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: codex.clone(), - event: EventId("evt-codex-start".to_string()), - actor_kind: ActorKind::Codex, - provenance: None, - }, - false, - ) - .expect("starting the Codex scope should succeed"); - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: claude.clone(), - event: EventId("evt-claude-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("starting the Claude scope should succeed"); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: codex.clone(), - event: EventId("evt-codex-close".to_string()), - actor_kind: ActorKind::Codex, - }, - false, - ) - .expect("closing the Codex scope should succeed"); - - let event = outcome - .mutation_event - .expect("a real tree change observed at the Codex Close should commit an event"); - assert_eq!(event.active_scopes, BTreeSet::from([codex, claude])); - assert_eq!(event.attribution, Attribution::AiContended); - - remove_test_db(&db_path); - } - - #[test] - fn an_unconfirmed_opencode_scope_makes_another_harness_boundary_ineligible() { - let (db, db_path) = test_db("opencode-unconfirmed-cross-harness"); - let worktree = WorktreeId("wt-1".to_string()); - let opencode = ScopeId("opencode-a".to_string()); - let claude = ScopeId("claude-c".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: opencode.clone(), - event: EventId("evt-opencode-start".to_string()), - actor_kind: ActorKind::OpenCode, - provenance: None, - }, - false, - ) - .expect("starting the OpenCode scope should succeed"); - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: claude.clone(), - event: EventId("evt-claude-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("starting the Claude scope should succeed"); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: claude.clone(), - event: EventId("evt-claude-close".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - false, - ) - .expect("closing the Claude scope should succeed"); - - let event = outcome - .mutation_event - .expect("a real tree change with two live scopes should still commit an event"); - assert_eq!( - event.active_scopes, - BTreeSet::from([opencode.clone(), claude.clone()]) - ); - assert_eq!(event.attribution, Attribution::IneligibleUnscoped); - assert_ne!(event.attribution, Attribution::AiContended); - - capture.push_success(TreeId("tree-c".to_string())); - let confirmed = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: opencode.clone(), - event: EventId("evt-opencode-close".to_string()), - actor_kind: ActorKind::OpenCode, - }, - false, - ) - .expect("closing the OpenCode scope should succeed"); - - let confirmed_event = confirmed - .mutation_event - .expect("the OpenCode Close should commit its own observed change"); - assert_eq!(confirmed_event.active_scopes, BTreeSet::from([opencode])); - assert_eq!( - confirmed_event.attribution, - Attribution::AiExclusive(ScopeId("opencode-a".to_string())) - ); - - remove_test_db(&db_path); - } - - #[test] - fn a_confirming_opencode_close_contends_with_a_live_non_opencode_scope() { - let (db, db_path) = test_db("opencode-confirmed-cross-harness"); - let worktree = WorktreeId("wt-1".to_string()); - let opencode = ScopeId("opencode-a".to_string()); - let claude = ScopeId("claude-c".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: opencode.clone(), - event: EventId("evt-opencode-start".to_string()), - actor_kind: ActorKind::OpenCode, - provenance: None, - }, - false, - ) - .expect("starting the OpenCode scope should succeed"); - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: claude.clone(), - event: EventId("evt-claude-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("starting the Claude scope should succeed"); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Close { - scope: opencode.clone(), - event: EventId("evt-opencode-close".to_string()), - actor_kind: ActorKind::OpenCode, - }, - false, - ) - .expect("closing the OpenCode scope should succeed"); - - let event = outcome - .mutation_event - .expect("a real tree change observed at the OpenCode Close should commit an event"); - assert_eq!(event.active_scopes, BTreeSet::from([opencode, claude])); - assert_eq!(event.attribution, Attribution::AiContended); - - remove_test_db(&db_path); - } - - #[test] - fn cas_conflict_reloads_and_recomputes_without_a_second_snapshot() { - const WRITERS: usize = 3; - - let db_path = unique_test_db_path("ac8-cas-conflict"); - let worktree = WorktreeId("wt-1".to_string()); - - { - let db = RepositoryAgentTraceDb::new_at(&db_path).expect("test db should open"); - let bootstrap_capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - coordinate_boundary( - &db, - &bootstrap_capture, - &worktree, - &RuntimeBoundary::Flush, - false, - ) - .expect("baseline flush should succeed"); - } - - let barrier = std::sync::Arc::new(std::sync::Barrier::new(WRITERS)); - let mut handles = Vec::new(); - for i in 0..WRITERS { - let db_path = db_path.clone(); - let worktree = worktree.clone(); - let barrier = std::sync::Arc::clone(&barrier); - handles.push(std::thread::spawn(move || { - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - .expect("writer handle should open"); - let capture = FakeSnapshotCapture::new(TreeId(format!("tree-writer-{i}"))); - barrier.wait(); - let outcome = - coordinate_boundary(&db, &capture, &worktree, &RuntimeBoundary::Flush, false) - .expect( - "each racing writer should eventually succeed after reload+recompute", - ); - ( - outcome.revision, - capture.capture_call_count(), - capture.pin_call_count(), - ) - })); - } - - let mut revisions = Vec::new(); - for handle in handles { - let (revision, captures, pins) = handle.join().expect("writer thread should not panic"); - assert_eq!( - captures, 1, - "each invocation must capture its Git snapshot exactly once, even across CAS retries" - ); - assert_eq!( - pins, 1, - "each invocation must pin exactly once, even across CAS retries" - ); - revisions.push(revision); - } - - revisions.sort_unstable(); - revisions.dedup(); - assert_eq!( - revisions.len(), - WRITERS, - "each racing writer's commit must land at a distinct revision, proving reload+recompute rather than a lost update" - ); - - remove_test_db(&db_path); - } - - #[test] - fn recovers_from_needs_rebaseline_preserving_live_scopes() { - let (db, db_path) = test_db("ac10-needs-rebaseline"); - let worktree = WorktreeId("wt-1".to_string()); - let live_scope = ScopeId("scope-live".to_string()); - let abandoned_scope = ScopeId("scope-abandoned".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: live_scope.clone(), - event: EventId("evt-start-live".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("starting the live scope should succeed"); - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: abandoned_scope.clone(), - event: EventId("evt-start-abandoned".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("starting the to-be-abandoned scope should succeed"); - - let store = MutationTraceStore::new(&db); - let state = store - .load_worktree(&worktree, Some(&abandoned_scope), None) - .expect("load should succeed") - .expect("worktree should exist") - .into_protocol_state(); - let abandoned_state = protocol::abandon(&state, &abandoned_scope); - let transition = DurableTransition::between(&state, &abandoned_state, &worktree) - .expect("diff should succeed") - .expect("abandon should durably change state"); - assert_eq!( - store - .commit(&transition) - .expect("abandon commit should not error"), - CasResult::Applied - ); - - let before_recovery = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!(before_recovery.worktree_state.needs_rebaseline); - - capture.push_success(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Advance { - scope: live_scope.clone(), - event: EventId("evt-advance-live".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - false, - ) - .expect("advance should trigger needs_rebaseline recovery first, then succeed"); - assert!( - outcome.mutation_event.is_none(), - "no evidence should be emitted for the just-discarded interval recovery rebaselined away" - ); - - let after = store - .load_worktree(&worktree, Some(&live_scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!( - !after.worktree_state.needs_rebaseline, - "recovery should clear needs_rebaseline" - ); - assert_eq!( - after.worktree_state.cursor_tree, - TreeId("tree-b".to_string()), - "recovery should rebaseline the cursor to the recovering invocation's own observed tree" - ); - assert_eq!( - after.scopes.get(&live_scope).map(|s| s.status), - Some(ScopeStatus::Active), - "needs_rebaseline recovery must preserve live scopes" - ); - - remove_test_db(&db_path); - } - - #[test] - fn recovers_from_snapshot_failure_taint_abandoning_live_scopes() { - let (db, db_path) = test_db("ac10-taint-recovery"); - let worktree = WorktreeId("wt-1".to_string()); - let live_scope = ScopeId("scope-live".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: live_scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("start should succeed"); - - let store = MutationTraceStore::new(&db); - let state = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("worktree should exist") - .into_protocol_state(); - let tainted_state = protocol::taint(&state, &worktree); - let transition = DurableTransition::between(&state, &tainted_state, &worktree) - .expect("diff should succeed") - .expect("taint should durably change state"); - assert_eq!( - store - .commit(&transition) - .expect("taint commit should not error"), - CasResult::Applied - ); - - let before_recovery = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!(before_recovery.worktree_state.tainted); - - capture.push_success(TreeId("tree-b".to_string())); - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Advance { - scope: live_scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - false, - ) - .expect("advance should trigger taint recovery first, then succeed"); - - let after = store - .load_worktree(&worktree, Some(&live_scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!( - !after.worktree_state.tainted, - "recovery should clear the taint" - ); - assert_eq!( - after.scopes.get(&live_scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "taint recovery must abandon live scopes, unlike needs_rebaseline recovery" - ); - - remove_test_db(&db_path); - } - - fn assert_recovery_at_revision_exhaustion_is_rejected( - label: &str, - tainted: bool, - failure_kind: &str, - needs_rebaseline: bool, - ) { - let (db, db_path) = test_db(label); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let event = EventId("evt-start".to_string()); - - insert_worktree_at_revision( - &db, - &worktree.0, - "tree-a", - u64::MAX, - tainted, - failure_kind, - needs_rebaseline, - ); - - let capture = FakeSnapshotCapture::new(TreeId("tree-b".to_string())); - let error = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: event.clone(), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect_err("recovery that cannot advance revision must reject the triggering boundary"); - - match error { - CoordinateError::RevisionExhausted { - worktree_id, - revision, - } => { - assert_eq!(worktree_id, worktree); - assert_eq!(revision, u64::MAX); - } - other => panic!("expected RevisionExhausted, got {other:?}"), - } - - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!(projection.worktree_state.revision, u64::MAX); - assert_eq!(projection.worktree_state.tainted, tainted); - assert_eq!(projection.worktree_state.needs_rebaseline, needs_rebaseline); - assert_eq!( - projection.worktree_state.cursor_tree, - TreeId("tree-a".to_string()), - "the cursor must never move when the triggering boundary was never evaluated" - ); - assert_eq!( - projection.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::NeverSeen), - "the triggering boundary must never have transitioned the scope" - ); - assert!( - projection.processed_events.is_empty(), - "the triggering boundary's event must never have been recorded as processed" - ); - - remove_test_db(&db_path); - } - - #[test] - fn start_registers_provenance_after_its_owning_scope_and_before_the_protocol_commits() { - let (db, db_path) = test_db("provenance-start-ordering"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::Codex, - provenance: Some(StartProvenance { - session_id: "cx_session-1".to_string(), - model_id: Some("gpt-5-codex".to_string()), - }), - }, - false, - ) - .expect("a start carrying provenance for a first-seen scope should succeed"); - - let store = MutationTraceStore::new(&db); - assert_eq!( - store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"), - Some(ScopeProvenance { - scope_id: scope.clone(), - session_id: "cx_session-1".to_string(), - model_id: Some("gpt-5-codex".to_string()), - }), - "provenance is registered against the scope row the same Start just created" - ); - - let projection = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - projection.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::Active), - "the pure protocol Start must have committed after provenance was registered" - ); - - remove_test_db(&db_path); - } - - #[test] - fn start_without_provenance_then_replay_with_provenance_does_not_backfill() { - let (db, db_path) = test_db("provenance-no-late-backfill"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let event = EventId("evt-start".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: event.clone(), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("a start without provenance should succeed"); - - let store = MutationTraceStore::new(&db); - let admitted = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - admitted.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::Active), - "the provenance-free start must have admitted the scope" - ); - assert_eq!( - store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"), - None, - "a start without provenance persists no row" - ); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: event.clone(), - actor_kind: ActorKind::ClaudeCode, - provenance: Some(StartProvenance { - session_id: "cc_session-1".to_string(), - model_id: Some("claude/opus".to_string()), - }), - }, - false, - ) - .expect("replaying the start with provenance follows normal replay semantics"); - - assert_eq!( - store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"), - None, - "provenance is admission-bounded: an already-admitted scope never gains it later" - ); - - let replayed = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - replayed.worktree_state.revision, admitted.worktree_state.revision, - "a replay must not advance the worktree revision" - ); - assert_eq!( - replayed.processed_events, admitted.processed_events, - "a replay must not change the processed-event set" - ); - assert_eq!( - replayed.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::Active) - ); - - remove_test_db(&db_path); - } - - #[test] - fn never_seen_scope_can_receive_provenance_before_successful_start() { - let (db, db_path) = test_db("provenance-retry-before-admission"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - let store = MutationTraceStore::new(&db); - store - .initialize_worktree(&worktree, &TreeId("tree-a".to_string())) - .expect("worktree initialization should succeed"); - let seeded = store - .register_scope(&scope, &worktree, ActorKind::ClaudeCode) - .expect("seeding the owning scope row should succeed"); - assert_eq!( - seeded.status, - ScopeStatus::NeverSeen, - "the seeded scope stands in for a first attempt that never reached the protocol" - ); - assert_eq!( - store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"), - None - ); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: Some(StartProvenance { - session_id: "cc_session-1".to_string(), - model_id: Some("claude/opus".to_string()), - }), - }, - false, - ) - .expect("retrying a start against a still-NeverSeen scope should succeed"); - - assert_eq!( - store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"), - Some(ScopeProvenance { - scope_id: scope.clone(), - session_id: "cc_session-1".to_string(), - model_id: Some("claude/opus".to_string()), - }), - "a scope that has not crossed admission may still gain provenance" - ); - - let projection = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - projection.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::Active), - "the retried start must commit the protocol transition" - ); - - remove_test_db(&db_path); - } - - #[test] - fn an_admitted_scope_with_provenance_still_rejects_a_different_session() { - let (db, db_path) = test_db("provenance-admitted-session-conflict"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let event = EventId("evt-start".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: event.clone(), - actor_kind: ActorKind::ClaudeCode, - provenance: Some(StartProvenance { - session_id: "cc_session-1".to_string(), - model_id: Some("claude/opus".to_string()), - }), - }, - false, - ) - .expect("the admitting start should succeed"); - - let store = MutationTraceStore::new(&db); - let admitted = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - admitted.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::Active) - ); - let stored = store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"); - - let failure = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event, - actor_kind: ActorKind::ClaudeCode, - provenance: Some(StartProvenance { - session_id: "cc_session-2".to_string(), - model_id: Some("claude/opus".to_string()), - }), - }, - false, - ) - .expect_err("an existing provenance row is checked even after admission"); - - assert!( - matches!(failure, CoordinateError::ScopeProvenanceRegistration(_)), - "unexpected error: {failure:?}" - ); - - assert_eq!( - store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"), - stored, - "a rejected replay must never rewrite the stored provenance row" - ); - - let after = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - after.worktree_state.revision, - admitted.worktree_state.revision - ); - assert_eq!(after.processed_events, admitted.processed_events); - - remove_test_db(&db_path); - } - - #[test] - fn a_provenance_registration_failure_rejects_the_start_before_the_protocol_commits() { - let (db, db_path) = test_db("provenance-registration-failure"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - let store = MutationTraceStore::new(&db); - store - .initialize_worktree(&worktree, &TreeId("tree-a".to_string())) - .expect("worktree initialization should succeed"); - store - .register_scope(&scope, &worktree, ActorKind::ClaudeCode) - .expect("seeding the owning scope row should succeed"); - let seeded = store - .register_scope_provenance(&ScopeProvenance { - scope_id: scope.clone(), - session_id: "cc_session-1".to_string(), - model_id: Some("claude/opus".to_string()), - }) - .expect("seeding provenance should succeed"); - - let failure = coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: Some(StartProvenance { - session_id: "cc_session-2".to_string(), - model_id: Some("claude/opus".to_string()), - }), - }, - false, - ) - .expect_err("a conflicting provenance session must reject the start"); - - assert!( - matches!(failure, CoordinateError::ScopeProvenanceRegistration(_)), - "unexpected error: {failure:?}" - ); - - assert_eq!( - store - .load_scope_provenance(&scope) - .expect("provenance load should succeed"), - Some(seeded), - "a rejected start must never rewrite the stored provenance row" - ); - - let projection = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - projection.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::NeverSeen), - "a failed provenance registration leaves at most the owning NeverSeen scope row" - ); - assert!( - projection.processed_events.is_empty(), - "the rejected start's event must never have been recorded as processed" - ); - assert_eq!(projection.worktree_state.revision, 0); - - remove_test_db(&db_path); - } - - #[test] - fn mandatory_recovery_that_cannot_advance_revision_rejects_the_triggering_boundary() { - assert_recovery_at_revision_exhaustion_is_rejected( - "revision-exhausted-tainted", - true, - "snapshot_failure", - false, - ); - assert_recovery_at_revision_exhaustion_is_rejected( - "revision-exhausted-needs-rebaseline", - false, - "healthy", - true, - ); - } - - #[test] - fn snapshot_failure_taints_an_existing_worktree() { - let (db, db_path) = test_db("ac11-taints-existing"); - let worktree = WorktreeId("wt-1".to_string()); - let bootstrap = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - coordinate_boundary(&db, &bootstrap, &worktree, &RuntimeBoundary::Flush, false) - .expect("baseline flush should materialize the worktree"); - - let failing = FakeSnapshotCapture::new(TreeId("tree-b".to_string())); - failing.push_failure("simulated git snapshot failure"); - - let error = coordinate_boundary(&db, &failing, &worktree, &RuntimeBoundary::Flush, false) - .expect_err("a capture failure against an existing worktree should be reported"); - match error { - CoordinateError::SnapshotFailure { - persisted_taint, .. - } => assert!(persisted_taint), - other => panic!("expected SnapshotFailure, got {other:?}"), - } - - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!(projection.worktree_state.tainted); - assert_eq!( - projection.worktree_state.failure_kind, - FailureKind::SnapshotFailure - ); - - remove_test_db(&db_path); - } - - #[test] - fn snapshot_failure_taint_survives_a_losing_cas_and_commits_on_retry() { - let (db, db_path) = test_db("ac11-taint-retry-succeeds"); - let worktree = WorktreeId("wt-1".to_string()); - let bootstrap = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - coordinate_boundary(&db, &bootstrap, &worktree, &RuntimeBoundary::Flush, false) - .expect("baseline flush should materialize the worktree"); - - let store = MutationTraceStore::new(&db); - let competing_scope = ScopeId("competing-scope".to_string()); - let mut interfered = false; - let persisted_taint = run_taint_retry_loop_inner(&store, &worktree, |attempt| { - if attempt == 0 && !interfered { - interfered = true; - commit_competing_advance( - &store, - &worktree, - &competing_scope, - "competing-event-1", - true, - ); - } - }) - .expect("taint retry loop should not error"); - - assert!( - persisted_taint, - "the taint should eventually be persisted after reloading past the losing CAS" - ); - - let projection = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!(projection.worktree_state.tainted); - assert_eq!( - projection.worktree_state.failure_kind, - FailureKind::SnapshotFailure - ); - - remove_test_db(&db_path); - } - - #[test] - fn snapshot_failure_taint_reports_not_persisted_after_retries_are_exhausted() { - let (db, db_path) = test_db("ac11-taint-exhaustion"); - let worktree = WorktreeId("wt-1".to_string()); - let bootstrap = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - coordinate_boundary(&db, &bootstrap, &worktree, &RuntimeBoundary::Flush, false) - .expect("baseline flush should materialize the worktree"); - - let store = MutationTraceStore::new(&db); - let competing_scope = ScopeId("competing-scope".to_string()); - let mut competing_calls = 0u32; - let persisted_taint = run_taint_retry_loop_inner(&store, &worktree, |_attempt| { - competing_calls += 1; - commit_competing_advance( - &store, - &worktree, - &competing_scope, - &format!("competing-event-{competing_calls}"), - competing_calls == 1, - ); - }) - .expect("taint retry loop should not error even when every attempt conflicts"); - - assert!( - !persisted_taint, - "taint must not be reported as persisted once every bounded attempt has conflicted" - ); - - let projection = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!( - !projection.worktree_state.tainted, - "the worktree must remain untainted when the taint itself was never actually committed" - ); - - remove_test_db(&db_path); - } - - #[test] - fn snapshot_failure_before_any_baseline_makes_no_durable_write() { - let (db, db_path) = test_db("ac11-bootstrap-failure"); - let worktree = WorktreeId("wt-1".to_string()); - let failing = FakeSnapshotCapture::new(TreeId("unused".to_string())); - failing.push_failure("simulated git snapshot failure before any baseline exists"); - - let error = coordinate_boundary(&db, &failing, &worktree, &RuntimeBoundary::Flush, false) - .expect_err("a capture failure with no prior worktree row should still be reported"); - match error { - CoordinateError::SnapshotFailure { - persisted_taint, .. - } => assert!(!persisted_taint), - other => panic!("expected SnapshotFailure, got {other:?}"), - } - - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree, None, None) - .expect("load should succeed"); - assert!( - projection.is_none(), - "no durable worktree row should ever be created for a bootstrap-time snapshot failure" - ); - - remove_test_db(&db_path); - } - - #[test] - fn snapshot_failure_taints_a_worktree_materialized_concurrently_during_capture() { - let (db, db_path) = test_db("ac11-concurrent-materialization"); - let worktree = WorktreeId("wt-1".to_string()); - - let capture = HookedFailingCapture::new( - "simulated git snapshot failure racing a concurrent materialization", - || { - let store = MutationTraceStore::new(&db); - store - .initialize_worktree( - &worktree, - &TreeId("tree-materialized-concurrently".to_string()), - ) - .expect("concurrent materialization should succeed"); - }, - ); - - let error = coordinate_boundary(&db, &capture, &worktree, &RuntimeBoundary::Flush, false) - .expect_err( - "a capture failure racing a concurrent materialization should still be reported", - ); - match error { - CoordinateError::SnapshotFailure { persisted_taint, .. } => assert!( - persisted_taint, - "the failure handler's fresh, post-failure read must still find and taint the concurrently materialized worktree" - ), - other => panic!("expected SnapshotFailure, got {other:?}"), - } - - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!(projection.worktree_state.tainted); - - remove_test_db(&db_path); - } - - #[test] - fn two_threads_on_the_same_worktree_serialize() { - let repo_root = unique_test_repo("t05-serialize"); - init_repo(&repo_root); - let db_path = repo_root.join("agent-trace.db"); - - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let held = acquire_inner(&git_dir, Duration::from_secs(5), || {}) - .expect("the test should hold a real WorktreeLock before the worker runs"); - - let (contention_tx, contention_rx) = mpsc::channel(); - let (result_tx, result_rx) = mpsc::channel(); - let repo_root_clone = repo_root.clone(); - let db_path_clone = db_path.clone(); - let worker = thread::spawn(move || { - let outcome = coordinate_inner( - &repo_root_clone, - &RuntimeBoundary::Flush, - || RepositoryAgentTraceDb::new_at(&db_path_clone), - move || { - contention_tx - .send(()) - .expect("contention signal channel should still be open"); - }, - |_attempt| {}, - |_attempt| Ok(()), - ); - result_tx - .send(()) - .expect("result signal channel should still be open"); - outcome - }); - - contention_rx.recv_timeout(Duration::from_secs(5)).expect( - "coordinate() should reach the WorktreeLock try_lock loop and observe \ - TryLockError::WouldBlock while the first guard is still held", - ); - - assert!( - result_rx.recv_timeout(Duration::from_millis(300)).is_err(), - "the worker's coordinate() call must not complete while the first worktree lock guard is still held" - ); - - drop(held); - - result_rx.recv_timeout(Duration::from_secs(5)).expect( - "the same coordinate() invocation should complete once the first guard is released", - ); - - let outcome = worker - .join() - .expect("worker thread should not panic") - .expect( - "coordinate() should succeed once it can acquire the worktree lock after release", - ); - assert_eq!( - outcome.revision, 0, - "the worker's first-observation flush should not advance the revision" - ); - - remove_test_repo(&repo_root); - } - - #[test] - fn public_coordinate_clears_marker_on_success() { - let repo_root = unique_test_repo("t02-success-clears-marker"); - init_repo(&repo_root); - let db_path = repo_root.join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("seed db should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - - let outcome = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("a first observation should succeed"); - assert_eq!( - outcome.revision, 0, - "a first-observation flush should not advance the revision" - ); - assert!( - !marker.exists().expect("marker existence should resolve"), - "a successful coordinate() must clear the marker it armed" - ); - - remove_test_repo(&repo_root); - } - - #[test] - fn public_coordinate_leaves_marker_after_a_snapshot_failure() { - let repo_root = unique_test_repo("t02-snapshot-failure-marker"); - init_repo(&repo_root); - let db_path = repo_root.join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("seed db should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - - coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect("the baseline observation should succeed"); - assert!( - !marker.exists().expect("marker existence should resolve"), - "the successful baseline must have cleared its marker" - ); - - let tmp_index_dir = git_dir.join("sce").join("tmp"); - let _ = std::fs::remove_dir_all(&tmp_index_dir); - std::fs::write(&tmp_index_dir, b"not a directory").expect( - "planting a file where the snapshot service expects its temp-index directory should succeed", - ); - - let error = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect_err("a Git snapshot failure after marker arming should be reported"); - assert!( - matches!(error, CoordinateError::SnapshotFailure { .. }), - "expected SnapshotFailure, got {error:?}" - ); - assert!( - marker.exists().expect("marker existence should resolve"), - "a snapshot failure after arming must leave the external-taint marker in place" - ); - - remove_test_repo(&repo_root); - } - - #[test] - fn public_coordinate_leaves_marker_after_a_non_snapshot_failure() { - let repo_root = unique_test_repo("t02-non-snapshot-failure-marker"); - init_repo(&repo_root); - let db_path = repo_root.join("agent-trace.db"); - let seed_db = RepositoryAgentTraceDb::new_at(&db_path).expect("seed db should open"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let worktree_id = resolve_worktree_id(&repo_root).expect("worktree id should resolve"); - insert_worktree_at_revision( - &seed_db, - &worktree_id.0, - "tree-a", - u64::MAX, - true, - "snapshot_failure", - false, - ); - drop(seed_db); - - let marker = ExternalTaintMarker::new(&git_dir); - let error = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect_err("mandatory recovery that cannot advance the revision must reject the boundary"); - assert!( - matches!(error, CoordinateError::RevisionExhausted { .. }), - "expected RevisionExhausted, got {error:?}" - ); - assert!( - marker.exists().expect("marker existence should resolve"), - "a non-snapshot failure after arming must leave the external-taint marker in place" - ); - - remove_test_repo(&repo_root); - } - - #[test] - fn public_coordinate_fails_closed_when_the_marker_cannot_be_armed() { - let repo_root = unique_test_repo("t02-marker-arm-failure"); - init_repo(&repo_root); - let db_path = repo_root.join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("seed db should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - - // Plant a directory exactly where the marker file must be created, so - // `ExternalTaintMarker::persist` fails deterministically regardless of uid. - std::fs::create_dir_all(git_dir.join("sce").join("mutation-cursor-tainted")) - .expect("planting a directory at the marker path should succeed"); - - let provider_called = std::sync::atomic::AtomicBool::new(false); - let error = coordinate(&repo_root, &RuntimeBoundary::Flush, || { - provider_called.store(true, Ordering::SeqCst); - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }) - .expect_err("an unarmed marker must fail coordinate() closed"); - assert!( - matches!( - error, - CoordinateError::ExternalTaintMarker { - operation: ExternalTaintOperation::Persist, - .. - } - ), - "expected an ExternalTaintMarker persist failure, got {error:?}" - ); - assert!( - !provider_called.load(Ordering::SeqCst), - "a marker-arming failure must return before DB-provider, snapshot, or protocol work" - ); - - remove_test_repo(&repo_root); - } - - #[test] - fn public_coordinate_fails_closed_when_marker_inspection_fails() { - let repo_root = unique_test_repo("t02-marker-inspect-failure"); - init_repo(&repo_root); - let db_path = repo_root.join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("seed db should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let sce_dir = git_dir.join("sce"); - std::fs::create_dir_all(&sce_dir).expect("the sce directory should be creatable"); - - let held = acquire_inner(&git_dir, Duration::from_secs(5), || {}) - .expect("the test should hold the runtime lock before the worker runs"); - - let provider_called = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let (contention_tx, contention_rx) = mpsc::channel(); - - let worker = { - let repo_root = repo_root.clone(); - let db_path = db_path.clone(); - let provider_called = std::sync::Arc::clone(&provider_called); - thread::spawn(move || { - coordinate_inner( - &repo_root, - &RuntimeBoundary::Flush, - || { - provider_called.store(true, Ordering::SeqCst); - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path) - }, - move || { - contention_tx - .send(()) - .expect("contention signal channel should still be open"); - }, - |_attempt| {}, - |_attempt| Ok(()), - ) - }) - }; - - contention_rx.recv_timeout(Duration::from_secs(5)).expect( - "the worker's coordinate_inner should reach the WorktreeLock contention branch", - ); - - std::fs::remove_dir_all(&sce_dir).expect("the sce directory should be removable"); - std::fs::write(&sce_dir, b"not a directory") - .expect("planting a file where the sce directory was should succeed"); - - drop(held); - - let error = worker - .join() - .expect("worker thread should not panic") - .expect_err("marker inspection failure must fail coordinate() closed"); - assert!( - matches!( - error, - CoordinateError::ExternalTaintMarker { - operation: ExternalTaintOperation::Inspect, - .. - } - ), - "expected an ExternalTaintMarker inspect failure, got {error:?}" - ); - assert!( - !provider_called.load(Ordering::SeqCst), - "marker inspection failing closed must precede DB-provider, checkout, snapshot, and protocol work" - ); - - remove_test_repo(&repo_root); - } - - #[test] - fn public_coordinate_leaves_marker_when_the_db_provider_fails() { - let repo_root = unique_test_repo("t02-db-provider-failure-marker"); - init_repo(&repo_root); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - - let error = coordinate( - &repo_root, - &RuntimeBoundary::Flush, - || -> anyhow::Result { - Err(anyhow::anyhow!("simulated Agent Trace DB open failure")) - }, - ) - .expect_err("a DB provider that returns Err must fail coordinate()"); - assert!( - matches!(error, CoordinateError::AgentTraceDbUnavailable(_)), - "expected AgentTraceDbUnavailable, got {error:?}" - ); - assert!( - marker.exists().expect("marker existence should resolve"), - "a DB-provider failure after arming must leave the external-taint marker present" - ); - - remove_test_repo(&repo_root); - } - - #[test] - fn inherited_external_taint_recovers_once_before_the_boundary() { - let (db, db_path) = test_db("t03-inherited-recovers-once"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-live".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &capture, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("start should establish the baseline and the live scope"); - - let advance_capture = FakeSnapshotCapture::new(TreeId("tree-b".to_string())); - let outcome = coordinate_boundary( - &db, - &advance_capture, - &worktree, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - true, - ) - .expect("an inherited external-taint marker must recover, then process the boundary"); - - assert!( - outcome.mutation_event.is_none(), - "no mutation evidence may span the interval the inherited marker fenced off" - ); - assert_eq!( - outcome.revision, 3, - "one recovery transition (rev 1 -> 2), then the triggering boundary (rev 2 -> 3)" - ); - assert_eq!( - advance_capture.capture_call_count(), - 1, - "recovery and the triggering boundary must share the single already-captured snapshot" - ); - assert_eq!(advance_capture.pin_call_count(), 1); - - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!(!projection.worktree_state.tainted); - assert!(!projection.worktree_state.needs_rebaseline); - assert_eq!(projection.worktree_state.failure_kind, FailureKind::Healthy); - assert_eq!( - projection.worktree_state.cursor_tree, - TreeId("tree-b".to_string()), - "recovery rebaselines the cursor to this invocation's observed tree" - ); - assert_eq!( - projection.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "inherited-taint recovery abandons the live scopes the fenced interval made untrustworthy" - ); - - remove_test_db(&db_path); - } - - #[test] - fn inherited_external_taint_with_no_worktree_row_baselines_without_evidence() { - let (db, db_path) = test_db("t03-inherited-no-row"); - let worktree = WorktreeId("wt-1".to_string()); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - let outcome = coordinate_boundary(&db, &capture, &worktree, &RuntimeBoundary::Flush, true) - .expect("a first-ever invocation carrying an inherited marker must still succeed"); - - assert!( - outcome.mutation_event.is_none(), - "a worktree with no prior durable row cannot produce evidence for the unknown interval" - ); - assert_eq!( - outcome.revision, 1, - "the freshly initialized worktree is baselined against the observed tree, then conservatively recovered once" - ); - - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree, None, None) - .expect("load should succeed") - .expect("the worktree row should now exist"); - assert!(!projection.worktree_state.tainted); - assert_eq!(projection.worktree_state.failure_kind, FailureKind::Healthy); - assert_eq!( - projection.worktree_state.cursor_tree, - TreeId("tree-a".to_string()) - ); - - remove_test_db(&db_path); - } - - #[test] - fn a_losing_recovery_cas_reinjects_external_taint_until_it_applies() { - let (db, db_path) = test_db("t03-recovery-cas-reinjection"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-live".to_string()); - let bootstrap = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &bootstrap, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("bootstrap start should establish the baseline and live scope"); - - let store = MutationTraceStore::new(&db); - let competing_scope = ScopeId("competing-scope".to_string()); - let interfered = Cell::new(false); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - let outcome = coordinate_boundary_inner( - &db, - &capture, - &worktree, - &RuntimeBoundary::Flush, - true, - |attempt| { - if attempt == 0 && !interfered.get() { - interfered.set(true); - commit_competing_advance( - &store, - &worktree, - &competing_scope, - "competing-event-1", - true, - ); - } - }, - |_attempt| Ok(()), - ) - .expect("recovery must recompute past the losing CAS and still succeed"); - - assert_eq!( - capture.capture_call_count(), - 1, - "a recovery CAS conflict must not trigger a second Git snapshot" - ); - assert_eq!( - outcome.revision, 3, - "bootstrap start (1) + competing advance (2) + exactly one landed recovery (3)" - ); - - let projection = store - .load_worktree(&worktree, Some(&scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert!(!projection.worktree_state.tainted); - assert_eq!( - projection.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "the re-injected recovery still abandons the fenced-off live scope" - ); - - let competing_projection = store - .load_worktree(&worktree, Some(&competing_scope), None) - .expect("load should succeed") - .expect("worktree should exist"); - assert_eq!( - competing_projection - .scopes - .get(&competing_scope) - .map(|s| s.status), - Some(ScopeStatus::Abandoned) - ); - - remove_test_db(&db_path); - } - - #[test] - fn a_landed_recovery_clears_the_flag_so_a_boundary_cas_retry_does_not_re_recover() { - let (db, db_path) = test_db("t03-flag-clears-after-recovery"); - let worktree = WorktreeId("wt-1".to_string()); - let scope = ScopeId("scope-live".to_string()); - let bootstrap = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - - coordinate_boundary( - &db, - &bootstrap, - &worktree, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - false, - ) - .expect("bootstrap start should establish the baseline and live scope"); - - let store = MutationTraceStore::new(&db); - let competing_scope = ScopeId("competing-scope".to_string()); - let interfered = Cell::new(false); - let capture = FakeSnapshotCapture::new(TreeId("tree-a".to_string())); - capture.push_success(TreeId("tree-b".to_string())); - - let outcome = coordinate_boundary_inner( - &db, - &capture, - &worktree, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - true, - |_attempt| {}, - |attempt| { - if attempt == 0 && !interfered.get() { - interfered.set(true); - commit_competing_advance( - &store, - &worktree, - &competing_scope, - "competing-event-1", - true, - ); - } - Ok(()) - }, - ) - .expect("the boundary CAS retry after a landed recovery must still succeed"); - - assert_eq!( - capture.capture_call_count(), - 1, - "neither the recovery nor the boundary retry may take a second snapshot" - ); - assert_eq!( - outcome.revision, 4, - "start (1) + one landed recovery (2) + competing advance (3) + the retried boundary (4); a re-triggered recovery on the retry would land at 5" - ); - - remove_test_db(&db_path); - } - - #[test] - #[allow(clippy::too_many_lines)] - fn a_failure_after_recovery_before_boundary_commit_leaves_marker_and_forces_later_recovery() { - let repo_root = unique_test_repo("t-ac8-recovery-then-fail"); - init_repo(&repo_root); - let db_path = unique_test_db_path("t-ac8-recovery-then-fail"); - RepositoryAgentTraceDb::new_at(&db_path).expect("seed db should open with schema"); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - let ok_db = || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path); - - std::fs::write(repo_root.join("work.txt"), b"a").expect("the baseline edit should write"); - let baseline = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should establish cursor A"); - let worktree_id = baseline.worktree_id.clone(); - let tree_a = baseline.observed_tree.clone(); - - let scope = ScopeId("scope-live".to_string()); - coordinate( - &repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting the live scope should succeed"); - - marker.persist().expect( - "simulating a prior crashed invocation that armed but never cleared the marker", - ); - std::fs::write(repo_root.join("work.txt"), b"b").expect("the A -> B edit should write"); - - let advance_event = EventId("evt-advance".to_string()); - let event_key = EventKey { - scope_id: scope.clone(), - event_id: advance_event.clone(), - }; - - let error = coordinate_inner( - &repo_root, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: advance_event.clone(), - actor_kind: ActorKind::ClaudeCode, - }, - ok_db, - || {}, - |_attempt| {}, - |_attempt| { - anyhow::bail!("injected failure after recovery, before the boundary commits") - }, - ) - .expect_err("the injected post-recovery failure must fail the invocation"); - assert!( - matches!(error, CoordinateError::Other(_)), - "expected CoordinateError::Other from the injected failure, got {error:?}" - ); - assert!( - marker.exists().expect("marker existence should resolve"), - "a failure after recovery but before the boundary commits must leave the marker armed" - ); - - let store_db = ok_db().expect("reopening the DB for assertions should succeed"); - let store = MutationTraceStore::new(&store_db); - let after_fail = store - .load_worktree(&worktree_id, Some(&scope), Some(&event_key)) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert!( - !after_fail.worktree_state.tainted, - "the recovery CAS committed durably before the injected failure" - ); - assert_eq!( - after_fail.worktree_state.failure_kind, - FailureKind::Healthy, - "recovery cleared the failure state before the injected failure" - ); - assert!( - after_fail.worktree_state.revision >= 2, - "the durable recovery advanced the revision past the start boundary" - ); - assert_ne!( - after_fail.worktree_state.cursor_tree, tree_a, - "recovery rebaselined the cursor away from A to the invocation's own observed tree" - ); - assert_eq!( - after_fail.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "the live scope was abandoned by the durable recovery" - ); - assert!( - !after_fail.processed_events.contains(&event_key), - "the triggering Advance must never have been processed" - ); - assert!( - store - .load_mutation_event(&worktree_id, after_fail.worktree_state.revision) - .expect("loading a mutation event should succeed") - .is_none(), - "no MutationEvent may be emitted for the boundary that never committed" - ); - drop(store_db); - - std::fs::write(repo_root.join("work.txt"), b"c").expect("the B -> C edit should write"); - let recovered = coordinate(&repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the later invocation inherits the still-armed marker and recovers again"); - assert!( - recovered.mutation_event.is_none(), - "no evidence may cross the interval the still-armed marker fenced off" - ); - assert_ne!( - recovered.observed_tree, tree_a, - "the later recovery rebaselines to the newer tree C" - ); - assert!( - !marker.exists().expect("marker existence should resolve"), - "the later successful recovery finally clears the marker" - ); - - let store_db = ok_db().expect("reopening the DB for assertions should succeed"); - let store = MutationTraceStore::new(&store_db); - let after_recover = store - .load_worktree(&worktree_id, Some(&scope), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - after_recover.worktree_state.cursor_tree, recovered.observed_tree, - "the later recovery rebaselines the cursor to its own observed tree" - ); - assert_eq!( - after_recover.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "the later invocation must not resurrect the abandoned scope" - ); - drop(store_db); - - remove_test_db(&db_path); - remove_test_repo(&repo_root); - } -} diff --git a/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs b/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs index f8bc7f91e..966cea930 100644 --- a/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs +++ b/cli/src/services/mutation_trace/runtime/external_mutation_guard.rs @@ -39,6 +39,8 @@ pub(crate) enum GuardError { ArmedDelivery(std::io::Error), Wait(std::io::Error), Finish(CoordinateError), + SupervisorWorker(tokio::task::JoinError), + #[allow(dead_code, reason = "constructed only under cfg(not(unix))")] UnsupportedPlatform, } @@ -59,6 +61,10 @@ impl std::fmt::Display for GuardError { write!(f, "failed to wait for the guarded shell: {source}") } GuardError::Finish(source) => write!(f, "{source}"), + GuardError::SupervisorWorker(source) => write!( + f, + "the guarded-shell supervisor worker did not complete: {source}" + ), GuardError::UnsupportedPlatform => { write!(f, "the external-mutation guard is Unix-only") } @@ -86,34 +92,19 @@ impl

ArmedExternalMutationGuard

{ } #[cfg(not(unix))] -pub(crate) fn arm_external_mutation_guard( +pub(crate) async fn arm_external_mutation_guard( _repository_root: &Path, _open_db: P, _on_armed: A, _cancel_rx: mpsc::Receiver<()>, ) -> Result, GuardError> where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, A: FnOnce() -> std::io::Result<()>, { Err(GuardError::UnsupportedPlatform) } -#[cfg(not(unix))] -pub(crate) fn run_external_mutation_guard( - _repository_root: &Path, - _request: &GuardRequest, - _open_db: P, - _on_event: E, - _cancel_rx: mpsc::Receiver<()>, -) -> Result -where - P: FnOnce() -> anyhow::Result, - E: FnMut(GuardEvent), -{ - Err(GuardError::UnsupportedPlatform) -} - #[cfg(unix)] mod unix_impl { use std::fs::File; @@ -138,11 +129,14 @@ mod unix_impl { const POLL_INTERVAL: Duration = Duration::from_millis(20); const STDIO_IDLE_GRACE: Duration = Duration::from_millis(100); const STDIO_FINALIZATION_LIMIT: Duration = Duration::from_secs(1); + const GUARD_EVENT_CHANNEL_CAPACITY: usize = 16; const SIGTERM: i32 = 15; const F_GETFD: i32 = 1; const F_SETFD: i32 = 2; const FD_CLOEXEC: i32 = 1; + #[cfg(target_os = "linux")] + const O_CLOEXEC: i32 = 0o200_0000; const POLLIN: i16 = 0x001; const POLLERR: i16 = 0x008; const POLLHUP: i16 = 0x010; @@ -180,11 +174,13 @@ mod unix_impl { Some(PathBuf::from(first_line)) } - fn resolve_execution_cwd( + async fn resolve_execution_cwd( repository_root: &Path, requested_cwd: Option<&str>, ) -> Result { - let worktree_root = resolve_worktree_root(repository_root).map_err(GuardError::Cwd)?; + let worktree_root = resolve_worktree_root(repository_root) + .await + .map_err(GuardError::Cwd)?; let Some(raw_cwd) = requested_cwd else { return Ok(worktree_root); @@ -227,25 +223,23 @@ mod unix_impl { mod raw { unsafe extern "C" { - pub(super) fn dup(fd: i32) -> i32; pub(super) fn fcntl(fd: i32, command: i32, ...) -> i32; pub(super) fn kill(pid: i32, sig: i32) -> i32; + #[cfg(target_os = "linux")] + pub(super) fn pipe2(fds: *mut i32, flags: i32) -> i32; + #[cfg(not(target_os = "linux"))] pub(super) fn pipe(fds: *mut i32) -> i32; pub(super) fn poll(fds: *mut super::PollFd, count: usize, timeout: i32) -> i32; } } - fn set_cloexec(fd: RawFd, enabled: bool) -> io::Result<()> { + #[cfg(not(target_os = "linux"))] + fn set_cloexec(fd: RawFd) -> io::Result<()> { let flags = unsafe { raw::fcntl(fd, F_GETFD, 0) }; if flags < 0 { return Err(io::Error::last_os_error()); } - let updated = if enabled { - flags | FD_CLOEXEC - } else { - flags & !FD_CLOEXEC - }; - if unsafe { raw::fcntl(fd, F_SETFD, updated) } < 0 { + if unsafe { raw::fcntl(fd, F_SETFD, flags | FD_CLOEXEC) } < 0 { return Err(io::Error::last_os_error()); } Ok(()) @@ -260,14 +254,22 @@ mod unix_impl { impl LifetimeToken { pub(super) fn new() -> io::Result { let mut fds = [-1_i32; 2]; + #[cfg(target_os = "linux")] + if unsafe { raw::pipe2(fds.as_mut_ptr(), O_CLOEXEC) } < 0 { + return Err(io::Error::last_os_error()); + } + #[cfg(not(target_os = "linux"))] if unsafe { raw::pipe(fds.as_mut_ptr()) } < 0 { return Err(io::Error::last_os_error()); } let reader = unsafe { File::from_raw_fd(fds[0]) }; let writer = unsafe { File::from_raw_fd(fds[1]) }; - set_cloexec(reader.as_raw_fd(), true)?; - set_cloexec(writer.as_raw_fd(), false)?; + #[cfg(not(target_os = "linux"))] + { + set_cloexec(reader.as_raw_fd())?; + set_cloexec(writer.as_raw_fd())?; + } Ok(Self { reader, @@ -301,14 +303,6 @@ mod unix_impl { lock_fd: RawFd, lifetime_fd: RawFd, ) -> std::io::Result { - let duplicated_lock_fd = unsafe { raw::dup(lock_fd) }; - if duplicated_lock_fd < 0 { - return Err(std::io::Error::last_os_error()); - } - let duplicated_lock_fd_guard = unsafe { File::from_raw_fd(duplicated_lock_fd) }; - set_cloexec(duplicated_lock_fd_guard.as_raw_fd(), false)?; - set_cloexec(lifetime_fd, false)?; - let mut command = Command::new(resolve_shell_executable()); command .arg("-c") @@ -320,9 +314,24 @@ mod unix_impl { .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let result = command.spawn(); - drop(duplicated_lock_fd_guard); - result + unsafe { + command.pre_exec(move || inherit_across_exec(&[lock_fd, lifetime_fd])); + } + + command.spawn() + } + + fn inherit_across_exec(fds: &[RawFd]) -> io::Result<()> { + for &fd in fds { + let flags = unsafe { raw::fcntl(fd, F_GETFD, 0) }; + if flags < 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { raw::fcntl(fd, F_SETFD, flags & !FD_CLOEXEC) } < 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) } fn poll_fds(fds: &mut [PollFd], timeout: Duration) -> io::Result<()> { @@ -535,20 +544,152 @@ mod unix_impl { hooks: GuardTestHooks, } + struct SupervisionOutcome { + ownership: GuardOwnership, + status: ExitStatus, + } + + struct EventForwarder { + event_tx: tokio::sync::mpsc::Sender, + consumer_gone: bool, + } + + impl EventForwarder { + fn forward(&mut self, event: GuardEvent) { + if self.consumer_gone { + return; + } + if self.event_tx.blocking_send(event).is_err() { + self.consumer_gone = true; + } + } + + fn consumer_gone(&mut self) -> bool { + if !self.consumer_gone && self.event_tx.is_closed() { + self.consumer_gone = true; + } + self.consumer_gone + } + } + + fn signal_process_group(pid: u32) { + #[allow(clippy::cast_possible_wrap)] + let group = -(pid as i32); + unsafe { + raw::kill(group, SIGTERM); + } + } + + #[allow(clippy::needless_pass_by_value)] + fn supervise_guarded_shell( + execution_cwd: PathBuf, + request: GuardRequest, + mut ownership: GuardOwnership, + mut lifetime: LifetimeToken, + cancel_rx: mpsc::Receiver<()>, + event_tx: tokio::sync::mpsc::Sender, + hooks: GuardTestHooks, + ) -> Result { + let lock_fd = ownership.protected().lock_raw_fd(); + let mut child = + match spawn_guarded_shell(&execution_cwd, &request, lock_fd, lifetime.writer_fd()) { + Ok(child) => child, + Err(source) => return Err(GuardError::Spawn(source)), + }; + ownership.mark_spawned(); + lifetime.close_writer(); + + let mut forwarder = EventForwarder { + event_tx, + consumer_gone: false, + }; + let mut stdout = child.stdout.take(); + let mut stderr = child.stderr.take(); + let pid = child.id(); + let mut status: Option = None; + let mut lifetime_complete = false; + let mut cancel_signaled = false; + let mut finalization_started = None; + let mut last_stream_activity = Instant::now(); + loop { + if status.is_none() { + status = match child.try_wait() { + Ok(status) => status, + Err(source) => { + ownership.abandon_after_spawn_without_unlock(); + return Err(GuardError::Wait(source)); + } + }; + } + + if should_finish_finalization( + status.as_ref(), + lifetime_complete, + stdout.as_ref(), + stderr.as_ref(), + &mut finalization_started, + &mut last_stream_activity, + ) { + break; + } + + if !cancel_signaled && (cancel_rx.try_recv().is_ok() || forwarder.consumer_gone()) { + cancel_signaled = true; + signal_process_group(pid); + } + + let poll_timeout = supervision_poll_timeout(finalization_started, last_stream_activity); + let stream_activity_before_poll = last_stream_activity; + if let Err(source) = poll_and_consume_streams( + &mut lifetime, + &mut stdout, + &mut stderr, + &mut |event| forwarder.forward(event), + poll_timeout, + &mut lifetime_complete, + &mut last_stream_activity, + ) { + ownership.abandon_after_spawn_without_unlock(); + return Err(GuardError::Wait(source)); + } + if lifetime_complete { + ownership.mark_lifetime_complete(); + } + if hooks.fail_after_first_output && last_stream_activity != stream_activity_before_poll + { + ownership.abandon_after_spawn_without_unlock(); + return Err(GuardError::Wait(io::Error::other( + "injected post-spawn supervision failure", + ))); + } + } + + ownership.mark_lifetime_complete(); + let status = status.expect("the guard loop only finishes after shell exit"); + Ok(SupervisionOutcome { ownership, status }) + } + impl

ArmedExternalMutationGuard

where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, { - #[allow(clippy::too_many_lines)] - pub(crate) fn exec( - mut self, + pub(crate) async fn exec( + self, request: &GuardRequest, mut on_event: E, ) -> Result where E: FnMut(GuardEvent), { - if self.cancel_rx.try_recv().is_ok() { + let ArmedExternalMutationGuard { + repository_root, + ownership, + lifetime, + open_db, + cancel_rx, + hooks, + } = self; + if cancel_rx.try_recv().is_ok() { return Err(GuardError::CancelledBeforeExec); } if request.command.trim().is_empty() { @@ -557,104 +698,55 @@ mod unix_impl { ))); } let execution_cwd = - resolve_execution_cwd(&self.repository_root, request.cwd.as_deref())?; - let lock_fd = self.ownership.protected().lock_raw_fd(); - let mut child = match spawn_guarded_shell( - &execution_cwd, - request, - lock_fd, - self.lifetime.writer_fd(), - ) { - Ok(child) => child, - Err(source) => return Err(GuardError::Spawn(source)), - }; - self.ownership.mark_spawned(); - self.lifetime.close_writer(); - - let mut stdout = child.stdout.take(); - let mut stderr = child.stderr.take(); - let pid = child.id(); - let mut status: Option = None; - let mut lifetime_complete = false; - let mut cancel_signaled = false; - let mut finalization_started = None; - let mut last_stream_activity = Instant::now(); - loop { - if status.is_none() { - status = match child.try_wait() { - Ok(status) => status, - Err(source) => { - self.ownership.abandon_after_spawn_without_unlock(); - return Err(GuardError::Wait(source)); - } - }; - } - - if should_finish_finalization( - status.as_ref(), - lifetime_complete, - stdout.as_ref(), - stderr.as_ref(), - &mut finalization_started, - &mut last_stream_activity, - ) { - break; - } - - if !cancel_signaled && self.cancel_rx.try_recv().is_ok() { - cancel_signaled = true; - #[allow(clippy::cast_possible_wrap)] - let group = -(pid as i32); - unsafe { - raw::kill(group, SIGTERM); - } - } + resolve_execution_cwd(&repository_root, request.cwd.as_deref()).await?; + + let (event_tx, mut event_rx) = + tokio::sync::mpsc::channel::(GUARD_EVENT_CHANNEL_CAPACITY); + let supervision_request = request.clone(); + let mut supervisor = tokio::task::spawn_blocking(move || { + supervise_guarded_shell( + execution_cwd, + supervision_request, + ownership, + lifetime, + cancel_rx, + event_tx, + hooks, + ) + }); - let poll_timeout = - supervision_poll_timeout(finalization_started, last_stream_activity); - let stream_activity_before_poll = last_stream_activity; - if let Err(source) = poll_and_consume_streams( - &mut self.lifetime, - &mut stdout, - &mut stderr, - &mut on_event, - poll_timeout, - &mut lifetime_complete, - &mut last_stream_activity, - ) { - self.ownership.abandon_after_spawn_without_unlock(); - return Err(GuardError::Wait(source)); - } - if lifetime_complete { - self.ownership.mark_lifetime_complete(); - } - if self.hooks.fail_after_first_output - && last_stream_activity != stream_activity_before_poll - { - self.ownership.abandon_after_spawn_without_unlock(); - return Err(GuardError::Wait(io::Error::other( - "injected post-spawn supervision failure", - ))); + let joined = loop { + tokio::select! { + biased; + Some(event) = event_rx.recv() => on_event(event), + joined = &mut supervisor => break joined, } + }; + while let Some(event) = event_rx.recv().await { + on_event(event); } + let SupervisionOutcome { + mut ownership, + status, + } = joined.map_err(GuardError::SupervisorWorker)??; - self.ownership.mark_lifetime_complete(); - let status = status.expect("the guard loop only finishes after shell exit"); - let worktree_id = self.ownership.protected().worktree_id().clone(); + let worktree_id = ownership.protected().worktree_id().clone(); + let pin_lease = ownership.protected().lock_lease(); if let Err(source) = coordinate_on_held_worktree( - &self.repository_root, + &repository_root, + pin_lease, &worktree_id, &RuntimeBoundary::Flush, - self.open_db - .take() - .expect("the guard database opener must be available before exec"), + open_db.expect("the guard database opener must be available before exec"), true, - ) { + ) + .await + { return Err(GuardError::Finish(source)); } - let protected = self.ownership.take_protected(); - let marker_clear_failed = protected.complete().is_err(); + let protected = ownership.take_protected(); + let marker_clear_failed = protected.complete().await.is_err(); Ok(GuardOutcome { exit_code: status.code(), @@ -663,7 +755,7 @@ mod unix_impl { } } - fn arm_external_mutation_guard_inner( + async fn arm_external_mutation_guard_inner( repository_root: &Path, open_db: P, on_armed: A, @@ -671,10 +763,12 @@ mod unix_impl { hooks: GuardTestHooks, ) -> Result, GuardError> where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, A: FnOnce() -> io::Result<()>, { - let protected = ProtectedWorktree::acquire(repository_root).map_err(GuardError::Acquire)?; + let protected = ProtectedWorktree::acquire(repository_root) + .await + .map_err(GuardError::Acquire)?; let ownership = GuardOwnership::new(protected); let lifetime = if hooks.fail_lifetime_token { Err(io::Error::other( @@ -696,14 +790,14 @@ mod unix_impl { }) } - pub(crate) fn arm_external_mutation_guard( + pub(crate) async fn arm_external_mutation_guard( repository_root: &Path, open_db: P, on_armed: A, cancel_rx: mpsc::Receiver<()>, ) -> Result, GuardError> where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, A: FnOnce() -> io::Result<()>, { arm_external_mutation_guard_inner( @@ -713,1060 +807,227 @@ mod unix_impl { cancel_rx, GuardTestHooks::default(), ) - } - - pub(crate) fn run_external_mutation_guard( - repository_root: &Path, - request: &GuardRequest, - open_db: P, - mut on_event: E, - cancel_rx: mpsc::Receiver<()>, - ) -> Result - where - P: FnOnce() -> anyhow::Result, - E: FnMut(GuardEvent), - { - let guard = arm_external_mutation_guard( - repository_root, - open_db, - || { - on_event(GuardEvent::Armed); - Ok(()) - }, - cancel_rx, - )?; - guard.exec(request, on_event) + .await } #[cfg(test)] - pub(super) fn run_external_mutation_guard_with_hooks( - repository_root: &Path, - request: &GuardRequest, - open_db: P, - mut on_event: E, - cancel_rx: mpsc::Receiver<()>, - hooks: GuardTestHooks, - ) -> Result - where - P: FnOnce() -> anyhow::Result, - E: FnMut(GuardEvent), - { - let guard = arm_external_mutation_guard_inner( - repository_root, - open_db, - || { - on_event(GuardEvent::Armed); - Ok(()) - }, - cancel_rx, - hooks, - )?; - guard.exec(request, on_event) - } + mod tests { + use std::path::{Path, PathBuf}; + use std::process::Command; + use std::sync::mpsc; + use std::time::{Duration, Instant}; - #[cfg(test)] - pub(super) fn arm_external_mutation_guard_with_hooks( - repository_root: &Path, - open_db: P, - on_armed: A, - cancel_rx: mpsc::Receiver<()>, - hooks: GuardTestHooks, - ) -> Result, GuardError> - where - P: FnOnce() -> anyhow::Result, - A: FnOnce() -> io::Result<()>, - { - arm_external_mutation_guard_inner(repository_root, open_db, on_armed, cancel_rx, hooks) - } -} - -#[cfg(unix)] -pub(crate) use unix_impl::{ - arm_external_mutation_guard, run_external_mutation_guard, ArmedExternalMutationGuard, -}; + use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; -#[cfg(all(unix, test))] -mod tests { - use std::fs; - use std::path::{Path, PathBuf}; - use std::process::Command; - use std::sync::mpsc; - use std::sync::{Arc, Mutex}; - use std::time::{Duration, Instant}; + use super::{ + arm_external_mutation_guard, GuardError, GuardEvent, GuardOutcome, GuardRequest, + }; - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage_at_state_root, AgentTraceStorageContext, - }; - use crate::services::mutation_trace::store::MutationTraceStore; + const WORKER_PROGRESS_LIMIT: Duration = Duration::from_millis(500); + const READY_TIMEOUT: Duration = Duration::from_secs(30); - use super::super::coordinator::{coordinate, RuntimeBoundary}; - use super::super::git_snapshot::GitSnapshotService; - use super::super::protected_worktree::ProtectedWorktree; - use super::super::worktree_lock::WorktreeLock; - use super::super::{resolve_git_dir, resolve_worktree_id}; - use super::unix_impl::{ - arm_external_mutation_guard_with_hooks, run_external_mutation_guard_with_hooks, - spawn_guarded_shell, GuardTestHooks, LifetimeToken, - }; - use super::*; - - fn git(dir: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } + async fn arm_and_exec_guard( + repository_root: &Path, + request: &GuardRequest, + open_db: P, + mut on_event: E, + cancel_rx: mpsc::Receiver<()>, + ) -> Result + where + P: std::ops::AsyncFnOnce() -> anyhow::Result, + E: FnMut(GuardEvent), + { + let guard = arm_external_mutation_guard( + repository_root, + open_db, + || { + on_event(GuardEvent::Armed); + Ok(()) + }, + cancel_rx, + ) + .await?; + guard.exec(request, on_event).await + } - struct TestRepo { - _temp: tempfile::TempDir, - root: PathBuf, - state_root: PathBuf, - } + fn git(repo_root: &Path, args: &[&str]) { + let output = Command::new("git") + .args(args) + .current_dir(repo_root) + .output() + .expect("git command should run"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } - impl TestRepo { - fn new(label: &str) -> Self { - let temp = tempfile::Builder::new() - .prefix(&format!("sce-external-mutation-guard-{label}-")) - .tempdir() - .expect("temp dir should be created"); - let root = temp.path().join("repo"); - fs::create_dir_all(&root).expect("repo dir should be created"); - git(&root, &["init", "-q"]); - git(&root, &["config", "user.email", "test@example.invalid"]); - git(&root, &["config", "user.name", "SCE Test"]); + fn init_repository() -> (tempfile::TempDir, PathBuf, PathBuf) { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + std::fs::create_dir_all(&repo_root).expect("repo dir"); + git(&repo_root, &["init", "-q"]); git( - &root, - &["remote", "add", "origin", "git@github.com:acme/widgets.git"], + &repo_root, + &["config", "user.email", "guard@example.invalid"], ); - fs::write(root.join("file.txt"), "one\n").expect("seed file should write"); - git(&root, &["add", "-A"]); - git(&root, &["commit", "-qm", "base"]); - - let state_root = temp.path().join("state"); - fs::create_dir_all(&state_root).expect("state root should be created"); - resolve_agent_trace_storage_at_state_root( - &AgentTraceStorageContext { - repository_root: &root, - explicit_repository_id: None, - repository_remote: "origin", - }, - &state_root, - ) - .expect("state-root storage should initialize the repository DB"); + git(&repo_root, &["config", "user.name", "Guard Test"]); + std::fs::write(repo_root.join("README.md"), "guard\n").expect("seed file"); + git(&repo_root, &["add", "README.md"]); + git(&repo_root, &["commit", "-q", "-m", "seed"]); + let db_path = dir.path().join("agent-trace.db"); + (dir, repo_root, db_path) + } - Self { - _temp: temp, - root, - state_root, + fn guard_request(command: &str) -> GuardRequest { + GuardRequest { + command: command.to_string(), + cwd: None, + env: Vec::new(), } } - fn git_dir(&self) -> PathBuf { - resolve_git_dir(&self.root).expect("git dir should resolve") + #[cfg(target_os = "linux")] + fn parse_targets(stdout: &[u8]) -> Vec { + String::from_utf8_lossy(stdout) + .lines() + .map(str::to_string) + .collect() } - fn nested_dir(&self, relative: &str) -> PathBuf { - let dir = self.root.join(relative); - fs::create_dir_all(&dir).expect("nested test directory should be created"); - dir.canonicalize() - .expect("nested test directory should canonicalize") + #[cfg(target_os = "linux")] + fn descriptor_target(fd: i32) -> String { + std::fs::read_link(format!("/proc/self/fd/{fd}")) + .expect("descriptor target") + .display() + .to_string() } - fn open_db(&self) -> anyhow::Result { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &self.root, - &self.state_root, - "external-mutation-guard test assertions", + #[cfg(target_os = "linux")] + #[test] + fn guard_descriptors_are_inherited_only_by_the_guarded_child() { + use std::os::fd::AsRawFd; + + const LIST_TARGETS: &str = "for fd in /proc/self/fd/*; do readlink \"$fd\"; done"; + + let dir = tempfile::tempdir().expect("temp dir"); + let lock_file = std::fs::File::create(dir.path().join("lock")).expect("lock file"); + let lock_fd = lock_file.as_raw_fd(); + let mut lifetime = super::LifetimeToken::new().expect("lifetime token"); + let lifetime_fd = lifetime.writer_fd(); + + let flags = unsafe { super::raw::fcntl(lifetime_fd, super::F_GETFD, 0) }; + assert_ne!(flags & super::FD_CLOEXEC, 0, "writer is close-on-exec"); + + let unrelated = Command::new("sh") + .arg("-c") + .arg(LIST_TARGETS) + .output() + .expect("unrelated listing should run"); + let lock_target = descriptor_target(lock_fd); + let lifetime_target = descriptor_target(lifetime_fd); + let unrelated_targets = parse_targets(&unrelated.stdout); + assert!(!unrelated_targets.contains(&lifetime_target)); + assert!(!unrelated_targets.contains(&lock_target)); + + let child = super::spawn_guarded_shell( + dir.path(), + &guard_request(LIST_TARGETS), + lock_fd, + lifetime_fd, ) + .expect("guarded shell should spawn"); + let output = child.wait_with_output().expect("guarded shell output"); + let guarded_targets = parse_targets(&output.stdout); + assert!(guarded_targets.contains(&lifetime_target)); + assert!(guarded_targets.contains(&lock_target)); + + let flags = unsafe { super::raw::fcntl(lock_fd, super::F_GETFD, 0) }; + assert_ne!( + flags & super::FD_CLOEXEC, + 0, + "parent lock stays close-on-exec" + ); + lifetime.close_writer(); } - } - - #[test] - fn lifetime_token_establishment_failure_cannot_emit_armed() { - let repo = TestRepo::new("lifetime-token-establishment-failure"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let events = Arc::new(Mutex::new(Vec::new())); - let captured_events = Arc::clone(&events); - - let result = run_external_mutation_guard_with_hooks( - &repo.root, - &request("true"), - || repo.open_db(), - move |event| captured_events.lock().expect("events mutex").push(event), - cancel_rx, - GuardTestHooks { - fail_lifetime_token: true, - fail_after_first_output: false, - }, - ); - - assert!(matches!(result, Err(GuardError::Spawn(_)))); - assert!( - events.lock().expect("events mutex").is_empty(), - "Armed must not be emitted before lifetime-token establishment" - ); - WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(200)) - .expect("pre-spawn lifetime failure must release the ordinary lock"); - assert!( - repo.git_dir() - .join("sce") - .join("mutation-cursor-tainted") - .exists(), - "the existing write-ahead marker remains armed after establishment failure" - ); - } - - #[test] - fn post_spawn_supervision_failure_abandons_without_unlocking_inherited_lock() { - let repo = TestRepo::new("post-spawn-supervision-failure"); - let release = repo.root.join("release"); - let descendant_done = repo.root.join("descendant-done"); - let command = format!( - "(while [ ! -f '{}' ]; do sleep 0.01; done; printf descendant > '{}'; touch '{}') & printf ready", - release.display(), - repo.root.join("file.txt").display(), - descendant_done.display(), - ); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let (ready_tx, ready_rx) = mpsc::channel(); - - let result = run_external_mutation_guard_with_hooks( - &repo.root, - &request(&command), - || repo.open_db(), - move |event| { - if let GuardEvent::Stdout(chunk) = event { - if String::from_utf8_lossy(&chunk).contains("ready") { - ready_tx.send(()).expect("ready channel should be open"); - } - } - }, - cancel_rx, - GuardTestHooks { - fail_lifetime_token: false, - fail_after_first_output: true, - }, - ); - - ready_rx - .recv_timeout(Duration::from_secs(5)) - .expect("the shell must have spawned before the injected failure"); - assert!(matches!(result, Err(GuardError::Wait(_)))); - - assert!( - WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(200)).is_err(), - "post-spawn abandonment must not execute flock(LOCK_UN) while the descendant lives" - ); - let marker = repo.git_dir().join("sce").join("mutation-cursor-tainted"); - assert!( - marker.exists(), - "the marker must remain armed on abandonment" - ); - - fs::write(&release, "release\n").expect("descendant release handshake should write"); - wait_for_path(&descendant_done); - WorktreeLock::acquire(&repo.git_dir(), Duration::from_secs(1)) - .expect("the inherited descriptor should release the flock naturally"); - - coordinate(&repo.root, &RuntimeBoundary::Flush, || repo.open_db()) - .expect("the next boundary should recover inherited external taint"); - assert!( - !marker.exists(), - "inherited-taint recovery should clear the marker" - ); - } - - #[test] - fn post_spawn_supervision_failure_without_descendants_uses_the_same_abandonment_path() { - let repo = TestRepo::new("post-spawn-no-descendant-failure"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let (ready_tx, ready_rx) = mpsc::channel(); - - let result = run_external_mutation_guard_with_hooks( - &repo.root, - &request("printf ready"), - || repo.open_db(), - move |event| { - if let GuardEvent::Stdout(chunk) = event { - if String::from_utf8_lossy(&chunk).contains("ready") { - ready_tx.send(()).expect("ready channel should be open"); - } - } - }, - cancel_rx, - GuardTestHooks { - fail_lifetime_token: false, - fail_after_first_output: true, - }, - ); - - ready_rx - .recv_timeout(Duration::from_secs(5)) - .expect("the shell must have spawned before the injected failure"); - assert!(matches!(result, Err(GuardError::Wait(_)))); - let marker = repo.git_dir().join("sce").join("mutation-cursor-tainted"); - assert!( - marker.exists(), - "the marker must remain armed on abandonment" - ); - WorktreeLock::acquire(&repo.git_dir(), Duration::from_secs(1)) - .expect("without descendants the shell's inherited fd closes naturally"); - - coordinate(&repo.root, &RuntimeBoundary::Flush, || repo.open_db()) - .expect("the next boundary should recover the still-armed marker"); - assert!( - !marker.exists(), - "inherited-taint recovery should clear the marker" - ); - } - - #[test] - fn lost_armed_acknowledgement_cannot_spawn_or_mutate() { - let repo = TestRepo::new("lost-armed-ack"); - let marker = repo.git_dir().join("sce").join("mutation-cursor-tainted"); - let target = repo.root.join("lost-armed-command-ran"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - - let result = arm_external_mutation_guard_with_hooks( - &repo.root, - || repo.open_db(), - || { - Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "injected lost Armed acknowledgement", - )) - }, - cancel_rx, - GuardTestHooks::default(), - ); - - assert!(matches!(result, Err(GuardError::ArmedDelivery(_)))); - assert!( - !target.exists(), - "the command must not run without Armed delivery" - ); - WorktreeLock::acquire(&repo.git_dir(), Duration::from_secs(1)) - .expect("pre-spawn acknowledgement failure must release the ordinary lock"); - assert!( - marker.exists(), - "ambiguous establishment remains conservatively tainted" - ); - } - - #[test] - fn armed_guard_waits_for_exec_and_drops_without_spawning_on_eof() { - let repo = TestRepo::new("armed-without-exec"); - let marker = repo.git_dir().join("sce").join("mutation-cursor-tainted"); - let target = repo.root.join("never-ran"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let armed = arm_external_mutation_guard_with_hooks( - &repo.root, - || repo.open_db(), - || Ok(()), - cancel_rx, - GuardTestHooks::default(), - ) - .expect("arm should succeed"); - - drop(armed); - assert!(!target.exists(), "EOF before exec must not create a shell"); - WorktreeLock::acquire(&repo.git_dir(), Duration::from_secs(1)) - .expect("ordinary pre-spawn cleanup must release the lock"); - assert!( - marker.exists(), - "pre-spawn EOF remains conservatively tainted" - ); - - coordinate(&repo.root, &RuntimeBoundary::Flush, || repo.open_db()) - .expect("the next boundary must self-heal the conservative marker"); - assert!(!marker.exists()); - } - - #[test] - fn cancellation_before_exec_cannot_spawn() { - let repo = TestRepo::new("cancel-before-exec"); - let target = repo.root.join("cancelled-command-ran"); - let (cancel_tx, cancel_rx) = mpsc::channel(); - let armed = arm_external_mutation_guard_with_hooks( - &repo.root, - || repo.open_db(), - || Ok(()), - cancel_rx, - GuardTestHooks::default(), - ) - .expect("arm should succeed"); - cancel_tx.send(()).expect("cancel should be received"); - - let result = armed.exec( - &request(&format!("touch '{}'", target.display())), - |_event| {}, - ); - assert!(matches!(result, Err(GuardError::CancelledBeforeExec))); - assert!(!target.exists()); - } - - #[test] - fn armed_guard_has_no_side_effect_before_exec_and_exec_runs_once() { - let repo = TestRepo::new("two-phase-happy-path"); - let target = repo.root.join("exec-ran"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let armed = arm_external_mutation_guard_with_hooks( - &repo.root, - || repo.open_db(), - || Ok(()), - cancel_rx, - GuardTestHooks::default(), - ) - .expect("arm should succeed"); - assert!(!target.exists(), "Armed must not execute the later command"); - - let request = request(&format!("touch '{}'", target.display())); - let outcome = armed - .exec(&request, |_event| {}) - .expect("the explicit exec request should run"); - assert_eq!(outcome.exit_code, Some(0)); - assert!(target.exists(), "the command must run after explicit exec"); - } - fn request(command: &str) -> GuardRequest { - GuardRequest { - command: command.to_string(), - cwd: None, - env: Vec::new(), - } - } + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn guarded_shell_supervision_leaves_the_runtime_worker_responsive() { + let (_dir, repo_root, db_path) = init_repository(); + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel::(); + let (_cancel_tx, cancel_rx) = mpsc::channel(); + + let guard_task = tokio::spawn(async move { + let mut ready_tx = Some(ready_tx); + arm_and_exec_guard( + &repo_root, + &guard_request("printf ready; sleep 2"), + async || RepositoryAgentTraceDb::new_at(&db_path).await, + |event| { + if matches!(event, GuardEvent::Stdout(_)) { + if let Some(ready_tx) = ready_tx.take() { + let _ = ready_tx.send(Instant::now()); + } + } + }, + cancel_rx, + ) + .await + }); - fn request_with_cwd(command: &str, cwd: &Path) -> GuardRequest { - GuardRequest { - command: command.to_string(), - cwd: Some(cwd.to_string_lossy().into_owned()), - env: Vec::new(), - } - } + let ready_at = tokio::time::timeout(READY_TIMEOUT, ready_rx) + .await + .expect("guarded shell should report readiness") + .expect("readiness sender should stay alive"); + let progressed_at = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(10)).await; + Instant::now() + }) + .await + .expect("unrelated task should complete"); - fn wait_for_path(path: &Path) { - let deadline = Instant::now() + Duration::from_secs(5); - while !path.exists() { assert!( - Instant::now() < deadline, - "timed out waiting for '{}'", - path.display() + progressed_at.duration_since(ready_at) < WORKER_PROGRESS_LIMIT, + "unrelated Tokio task waited {:?} while the guarded shell was running", + progressed_at.duration_since(ready_at) ); - std::thread::yield_now(); - } - } - fn wait_for_output(output: &Mutex>, expected: &str) { - let deadline = Instant::now() + Duration::from_secs(5); - while !String::from_utf8_lossy(&output.lock().expect("output mutex")).contains(expected) { - assert!( - Instant::now() < deadline, - "timed out waiting for output '{expected}'" - ); - std::thread::yield_now(); + let outcome = guard_task + .await + .expect("guard task should join") + .expect("guard should succeed"); + assert_eq!(outcome.exit_code, Some(0)); } - } - #[test] - fn a_concurrent_foreign_lock_attempt_times_out_while_the_guard_is_active() { - let repo = TestRepo::new("foreign-contention"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - - let root = repo.root.clone(); - let state_root = repo.state_root.clone(); - let handle = std::thread::spawn(move || { - run_external_mutation_guard( - &root, - &request("sleep 1"), - || { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &root, - &state_root, - "external-mutation-guard test assertions", - ) - }, - |_event| {}, + #[tokio::test] + async fn guarded_shell_streams_output_and_finishes_with_exit_code() { + let (_dir, repo_root, db_path) = init_repository(); + let (_cancel_tx, cancel_rx) = mpsc::channel(); + let mut events = Vec::new(); + + let outcome = arm_and_exec_guard( + &repo_root, + &guard_request("printf out; printf err >&2; exit 7"), + async || RepositoryAgentTraceDb::new_at(&db_path).await, + |event| events.push(event), cancel_rx, ) - }); - - std::thread::sleep(Duration::from_millis(200)); - - let foreign = WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(200)); - assert!( - foreign.is_err(), - "a foreign lock attempt must fail closed while the guard holds the lock" - ); - - let outcome = handle - .join() - .expect("guard thread should not panic") - .expect("the guard should reach its finish step"); - assert_eq!(outcome.exit_code, Some(0)); - - WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(500)) - .expect("the lock must free once the guard's own finish step completes"); - } - - #[test] - fn graceful_completion_waits_for_an_inherited_background_descendant() { - let repo = TestRepo::new("graceful-background-descendant"); - let release = repo.root.join("release"); - let foreground_exited = repo.root.join("foreground-exited"); - let descendant_done = repo.root.join("descendant-done"); - let command = format!( - "(while [ ! -f '{}' ]; do sleep 0.01; done; printf descendant > '{}'; touch '{}') >/dev/null 2>&1 & touch '{}'", - release.display(), - repo.root.join("file.txt").display(), - descendant_done.display(), - foreground_exited.display(), - ); - let (cancel_tx, cancel_rx) = mpsc::channel(); - drop(cancel_tx); - let (armed_tx, armed_rx) = mpsc::channel(); - let (result_tx, result_rx) = mpsc::channel(); - let root = repo.root.clone(); - let state_root = repo.state_root.clone(); - let handle = std::thread::spawn(move || { - let result = run_external_mutation_guard( - &root, - &request(&command), - || { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &root, - &state_root, - "external-mutation-guard graceful-descendant test", - ) - }, - |event| { - if matches!(event, GuardEvent::Armed) { - armed_tx.send(()).expect("armed channel should be open"); - } - }, - cancel_rx, - ); - result_tx - .send(result) - .expect("result channel should be open"); - }); - - armed_rx - .recv_timeout(Duration::from_secs(5)) - .expect("guard should arm before spawning the shell"); - wait_for_path(&foreground_exited); - assert!( - result_rx.try_recv().is_err(), - "foreground shell exit must not complete the guard while its descendant is alive" - ); - let marker = repo.git_dir().join("sce").join("mutation-cursor-tainted"); - assert!( - marker.exists(), - "the external-taint marker must remain armed" - ); - assert!( - WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(200)).is_err(), - "the real worktree lock must exclude foreign acquirers until descendant completion" - ); - - fs::write(&release, "release\n").expect("descendant release handshake should write"); - wait_for_path(&descendant_done); - let outcome = result_rx - .recv_timeout(Duration::from_secs(5)) - .expect("guard should finish after the descendant releases the token") - .expect("guard should recover successfully"); - assert_eq!(outcome.exit_code, Some(0)); - handle.join().expect("guard thread should not panic"); - - assert_eq!( - fs::read_to_string(repo.root.join("file.txt")).expect("mutated file should read"), - "descendant" - ); - let final_tree = GitSnapshotService::new(&repo.root) - .expect("snapshot service should construct") - .capture_tree() - .expect("final tree should capture"); - let worktree_id = resolve_worktree_id(&repo.root).expect("worktree id should resolve"); - let db = repo - .open_db() - .expect("database should reopen for assertions"); - let projection = MutationTraceStore::new(&db) - .load_worktree(&worktree_id, None, None) - .expect("worktree state should load") - .expect("guard recovery should initialize worktree state"); - assert_eq!( - projection.worktree_state.cursor_tree, final_tree, - "final recovery must rebaseline to the descendant mutation" - ); - assert!(!marker.exists(), "marker clears only after final recovery"); - WorktreeLock::acquire(&repo.git_dir(), Duration::from_secs(1)) - .expect("worktree lock should be released after final recovery"); - } - - #[test] - fn output_is_consumed_while_a_background_descendant_holds_the_lifetime_token() { - let repo = TestRepo::new("background-output"); - let release = repo.root.join("release"); - let foreground_exited = repo.root.join("foreground-exited"); - let start_output = repo.root.join("start-output"); - let output_ready = repo.root.join("output-ready"); - let descendant_done = repo.root.join("descendant-done"); - let command = format!( - "(while [ ! -f '{}' ]; do sleep 0.01; done; i=0; while [ $i -lt 20 ]; do printf 'stdout-%s\\n' $i; printf 'stderr-%s\\n' $i >&2; i=$((i+1)); done; touch '{}'; while [ ! -f '{}' ]; do sleep 0.01; done; i=20; while [ $i -lt 40 ]; do printf 'stdout-%s\\n' $i; printf 'stderr-%s\\n' $i >&2; i=$((i+1)); done; touch '{}') & touch '{}'", - start_output.display(), - output_ready.display(), - release.display(), - descendant_done.display(), - foreground_exited.display(), - ); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let (armed_tx, armed_rx) = mpsc::channel(); - let (result_tx, result_rx) = mpsc::channel(); - let stdout = Arc::new(Mutex::new(Vec::new())); - let stderr = Arc::new(Mutex::new(Vec::new())); - let captured_stdout = Arc::clone(&stdout); - let captured_stderr = Arc::clone(&stderr); - let root = repo.root.clone(); - let state_root = repo.state_root.clone(); - let handle = std::thread::spawn(move || { - let result = run_external_mutation_guard( - &root, - &request(&command), - || { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &root, - &state_root, - "external-mutation-guard background-output test", - ) - }, - move |event| match event { - GuardEvent::Armed => armed_tx.send(()).expect("armed channel should be open"), - GuardEvent::Stdout(chunk) => { - captured_stdout.lock().expect("stdout mutex").extend(chunk); - } - GuardEvent::Stderr(chunk) => { - captured_stderr.lock().expect("stderr mutex").extend(chunk); - } - }, - cancel_rx, + .await + .expect("guard should finish"); + + assert_eq!(outcome.exit_code, Some(7)); + assert!(!outcome.marker_clear_failed); + assert_eq!(events.first(), Some(&GuardEvent::Armed)); + assert!(events.contains(&GuardEvent::Stdout(b"out".to_vec()))); + assert!(events.contains(&GuardEvent::Stderr(b"err".to_vec()))); + assert!( + !repo_root.join(".git/sce/mutation-cursor-tainted").exists(), + "a durable Flush finish should clear the external-taint marker" ); - result_tx - .send(result) - .expect("result channel should be open"); - }); - - armed_rx - .recv_timeout(Duration::from_secs(5)) - .expect("guard should arm before spawning the shell"); - wait_for_path(&foreground_exited); - assert!( - result_rx.try_recv().is_err(), - "guard completion must wait for the lifetime token, not stream timing" - ); - fs::write(&start_output, "start\n").expect("output handshake should write"); - wait_for_path(&output_ready); - wait_for_output(&stdout, "stdout-19"); - wait_for_output(&stderr, "stderr-19"); - assert!( - result_rx.try_recv().is_err(), - "guard completion must wait for the lifetime token, not stream timing" - ); - - fs::write(&release, "release\n").expect("descendant release handshake should write"); - wait_for_path(&descendant_done); - result_rx - .recv_timeout(Duration::from_secs(5)) - .expect("guard should finish after descendant output and exit") - .expect("guard should recover successfully"); - handle.join().expect("guard thread should not panic"); - - let stdout_text = String::from_utf8(stdout.lock().expect("stdout mutex").clone()) - .expect("stdout should be valid UTF-8"); - let stderr_text = String::from_utf8(stderr.lock().expect("stderr mutex").clone()) - .expect("stderr should be valid UTF-8"); - assert!(stdout_text.contains("stdout-39")); - assert!(stderr_text.contains("stderr-39")); - } - - #[test] - fn the_spawned_shells_parent_is_the_calling_process() { - let repo = TestRepo::new("parent-pid"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let captured_stdout: Mutex> = Mutex::new(Vec::new()); - - let outcome = run_external_mutation_guard( - &repo.root, - &request("echo $PPID"), - || repo.open_db(), - |event| { - if let GuardEvent::Stdout(chunk) = event { - captured_stdout.lock().expect("stdout mutex").extend(chunk); - } - }, - cancel_rx, - ) - .expect("guard should succeed"); - assert_eq!(outcome.exit_code, Some(0)); - - let reported = String::from_utf8(captured_stdout.into_inner().expect("stdout mutex")) - .expect("$PPID output should be valid UTF-8"); - let ppid: u32 = reported - .trim() - .parse() - .expect("$PPID should parse as an integer"); - assert_eq!(ppid, std::process::id()); - } - - unsafe extern "C" { - fn close(fd: i32) -> i32; - } - - #[test] - fn a_supervisor_killed_without_unlocking_leaves_the_flock_held_by_the_spawned_shell() { - let repo = TestRepo::new("kill-9-simulated"); - - let protected = ProtectedWorktree::acquire(&repo.root).expect("acquire should succeed"); - let lock_fd = protected.lock_raw_fd(); - let lifetime = LifetimeToken::new().expect("lifetime token should be created"); - let mut child = spawn_guarded_shell( - &repo.root, - &request("sleep 1"), - lock_fd, - lifetime.writer_fd(), - ) - .expect("the guarded shell should spawn"); - drop(lifetime); - - unsafe { - close(lock_fd); } - std::mem::forget(protected); - - let foreign_while_shell_alive = - WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(200)); - assert!( - foreign_while_shell_alive.is_err(), - "an implicit close (as a killed process's fd table teardown performs, never an \ - explicit flock unlock) must not release the flock while the spawned shell still \ - holds its own inherited descriptor" - ); - - child.wait().expect("the shell should terminate"); - std::thread::sleep(Duration::from_millis(100)); - - WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(500)) - .expect("the lock must free once the spawned shell itself exits"); - } - - #[test] - fn closing_the_control_channel_does_not_trigger_finish_or_signal_the_shell() { - let repo = TestRepo::new("control-channel-death"); - let (cancel_tx, cancel_rx) = mpsc::channel(); - drop(cancel_tx); - - let outcome = run_external_mutation_guard( - &repo.root, - &request("exit 7"), - || repo.open_db(), - |_event| {}, - cancel_rx, - ) - .expect("a disconnected cancel channel must not itself trigger anything abnormal"); - assert_eq!(outcome.exit_code, Some(7)); - } - - #[test] - fn a_failed_finish_commit_leaves_the_marker_armed_and_reports_failure() { - let repo = TestRepo::new("finish-failure"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - - let result = run_external_mutation_guard( - &repo.root, - &request("true"), - || Err(anyhow::anyhow!("injected DB-unavailable failure")), - |_event| {}, - cancel_rx, - ); - assert!(matches!(result, Err(GuardError::Finish(_)))); - - let marker = repo.git_dir().join("sce").join("mutation-cursor-tainted"); - assert!( - marker.exists(), - "a failed finish commit must leave the external-taint marker armed" - ); - } - - #[test] - fn a_cancel_request_signals_the_shells_process_group_and_finish_still_waits_for_real_exit() { - let repo = TestRepo::new("cancel-request"); - let (cancel_tx, cancel_rx) = mpsc::channel(); - - let (ready_tx, ready_rx) = mpsc::channel(); - - let root = repo.root.clone(); - let state_root = repo.state_root.clone(); - let handle = std::thread::spawn(move || { - run_external_mutation_guard( - &root, - &request("trap 'exit 9' TERM; printf ready; sleep 30"), - || { - crate::services::hooks::open_agent_trace_db_for_hook_runtime_at_state_root( - &root, - &state_root, - "external-mutation-guard test assertions", - ) - }, - move |event| { - if let GuardEvent::Stdout(chunk) = event { - if String::from_utf8_lossy(&chunk).contains("ready") { - ready_tx.send(()).ok(); - } - } - }, - cancel_rx, - ) - }); - - ready_rx - .recv_timeout(Duration::from_secs(30)) - .expect("the shell should report readiness after installing its trap"); - cancel_tx - .send(()) - .expect("cancel channel should still be open"); - - let outcome = handle - .join() - .expect("guard thread should not panic") - .expect("the guard should reach its finish step after the signaled shell exits"); - assert_eq!(outcome.exit_code, Some(9)); - } - - fn run_and_capture_stdout(repo: &TestRepo, req: &GuardRequest) -> (GuardOutcome, String) { - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let captured_stdout: Mutex> = Mutex::new(Vec::new()); - let outcome = run_external_mutation_guard( - &repo.root, - req, - || repo.open_db(), - |event| { - if let GuardEvent::Stdout(chunk) = event { - captured_stdout.lock().expect("stdout mutex").extend(chunk); - } - }, - cancel_rx, - ) - .expect("guard should succeed"); - let output = String::from_utf8(captured_stdout.into_inner().expect("stdout mutex")) - .expect("stdout should be valid UTF-8"); - (outcome, output) - } - - #[test] - fn a_root_cwd_request_is_preserved_exactly() { - let repo = TestRepo::new("cwd-root"); - let canonical_root = repo - .root - .canonicalize() - .expect("repo root should canonicalize"); - let (outcome, output) = - run_and_capture_stdout(&repo, &request_with_cwd("pwd", &canonical_root)); - assert_eq!(outcome.exit_code, Some(0)); - assert_eq!(output.trim(), canonical_root.to_string_lossy()); - } - - #[test] - fn an_absent_cwd_defaults_to_the_worktree_root() { - let repo = TestRepo::new("cwd-default"); - let canonical_root = repo - .root - .canonicalize() - .expect("repo root should canonicalize"); - let (outcome, output) = run_and_capture_stdout(&repo, &request("pwd")); - assert_eq!(outcome.exit_code, Some(0)); - assert_eq!(output.trim(), canonical_root.to_string_lossy()); - } - - #[test] - fn a_nested_cwd_request_is_preserved_exactly() { - let repo = TestRepo::new("cwd-nested"); - let nested = repo.nested_dir("crates/foo"); - let (outcome, output) = run_and_capture_stdout(&repo, &request_with_cwd("pwd", &nested)); - assert_eq!(outcome.exit_code, Some(0)); - assert_eq!(output.trim(), nested.to_string_lossy()); - } - - #[test] - fn a_relative_exec_cwd_is_rejected_fail_closed() { - let repo = TestRepo::new("cwd-relative"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - - let result = run_external_mutation_guard( - &repo.root, - &request_with_cwd("true", Path::new("relative")), - || repo.open_db(), - |_event| {}, - cancel_rx, - ); - assert!(matches!(result, Err(GuardError::Cwd(_)))); - } - - #[test] - fn a_parent_traversal_cwd_cannot_escape_the_checkout() { - let repo = TestRepo::new("cwd-traversal"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let escaping_cwd = repo.root.join("..").to_string_lossy().into_owned(); - - let result = run_external_mutation_guard( - &repo.root, - &GuardRequest { - command: "true".to_string(), - cwd: Some(escaping_cwd), - env: Vec::new(), - }, - || repo.open_db(), - |_event| {}, - cancel_rx, - ); - assert!(matches!(result, Err(GuardError::Cwd(_)))); - } - - #[test] - fn an_absolute_cwd_outside_the_checkout_is_rejected() { - let repo = TestRepo::new("cwd-outside"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - - let result = run_external_mutation_guard( - &repo.root, - &request_with_cwd("true", &repo.state_root), - || repo.open_db(), - |_event| {}, - cancel_rx, - ); - assert!(matches!(result, Err(GuardError::Cwd(_)))); - } - - #[test] - fn a_nonexistent_cwd_is_rejected_fail_closed() { - let repo = TestRepo::new("cwd-missing"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let missing = repo.root.join("does-not-exist"); - - let result = run_external_mutation_guard( - &repo.root, - &request_with_cwd("true", &missing), - || repo.open_db(), - |_event| {}, - cancel_rx, - ); - assert!(matches!(result, Err(GuardError::Cwd(_)))); - } - - #[test] - fn a_non_directory_cwd_is_rejected_fail_closed() { - let repo = TestRepo::new("cwd-file"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let file_path = repo.root.join("file.txt"); - - let result = run_external_mutation_guard( - &repo.root, - &request_with_cwd("true", &file_path), - || repo.open_db(), - |_event| {}, - cancel_rx, - ); - assert!(matches!(result, Err(GuardError::Cwd(_)))); - } - - #[test] - fn a_rejected_exec_cwd_never_spawns_and_keeps_the_marker_conservative() { - let repo = TestRepo::new("cwd-rejected-no-lock"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - - let result = run_external_mutation_guard( - &repo.root, - &request_with_cwd("true", &repo.state_root), - || repo.open_db(), - |_event| {}, - cancel_rx, - ); - assert!(matches!(result, Err(GuardError::Cwd(_)))); - - WorktreeLock::acquire(&repo.git_dir(), Duration::from_millis(200)) - .expect("a rejected pre-spawn exec must release the ordinary worktree lock"); - - let marker = repo.git_dir().join("sce").join("mutation-cursor-tainted"); - assert!( - marker.exists(), - "an armed guard that rejects its exec request must keep the marker conservative" - ); - } - - #[test] - fn multiple_stdout_chunks_immediately_before_exit_are_not_truncated() { - let repo = TestRepo::new("stdout-chunks"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let captured_stdout: Mutex> = Mutex::new(Vec::new()); - - let outcome = run_external_mutation_guard( - &repo.root, - &request( - "i=0; while [ $i -lt 4000 ]; do echo \"line-$i-0123456789ABCDEF\"; i=$((i+1)); done", - ), - || repo.open_db(), - |event| { - if let GuardEvent::Stdout(chunk) = event { - captured_stdout.lock().expect("stdout mutex").extend(chunk); - } - }, - cancel_rx, - ) - .expect("guard should succeed"); - assert_eq!(outcome.exit_code, Some(0)); - - let output = String::from_utf8(captured_stdout.into_inner().expect("stdout mutex")) - .expect("stdout should be valid UTF-8"); - let lines: Vec<&str> = output.lines().collect(); - assert_eq!( - lines.len(), - 4000, - "every line must be delivered, none dropped at process exit" - ); - assert_eq!(lines[0], "line-0-0123456789ABCDEF"); - assert_eq!(lines[3999], "line-3999-0123456789ABCDEF"); - } - - #[test] - fn stderr_output_immediately_before_exit_is_not_truncated() { - let repo = TestRepo::new("stderr-chunks"); - let (_cancel_tx, cancel_rx) = mpsc::channel(); - let captured_stderr: Mutex> = Mutex::new(Vec::new()); - - let outcome = run_external_mutation_guard( - &repo.root, - &request( - "i=0; while [ $i -lt 4000 ]; do echo \"err-$i-0123456789ABCDEF\" >&2; i=$((i+1)); done", - ), - || repo.open_db(), - |event| { - if let GuardEvent::Stderr(chunk) = event { - captured_stderr.lock().expect("stderr mutex").extend(chunk); - } - }, - cancel_rx, - ) - .expect("guard should succeed"); - assert_eq!(outcome.exit_code, Some(0)); - - let output = String::from_utf8(captured_stderr.into_inner().expect("stderr mutex")) - .expect("stderr should be valid UTF-8"); - let lines: Vec<&str> = output.lines().collect(); - assert_eq!(lines.len(), 4000); - assert_eq!(lines[3999], "err-3999-0123456789ABCDEF"); } } + +#[cfg(unix)] +pub(crate) use unix_impl::{arm_external_mutation_guard, ArmedExternalMutationGuard}; diff --git a/cli/src/services/mutation_trace/runtime/external_taint.rs b/cli/src/services/mutation_trace/runtime/external_taint.rs index cfccb8f70..9c2f89d88 100644 --- a/cli/src/services/mutation_trace/runtime/external_taint.rs +++ b/cli/src/services/mutation_trace/runtime/external_taint.rs @@ -147,111 +147,3 @@ impl ExternalTaintMarker { #[cfg(not(unix))] fn best_effort_sync_marker_dir(&self) {} } - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-external-taint-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - #[test] - fn marker_is_worktree_scoped() { - let git_dir_a = unique_test_git_dir("scoped-a"); - let git_dir_b = unique_test_git_dir("scoped-b"); - std::fs::create_dir_all(&git_dir_a).expect("git dir a should be created"); - std::fs::create_dir_all(&git_dir_b).expect("git dir b should be created"); - - let marker_a = ExternalTaintMarker::new(&git_dir_a); - let marker_b = ExternalTaintMarker::new(&git_dir_b); - - assert_ne!( - marker_a.marker_path, marker_b.marker_path, - "each git dir must derive an independent marker path" - ); - - marker_a.persist().expect("marker a should persist"); - - assert!( - marker_a - .exists() - .expect("marker a existence should resolve"), - "the armed marker must be visible in its own worktree" - ); - assert!( - !marker_b - .exists() - .expect("marker b existence should resolve"), - "arming worktree A must not arm worktree B" - ); - - remove_test_git_dir(&git_dir_a); - remove_test_git_dir(&git_dir_b); - } - - #[test] - fn marker_persists_until_explicitly_cleared() { - let git_dir = unique_test_git_dir("persists"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - ExternalTaintMarker::new(&git_dir) - .persist() - .expect("marker should persist"); - - let reloaded = ExternalTaintMarker::new(&git_dir); - assert!( - reloaded.exists().expect("marker existence should resolve"), - "the marker must survive reconstruction of the handle" - ); - - reloaded.clear().expect("marker should clear"); - assert!( - !ExternalTaintMarker::new(&git_dir) - .exists() - .expect("marker existence should resolve"), - "an explicitly cleared marker must be gone" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn persist_and_clear_are_idempotent() { - let git_dir = unique_test_git_dir("idempotent"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - let marker = ExternalTaintMarker::new(&git_dir); - - marker - .clear() - .expect("clear on absent marker should succeed"); - - marker.persist().expect("first persist should succeed"); - marker.persist().expect("second persist should succeed"); - assert!( - marker.exists().expect("marker existence should resolve"), - "the marker must remain armed after repeated persist" - ); - - marker.clear().expect("first clear should succeed"); - marker.clear().expect("second clear should succeed"); - assert!( - !marker.exists().expect("marker existence should resolve"), - "the marker must remain cleared after repeated clear" - ); - - remove_test_git_dir(&git_dir); - } -} diff --git a/cli/src/services/mutation_trace/runtime/git_snapshot.rs b/cli/src/services/mutation_trace/runtime/git_snapshot.rs index 0a0379bc2..0c64a8c9d 100644 --- a/cli/src/services/mutation_trace/runtime/git_snapshot.rs +++ b/cli/src/services/mutation_trace/runtime/git_snapshot.rs @@ -1,21 +1,19 @@ use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::Stdio; use anyhow::{anyhow, bail, Context, Result}; +use tokio::process::Command; use uuid::Uuid; use crate::services::mutation_trace::types::{TreeId, WorktreeId}; +use super::worktree_lock::WorktreeLockLease; + const SCE_RUNTIME_DIR: &str = "sce"; const TMP_INDEX_DIR: &str = "tmp"; const REF_NAMESPACE: &str = "refs/sce/mutation-cursor"; -/// `git for-each-ref` format for pin inventory: four `%00`-separated fields — -/// refname, target object name, target object type, and the symbolic-ref -/// target (empty for a direct ref). NUL-separated so no field can be split or -/// trimmed ambiguously; the trailing symref field is always present (possibly -/// empty), so every well-formed line has exactly four fields. const FOR_EACH_REF_PIN_FORMAT: &str = "--format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)"; @@ -25,44 +23,86 @@ pub struct GitSnapshotService { } impl GitSnapshotService { - pub fn new(repository_root: &Path) -> Result { - let repository_root = resolve_worktree_root(repository_root)?; - let git_dir = resolve_git_dir(&repository_root)?; + pub async fn new(repository_root: &Path) -> Result { + let repository_root = resolve_worktree_root(repository_root).await?; + let git_dir = resolve_git_dir(&repository_root).await?; Ok(GitSnapshotService { git_dir, repository_root, }) } - pub fn capture_tree(&self) -> Result { + pub async fn capture_tree(&self) -> Result { + self.capture_tree_inner(|_| {}).await + } + + async fn capture_tree_inner(&self, after_read_tree: impl FnOnce(&Path)) -> Result { let tmp_dir = self.git_dir.join(SCE_RUNTIME_DIR).join(TMP_INDEX_DIR); - std::fs::create_dir_all(&tmp_dir).with_context(|| { + tokio::fs::create_dir_all(&tmp_dir).await.with_context(|| { format!( "Failed to create temporary index directory '{}'", tmp_dir.display() ) })?; - let index_guard = TempIndexGuard::reserve(&tmp_dir); + let mut index_guard = TempIndexGuard::reserve(&tmp_dir); + + let result = self + .capture_tree_with_index(index_guard.path(), after_read_tree) + .await; + index_guard.cleanup().await; + result + } - if self.head_exists()? { - self.run_git(&["read-tree", "HEAD"], Some(&index_guard.path))?; + async fn capture_tree_with_index( + &self, + index_file: &Path, + after_read_tree: impl FnOnce(&Path), + ) -> Result { + if self.head_exists().await? { + self.run_git(&["read-tree", "HEAD"], Some(index_file)) + .await?; } else { - self.run_git(&["read-tree", "--empty"], Some(&index_guard.path))?; + self.run_git(&["read-tree", "--empty"], Some(index_file)) + .await?; } - self.run_git(&["add", "-A", "--", "."], Some(&index_guard.path))?; + after_read_tree(index_file); + + self.run_git(&["add", "-A", "--", "."], Some(index_file)) + .await?; - let tree_sha = self.run_git(&["write-tree"], Some(&index_guard.path))?; + let tree_sha = self.run_git(&["write-tree"], Some(index_file)).await?; Ok(TreeId(tree_sha.trim().to_string())) } - pub fn pin_tree(&self, worktree_id: &WorktreeId, tree: &TreeId) -> Result<()> { + pub async fn pin_tree( + &self, + lease: WorktreeLockLease, + worktree_id: &WorktreeId, + tree: &TreeId, + ) -> Result<()> { + self.pin_tree_inner(lease, worktree_id, tree, || {}).await + } + + async fn pin_tree_inner( + &self, + lease: WorktreeLockLease, + worktree_id: &WorktreeId, + tree: &TreeId, + on_worker_entered: impl FnOnce() + Send + 'static, + ) -> Result<()> { let ref_name = pin_ref_name(worktree_id, tree); - self.run_git(&["update-ref", &ref_name, &tree.0], None)?; + self.run_ref_mutation_inner( + lease, + vec!["update-ref".to_string(), ref_name, tree.0.clone()], + None, + on_worker_entered, + ) + .await?; Ok(()) } - pub fn diff_trees(&self, before: &TreeId, after: &TreeId) -> Result { + pub async fn diff_trees(&self, before: &TreeId, after: &TreeId) -> Result { self.run_git( &[ "diff", @@ -75,20 +115,20 @@ impl GitSnapshotService { ], None, ) + .await } - pub fn head_tree(&self) -> Result { - let tree = self.run_git(&["rev-parse", "HEAD^{tree}"], None)?; + pub async fn head_tree(&self) -> Result { + let tree = self.run_git(&["rev-parse", "HEAD^{tree}"], None).await?; Ok(TreeId(tree.trim().to_string())) } - pub fn file_at_tree(&self, tree: &TreeId, path: &str) -> Result> { + pub async fn file_at_tree(&self, tree: &TreeId, path: &str) -> Result> { let spec = format!("{}:{}", tree.0, path); - let output = Command::new("git") - .args(["cat-file", "blob", &spec]) - .current_dir(&self.repository_root) - .env("GIT_DIR", &self.git_dir) + let output = self + .cancellable_git_command(&["cat-file", "blob", &spec], None) .output() + .await .with_context(|| { format!( "Failed to run git cat-file blob '{spec}' in '{}'", @@ -102,26 +142,14 @@ impl GitSnapshotService { Ok(String::from_utf8(output.stdout).ok()) } - /// Inventory every SCE snapshot pin owned by `worktree_id`. - /// - /// Runs `git for-each-ref` constrained to the single path prefix - /// `refs/sce/mutation-cursor//`, so a ref owned by any other - /// worktree or in an unrelated namespace is never returned. Each line is - /// validated against the shape `pin_tree` produces: a **direct** ref (never - /// a symbolic ref) whose target is a tree object and whose final path - /// component equals the target SHA. A symbolic ref anywhere in the - /// namespace is malformed state — it would let one worktree's pin resolve - /// through another worktree's ref — and is rejected rather than followed. A - /// `git for-each-ref` execution or exit failure is - /// [`PinInventoryError::Git`]; anything malformed inside the namespace is - /// [`PinInventoryError::MalformedRef`], matchable separately. - pub fn list_pins( + pub async fn list_pins( &self, worktree_id: &WorktreeId, ) -> std::result::Result, PinInventoryError> { let prefix = pin_ref_prefix(worktree_id); let raw = self .run_git(&["for-each-ref", FOR_EACH_REF_PIN_FORMAT, &prefix], None) + .await .map_err(PinInventoryError::Git)?; raw.lines() @@ -131,47 +159,22 @@ impl GitSnapshotService { .collect() } - /// Delete exactly `pins` in one atomic, no-dereference - /// `git update-ref --no-deref --stdin` transaction, each `delete` - /// conditioned on the tree SHA recorded in the [`PinnedRef`]. - /// - /// Two independent safety properties: - /// - /// - **Atomic** — `git update-ref --stdin` commits every command together - /// at end of input; if any command fails (including a failed old-value - /// check) the whole transaction aborts and no ref is changed. - /// - **No dereference** — `--no-deref` makes every `delete` operate on the - /// exact ref name given, never on a ref reached by resolving a symbolic - /// ref. Combined with a fail-closed re-check (below), a - /// direct-ref → symbolic-ref race between inventory and deletion can - /// never cause this call to touch the symref's target (for example a ref - /// owned by another worktree). - /// - /// Before issuing the transaction, each supplied ref is re-inventoried: it - /// must still exist, still be a direct ref to a tree, and still point at - /// the inventoried SHA. If any has changed — deleted, retargeted, or turned - /// into a symbolic ref — this returns `Err` and deletes nothing, preferring - /// failure over acting on unexpected namespace state. An empty slice is a - /// successful no-op. - pub fn delete_pins(&self, pins: &[PinnedRef]) -> Result<()> { - self.delete_pins_inner(pins, || {}) + pub async fn delete_pins(&self, lease: WorktreeLockLease, pins: &[PinnedRef]) -> Result<()> { + self.delete_pins_inner(lease, pins, || {}, || {}).await } - /// Body of [`delete_pins`] with a deterministic test seam that fires - /// **after** the fail-closed preflight re-inventory and **before** the - /// `git update-ref --no-deref --stdin` transaction is spawned. Production - /// calls it with a no-op hook; the inline atomicity test uses the hook to - /// mutate a ref *after* it has passed preflight, so the transaction is - /// actually issued and the per-`delete` expected-old-value check — not the - /// preflight — is what aborts the batch. This is the only proof that the - /// Git transaction itself is atomic; the preflight proves a different - /// property (unexpected ref state before the transaction is even attempted). - fn delete_pins_inner(&self, pins: &[PinnedRef], after_preflight: impl FnOnce()) -> Result<()> { + async fn delete_pins_inner( + &self, + lease: WorktreeLockLease, + pins: &[PinnedRef], + after_preflight: impl FnOnce(), + on_worker_entered: impl FnOnce() + Send + 'static, + ) -> Result<()> { if pins.is_empty() { return Ok(()); } - self.assert_pins_are_unchanged_direct_refs(pins)?; + self.assert_pins_are_unchanged_direct_refs(pins).await?; after_preflight(); @@ -184,58 +187,44 @@ impl GitSnapshotService { stdin_payload.push('\n'); } - let mut child = Command::new("git") - .args(["update-ref", "--no-deref", "--stdin"]) - .current_dir(&self.repository_root) - .env("GIT_DIR", &self.git_dir) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .with_context(|| { - format!( - "Failed to run git update-ref --no-deref --stdin in '{}'", - self.repository_root.display() - ) - })?; - - child - .stdin - .take() - .ok_or_else(|| anyhow!("Failed to open stdin for git update-ref --no-deref --stdin"))? - .write_all(stdin_payload.as_bytes()) - .with_context(|| { - "Failed to write the delete transaction to git update-ref --no-deref --stdin" - })?; - - let output = child - .wait_with_output() - .with_context(|| "Failed to wait for git update-ref --no-deref --stdin")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let detail = if stderr.is_empty() { stdout } else { stderr }; - return Err(anyhow!( - "git update-ref --no-deref --stdin failed: {detail}" - )); - } + self.run_ref_mutation_inner( + lease, + vec![ + "update-ref".to_string(), + "--no-deref".to_string(), + "--stdin".to_string(), + ], + Some(stdin_payload.into_bytes()), + on_worker_entered, + ) + .await?; Ok(()) } - /// Fail closed unless every supplied pin is still exactly the direct ref - /// that was inventoried: present, a direct (non-symbolic) ref, targeting a - /// tree, and pointing at the recorded SHA. Re-inventoried in a single - /// `git for-each-ref` over the exact ref names, so no enumeration order is - /// relied on. This closes the common inventory→delete race cleanly; the - /// residual sub-transaction race is still contained by `--no-deref` plus - /// the per-`delete` old-value condition, which together cannot follow a - /// symbolic ref or mutate a ref the caller did not name. - fn assert_pins_are_unchanged_direct_refs(&self, pins: &[PinnedRef]) -> Result<()> { + async fn run_ref_mutation_inner( + &self, + lease: WorktreeLockLease, + args: Vec, + stdin: Option>, + on_worker_entered: impl FnOnce() + Send + 'static, + ) -> Result { + let repository_root = self.repository_root.clone(); + let git_dir = self.git_dir.clone(); + + tokio::task::spawn_blocking(move || { + let _lease = lease; + on_worker_entered(); + run_ref_mutation_blocking(&repository_root, &git_dir, &args, stdin.as_deref()) + }) + .await + .map_err(|source| anyhow!("Git ref mutation worker failed: {source}"))? + } + + async fn assert_pins_are_unchanged_direct_refs(&self, pins: &[PinnedRef]) -> Result<()> { let mut args: Vec<&str> = vec!["for-each-ref", FOR_EACH_REF_PIN_FORMAT]; args.extend(pins.iter().map(|pin| pin.ref_name.as_str())); - let raw = self.run_git(&args, None)?; + let raw = self.run_git(&args, None).await?; let current: Vec<[&str; 4]> = raw .lines() @@ -284,12 +273,11 @@ impl GitSnapshotService { Ok(()) } - fn head_exists(&self) -> Result { - let output = Command::new("git") - .args(["rev-parse", "--verify", "--quiet", "HEAD"]) - .current_dir(&self.repository_root) - .env("GIT_DIR", &self.git_dir) + async fn head_exists(&self) -> Result { + let output = self + .cancellable_git_command(&["rev-parse", "--verify", "--quiet", "HEAD"], None) .output() + .await .with_context(|| { format!( "Failed to run git rev-parse --verify --quiet HEAD in '{}'", @@ -312,23 +300,31 @@ impl GitSnapshotService { } } - fn run_git(&self, args: &[&str], index_file: Option<&Path>) -> Result { + fn cancellable_git_command(&self, args: &[&str], index_file: Option<&Path>) -> Command { let mut command = Command::new("git"); command .args(args) .current_dir(&self.repository_root) - .env("GIT_DIR", &self.git_dir); + .env("GIT_DIR", &self.git_dir) + .kill_on_drop(true); if let Some(index_file) = index_file { command.env("GIT_INDEX_FILE", index_file); } + command + } - let output = command.output().with_context(|| { - format!( - "Failed to run git command {:?} in '{}'", - args, - self.repository_root.display() - ) - })?; + async fn run_git(&self, args: &[&str], index_file: Option<&Path>) -> Result { + let output = self + .cancellable_git_command(args, index_file) + .output() + .await + .with_context(|| { + format!( + "Failed to run git command {:?} in '{}'", + args, + self.repository_root.display() + ) + })?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); @@ -342,15 +338,12 @@ impl GitSnapshotService { } } -/// One SCE-owned snapshot pin: a ref under -/// `refs/sce/mutation-cursor//` and the tree object it protects. #[derive(Clone, Debug, Eq, PartialEq)] pub struct PinnedRef { pub ref_name: String, pub tree: TreeId, } -/// Why a worktree's pin inventory could not be produced. #[derive(Debug)] pub enum PinInventoryError { /// `git for-each-ref` itself failed to execute or exited non-zero. @@ -446,11 +439,62 @@ fn pin_ref_name(worktree_id: &WorktreeId, tree: &TreeId) -> String { format!("{REF_NAMESPACE}/{}/{}", worktree_id.0, tree.0) } -pub(crate) fn resolve_git_dir(repository_root: &Path) -> Result { - let git_dir = PathBuf::from(run_rev_parse( - repository_root, - &["rev-parse", "--absolute-git-dir"], - )?); +fn run_ref_mutation_blocking( + repository_root: &Path, + git_dir: &Path, + args: &[String], + stdin: Option<&[u8]>, +) -> Result { + let mut command = std::process::Command::new("git"); + command + .args(args) + .current_dir(repository_root) + .env("GIT_DIR", git_dir); + + let output = match stdin { + None => command.output().with_context(|| { + format!( + "Failed to run git command {args:?} in '{}'", + repository_root.display() + ) + })?, + Some(payload) => { + let mut child = command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| { + format!( + "Failed to run git command {args:?} in '{}'", + repository_root.display() + ) + })?; + + let write_result = match child.stdin.take() { + Some(mut child_stdin) => child_stdin.write_all(payload), + None => Err(std::io::Error::other("child stdin was not piped")), + }; + let output = child.wait_with_output(); + + write_result.with_context(|| format!("Failed to write stdin to git {args:?}"))?; + output.with_context(|| format!("Failed to wait for git {args:?}"))? + } + }; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let detail = if stderr.is_empty() { stdout } else { stderr }; + return Err(anyhow!("git {args:?} failed: {detail}")); + } + + String::from_utf8(output.stdout).with_context(|| format!("git {args:?} emitted invalid UTF-8")) +} + +pub(crate) async fn resolve_git_dir(repository_root: &Path) -> Result { + let git_dir = + PathBuf::from(run_rev_parse(repository_root, &["rev-parse", "--absolute-git-dir"]).await?); debug_assert!( git_dir.is_absolute(), @@ -461,11 +505,14 @@ pub(crate) fn resolve_git_dir(repository_root: &Path) -> Result { Ok(git_dir) } -fn resolve_git_common_dir(repository_root: &Path) -> Result { - let git_common_dir = PathBuf::from(run_rev_parse( - repository_root, - &["rev-parse", "--path-format=absolute", "--git-common-dir"], - )?); +async fn resolve_git_common_dir(repository_root: &Path) -> Result { + let git_common_dir = PathBuf::from( + run_rev_parse( + repository_root, + &["rev-parse", "--path-format=absolute", "--git-common-dir"], + ) + .await?, + ); debug_assert!( git_common_dir.is_absolute(), @@ -476,11 +523,12 @@ fn resolve_git_common_dir(repository_root: &Path) -> Result { Ok(git_common_dir) } -pub(crate) fn resolve_worktree_root(repository_root: &Path) -> Result { +pub(crate) async fn resolve_worktree_root(repository_root: &Path) -> Result { let reported_root = run_rev_parse( repository_root, &["rev-parse", "--path-format=absolute", "--show-toplevel"], - )?; + ) + .await?; let root_path = PathBuf::from(reported_root); debug_assert!( @@ -489,13 +537,16 @@ pub(crate) fn resolve_worktree_root(repository_root: &Path) -> Result { root_path.display() ); - let canonical_root = std::fs::canonicalize(&root_path).with_context(|| { + let canonical_root = tokio::fs::canonicalize(&root_path).await.with_context(|| { format!( "failed to canonicalize the worktree root '{}'", root_path.display() ) })?; - if !canonical_root.is_dir() { + let is_directory = tokio::fs::metadata(&canonical_root) + .await + .is_ok_and(|metadata| metadata.is_dir()); + if !is_directory { bail!( "resolved worktree root '{}' is not a directory", canonical_root.display() @@ -505,9 +556,9 @@ pub(crate) fn resolve_worktree_root(repository_root: &Path) -> Result { Ok(canonical_root) } -pub(crate) fn resolve_worktree_id(repository_root: &Path) -> Result { - let git_dir = resolve_git_dir(repository_root)?; - let git_common_dir = resolve_git_common_dir(repository_root)?; +pub(crate) async fn resolve_worktree_id(repository_root: &Path) -> Result { + let git_dir = resolve_git_dir(repository_root).await?; + let git_common_dir = resolve_git_common_dir(repository_root).await?; if git_dir == git_common_dir { return Ok(WorktreeId("main".to_string())); @@ -538,11 +589,13 @@ fn sanitize_ref_component(raw: &str) -> String { .collect() } -fn run_rev_parse(repository_root: &Path, args: &[&str]) -> Result { +async fn run_rev_parse(repository_root: &Path, args: &[&str]) -> Result { let output = Command::new("git") .args(args) .current_dir(repository_root) + .kill_on_drop(true) .output() + .await .with_context(|| { format!( "Failed to run git {args:?} in '{}'", @@ -566,954 +619,483 @@ fn run_rev_parse(repository_root: &Path, args: &[&str]) -> Result { } struct TempIndexGuard { - path: PathBuf, + path: Option, } impl TempIndexGuard { fn reserve(tmp_dir: &Path) -> TempIndexGuard { let path = tmp_dir.join(format!("index-{}", Uuid::new_v4())); - TempIndexGuard { path } + TempIndexGuard { path: Some(path) } + } + + fn path(&self) -> &Path { + self.path + .as_deref() + .expect("temporary index guard is still armed") + } + + async fn cleanup(&mut self) { + let Some(path) = self.path.as_deref() else { + return; + }; + let _ = tokio::fs::remove_file(path).await; + self.path = None; } } impl Drop for TempIndexGuard { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); + let Some(path) = self.path.take() else { + return; + }; + if let Ok(handle) = tokio::runtime::Handle::try_current() { + drop(handle.spawn_blocking(move || { + let _ = std::fs::remove_file(path); + })); + } else { + let _ = std::fs::remove_file(path); + } } } #[cfg(test)] mod tests { - use super::*; - - struct TestRepo { - _temp_dir: tempfile::TempDir, - root: PathBuf, - } - - impl TestRepo { - fn root(&self) -> &Path { - &self.root - } - } - - fn test_repo(label: &str) -> TestRepo { - let temp_dir = tempfile::Builder::new() - .prefix(&format!("sce-git-snapshot-{label}-")) - .tempdir() - .expect("test repository temp directory should be created"); - - let root = temp_dir.path().to_path_buf(); - - TestRepo { - _temp_dir: temp_dir, - root, - } - } - - fn unique_test_repo(label: &str) -> PathBuf { - let path = - std::env::temp_dir().join(format!("sce-git-snapshot-{label}-{}", Uuid::new_v4())); - std::fs::create_dir_all(&path).expect("repo root should be created"); - path - } - - fn remove_test_repo(repo_root: &Path) { - std::fs::remove_dir_all(repo_root).expect("test repository should be removable"); - } + use std::path::Path; + use std::process::Command; + use std::time::Duration; + + use super::super::worktree_lock::{WorktreeLock, WorktreeLockError}; + use super::{ + resolve_git_dir, resolve_worktree_id, GitSnapshotService, PinnedRef, TempIndexGuard, + SCE_RUNTIME_DIR, TMP_INDEX_DIR, + }; + use crate::services::mutation_trace::types::WorktreeId; - fn init_repo(repo_root: &Path) { - std::fs::create_dir_all(repo_root).expect("repo root should be created"); - run(repo_root, &["init", "--quiet"]); - run(repo_root, &["config", "user.email", "test@example.com"]); - run(repo_root, &["config", "user.name", "Test"]); - } + const LOCK_CONTENDED_TIMEOUT: Duration = Duration::from_millis(300); + const LOCK_RELEASE_TIMEOUT: Duration = Duration::from_secs(10); + const GUARD_DROP_CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); - fn run(repo_root: &Path, args: &[&str]) -> String { + fn git(cwd: &Path, args: &[&str]) { let output = Command::new("git") .args(args) - .current_dir(repo_root) + .current_dir(cwd) .output() - .expect("git command should spawn"); + .expect("git command should run"); assert!( output.status.success(), "git {args:?} failed: {}", String::from_utf8_lossy(&output.stderr) ); - String::from_utf8(output.stdout).expect("git output should be valid UTF-8") } - fn commit_all(repo_root: &Path, message: &str) { - run(repo_root, &["add", "-A"]); - run(repo_root, &["commit", "--quiet", "-m", message]); + fn init_committed_repository(root: &Path) { + std::fs::create_dir_all(root).expect("repo dir"); + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "snapshot@example.invalid"]); + git(root, &["config", "user.name", "Snapshot Test"]); + git(root, &["config", "commit.gpgsign", "false"]); + std::fs::write(root.join("tracked.txt"), "tracked\n").expect("write tracked file"); + git(root, &["add", "tracked.txt"]); + git(root, &["commit", "-q", "-m", "initial"]); + } + + fn temp_index_files(git_dir: &Path) -> Vec { + let tmp_dir = git_dir.join(SCE_RUNTIME_DIR).join(TMP_INDEX_DIR); + let mut names: Vec = std::fs::read_dir(&tmp_dir) + .expect("read temporary index dir") + .map(|entry| { + entry + .expect("temporary index dir entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + names.sort(); + names } - fn worktree_id() -> WorktreeId { - WorktreeId("test-worktree".to_string()) - } + #[tokio::test] + async fn capture_tree_removes_temporary_index_before_returning_and_keeps_real_index() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + std::fs::write(repo_root.join("untracked.txt"), "untracked\n").expect("write file"); - #[test] - fn capture_preserves_real_index_and_working_tree_state() { - let repo = test_repo("preserves-index"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("committed.txt"), b"original\n") - .expect("committed file should be writable"); - commit_all(&repo_root, "initial commit"); - - std::fs::write(repo_root.join("staged.txt"), b"staged\n") - .expect("staged file should be writable"); - run(&repo_root, &["add", "staged.txt"]); - std::fs::write(repo_root.join("committed.txt"), b"modified\n") - .expect("unstaged modification should be writable"); - std::fs::write(repo_root.join("untracked.txt"), b"untracked\n") - .expect("untracked file should be writable"); - - let status_before = run(&repo_root, &["status", "--porcelain"]); - let diff_before = run(&repo_root, &["diff"]); - let diff_cached_before = run(&repo_root, &["diff", "--cached"]); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service - .capture_tree() - .expect("capture should succeed with mixed staged/unstaged/untracked state"); - assert!(!tree.0.is_empty()); - - let status_after = run(&repo_root, &["status", "--porcelain"]); - let diff_after = run(&repo_root, &["diff"]); - let diff_cached_after = run(&repo_root, &["diff", "--cached"]); - - assert_eq!(status_before, status_after); - assert_eq!(diff_before, diff_after); - assert_eq!(diff_cached_before, diff_cached_after); - - let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); - assert!(ls_tree.contains("staged.txt")); - assert!(ls_tree.contains("untracked.txt")); - let committed_blob = run(&repo_root, &["show", &format!("{}:committed.txt", tree.0)]); - assert_eq!(committed_blob, "modified\n"); - } + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let real_index_before = std::fs::read(git_dir.join("index")).expect("read real index"); - #[test] - fn capture_excludes_ignored_files() { - let repo = test_repo("ignored-files"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join(".gitignore"), b"ignored.txt\n") - .expect(".gitignore should be writable"); - commit_all(&repo_root, "add gitignore"); - std::fs::write(repo_root.join("ignored.txt"), b"should not appear\n") - .expect("ignored file should be writable"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service - .capture_tree() - .expect("capture should succeed with an ignored file present"); - - let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); - assert!(!ls_tree.contains("ignored.txt")); - } + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("snapshot service"); + let tree = snapshot.capture_tree().await.expect("capture tree"); - #[test] - fn capture_from_nested_directory_uses_worktree_root_for_gitignore_rules() { - let repo = test_repo("nested-directory-ignore"); - let repo_root = repo.root().to_path_buf(); - let nested_root = repo_root.join("cli"); - init_repo(&repo_root); - std::fs::write(repo_root.join(".gitignore"), b"cli/target/\n") - .expect(".gitignore should be writable"); - std::fs::write(repo_root.join("README.md"), b"tracked\n") - .expect("README should be writable"); - commit_all(&repo_root, "add ignore rule"); - - std::fs::create_dir_all(nested_root.join("target")) - .expect("nested target directory should be creatable"); - std::fs::write(nested_root.join("target/artifact"), b"ignored\n") - .expect("ignored artifact should be writable"); - - let service = GitSnapshotService::new(&nested_root) - .expect("service should resolve the containing worktree root"); - let tree = service - .capture_tree() - .expect("capture should succeed from a nested directory"); - - let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); - assert!(ls_tree.contains("README.md")); - assert!( - !ls_tree.lines().any(|path| path == "cli/target/artifact"), - "ignored files below cli/target must not enter a snapshot: {ls_tree}" + assert!(temp_index_files(&git_dir).is_empty()); + assert_eq!( + std::fs::read(git_dir.join("index")).expect("read real index after capture"), + real_index_before ); - } - - #[test] - fn capture_reflects_deletion_of_a_committed_file() { - let repo = test_repo("deletion"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("to-delete.txt"), b"will be removed\n") - .expect("file should be writable"); - commit_all(&repo_root, "add file to delete"); - std::fs::remove_file(repo_root.join("to-delete.txt")).expect("file should be removable"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service - .capture_tree() - .expect("capture should succeed with a deletion pending"); - - let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); - assert!(!ls_tree.contains("to-delete.txt")); - } - - #[test] - fn capture_on_unborn_head_with_a_file_produces_a_valid_tree() { - let repo = test_repo("unborn-head-with-file"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("untracked.txt"), b"before any commit\n") - .expect("untracked file should be writable"); - - let head_probe = Command::new("git") - .args(["rev-parse", "--verify", "--quiet", "HEAD"]) - .current_dir(&repo_root) - .output() - .expect("git rev-parse should spawn"); - assert!( - !head_probe.status.success(), - "HEAD should be unborn in a freshly initialized repository" + assert_eq!( + snapshot + .file_at_tree(&tree, "tracked.txt") + .await + .expect("read tracked file"), + Some("tracked\n".to_string()) ); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service - .capture_tree() - .expect("capture should succeed on an unborn HEAD with a file present"); - - let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); - assert!(ls_tree.contains("untracked.txt")); - } - - #[test] - fn an_unexpected_head_probe_failure_propagates_instead_of_using_read_tree_empty() { - let repo = test_repo("head-probe-failure"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); - commit_all(&repo_root, "initial commit"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - std::fs::remove_file(service.git_dir.join("HEAD")) - .expect("HEAD file should be removable to simulate an unexpected failure"); - - let result = service.capture_tree(); - assert!( - result.is_err(), - "an unexpected HEAD-probe failure must surface as an error, not a false empty-baseline capture" + assert_eq!( + snapshot + .file_at_tree(&tree, "untracked.txt") + .await + .expect("read untracked file"), + Some("untracked\n".to_string()) ); } - fn relative_path_from(base: &Path, target: &Path) -> PathBuf { - let base_components: Vec<_> = base.components().collect(); - let target_components: Vec<_> = target.components().collect(); - - let common_len = base_components - .iter() - .zip(target_components.iter()) - .take_while(|(a, b)| a == b) - .count(); - - let mut relative = PathBuf::new(); - for _ in common_len..base_components.len() { - relative.push(".."); - } - for component in &target_components[common_len..] { - relative.push(component); - } - relative - } + #[tokio::test] + async fn capture_tree_removes_temporary_index_and_keeps_original_git_error() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + std::fs::write(repo_root.join("untracked.txt"), "untracked\n").expect("write file"); + + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("snapshot service"); + let mut index_name = None; + + let error = snapshot + .capture_tree_inner(|index_file| { + assert!(index_file.is_file(), "read-tree should create the index"); + let lock_file = index_file.with_file_name(format!( + "{}.lock", + index_file + .file_name() + .expect("index name") + .to_string_lossy() + )); + std::fs::write(&lock_file, "").expect("hold temporary index lock"); + index_name = Some(index_file.file_name().expect("index name").to_owned()); + }) + .await + .expect_err("git add must fail while the index lock is held"); - #[test] - fn resolves_an_absolute_git_dir_from_a_relative_repository_root() { - let repo = test_repo("relative-root"); - let repo_root_abs = repo.root().to_path_buf(); - init_repo(&repo_root_abs); - std::fs::write(repo_root_abs.join("file.txt"), b"relative content\n") - .expect("file should be writable"); - - let cwd = std::env::current_dir().expect("current dir should be readable"); - let canonical_repo_root = repo_root_abs - .canonicalize() - .expect("temp repo root should canonicalize"); - let relative_repo_root = relative_path_from(&cwd, &canonical_repo_root); + let index_name = index_name.expect("hook ran").to_string_lossy().into_owned(); + let message = format!("{error:#}"); assert!( - relative_repo_root.is_relative(), - "test setup should produce a genuinely relative repository root" + message.starts_with("git [\"add\", \"-A\", \"--\", \".\"] failed:"), + "unexpected error: {message}" ); - - let service = GitSnapshotService::new(&relative_repo_root) - .expect("service should resolve git-dir from a relative repository root"); - assert!( - service.git_dir.is_absolute(), - "GitSnapshotService must always store an absolute git_dir, even from a relative repository_root" + assert_eq!( + temp_index_files(&git_dir), + vec![format!("{index_name}.lock")] ); - - let tree = service - .capture_tree() - .expect("capture should succeed with a relative repository root"); - let ls_tree = run(&repo_root_abs, &["ls-tree", "-r", "--name-only", &tree.0]); - assert!(ls_tree.contains("file.txt")); - - service - .pin_tree(&worktree_id(), &tree) - .expect("pin should succeed with a relative repository root"); - let diff = service - .diff_trees(&tree, &tree) - .expect("diff should succeed with a relative repository root"); - assert!(diff.is_empty()); } - #[test] - fn capture_on_unborn_head_with_no_files_produces_an_empty_tree() { - let repo = test_repo("unborn-head-no-files"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); + #[tokio::test] + async fn dropped_temp_index_guard_schedules_cleanup_off_the_runtime_worker() { + let dir = tempfile::tempdir().expect("temp dir"); + let guard = TempIndexGuard::reserve(dir.path()); + let index_file = guard.path().to_path_buf(); + std::fs::write(&index_file, "index").expect("create temporary index"); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service - .capture_tree() - .expect("capture should succeed on an unborn HEAD with no files at all"); + drop(guard); - let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); - assert_eq!(ls_tree.trim(), ""); - } - - #[test] - fn snapshot_survives_a_fresh_process_and_temp_index_deletion() { - let repo = test_repo("survives-process-exit"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service.capture_tree().expect("capture should succeed"); - - let tmp_dir = service.git_dir.join(SCE_RUNTIME_DIR).join(TMP_INDEX_DIR); - assert!( - std::fs::read_dir(&tmp_dir).map_or(true, |mut entries| entries.next().is_none()), - "temp index directory should not retain the completed capture's index file" - ); - - let resolved = run(&repo_root, &["cat-file", "-t", &tree.0]); - assert_eq!(resolved.trim(), "tree"); + assert_eq!(tokio::spawn(async { 7 }).await.expect("unrelated task"), 7); + tokio::time::timeout(GUARD_DROP_CLEANUP_TIMEOUT, async { + while tokio::fs::try_exists(&index_file) + .await + .expect("check temporary index") + { + tokio::task::yield_now().await; + } + }) + .await + .expect("dropped guard should remove the temporary index"); } - #[test] - fn pinned_snapshot_survives_git_gc_prune_now() { - let repo = test_repo("gc-prune-now"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("pinned.txt"), b"pinned content\n") - .expect("file should be writable"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let pinned_tree = service.capture_tree().expect("capture should succeed"); - service - .pin_tree(&worktree_id(), &pinned_tree) - .expect("pin should succeed"); - - std::fs::write( - repo_root.join("unreachable.txt"), - b"distinct unreachable content\n", - ) - .expect("file should be writable"); - let unpinned_tree = service.capture_tree().expect("capture should succeed"); - assert_ne!(pinned_tree.0, unpinned_tree.0); - - run(&repo_root, &["gc", "--prune=now"]); + #[tokio::test] + async fn async_snapshot_service_captures_pins_inventories_and_deletes_tree() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + std::fs::write(repo_root.join("untracked.txt"), "untracked\n").expect("write file"); - let pinned_resolved = run(&repo_root, &["cat-file", "-t", &pinned_tree.0]); - assert_eq!(pinned_resolved.trim(), "tree"); + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("snapshot service"); + let worktree_id = WorktreeId("main".to_string()); - let unpinned_probe = Command::new("git") - .args(["cat-file", "-t", &unpinned_tree.0]) - .current_dir(&repo_root) - .output() - .expect("git cat-file should spawn"); - assert!( - !unpinned_probe.status.success(), - "an unpinned, unreachable tree should be reclaimed by git gc --prune=now" + let tree = snapshot.capture_tree().await.expect("capture tree"); + assert_eq!( + snapshot + .file_at_tree(&tree, "untracked.txt") + .await + .expect("read captured file"), + Some("untracked\n".to_string()) ); - } - #[test] - fn pinned_snapshot_survives_git_prune_expire_now() { - let repo = test_repo("prune-expire-now"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("pinned.txt"), b"pinned content\n") - .expect("file should be writable"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let pinned_tree = service.capture_tree().expect("capture should succeed"); - service - .pin_tree(&worktree_id(), &pinned_tree) - .expect("pin should succeed"); - - std::fs::write( - repo_root.join("unreachable-2.txt"), - b"another distinct unreachable content\n", - ) - .expect("file should be writable"); - let unpinned_tree = service.capture_tree().expect("capture should succeed"); - assert_ne!(pinned_tree.0, unpinned_tree.0); - - run(&repo_root, &["prune", "--expire=now"]); + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let lock = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("worktree lock"); - let pinned_resolved = run(&repo_root, &["cat-file", "-t", &pinned_tree.0]); - assert_eq!(pinned_resolved.trim(), "tree"); - - let unpinned_probe = Command::new("git") - .args(["cat-file", "-t", &unpinned_tree.0]) - .current_dir(&repo_root) - .output() - .expect("git cat-file should spawn"); - assert!( - !unpinned_probe.status.success(), - "an unpinned, unreachable tree should be reclaimed by git prune --expire=now" + snapshot + .pin_tree(lock.lease(), &worktree_id, &tree) + .await + .expect("pin tree"); + let pins = snapshot.list_pins(&worktree_id).await.expect("list pins"); + assert_eq!( + pins, + vec![PinnedRef { + ref_name: format!("refs/sce/mutation-cursor/main/{}", tree.0), + tree: tree.clone(), + }] ); - } - - #[test] - fn pin_tree_is_idempotent_for_the_same_worktree_and_tree() { - let repo = test_repo("pin-idempotent"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service.capture_tree().expect("capture should succeed"); - - service - .pin_tree(&worktree_id(), &tree) - .expect("first pin should succeed"); - service - .pin_tree(&worktree_id(), &tree) - .expect("second, identical pin should be a harmless no-op"); - - let ref_name = pin_ref_name(&worktree_id(), &tree); - let resolved = run(&repo_root, &["rev-parse", &ref_name]); - assert_eq!(resolved.trim(), tree.0); - } - - #[test] - fn diff_trees_returns_parseable_git_diff_output() { - let repo = test_repo("diff-trees"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("file.txt"), b"before\n").expect("file should be writable"); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let before = service.capture_tree().expect("capture should succeed"); - - std::fs::write(repo_root.join("file.txt"), b"after\n").expect("file should be writable"); - let after = service.capture_tree().expect("capture should succeed"); - - let diff = service - .diff_trees(&before, &after) - .expect("diff should succeed"); - assert!(diff.contains("file.txt")); - assert!(diff.contains("-before")); - assert!(diff.contains("+after")); - } - #[test] - fn capture_and_pin_work_against_a_sha256_repository_when_supported() { - let repo = test_repo("sha256"); - let repo_root = repo.root().to_path_buf(); - std::fs::create_dir_all(&repo_root).expect("repo root should be created"); - let init = Command::new("git") - .args(["init", "--quiet", "--object-format=sha256"]) - .current_dir(&repo_root) - .output() - .expect("git init should spawn"); - if !init.status.success() { - return; - } - run(&repo_root, &["config", "user.email", "test@example.com"]); - run(&repo_root, &["config", "user.name", "Test"]); - std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); - - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - let tree = service - .capture_tree() - .expect("capture should succeed against a SHA-256 repository"); - service - .pin_tree(&worktree_id(), &tree) - .expect("pin should succeed against a SHA-256 repository"); - - let ls_tree = run(&repo_root, &["ls-tree", "-r", "--name-only", &tree.0]); - assert!(ls_tree.contains("file.txt")); - } - - #[test] - fn resolve_worktree_id_is_stable_and_named_main_for_a_normal_repository() { - let repo_root = unique_test_repo("worktree-id-main"); - init_repo(&repo_root); + snapshot + .delete_pins(lock.lease(), &pins) + .await + .expect("delete pins"); + assert!(snapshot + .list_pins(&worktree_id) + .await + .expect("list pins after delete") + .is_empty()); + } + + #[tokio::test] + async fn cancelled_pin_keeps_worktree_lock_until_git_ref_mutation_completes() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + std::fs::write(repo_root.join("untracked.txt"), "untracked\n").expect("write file"); + + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let worktree_id = WorktreeId("main".to_string()); + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("snapshot service"); + let tree = snapshot.capture_tree().await.expect("capture tree"); + let pin_ref = format!("refs/sce/mutation-cursor/main/{}", tree.0); + + let lock_a = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("worktree lock A"); + let lease = lock_a.lease(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + + let caller = tokio::spawn({ + let repo_root = repo_root.clone(); + let worktree_id = worktree_id.clone(); + let tree = tree.clone(); + async move { + let _lock_a = lock_a; + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("caller snapshot service"); + snapshot + .pin_tree_inner(lease, &worktree_id, &tree, move || { + entered_tx.send(()).expect("signal worker entered"); + release_rx.recv().expect("release worker"); + }) + .await + } + }); - let first = resolve_worktree_id(&repo_root).expect("worktree id should resolve"); - let second = resolve_worktree_id(&repo_root).expect("worktree id should resolve again"); + entered_rx.await.expect("ref mutation worker entered"); + caller.abort(); + assert!(caller + .await + .expect_err("caller must be cancelled") + .is_cancelled()); - assert_eq!(first, WorktreeId("main".to_string())); + assert!(matches!( + WorktreeLock::acquire_async(&git_dir, LOCK_CONTENDED_TIMEOUT).await, + Err(WorktreeLockError::TimedOut { .. }) + )); + assert!(snapshot + .list_pins(&worktree_id) + .await + .expect("list pins before release") + .is_empty()); + + release_tx.send(()).expect("release worker"); + + let lock_b = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("worktree lock B after the ref mutation completes"); + let pins = snapshot.list_pins(&worktree_id).await.expect("list pins"); assert_eq!( - first, second, - "repeated resolution of the same worktree must return the same identity" + pins, + vec![PinnedRef { + ref_name: pin_ref.clone(), + tree: tree.clone(), + }] ); + assert!(!git_dir.join(format!("{pin_ref}.lock")).exists()); + assert!(!git_dir.join("packed-refs.lock").exists()); - remove_test_repo(&repo_root); - } - - #[test] - fn resolve_worktree_id_distinguishes_main_and_linked_worktrees() { - let main_root = unique_test_repo("worktree-id-main-with-links"); - init_repo(&main_root); - let linked_root_a = unique_test_repo("worktree-id-linked-a"); - let linked_root_b = unique_test_repo("worktree-id-linked-b"); - run( - &main_root, - &[ - "worktree", - "add", - "--quiet", - linked_root_a - .to_str() - .expect("linked worktree path a should be UTF-8"), - ], - ); - run( - &main_root, - &[ - "worktree", - "add", - "--quiet", - linked_root_b - .to_str() - .expect("linked worktree path b should be UTF-8"), - ], + git( + &repo_root, + &["update-ref", "refs/sce-test/after-cancelled-pin", "HEAD"], ); - - let main_id = resolve_worktree_id(&main_root).expect("main worktree id should resolve"); - let linked_id_a = - resolve_worktree_id(&linked_root_a).expect("linked worktree a id should resolve"); - let linked_id_b = - resolve_worktree_id(&linked_root_b).expect("linked worktree b id should resolve"); - - assert_eq!(main_id, WorktreeId("main".to_string())); - assert_ne!(main_id, linked_id_a); - assert_ne!(main_id, linked_id_b); - assert_ne!(linked_id_a, linked_id_b); - + snapshot + .delete_pins(lock_b.lease(), &pins) + .await + .expect("delete pins after cancelled pin"); + assert!(snapshot + .list_pins(&worktree_id) + .await + .expect("list pins after delete") + .is_empty()); + } + + #[tokio::test] + async fn cancelled_delete_keeps_worktree_lock_until_git_ref_deletion_completes() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + std::fs::write(repo_root.join("untracked.txt"), "untracked\n").expect("write file"); + + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let worktree_id = WorktreeId("main".to_string()); + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("snapshot service"); + let tree = snapshot.capture_tree().await.expect("capture tree"); + let pin_ref = format!("refs/sce/mutation-cursor/main/{}", tree.0); + + let pin_lock = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("pin lock"); + snapshot + .pin_tree(pin_lock.lease(), &worktree_id, &tree) + .await + .expect("pin orphan tree"); + drop(pin_lock); + let pins = snapshot.list_pins(&worktree_id).await.expect("list pins"); assert_eq!( - resolve_worktree_id(&linked_root_a).expect("linked worktree a id should resolve again"), - linked_id_a, - "repeated resolution of the same linked worktree must return the same identity" - ); - - remove_test_repo(&linked_root_a); - remove_test_repo(&linked_root_b); - remove_test_repo(&main_root); - } - - #[test] - fn resolve_worktree_id_persists_no_sce_identity_file() { - let repo_root = unique_test_repo("worktree-id-no-persistence"); - init_repo(&repo_root); - - let _ = resolve_worktree_id(&repo_root).expect("worktree id should resolve"); - let _ = resolve_worktree_id(&repo_root).expect("worktree id should resolve again"); - - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let sce_dir = git_dir.join(SCE_RUNTIME_DIR); - assert!( - !sce_dir.exists(), - "resolving a worktree id must never create an SCE identity directory or file" + pins, + vec![PinnedRef { + ref_name: pin_ref.clone(), + tree: tree.clone(), + }] ); - remove_test_repo(&repo_root); - } - - fn other_worktree_id() -> WorktreeId { - WorktreeId("other-worktree".to_string()) - } - - fn capture_with_file( - service: &GitSnapshotService, - repo_root: &Path, - name: &str, - body: &[u8], - ) -> TreeId { - std::fs::write(repo_root.join(name), body).expect("file should be writable"); - service.capture_tree().expect("capture should succeed") - } - - fn ref_target(repo_root: &Path, ref_name: &str) -> String { - run(repo_root, &["rev-parse", "--verify", ref_name]) - .trim() - .to_string() - } - - fn ref_exists(repo_root: &Path, ref_name: &str) -> bool { - Command::new("git") - .args(["rev-parse", "--verify", "--quiet", ref_name]) - .current_dir(repo_root) - .output() - .expect("git rev-parse should spawn") - .status - .success() - } - - #[test] - fn list_pins_returns_only_the_target_worktree_prefix() { - let repo = test_repo("list-pins-scoped"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); - let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); - - service - .pin_tree(&worktree_id(), &tree_a) - .expect("pin should succeed"); - service - .pin_tree(&worktree_id(), &tree_b) - .expect("pin should succeed"); - service - .pin_tree(&other_worktree_id(), &tree_a) - .expect("pin should succeed"); - - let pins = service - .list_pins(&worktree_id()) - .expect("inventory should succeed"); - assert_eq!(pins.len(), 2, "only the target worktree's pins are listed"); - - let mut trees: Vec = pins.iter().map(|pin| pin.tree.0.clone()).collect(); - trees.sort(); - let mut expected = vec![tree_a.0.clone(), tree_b.0.clone()]; - expected.sort(); - assert_eq!(trees, expected); - - let prefix = format!("{REF_NAMESPACE}/{}/", worktree_id().0); - for pin in &pins { - assert!(pin.ref_name.starts_with(&prefix)); - assert_eq!(pin.ref_name, format!("{prefix}{}", pin.tree.0)); - } - - assert!(ref_exists( - &repo_root, - &format!("{REF_NAMESPACE}/{}/{}", other_worktree_id().0, tree_a.0) - )); - } - - #[test] - fn list_pins_is_empty_when_the_worktree_has_no_pins() { - let repo = test_repo("list-pins-empty"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let pins = service - .list_pins(&worktree_id()) - .expect("inventory should succeed"); - assert!(pins.is_empty()); - } - - #[test] - fn list_pins_rejects_a_ref_whose_target_is_not_a_tree() { - let repo = test_repo("list-pins-non-tree"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - std::fs::write(repo_root.join("file.txt"), b"content\n").expect("file should be writable"); - commit_all(&repo_root, "initial commit"); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let head = run(&repo_root, &["rev-parse", "HEAD"]).trim().to_string(); - let ref_name = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, head); - run(&repo_root, &["update-ref", &ref_name, &head]); - - match service.list_pins(&worktree_id()) { - Err(PinInventoryError::MalformedRef { - ref_name: rn, - reason, - }) => { - assert_eq!(rn, ref_name); - assert!( - reason.contains("commit"), - "reason names the wrong object type: {reason}" - ); + let lock_a = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("worktree lock A"); + let lease = lock_a.lease(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel::<()>(); + let (release_tx, release_rx) = std::sync::mpsc::channel::<()>(); + + let caller = tokio::spawn({ + let repo_root = repo_root.clone(); + let pins = pins.clone(); + async move { + let _lock_a = lock_a; + let snapshot = GitSnapshotService::new(&repo_root) + .await + .expect("caller snapshot service"); + snapshot + .delete_pins_inner( + lease, + &pins, + || {}, + move || { + entered_tx.send(()).expect("signal worker entered"); + release_rx + .recv_timeout(LOCK_RELEASE_TIMEOUT) + .expect("release worker"); + }, + ) + .await } - other => panic!("expected MalformedRef for a non-tree target, got {other:?}"), - } - } - - #[test] - fn list_pins_rejects_a_ref_whose_name_disagrees_with_its_target() { - let repo = test_repo("list-pins-name-mismatch"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); - let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); - - let ref_name = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0); - run(&repo_root, &["update-ref", &ref_name, &tree_b.0]); - - match service.list_pins(&worktree_id()) { - Err(PinInventoryError::MalformedRef { - ref_name: rn, - reason, - }) => { - assert_eq!(rn, ref_name); - assert!(reason.contains("disagrees"), "unexpected reason: {reason}"); - } - other => panic!("expected MalformedRef for a name/target mismatch, got {other:?}"), - } - } - - #[test] - fn list_pins_rejects_a_ref_with_an_unexpected_extra_path_segment() { - let repo = test_repo("list-pins-extra-segment"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); - let ref_name = format!("{REF_NAMESPACE}/{}/nested/{}", worktree_id().0, tree_a.0); - run(&repo_root, &["update-ref", &ref_name, &tree_a.0]); - - match service.list_pins(&worktree_id()) { - Err(PinInventoryError::MalformedRef { reason, .. }) => { - assert!( - reason.contains("path segment"), - "unexpected reason: {reason}" - ); - } - other => panic!("expected MalformedRef for an extra path segment, got {other:?}"), - } - } - - #[test] - fn list_pins_reports_a_for_each_ref_execution_failure_as_the_git_variant() { - let repo = test_repo("list-pins-git-failure"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - std::fs::remove_dir_all(&service.git_dir).expect("git-dir should be removable"); - - match service.list_pins(&worktree_id()) { - Err(PinInventoryError::Git(_)) => {} - other => panic!( - "expected the Git variant for a for-each-ref execution failure, got {other:?}" - ), - } - } - - #[test] - fn delete_pins_removes_exactly_the_supplied_refs() { - let repo = test_repo("delete-pins-exact"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); - let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); - service - .pin_tree(&worktree_id(), &tree_a) - .expect("pin should succeed"); - service - .pin_tree(&worktree_id(), &tree_b) - .expect("pin should succeed"); - - let inventory = service - .list_pins(&worktree_id()) - .expect("inventory should succeed"); - let stale: Vec = inventory - .into_iter() - .filter(|pin| pin.tree == tree_a) - .collect(); - service.delete_pins(&stale).expect("delete should succeed"); - - let remaining = service - .list_pins(&worktree_id()) - .expect("inventory should succeed"); - assert_eq!(remaining.len(), 1); - assert_eq!(remaining[0].tree, tree_b); - assert!(!ref_exists( - &repo_root, - &format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0) - )); - } - - #[test] - fn delete_pins_is_a_successful_noop_for_an_empty_slice() { - let repo = test_repo("delete-pins-empty"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); - service - .pin_tree(&worktree_id(), &tree_a) - .expect("pin should succeed"); - - service - .delete_pins(&[]) - .expect("empty delete should be a successful no-op"); - - assert!(ref_exists( - &repo_root, - &format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0) - )); - } - - #[test] - fn delete_pins_atomically_aborts_when_a_ref_changes_after_preflight() { - let repo = test_repo("delete-pins-atomic-abort"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_a = capture_with_file(&service, &repo_root, "a.txt", b"a\n"); - let tree_b = capture_with_file(&service, &repo_root, "b.txt", b"b\n"); - let tree_c = capture_with_file(&service, &repo_root, "c.txt", b"c\n"); - service - .pin_tree(&worktree_id(), &tree_a) - .expect("pin should succeed"); - service - .pin_tree(&worktree_id(), &tree_b) - .expect("pin should succeed"); - - let ref_a = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_a.0); - let ref_b = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_b.0); - - let valid_ref = PinnedRef { - ref_name: ref_a.clone(), - tree: tree_a.clone(), - }; - let mismatched_ref = PinnedRef { - ref_name: ref_b.clone(), - tree: tree_b.clone(), - }; - - let pins = [valid_ref, mismatched_ref]; - - let result = service.delete_pins_inner(&pins, || { - run(&repo_root, &["update-ref", &ref_b, &tree_c.0]); }); - result.expect_err( - "the transaction is issued after preflight; git's per-delete expected-old-value \ - check on the second ref must abort the whole batch", + entered_rx.await.expect("ref deletion worker entered"); + assert_eq!( + snapshot.list_pins(&worktree_id).await.expect("list pins"), + pins, + "the parked worker must not have deleted the pin yet" ); - assert!( - ref_exists(&repo_root, &ref_a), - "the first (valid) delete must not have been applied — the batch is all-or-nothing" - ); - assert_eq!(ref_target(&repo_root, &ref_a), tree_a.0); + caller.abort(); + assert!(caller + .await + .expect_err("caller must be cancelled") + .is_cancelled()); - assert!( - ref_exists(&repo_root, &ref_b), - "the mismatched ref still exists" - ); + assert!(matches!( + WorktreeLock::acquire_async(&git_dir, LOCK_CONTENDED_TIMEOUT).await, + Err(WorktreeLockError::TimedOut { .. }) + )); assert_eq!( - ref_target(&repo_root, &ref_b), - tree_c.0, - "the mismatched ref keeps the value it was given after preflight" + snapshot + .list_pins(&worktree_id) + .await + .expect("list pins while the worker is still parked"), + pins ); - } - #[test] - fn list_pins_rejects_a_symbolic_ref_inside_the_mutation_cursor_namespace() { - let repo = test_repo("list-pins-symref"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_t = capture_with_file(&service, &repo_root, "t.txt", b"t\n"); - - let b_ref = format!("{REF_NAMESPACE}/{}/{}", other_worktree_id().0, tree_t.0); - let a_ref = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_t.0); - run(&repo_root, &["update-ref", &b_ref, &tree_t.0]); - run(&repo_root, &["symbolic-ref", &a_ref, &b_ref]); - - match service.list_pins(&worktree_id()) { - Err(PinInventoryError::MalformedRef { - ref_name: rn, - reason, - }) => { - assert_eq!(rn, a_ref); - assert!( - reason.contains("symbolic ref"), - "unexpected reason: {reason}" - ); - } - other => { - panic!("expected MalformedRef for a symbolic ref in the namespace, got {other:?}") - } - } - - assert!(ref_exists(&repo_root, &b_ref)); - assert_eq!(ref_target(&repo_root, &b_ref), tree_t.0); + release_tx.send(()).expect("release worker"); + + let lock_b = WorktreeLock::acquire_async(&git_dir, LOCK_RELEASE_TIMEOUT) + .await + .expect("worktree lock B after the ref deletion completes"); + assert!(snapshot + .list_pins(&worktree_id) + .await + .expect("list pins after the released worker finished") + .is_empty()); + assert!(!git_dir.join(format!("{pin_ref}.lock")).exists()); + assert!(!git_dir.join("packed-refs.lock").exists()); + + snapshot + .pin_tree(lock_b.lease(), &worktree_id, &tree) + .await + .expect("a legitimate pin proceeds after the cancelled deletion"); assert_eq!( - run(&repo_root, &["symbolic-ref", &a_ref]).trim(), - b_ref, - "A/T must still be the untouched symbolic ref" + snapshot + .list_pins(&worktree_id) + .await + .expect("list pins after re-pin"), + pins ); } - #[test] - fn delete_pins_refuses_to_act_when_an_inventoried_direct_ref_became_a_symbolic_ref() { - let repo = test_repo("delete-pins-symref-race"); - let repo_root = repo.root().to_path_buf(); - init_repo(&repo_root); - let service = GitSnapshotService::new(&repo_root).expect("service should resolve git-dir"); - - let tree_t = capture_with_file(&service, &repo_root, "t.txt", b"t\n"); - let a_ref = format!("{REF_NAMESPACE}/{}/{}", worktree_id().0, tree_t.0); - let b_ref = format!("{REF_NAMESPACE}/{}/{}", other_worktree_id().0, tree_t.0); - - service - .pin_tree(&worktree_id(), &tree_t) - .expect("pin should succeed"); - let inventory = service - .list_pins(&worktree_id()) - .expect("inventory should succeed"); - assert_eq!( - inventory, - vec![PinnedRef { - ref_name: a_ref.clone(), - tree: tree_t.clone(), - }] + #[tokio::test] + async fn async_identity_resolution_distinguishes_main_and_linked_worktrees() { + let dir = tempfile::tempdir().expect("temp dir"); + let main_root = dir.path().join("main"); + let linked_root = dir.path().join("linked"); + init_committed_repository(&main_root); + git( + &main_root, + &[ + "worktree", + "add", + "-q", + "-b", + "linked-branch", + linked_root.to_str().expect("utf-8 path"), + ], ); - run(&repo_root, &["update-ref", &b_ref, &tree_t.0]); - run(&repo_root, &["update-ref", "-d", &a_ref]); - run(&repo_root, &["symbolic-ref", &a_ref, &b_ref]); - - let error = service - .delete_pins(&inventory) - .expect_err("delete must fail closed once an inventoried direct ref became a symref"); - assert!( - error.to_string().contains("symbolic ref"), - "unexpected error: {error}" - ); + let main_git_dir = resolve_git_dir(&main_root).await.expect("main git dir"); + let linked_git_dir = resolve_git_dir(&linked_root).await.expect("linked git dir"); + assert_ne!(main_git_dir, linked_git_dir); - assert!(ref_exists(&repo_root, &b_ref)); - assert_eq!(ref_target(&repo_root, &b_ref), tree_t.0); assert_eq!( - run(&repo_root, &["symbolic-ref", &a_ref]).trim(), - b_ref, - "A/T is not touched — delete_pins prefers failure over acting on a symref" + resolve_worktree_id(&main_root).await.expect("main id"), + WorktreeId("main".to_string()) + ); + let linked_id = resolve_worktree_id(&linked_root).await.expect("linked id"); + assert!( + linked_id.0.starts_with("worktrees/"), + "unexpected linked worktree id {linked_id:?}" ); } } diff --git a/cli/src/services/mutation_trace/runtime/maintenance_state.rs b/cli/src/services/mutation_trace/runtime/maintenance_state.rs new file mode 100644 index 000000000..e094a536b --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/maintenance_state.rs @@ -0,0 +1,740 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +pub(super) const RECONCILIATION_ADVISORY_AFTER_MS: i64 = 24 * 60 * 60 * 1000; +pub(super) const MAINTENANCE_STATE_MAX_BYTES: u64 = 4096; +pub(super) const FUTURE_SKEW_TOLERANCE_MS: i64 = 5 * 60 * 1000; +pub(super) const MAX_FAILURE_MESSAGE_CHARS: usize = 256; + +const MAINTENANCE_STATE_VERSION: u32 = 1; +const SCE_RUNTIME_DIR: &str = "sce"; +const MAINTENANCE_STATE_FILE: &str = "ref-maintenance.json"; +const STAGING_ATTEMPTS: u16 = 1000; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(super) enum AttemptOutcome { + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct StoredReport { + pub retained: usize, + pub deleted: usize, + pub local_required: usize, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct StoredFailure { + pub kind: String, + pub message: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub(super) struct MaintenanceState { + pub version: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub anchor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_success: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_attempt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_advised: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_attempt_outcome: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_report: Option, + #[serde(default)] + pub consecutive_failures: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_failure: Option, +} + +impl Default for MaintenanceState { + fn default() -> Self { + Self { + version: MAINTENANCE_STATE_VERSION, + anchor: None, + last_success: None, + last_attempt: None, + last_advised: None, + last_attempt_outcome: None, + last_report: None, + consecutive_failures: 0, + last_failure: None, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) enum StateRead { + Absent, + Unusable, + Valid(MaintenanceState), +} + +impl StateRead { + pub(super) fn into_state(self) -> MaintenanceState { + match self { + StateRead::Valid(state) => state, + StateRead::Absent | StateRead::Unusable => MaintenanceState::default(), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct Evaluation { + pub recommended: bool, + pub advise: bool, + pub invalid_timestamps: bool, + pub reference_age_ms: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct Recommendation { + pub recommended: bool, + pub invalid_timestamps: bool, + pub last_success_age_ms: Option, + pub last_attempt_age_ms: Option, + pub last_advised_age_ms: Option, + pub last_attempt_outcome: Option, + pub consecutive_failures: u32, +} + +pub(super) fn system_now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|elapsed| i64::try_from(elapsed.as_millis()).ok()) + .unwrap_or(0) +} + +pub(super) fn state_path(git_dir: &Path) -> PathBuf { + git_dir.join(SCE_RUNTIME_DIR).join(MAINTENANCE_STATE_FILE) +} + +fn is_valid_timestamp(timestamp: i64, now: i64) -> bool { + timestamp <= now.saturating_add(FUTURE_SKEW_TOLERANCE_MS) +} + +fn age_ms(timestamp: i64, now: i64) -> i64 { + now.saturating_sub(timestamp).max(0) +} + +fn usable(timestamp: Option, now: i64) -> Option { + timestamp.filter(|value| is_valid_timestamp(*value, now)) +} + +pub(super) fn read_state(path: &Path) -> std::io::Result { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(StateRead::Absent); + } + Err(error) => return Err(error), + }; + + let mut bytes = Vec::new(); + file.take(MAINTENANCE_STATE_MAX_BYTES + 1) + .read_to_end(&mut bytes)?; + if u64::try_from(bytes.len()).map_or(true, |len| len > MAINTENANCE_STATE_MAX_BYTES) { + return Ok(StateRead::Unusable); + } + + Ok(match serde_json::from_slice::(&bytes) { + Ok(state) if state.version == MAINTENANCE_STATE_VERSION => StateRead::Valid(state), + Ok(_) | Err(_) => StateRead::Unusable, + }) +} + +fn failure_outstanding(state: &MaintenanceState) -> bool { + state.last_attempt_outcome == Some(AttemptOutcome::Failed) || state.consecutive_failures > 0 +} + +pub(super) fn evaluate(state: &MaintenanceState, now: i64) -> Evaluation { + let timestamps = [ + state.anchor, + state.last_success, + state.last_attempt, + state.last_advised, + ]; + let invalid_timestamps = timestamps + .iter() + .flatten() + .any(|timestamp| !is_valid_timestamp(*timestamp, now)); + + let reference = usable(state.last_success, now).or_else(|| usable(state.anchor, now)); + let reference_lost = + reference.is_none() && (state.last_success.is_some() || state.anchor.is_some()); + let reference_age_ms = reference.map(|timestamp| age_ms(timestamp, now)); + let due = reference_lost + || reference_age_ms.is_some_and(|age| age >= RECONCILIATION_ADVISORY_AFTER_MS); + let advised = usable(state.last_advised, now) + .filter(|timestamp| age_ms(*timestamp, now) < RECONCILIATION_ADVISORY_AFTER_MS); + let failure_outstanding = failure_outstanding(state); + let failure_reminded = advised.is_some_and(|advised_at| { + usable(state.last_attempt, now).is_none_or(|attempt| advised_at >= attempt) + }); + + Evaluation { + recommended: due || failure_outstanding || invalid_timestamps, + advise: (due && advised.is_none()) || (failure_outstanding && !failure_reminded), + invalid_timestamps, + reference_age_ms, + } +} + +pub(super) fn evaluate_recommendation(state: &MaintenanceState, now: i64) -> Recommendation { + let evaluation = evaluate(state, now); + let age_of = |timestamp: Option| usable(timestamp, now).map(|value| age_ms(value, now)); + + Recommendation { + recommended: evaluation.recommended, + invalid_timestamps: evaluation.invalid_timestamps, + last_success_age_ms: age_of(state.last_success), + last_attempt_age_ms: age_of(state.last_attempt), + last_advised_age_ms: age_of(state.last_advised), + last_attempt_outcome: state.last_attempt_outcome, + consecutive_failures: state.consecutive_failures, + } +} + +pub(super) fn normalize(state: &MaintenanceState, now: i64) -> MaintenanceState { + let mut normalized = state.clone(); + normalized.anchor = usable(state.anchor, now); + normalized.last_success = usable(state.last_success, now); + normalized.last_attempt = usable(state.last_attempt, now); + normalized.last_advised = usable(state.last_advised, now); + normalized +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum AdvisoryDecision { + Anchored, + NoAction, + Advised, +} + +pub(super) fn plan_advisory( + read: &StateRead, + now: i64, +) -> (AdvisoryDecision, Option) { + let StateRead::Valid(state) = read else { + let anchored = MaintenanceState { + anchor: Some(now), + ..MaintenanceState::default() + }; + return (AdvisoryDecision::Anchored, Some(anchored)); + }; + + let evaluation = evaluate(state, now); + let mut next = normalize(state, now); + let decision = if evaluation.advise { + next.last_advised = Some(now); + if next.anchor.is_none() && next.last_success.is_none() { + next.anchor = Some(now); + } + AdvisoryDecision::Advised + } else if next.anchor.is_none() && next.last_success.is_none() { + next.anchor = Some(now); + AdvisoryDecision::Anchored + } else { + AdvisoryDecision::NoAction + }; + + let changed = next != *state; + (decision, changed.then_some(next)) +} + +pub(super) fn record_success( + state: &MaintenanceState, + now: i64, + report: StoredReport, +) -> MaintenanceState { + let mut next = normalize(state, now); + next.last_attempt = Some(now); + next.last_success = Some(now); + next.last_attempt_outcome = Some(AttemptOutcome::Completed); + next.last_report = Some(report); + next.consecutive_failures = 0; + next.last_failure = None; + next +} + +pub(super) fn record_failure( + state: &MaintenanceState, + now: i64, + kind: &str, + message: &str, +) -> MaintenanceState { + let mut next = normalize(state, now); + next.last_attempt = Some(now); + next.last_attempt_outcome = Some(AttemptOutcome::Failed); + next.consecutive_failures = state.consecutive_failures.saturating_add(1); + next.last_failure = Some(StoredFailure { + kind: kind.to_string(), + message: message.chars().take(MAX_FAILURE_MESSAGE_CHARS).collect(), + }); + next +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum PersistPhase { + Serialize, + CreateStaging, + WriteStaging, + SyncStaging, + Rename, + OpenParentDirectory, + SyncParentDirectory, +} + +#[derive(Debug)] +pub(super) enum PersistFailure { + NotApplied { + phase: PersistPhase, + source: std::io::Error, + staging_cleanup: Option, + }, + DurabilityUncertain { + phase: PersistPhase, + source: std::io::Error, + }, +} + +impl std::fmt::Display for PersistFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PersistFailure::NotApplied { + phase, + source, + staging_cleanup, + } => { + write!( + f, + "maintenance state was not recorded ({phase:?}): {source}; previous state preserved" + )?; + if let Some(cleanup) = staging_cleanup { + write!(f, "; staging file cleanup failed: {cleanup}")?; + } + Ok(()) + } + PersistFailure::DurabilityUncertain { phase, source } => write!( + f, + "maintenance state was renamed into place but crash durability is unconfirmed ({phase:?}): {source}; the new state may be visible" + ), + } + } +} + +impl std::error::Error for PersistFailure {} + +pub(super) trait StateFilesystem { + fn create_staging(&self, directory: &Path) -> std::io::Result; + fn write_staging(&self, staging: &Path, bytes: &[u8]) -> std::io::Result<()>; + fn sync_staging(&self, staging: &Path) -> std::io::Result<()>; + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()>; + fn open_directory(&self, directory: &Path) -> std::io::Result; + fn sync_directory(&self, directory: &std::fs::File) -> std::io::Result<()>; + fn remove_staging(&self, staging: &Path) -> std::io::Result<()>; +} + +pub(super) struct RealStateFilesystem; + +impl StateFilesystem for RealStateFilesystem { + fn create_staging(&self, directory: &Path) -> std::io::Result { + create_staging_file(directory) + } + + fn write_staging(&self, staging: &Path, bytes: &[u8]) -> std::io::Result<()> { + let mut file = std::fs::OpenOptions::new().write(true).open(staging)?; + file.write_all(bytes) + } + + fn sync_staging(&self, staging: &Path) -> std::io::Result<()> { + std::fs::OpenOptions::new() + .write(true) + .open(staging)? + .sync_data() + } + + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + std::fs::rename(from, to) + } + + fn open_directory(&self, directory: &Path) -> std::io::Result { + std::fs::File::open(directory) + } + + fn sync_directory(&self, directory: &std::fs::File) -> std::io::Result<()> { + directory.sync_all() + } + + fn remove_staging(&self, staging: &Path) -> std::io::Result<()> { + std::fs::remove_file(staging) + } +} + +pub(super) fn write_state_atomically( + path: &Path, + state: &MaintenanceState, +) -> std::result::Result<(), PersistFailure> { + write_state_atomically_with(&RealStateFilesystem, path, state) +} + +fn not_applied( + phase: PersistPhase, + source: std::io::Error, + staging_cleanup: Option, +) -> PersistFailure { + PersistFailure::NotApplied { + phase, + source, + staging_cleanup, + } +} + +pub(super) fn write_state_atomically_with( + fs: &F, + path: &Path, + state: &MaintenanceState, +) -> std::result::Result<(), PersistFailure> { + let bytes = serde_json::to_vec(state) + .map_err(|error| not_applied(PersistPhase::Serialize, error.into(), None))?; + let directory = path.parent().ok_or_else(|| { + not_applied( + PersistPhase::CreateStaging, + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "maintenance state path has no parent directory", + ), + None, + ) + })?; + + let staging = fs + .create_staging(directory) + .map_err(|error| not_applied(PersistPhase::CreateStaging, error, None))?; + + let staged = fs + .write_staging(&staging, &bytes) + .map_err(|error| (PersistPhase::WriteStaging, error)) + .and_then(|()| { + fs.sync_staging(&staging) + .map_err(|error| (PersistPhase::SyncStaging, error)) + }) + .and_then(|()| { + fs.rename(&staging, path) + .map_err(|error| (PersistPhase::Rename, error)) + }); + if let Err((phase, source)) = staged { + return Err(not_applied( + phase, + source, + fs.remove_staging(&staging).err(), + )); + } + + let directory_handle = + fs.open_directory(directory) + .map_err(|source| PersistFailure::DurabilityUncertain { + phase: PersistPhase::OpenParentDirectory, + source, + })?; + fs.sync_directory(&directory_handle) + .map_err(|source| PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + source, + }) +} + +fn create_staging_file(directory: &Path) -> std::io::Result { + let epoch_nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(std::io::Error::other)? + .as_nanos(); + + for attempt in 0..STAGING_ATTEMPTS { + let candidate = directory.join(format!( + ".{MAINTENANCE_STATE_FILE}.staging-{epoch_nanos}-{}-{attempt}", + std::process::id() + )); + match std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&candidate) + { + Ok(_) => return Ok(candidate), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not allocate a unique maintenance state staging file", + )) +} + +pub(super) fn derive_git_dir(start: &Path) -> Option { + if ["GIT_DIR", "GIT_COMMON_DIR", "GIT_WORK_TREE"] + .iter() + .any(|name| std::env::var_os(name).is_some()) + { + return None; + } + + for ancestor in start.ancestors() { + let dot_git = ancestor.join(".git"); + let Ok(metadata) = std::fs::symlink_metadata(&dot_git) else { + continue; + }; + + let git_dir = if metadata.is_dir() { + dot_git + } else if metadata.is_file() { + let contents = std::fs::read_to_string(&dot_git).ok()?; + let target = contents.lines().next()?.strip_prefix("gitdir:")?.trim(); + let target = Path::new(target); + if target.is_absolute() { + target.to_path_buf() + } else { + ancestor.join(target) + } + } else { + return None; + }; + + return git_dir + .is_dir() + .then(|| std::fs::canonicalize(&git_dir).ok()) + .flatten(); + } + + None +} + +#[cfg(test)] +pub(super) struct FaultInjectingFilesystem { + pub fail_at: Option, + pub fail_staging_cleanup: bool, +} + +#[cfg(test)] +impl FaultInjectingFilesystem { + pub(super) const fn failing_at(phase: PersistPhase) -> Self { + Self { + fail_at: Some(phase), + fail_staging_cleanup: false, + } + } + + fn check(&self, phase: PersistPhase) -> std::io::Result<()> { + if self.fail_at == Some(phase) { + return Err(std::io::Error::other(format!("injected {phase:?} failure"))); + } + Ok(()) + } +} + +#[cfg(test)] +impl StateFilesystem for FaultInjectingFilesystem { + fn create_staging(&self, directory: &Path) -> std::io::Result { + self.check(PersistPhase::CreateStaging)?; + RealStateFilesystem.create_staging(directory) + } + + fn write_staging(&self, staging: &Path, bytes: &[u8]) -> std::io::Result<()> { + self.check(PersistPhase::WriteStaging)?; + RealStateFilesystem.write_staging(staging, bytes) + } + + fn sync_staging(&self, staging: &Path) -> std::io::Result<()> { + self.check(PersistPhase::SyncStaging)?; + RealStateFilesystem.sync_staging(staging) + } + + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self.check(PersistPhase::Rename)?; + RealStateFilesystem.rename(from, to) + } + + fn open_directory(&self, directory: &Path) -> std::io::Result { + self.check(PersistPhase::OpenParentDirectory)?; + RealStateFilesystem.open_directory(directory) + } + + fn sync_directory(&self, directory: &std::fs::File) -> std::io::Result<()> { + self.check(PersistPhase::SyncParentDirectory)?; + RealStateFilesystem.sync_directory(directory) + } + + fn remove_staging(&self, staging: &Path) -> std::io::Result<()> { + if self.fail_staging_cleanup { + return Err(std::io::Error::other("injected staging cleanup failure")); + } + RealStateFilesystem.remove_staging(staging) + } +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use super::{ + read_state, record_success, write_state_atomically, write_state_atomically_with, + FaultInjectingFilesystem, MaintenanceState, PersistFailure, PersistPhase, StateRead, + StoredReport, + }; + + const NOW: i64 = 1_000_000_000_000; + + fn state(marker: usize) -> MaintenanceState { + record_success( + &MaintenanceState::default(), + NOW, + StoredReport { + retained: marker, + deleted: 0, + local_required: 0, + }, + ) + } + + fn state_file(dir: &Path) -> PathBuf { + dir.join("ref-maintenance.json") + } + + fn stored_marker(path: &Path) -> Option { + match read_state(path).expect("read state") { + StateRead::Valid(state) => state.last_report.map(|report| report.retained), + StateRead::Absent => None, + StateRead::Unusable => panic!("state unusable"), + } + } + + fn staging_leftovers(dir: &Path) -> usize { + std::fs::read_dir(dir) + .expect("read dir") + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().contains("staging")) + .count() + } + + fn assert_not_applied(phase: PersistPhase, previous_marker: Option) { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + if let Some(marker) = previous_marker { + write_state_atomically(&path, &state(marker)).expect("seed"); + } + let previous_bytes = std::fs::read(&path).ok(); + + let fs = FaultInjectingFilesystem::failing_at(phase); + let failure = + write_state_atomically_with(&fs, &path, &state(99)).expect_err("injected failure"); + let PersistFailure::NotApplied { + phase: reported, + staging_cleanup, + .. + } = failure + else { + panic!("expected NotApplied for {phase:?}, got {failure:?}"); + }; + assert_eq!(reported, phase); + assert!(staging_cleanup.is_none()); + assert_eq!(std::fs::read(&path).ok(), previous_bytes); + assert_eq!(path.exists(), previous_marker.is_some()); + assert_eq!(stored_marker(&path), previous_marker); + assert_eq!(staging_leftovers(dir.path()), 0); + + write_state_atomically(&path, &state(100)).expect("recovery write"); + assert_eq!(stored_marker(&path), Some(100)); + assert_eq!(staging_leftovers(dir.path()), 0); + } + + #[test] + fn maintenance_state_write_succeeds_end_to_end() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + write_state_atomically(&path, &state(1)).expect("first write"); + write_state_atomically(&path, &state(2)).expect("second write"); + assert_eq!(stored_marker(&path), Some(2)); + assert_eq!(staging_leftovers(dir.path()), 0); + } + + #[test] + fn maintenance_state_pre_rename_failures_preserve_previous_state() { + for (phase, previous_marker) in [ + (PersistPhase::CreateStaging, Some(1)), + (PersistPhase::WriteStaging, Some(1)), + (PersistPhase::SyncStaging, Some(1)), + (PersistPhase::Rename, Some(1)), + (PersistPhase::Rename, None), + ] { + assert_not_applied(phase, previous_marker); + } + } + + #[test] + fn maintenance_state_post_rename_failures_report_uncertain_durability() { + for phase in [ + PersistPhase::OpenParentDirectory, + PersistPhase::SyncParentDirectory, + ] { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + write_state_atomically(&path, &state(1)).expect("seed"); + + let fs = FaultInjectingFilesystem::failing_at(phase); + let failure = write_state_atomically_with(&fs, &path, &state(2)).expect_err("failure"); + let PersistFailure::DurabilityUncertain { + phase: reported, .. + } = failure + else { + panic!("expected DurabilityUncertain for {phase:?}, got {failure:?}"); + }; + assert_eq!(reported, phase); + let rendered = failure.to_string(); + assert!(rendered.contains("may be visible"), "{phase:?}: {rendered}"); + assert!( + !rendered.contains("previous state preserved"), + "{phase:?}: {rendered}" + ); + assert_eq!(stored_marker(&path), Some(2), "{phase:?}"); + assert_eq!(staging_leftovers(dir.path()), 0, "{phase:?}"); + + write_state_atomically(&path, &state(3)).expect("recovery write"); + assert_eq!(stored_marker(&path), Some(3), "{phase:?}"); + } + } + + #[test] + fn maintenance_state_recovers_after_interrupted_pre_rename_write() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = state_file(dir.path()); + write_state_atomically(&path, &state(1)).expect("seed"); + + let fs = FaultInjectingFilesystem { + fail_at: Some(PersistPhase::Rename), + fail_staging_cleanup: true, + }; + let failure = write_state_atomically_with(&fs, &path, &state(2)).expect_err("failure"); + let PersistFailure::NotApplied { + staging_cleanup, .. + } = failure + else { + panic!("expected NotApplied"); + }; + assert!(staging_cleanup.is_some()); + assert_eq!(staging_leftovers(dir.path()), 1); + assert_eq!(stored_marker(&path), Some(1)); + + write_state_atomically(&path, &state(3)).expect("recovery write"); + assert_eq!(stored_marker(&path), Some(3)); + } +} diff --git a/cli/src/services/mutation_trace/runtime/mod.rs b/cli/src/services/mutation_trace/runtime/mod.rs index d7961232d..4722da2da 100644 --- a/cli/src/services/mutation_trace/runtime/mod.rs +++ b/cli/src/services/mutation_trace/runtime/mod.rs @@ -2,15 +2,16 @@ mod coordinator; mod external_mutation_guard; mod external_taint; mod git_snapshot; +mod maintenance_state; mod mutation_attribution; mod protected_worktree; +mod ref_advisory; +mod ref_doctor; +mod ref_maintenance; mod ref_reconciliation; mod scope_runtime; mod worktree_lock; -#[cfg(test)] -mod tests; - #[allow(unused_imports)] pub(crate) use coordinator::{ coordinate, CoordinateError, CoordinateOutcome, ExternalTaintOperation, RuntimeBoundary, @@ -18,8 +19,8 @@ pub(crate) use coordinator::{ }; #[allow(unused_imports)] pub(crate) use external_mutation_guard::{ - arm_external_mutation_guard, run_external_mutation_guard, ArmedExternalMutationGuard, - GuardError, GuardEvent, GuardOutcome, GuardRequest, + arm_external_mutation_guard, ArmedExternalMutationGuard, GuardError, GuardEvent, GuardOutcome, + GuardRequest, }; #[allow(unused_imports)] pub(crate) use git_snapshot::{resolve_git_dir, resolve_worktree_id}; @@ -29,6 +30,11 @@ pub(crate) use mutation_attribution::{ BoundedMutationAttribution, MutationAttributionBarrier, MutationEventPageSource, TreeReadSource, MAX_MUTATION_ATTRIBUTION_EVENTS, }; +pub(crate) use ref_advisory::{advise_after_completed_boundary, AdvisoryReport, AdvisorySeverity}; +pub(crate) use ref_doctor::{ + inspect_reconciliation_recommendation, run_explicit_reconciliation, ReconciliationCounts, + ReconciliationFix, ReconciliationRecommendation, StateWarning, +}; #[allow(unused_imports)] pub(crate) use scope_runtime::{ abandon_scope, AbandonRecoveryReason, AbandonScopeError, AbandonScopeOutcome, diff --git a/cli/src/services/mutation_trace/runtime/mutation_attribution.rs b/cli/src/services/mutation_trace/runtime/mutation_attribution.rs index fc54a9152..cf2f18af1 100644 --- a/cli/src/services/mutation_trace/runtime/mutation_attribution.rs +++ b/cli/src/services/mutation_trace/runtime/mutation_attribution.rs @@ -26,21 +26,21 @@ pub const MAX_MUTATION_ATTRIBUTION_EVENTS: usize = 128; const REVISION_CUT_LOCK_TIMEOUT: Duration = Duration::from_secs(10); pub trait MutationEventPageSource { - fn load_mutation_event_page( + async fn load_mutation_event_page( &self, worktree: &WorktreeId, revision_cursor: Option, requested_limit: usize, ) -> Result>; - fn load_scope_provenance(&self, scope_id: &ScopeId) -> Result> { + async fn load_scope_provenance(&self, scope_id: &ScopeId) -> Result> { let _ = scope_id; Ok(None) } } impl MutationEventPageSource for MutationTraceStore<'_> { - fn load_mutation_event_page( + async fn load_mutation_event_page( &self, worktree: &WorktreeId, revision_cursor: Option, @@ -52,25 +52,26 @@ impl MutationEventPageSource for MutationTraceStore<'_> { revision_cursor, requested_limit, ) + .await } - fn load_scope_provenance(&self, scope_id: &ScopeId) -> Result> { - MutationTraceStore::load_scope_provenance(self, scope_id) + async fn load_scope_provenance(&self, scope_id: &ScopeId) -> Result> { + MutationTraceStore::load_scope_provenance(self, scope_id).await } } pub trait TreeReadSource { - fn diff_trees(&self, before: &TreeId, after: &TreeId) -> Result; - fn file_at_tree(&self, tree: &TreeId, path: &str) -> Result>; + async fn diff_trees(&self, before: &TreeId, after: &TreeId) -> Result; + async fn file_at_tree(&self, tree: &TreeId, path: &str) -> Result>; } impl TreeReadSource for GitSnapshotService { - fn diff_trees(&self, before: &TreeId, after: &TreeId) -> Result { - GitSnapshotService::diff_trees(self, before, after) + async fn diff_trees(&self, before: &TreeId, after: &TreeId) -> Result { + GitSnapshotService::diff_trees(self, before, after).await } - fn file_at_tree(&self, tree: &TreeId, path: &str) -> Result> { - GitSnapshotService::file_at_tree(self, tree, path) + async fn file_at_tree(&self, tree: &TreeId, path: &str) -> Result> { + GitSnapshotService::file_at_tree(self, tree, path).await } } @@ -115,7 +116,7 @@ fn empty_patch() -> ParsedPatch { } #[allow(clippy::too_many_arguments)] -pub fn resolve_bounded_mutation_attribution( +pub async fn resolve_bounded_mutation_attribution( page_source: &P, tree_source: &R, worktree: &WorktreeId, @@ -126,7 +127,7 @@ pub fn resolve_bounded_mutation_attribution( ) -> BoundedMutationAttribution where P: MutationEventPageSource + ?Sized, - R: TreeReadSource + ?Sized, + R: TreeReadSource, { let target = exclude_direct_coverage(committed_patch, direct_coverage); let target_paths = target_logical_paths(&target); @@ -143,19 +144,13 @@ where barrier: None, }; - let events = load_event_window(page_source, worktree, revision_ceiling, &mut state); + let events = load_event_window(page_source, worktree, revision_ceiling, &mut state).await; let lineage = if events.is_empty() { None } else { - Some(replay( - &events, - &target_paths, - commit_tree, - tree_source, - &mut state, - )) + Some(replay(&events, &target_paths, commit_tree, tree_source, &mut state).await) }; - finish(&target, lineage.as_ref(), &state, page_source) + finish(&target, lineage.as_ref(), &state, page_source).await } struct ReplayState { @@ -167,7 +162,7 @@ struct ReplayState { barrier: Option, } -fn finish

( +async fn finish

( target: &ParsedPatch, lineage: Option<&MutationLineage>, state: &ReplayState, @@ -177,7 +172,7 @@ where P: MutationEventPageSource + ?Sized, { let result = match lineage { - Some(lineage) => project(target, lineage, page_source), + Some(lineage) => project(target, lineage, page_source).await, None => MutationAttributionResult { mutation_ai_patch: empty_patch(), resolved_non_ai_patch: empty_patch(), @@ -205,7 +200,7 @@ fn target_logical_paths(target: &ParsedPatch) -> BTreeSet { .collect() } -fn load_event_window

( +async fn load_event_window

( page_source: &P, worktree: &WorktreeId, revision_ceiling: Option, @@ -223,7 +218,10 @@ where } let want = MUTATION_ATTRIBUTION_PAGE_SIZE.min(MAX_MUTATION_ATTRIBUTION_EVENTS - rows.len()); - let Ok(page) = page_source.load_mutation_event_page(worktree, cursor, want) else { + let Ok(page) = page_source + .load_mutation_event_page(worktree, cursor, want) + .await + else { state.barrier = Some(MutationAttributionBarrier::PageQuery); break; }; @@ -246,7 +244,7 @@ where rows } -fn replay( +async fn replay( events: &[MutationEventPageRow], target_paths: &BTreeSet, commit_tree: &TreeId, @@ -254,25 +252,24 @@ fn replay( state: &mut ReplayState, ) -> MutationLineage where - R: TreeReadSource + ?Sized, + R: TreeReadSource, { - let mut lineage = MutationLineage::from_baseline(&load_baseline( - tree_source, - &events[0].before_tree, - target_paths, - )); + let mut lineage = MutationLineage::from_baseline( + &load_baseline(tree_source, &events[0].before_tree, target_paths).await, + ); let mut prev_after = events[0].before_tree.clone(); for row in events { state.inspected_events += 1; if row.before_tree != prev_after { - lineage.reset_all(&load_baseline(tree_source, &row.before_tree, target_paths)); + lineage.reset_all(&load_baseline(tree_source, &row.before_tree, target_paths).await); state.gap_resets += 1; } let reconstructed = tree_source .diff_trees(&row.before_tree, &row.after_tree) + .await .ok() .and_then(|text| parse_patch(&text, None).ok()); if reconstructed.is_some() { @@ -287,7 +284,9 @@ where &row.after_tree, target_paths, tree_source, - ) { + ) + .await + { state.barrier = Some(MutationAttributionBarrier::EventReconstruction); state.gap_resets += 1; } @@ -298,6 +297,7 @@ where if prev_after != *commit_tree { let tail = tree_source .diff_trees(&prev_after, commit_tree) + .await .ok() .and_then(|text| parse_patch(&text, None).ok()); if apply_or_reset( @@ -307,7 +307,9 @@ where commit_tree, target_paths, tree_source, - ) { + ) + .await + { state.barrier = Some(MutationAttributionBarrier::Tail); state.gap_resets += 1; } @@ -316,7 +318,7 @@ where lineage } -fn apply_or_reset( +async fn apply_or_reset( lineage: &mut MutationLineage, patch: Option<&ParsedPatch>, origin: &TransitionOrigin, @@ -325,30 +327,29 @@ fn apply_or_reset( tree_source: &R, ) -> bool where - R: TreeReadSource + ?Sized, + R: TreeReadSource, { let applied = patch.is_some_and(|patch| lineage.apply(patch, origin).is_ok()); if !applied { - lineage.reset_all(&load_baseline(tree_source, fallback_tree, target_paths)); + lineage.reset_all(&load_baseline(tree_source, fallback_tree, target_paths).await); } !applied } -fn load_baseline( +async fn load_baseline( tree_source: &R, tree: &TreeId, target_paths: &BTreeSet, ) -> std::collections::BTreeMap> where - R: TreeReadSource + ?Sized, + R: TreeReadSource, { - target_paths - .iter() - .map(|path| { - let content = tree_source.file_at_tree(tree, path).unwrap_or(None); - (path.clone(), content) - }) - .collect() + let mut baseline = std::collections::BTreeMap::new(); + for path in target_paths { + let content = tree_source.file_at_tree(tree, path).await.unwrap_or(None); + baseline.insert(path.clone(), content); + } + baseline } fn transition_origin(row: &MutationEventPageRow) -> TransitionOrigin { @@ -361,7 +362,7 @@ fn transition_origin(row: &MutationEventPageRow) -> TransitionOrigin { } } -fn project

( +async fn project

( target: &ParsedPatch, lineage: &MutationLineage, page_source: &P, @@ -401,15 +402,15 @@ where } } - let scope_provenance = ai - .values() - .collect::>() - .into_iter() - .map(|scope_id| { - let provenance = page_source.load_scope_provenance(scope_id).ok().flatten(); - (scope_id.clone(), provenance) - }) - .collect::>(); + let mut scope_provenance = BTreeMap::new(); + for scope_id in ai.values().collect::>() { + let provenance = page_source + .load_scope_provenance(scope_id) + .await + .ok() + .flatten(); + scope_provenance.insert(scope_id.clone(), provenance); + } let ai_with_provenance = ai .into_iter() .map(|(location, scope_id)| { @@ -431,27 +432,27 @@ where } } -pub(crate) fn resolve_post_commit_mutation_ai_patch( +pub(crate) async fn resolve_post_commit_mutation_ai_patch( repository_root: &Path, - db: &RepositoryAgentTraceDb, + db: &mut RepositoryAgentTraceDb, direct_coverage: &ParsedPatch, committed_patch: &ParsedPatch, ) -> ParsedPatch { - let Ok(git_dir) = resolve_git_dir(repository_root) else { + let Ok(git_dir) = resolve_git_dir(repository_root).await else { return empty_patch(); }; - let Ok(worktree) = resolve_worktree_id(repository_root) else { + let Ok(worktree) = resolve_worktree_id(repository_root).await else { return empty_patch(); }; - let Ok(snapshot) = GitSnapshotService::new(repository_root) else { + let Ok(snapshot) = GitSnapshotService::new(repository_root).await else { return empty_patch(); }; - let Ok(commit_tree) = snapshot.head_tree() else { + let Ok(commit_tree) = snapshot.head_tree().await else { return empty_patch(); }; let store = MutationTraceStore::new(db); - let Some(revision_ceiling) = capture_revision_cut(&git_dir, &store, &worktree) else { + let Some(revision_ceiling) = capture_revision_cut(&git_dir, &store, &worktree).await else { return empty_patch(); }; @@ -464,22 +465,22 @@ pub(crate) fn resolve_post_commit_mutation_ai_patch( &commit_tree, Some(revision_ceiling), ) + .await .result .mutation_ai_patch } -fn capture_revision_cut( +async fn capture_revision_cut( git_dir: &Path, store: &MutationTraceStore<'_>, worktree: &WorktreeId, ) -> Option { - let _lock = WorktreeLock::acquire(git_dir, REVISION_CUT_LOCK_TIMEOUT).ok()?; + let _lock = WorktreeLock::acquire_async(git_dir, REVISION_CUT_LOCK_TIMEOUT) + .await + .ok()?; store .latest_mutation_event_revision(worktree) + .await .ok() .flatten() } - -#[cfg(test)] -#[path = "mutation_attribution/tests.rs"] -mod tests; diff --git a/cli/src/services/mutation_trace/runtime/mutation_attribution/tests.rs b/cli/src/services/mutation_trace/runtime/mutation_attribution/tests.rs deleted file mode 100644 index 72e690827..000000000 --- a/cli/src/services/mutation_trace/runtime/mutation_attribution/tests.rs +++ /dev/null @@ -1,1159 +0,0 @@ -use std::cell::RefCell; -use std::collections::HashMap; - -use super::*; -use crate::services::mutation_trace::store::{AttributionKind, ScopeProvenance}; -use crate::services::mutation_trace::types::{FailureKind, ScopeId}; -use crate::services::patch::{FileChangeKind, PatchFileChange, PatchHunk, TouchedLine}; - -fn tree(id: &str) -> TreeId { - TreeId(id.to_owned()) -} - -fn worktree() -> WorktreeId { - WorktreeId("wt".to_owned()) -} - -fn empty() -> ParsedPatch { - ParsedPatch { files: Vec::new() } -} - -fn added(number: u64, content: &str) -> TouchedLine { - TouchedLine { - kind: TouchedLineKind::Added, - line_number: number, - content: content.to_owned(), - session_id: None, - } -} - -fn committed( - path: &str, - old_start: u64, - old_count: u64, - new_start: u64, - lines: Vec, -) -> ParsedPatch { - ParsedPatch { - files: vec![PatchFileChange { - old_path: path.to_owned(), - new_path: path.to_owned(), - kind: FileChangeKind::Modified, - hunks: vec![PatchHunk { - old_start, - old_count, - new_start, - new_count: lines.len() as u64, - model_id: None, - lines, - }], - }], - } -} - -fn page_row( - revision: u64, - before: &str, - after: &str, - kind: AttributionKind, - scope: Option<&str>, -) -> MutationEventPageRow { - MutationEventPageRow { - revision, - before_tree: tree(before), - after_tree: tree(after), - failure_kind: FailureKind::Healthy, - attribution_kind: kind, - attribution_scope_id: scope.map(|id| ScopeId(id.to_owned())), - } -} - -fn ai_row(revision: u64, before: &str, after: &str, scope: &str) -> MutationEventPageRow { - page_row( - revision, - before, - after, - AttributionKind::AiExclusive, - Some(scope), - ) -} - -fn non_ai_row(revision: u64, before: &str, after: &str) -> MutationEventPageRow { - page_row(revision, before, after, AttributionKind::AiContended, None) -} - -fn provenance(scope_id: &str, session_id: &str, model_id: Option<&str>) -> ScopeProvenance { - ScopeProvenance { - scope_id: ScopeId(scope_id.to_owned()), - session_id: session_id.to_owned(), - model_id: model_id.map(str::to_owned), - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct PageRequest { - cursor: Option, - limit: usize, -} - -struct FakePageSource { - events: Vec, - provenance: HashMap, - fail_on_page: Option, - requests: RefCell>, - provenance_reads: RefCell>, -} - -impl FakePageSource { - fn new(events: Vec) -> Self { - Self { - events, - provenance: HashMap::new(), - fail_on_page: None, - requests: RefCell::new(Vec::new()), - provenance_reads: RefCell::new(Vec::new()), - } - } - - fn with_provenance(mut self, value: ScopeProvenance) -> Self { - self.provenance.insert(value.scope_id.clone(), value); - self - } - - fn failing_on_page(mut self, page: usize) -> Self { - self.fail_on_page = Some(page); - self - } - - fn requests(&self) -> Vec { - self.requests.borrow().clone() - } - - fn provenance_reads(&self) -> Vec { - self.provenance_reads.borrow().clone() - } -} - -impl MutationEventPageSource for FakePageSource { - fn load_mutation_event_page( - &self, - _worktree: &WorktreeId, - revision_cursor: Option, - requested_limit: usize, - ) -> Result> { - let page_number = self.requests.borrow().len() + 1; - self.requests.borrow_mut().push(PageRequest { - cursor: revision_cursor, - limit: requested_limit, - }); - if self.fail_on_page == Some(page_number) { - anyhow::bail!("injected page failure"); - } - let start = match revision_cursor { - None => 0, - Some(cursor) => self - .events - .iter() - .position(|event| event.revision < cursor) - .unwrap_or(self.events.len()), - }; - Ok(self - .events - .iter() - .skip(start) - .take(requested_limit) - .cloned() - .collect()) - } - - fn load_scope_provenance(&self, scope_id: &ScopeId) -> Result> { - self.provenance_reads.borrow_mut().push(scope_id.clone()); - Ok(self.provenance.get(scope_id).cloned()) - } -} - -#[derive(Default)] -struct FakeTreeSource { - diffs: HashMap<(String, String), String>, - files: HashMap<(String, String), String>, - fail_diff_on: Option, - diff_calls: RefCell, -} - -impl FakeTreeSource { - fn new() -> Self { - Self::default() - } - - fn with_diff(mut self, before: &str, after: &str, text: &str) -> Self { - self.diffs - .insert((before.to_owned(), after.to_owned()), text.to_owned()); - self - } - - fn with_file(mut self, tree: &str, path: &str, content: &str) -> Self { - self.files - .insert((tree.to_owned(), path.to_owned()), content.to_owned()); - self - } - - fn failing_diff_on(mut self, call: usize) -> Self { - self.fail_diff_on = Some(call); - self - } - - fn diff_calls(&self) -> usize { - *self.diff_calls.borrow() - } -} - -impl TreeReadSource for FakeTreeSource { - fn diff_trees(&self, before: &TreeId, after: &TreeId) -> Result { - let call = { - let mut calls = self.diff_calls.borrow_mut(); - *calls += 1; - *calls - }; - if self.fail_diff_on == Some(call) { - anyhow::bail!("injected diff failure"); - } - if before == after { - return Ok(String::new()); - } - self.diffs - .get(&(before.0.clone(), after.0.clone())) - .cloned() - .ok_or_else(|| anyhow::anyhow!("no canned diff {before:?} -> {after:?}")) - } - - fn file_at_tree(&self, tree: &TreeId, path: &str) -> Result> { - Ok(self.files.get(&(tree.0.clone(), path.to_owned())).cloned()) - } -} - -fn ai_contents(attr: &BoundedMutationAttribution) -> Vec { - attr.result - .mutation_ai_patch - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .flat_map(|hunk| hunk.lines.iter()) - .map(|line| line.content.clone()) - .collect() -} - -fn unresolved_contents(attr: &BoundedMutationAttribution) -> Vec { - attr.result - .unresolved_patch - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .flat_map(|hunk| hunk.lines.iter()) - .map(|line| line.content.clone()) - .collect() -} - -#[test] -fn no_events_leaves_every_target_line_unresolved() { - let page_source = FakePageSource::new(Vec::new()); - let tree_source = FakeTreeSource::new(); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo")]), - &tree("commit"), - Some(5), - ); - - assert!(ai_contents(&attr).is_empty()); - assert_eq!(unresolved_contents(&attr), vec!["foo".to_owned()]); - assert_eq!( - attr.loaded_pages, 0, - "an empty page is not counted as loaded" - ); - assert_eq!(page_source.requests().len(), 1); -} - -#[test] -fn fully_direct_covered_target_does_zero_mutation_history_work() { - let page_source = FakePageSource::new(vec![ai_row(1, "b", "a", "s")]); - let tree_source = FakeTreeSource::new(); - - let direct = committed("f.rs", 1, 1, 1, vec![added(2, "foo")]); - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &direct, - &direct, - &tree("commit"), - Some(1), - ); - - assert!(page_source.requests().is_empty()); - assert_eq!(tree_source.diff_calls(), 0); - assert_eq!(attr.inspected_events, 0); - assert!(attr.result.mutation_ai_patch.files.is_empty()); -} - -#[test] -fn a_surviving_ai_mutation_line_is_attributed() { - let page_source = FakePageSource::new(vec![ai_row(1, "t0", "t1", "scope-1")]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo")]), - &tree("t1"), - Some(1), - ); - - assert_eq!(ai_contents(&attr), vec!["foo".to_owned()]); - assert!(unresolved_contents(&attr).is_empty()); - assert_eq!(attr.reconstructed_events, 1); - assert_eq!(attr.barrier, None); - let hunk = &attr.result.mutation_ai_patch.files[0].hunks[0]; - assert_eq!(hunk.model_id, None); - assert_eq!(hunk.lines[0].session_id, None); -} - -#[test] -fn a_mutation_ai_hunk_carries_scope_session_and_model_provenance() { - let page_source = FakePageSource::new(vec![ai_row(1, "t0", "t1", "scope-1")]) - .with_provenance(provenance("scope-1", "cx_session-1", Some("codex/gpt-5"))); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo")]), - &tree("t1"), - Some(1), - ); - - let hunk = &attr.result.mutation_ai_patch.files[0].hunks[0]; - assert_eq!(hunk.model_id.as_deref(), Some("codex/gpt-5")); - assert_eq!(hunk.lines[0].session_id.as_deref(), Some("cx_session-1")); - assert_eq!( - page_source.provenance_reads(), - vec![ScopeId("scope-1".to_owned())] - ); -} - -#[test] -fn mutation_ai_lines_keep_their_sessions_and_agreeing_models_across_scopes() { - let page_source = FakePageSource::new(vec![ - ai_row(2, "t1", "t2", "scope-2"), - ai_row(1, "t0", "t1", "scope-1"), - ]) - .with_provenance(provenance("scope-1", "cx_session-1", Some("codex/gpt-5"))) - .with_provenance(provenance("scope-2", "cx_session-2", Some("codex/gpt-5"))); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ) - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +3,1 @@\n+bar\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo"), added(3, "bar")]), - &tree("t2"), - Some(2), - ); - - let hunk = &attr.result.mutation_ai_patch.files[0].hunks[0]; - assert_eq!(hunk.model_id.as_deref(), Some("codex/gpt-5")); - assert_eq!( - hunk.lines - .iter() - .map(|line| line.session_id.as_deref()) - .collect::>(), - vec![Some("cx_session-1"), Some("cx_session-2")] - ); - assert_eq!(page_source.provenance_reads().len(), 2); -} - -#[test] -fn mutation_ai_hunk_omits_a_conflicting_model_but_keeps_line_sessions() { - let page_source = FakePageSource::new(vec![ - ai_row(2, "t1", "t2", "scope-2"), - ai_row(1, "t0", "t1", "scope-1"), - ]) - .with_provenance(provenance("scope-1", "cc_session-1", Some("claude/sonnet"))) - .with_provenance(provenance("scope-2", "cc_session-2", Some("claude/opus"))); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ) - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +3,1 @@\n+bar\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo"), added(3, "bar")]), - &tree("t2"), - Some(2), - ); - - let hunk = &attr.result.mutation_ai_patch.files[0].hunks[0]; - assert_eq!(hunk.model_id, None); - assert_eq!( - hunk.lines - .iter() - .map(|line| line.session_id.as_deref()) - .collect::>(), - vec![Some("cc_session-1"), Some("cc_session-2")] - ); -} - -#[test] -fn missing_or_unknown_scope_provenance_does_not_downgrade_ai_lines() { - let page_source = FakePageSource::new(vec![ - ai_row(2, "t1", "t2", "scope-2"), - ai_row(1, "t0", "t1", "scope-1"), - ]) - .with_provenance(provenance("scope-1", "cx_session-1", None)) - .with_provenance(provenance("scope-2", "cx_session-2", Some("codex/gpt-5"))); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ) - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +3,1 @@\n+bar\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo"), added(3, "bar")]), - &tree("t2"), - Some(2), - ); - - let hunk = &attr.result.mutation_ai_patch.files[0].hunks[0]; - assert_eq!(ai_contents(&attr), vec!["foo".to_owned(), "bar".to_owned()]); - assert_eq!(hunk.model_id, None); - assert_eq!(hunk.lines[0].session_id.as_deref(), Some("cx_session-1")); - assert_eq!(hunk.lines[1].session_id.as_deref(), Some("cx_session-2")); -} - -#[test] -fn an_ineligible_unscoped_transition_never_becomes_ai_mutation_lineage() { - let page_source = FakePageSource::new(vec![page_row( - 1, - "t0", - "t1", - AttributionKind::IneligibleUnscoped, - None, - )]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo")]), - &tree("t1"), - Some(1), - ); - - assert_eq!(attr.reconstructed_events, 1); - assert!(ai_contents(&attr).is_empty()); - assert!(unresolved_contents(&attr).is_empty()); -} - -#[test] -fn ai_mutation_survives_an_unrelated_later_mutation() { - let page_source = FakePageSource::new(vec![ - non_ai_row(2, "t1", "t2"), - ai_row(1, "t0", "t1", "scope-1"), - ]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ) - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +3,1 @@\n+bar\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo"), added(3, "bar")]), - &tree("t2"), - Some(2), - ); - - assert_eq!(ai_contents(&attr), vec!["foo".to_owned()]); - let non_ai: Vec = attr - .result - .resolved_non_ai_patch - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .flat_map(|hunk| hunk.lines.iter()) - .map(|line| line.content.clone()) - .collect(); - assert_eq!(non_ai, vec!["bar".to_owned()]); -} - -#[test] -fn a_stale_ai_mutation_cannot_resurrect_through_an_unobserved_tail() { - let page_source = FakePageSource::new(vec![ - non_ai_row(2, "t1", "t2"), - ai_row(1, "t0", "t1", "scope-1"), - ]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ) - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,1 +1,0 @@\n-foo\n", - ) - .with_diff( - "t2", - "commit", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo")]), - &tree("commit"), - Some(2), - ); - - assert!( - ai_contents(&attr).is_empty(), - "the re-added foo must not inherit E1's dead provenance" - ); - assert_eq!(unresolved_contents(&attr), vec!["foo".to_owned()]); -} - -#[test] -fn a_non_ai_replacement_of_an_ai_line_owns_the_new_line() { - let page_source = FakePageSource::new(vec![ - non_ai_row(2, "t1", "t2"), - ai_row(1, "t0", "t1", "scope-1"), - ]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "head\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n head\n+foo = 1\n", - ) - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,1 +2,1 @@\n-foo = 1\n+foo = 2\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo = 2")]), - &tree("t2"), - Some(2), - ); - - assert!(ai_contents(&attr).is_empty()); - let non_ai: Vec = attr - .result - .resolved_non_ai_patch - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .flat_map(|hunk| hunk.lines.iter()) - .map(|line| line.content.clone()) - .collect(); - assert_eq!(non_ai, vec!["foo = 2".to_owned()]); -} - -#[test] -fn a_suffix_related_mutation_file_never_pairs_with_the_committed_target() { - let page_source = FakePageSource::new(vec![ai_row(1, "t0", "t1", "scope-1")]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "src/lib.rs", "a\nunique_line\n") - .with_file("t0", "packages/foo/src/lib.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/packages/foo/src/lib.rs b/packages/foo/src/lib.rs\n\ - --- a/packages/foo/src/lib.rs\n\ - +++ b/packages/foo/src/lib.rs\n\ - @@ -1,1 +1,2 @@\n a\n+unique_line\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("src/lib.rs", 1, 1, 1, vec![added(2, "unique_line")]), - &tree("t1"), - Some(1), - ); - - assert!( - ai_contents(&attr).is_empty(), - "a suffix-related mutation path must not attribute the committed line" - ); - assert_eq!(unresolved_contents(&attr), vec!["unique_line".to_owned()]); -} - -#[test] -fn an_exact_nested_repository_path_still_attributes() { - let page_source = FakePageSource::new(vec![ai_row(1, "t0", "t1", "scope-1")]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "src/lib.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/src/lib.rs b/src/lib.rs\n\ - --- a/src/lib.rs\n\ - +++ b/src/lib.rs\n\ - @@ -1,1 +1,2 @@\n a\n+unique_line\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("src/lib.rs", 1, 1, 1, vec![added(2, "unique_line")]), - &tree("t1"), - Some(1), - ); - - assert_eq!(ai_contents(&attr), vec!["unique_line".to_owned()]); - assert!(unresolved_contents(&attr).is_empty()); -} - -#[test] -fn multiple_similar_nested_mutation_paths_stay_independent_of_the_target() { - let page_source = FakePageSource::new(vec![ai_row(1, "t0", "t1", "scope-1")]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "src/lib.rs", "a\nunique_line\n") - .with_file("t0", "packages/a/src/lib.rs", "a\n") - .with_file("t0", "packages/b/src/lib.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/packages/a/src/lib.rs b/packages/a/src/lib.rs\n\ - --- a/packages/a/src/lib.rs\n\ - +++ b/packages/a/src/lib.rs\n\ - @@ -1,1 +1,2 @@\n a\n+unique_line\n\ - diff --git a/packages/b/src/lib.rs b/packages/b/src/lib.rs\n\ - --- a/packages/b/src/lib.rs\n\ - +++ b/packages/b/src/lib.rs\n\ - @@ -1,1 +1,2 @@\n a\n+unique_line\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("src/lib.rs", 1, 1, 1, vec![added(2, "unique_line")]), - &tree("t1"), - Some(1), - ); - - assert!(ai_contents(&attr).is_empty()); - assert_eq!(unresolved_contents(&attr), vec!["unique_line".to_owned()]); -} - -#[test] -fn a_shared_basename_between_mutation_and_target_is_not_a_match() { - let page_source = FakePageSource::new(vec![ai_row(1, "t0", "t1", "scope-1")]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "bar/config.rs", "a\nunique_line\n") - .with_file("t0", "foo/config.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/foo/config.rs b/foo/config.rs\n\ - --- a/foo/config.rs\n\ - +++ b/foo/config.rs\n\ - @@ -1,1 +1,2 @@\n a\n+unique_line\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("bar/config.rs", 1, 1, 1, vec![added(2, "unique_line")]), - &tree("t1"), - Some(1), - ); - - assert!(ai_contents(&attr).is_empty()); - assert_eq!(unresolved_contents(&attr), vec!["unique_line".to_owned()]); -} - -#[test] -fn a_history_gap_is_not_crossed_by_older_provenance() { - let page_source = FakePageSource::new(vec![ - ai_row(2, "t9", "commit", "scope-2"), - ai_row(1, "t0", "t1", "scope-1"), - ]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_file("t9", "f.rs", "a\nfoo\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ) - .with_diff( - "t9", - "commit", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +3,1 @@\n+bar\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 2, 1, vec![added(2, "foo"), added(3, "bar")]), - &tree("commit"), - Some(2), - ); - - assert_eq!(ai_contents(&attr), vec!["bar".to_owned()]); - assert_eq!(unresolved_contents(&attr), vec!["foo".to_owned()]); - assert_eq!(attr.gap_resets, 1); -} - -#[test] -fn bounded_history_baseline_starts_unknown() { - let page_source = FakePageSource::new(vec![non_ai_row(2, "t1", "t2")]); - let tree_source = FakeTreeSource::new() - .with_file("t1", "f.rs", "a\nfoo\n") - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +3,1 @@\n+bar\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 2, 1, vec![added(2, "foo")]), - &tree("t2"), - Some(2), - ); - - assert!(ai_contents(&attr).is_empty()); - assert_eq!(unresolved_contents(&attr), vec!["foo".to_owned()]); -} - -#[test] -fn an_unobserved_tail_adds_unknown_lines_but_keeps_surviving_ai_lines() { - let page_source = FakePageSource::new(vec![ai_row(1, "t0", "t1", "scope-1")]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+ai_line\n", - ) - .with_diff( - "t1", - "commit", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +3,1 @@\n+human_line\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed( - "f.rs", - 1, - 1, - 1, - vec![added(2, "ai_line"), added(3, "human_line")], - ), - &tree("commit"), - Some(1), - ); - - assert_eq!(ai_contents(&attr), vec!["ai_line".to_owned()]); - assert_eq!(unresolved_contents(&attr), vec!["human_line".to_owned()]); -} - -#[test] -fn an_event_after_the_commit_cut_has_no_influence() { - let page_source = FakePageSource::new(vec![ - ai_row(2, "t0", "t1", "scope-after"), - non_ai_row(1, "t0", "t0"), - ]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\nfoo\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -2,0 +2,1 @@\n+foo\n", - ); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 2, 1, vec![added(2, "foo")]), - &tree("t0"), - Some(1), - ); - - assert!( - ai_contents(&attr).is_empty(), - "an event past the cut must not attribute" - ); - assert_eq!(page_source.requests()[0].cursor, Some(2)); -} - -#[test] -fn event_128_within_the_horizon_contributes_and_event_129_is_never_loaded() { - let total = 200u64; - let relevant = total - 127; - let mut events = Vec::new(); - for revision in (1..=total).rev() { - if revision == relevant { - events.push(ai_row(revision, "t0", "t1", "scope-128")); - } else { - events.push(non_ai_row(revision, "t1", "t1")); - } - } - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_diff( - "t0", - "t1", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+target\n", - ); - let page_source = FakePageSource::new(events); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "target")]), - &tree("t1"), - None, - ); - - assert_eq!(ai_contents(&attr), vec!["target".to_owned()]); - assert_eq!(attr.inspected_events, 128); - assert_eq!(attr.loaded_pages, 4); - assert_eq!(attr.loaded_rows, 128); - assert_eq!(attr.barrier, None); -} - -#[test] -fn a_page_query_failure_is_a_conservative_barrier() { - let mut events = Vec::new(); - for revision in (1..=40).rev() { - events.push(non_ai_row(revision, "t1", "t1")); - } - let tree_source = FakeTreeSource::new().with_file("t1", "f.rs", "a\n"); - let page_source = FakePageSource::new(events).failing_on_page(2); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo")]), - &tree("t1"), - None, - ); - - assert_eq!(attr.barrier, Some(MutationAttributionBarrier::PageQuery)); - assert_eq!(attr.loaded_rows, 32); - assert_eq!(unresolved_contents(&attr), vec!["foo".to_owned()]); -} - -#[test] -fn a_reconstruction_failure_reloads_a_conservative_baseline_and_keeps_going() { - let page_source = FakePageSource::new(vec![ - ai_row(2, "t1", "t2", "scope-2"), - ai_row(1, "t0", "t1", "scope-1"), - ]); - let tree_source = FakeTreeSource::new() - .with_file("t0", "f.rs", "a\n") - .with_file("t1", "f.rs", "a\n") - .with_diff( - "t1", - "t2", - "diff --git a/f.rs b/f.rs\n--- a/f.rs\n+++ b/f.rs\n@@ -1,1 +1,2 @@\n a\n+foo\n", - ) - .failing_diff_on(1); - - let attr = resolve_bounded_mutation_attribution( - &page_source, - &tree_source, - &worktree(), - &empty(), - &committed("f.rs", 1, 1, 1, vec![added(2, "foo")]), - &tree("t2"), - Some(2), - ); - - assert_eq!( - attr.barrier, - Some(MutationAttributionBarrier::EventReconstruction) - ); - assert_eq!(ai_contents(&attr), vec!["foo".to_owned()]); -} - -#[test] -fn real_git_snapshot_and_store_satisfy_the_consumer_seams() { - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use crate::services::mutation_trace::store::encode_revision; - use std::process::Command; - - let temp = tempfile::Builder::new() - .prefix("sce-lineage-consumer-") - .tempdir() - .expect("temp dir"); - let repo_root = temp.path().join("repo"); - std::fs::create_dir_all(&repo_root).expect("repo dir"); - let git = |args: &[&str]| { - let output = Command::new("git") - .args(args) - .current_dir(&repo_root) - .output() - .expect("git spawns"); - assert!(output.status.success(), "git {args:?} failed"); - }; - git(&["init", "--quiet"]); - git(&["config", "user.email", "t@example.com"]); - git(&["config", "user.name", "Test"]); - git(&["commit", "--allow-empty", "--quiet", "-m", "init"]); - - let snapshot = GitSnapshotService::new(&repo_root).expect("snapshot service"); - std::fs::write(repo_root.join("file.rs"), b"one\n").expect("write"); - let before = snapshot.capture_tree().expect("capture before"); - std::fs::write(repo_root.join("file.rs"), b"one\ntwo\n").expect("write"); - let after = snapshot.capture_tree().expect("capture after"); - - let db = RepositoryAgentTraceDb::new_at(temp.path().join("agent-trace.db")).expect("db opens"); - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ai_exclusive', 'scope-real', 'flush', NULL, NULL)", - ( - "wt-real", - encode_revision(1).as_slice(), - before.0.as_str(), - after.0.as_str(), - ), - ) - .expect("event insert"); - let store = MutationTraceStore::new(&db); - - let attr = resolve_bounded_mutation_attribution( - &store, - &snapshot, - &WorktreeId("wt-real".to_owned()), - &empty(), - &committed("file.rs", 1, 1, 1, vec![added(2, "two")]), - &after, - Some(1), - ); - - assert_eq!(ai_contents(&attr), vec!["two".to_owned()]); - assert_eq!(attr.reconstructed_events, 1); - assert_eq!(attr.barrier, None); -} - -fn init_repo_with_commit(repo_root: &std::path::Path) { - use std::process::Command; - std::fs::create_dir_all(repo_root).expect("repo dir"); - let git = |args: &[&str]| { - let output = Command::new("git") - .args(args) - .current_dir(repo_root) - .output() - .expect("git spawns"); - assert!(output.status.success(), "git {args:?} failed"); - }; - git(&["init", "--quiet"]); - git(&["config", "user.email", "t@example.com"]); - git(&["config", "user.name", "Test"]); - std::fs::write(repo_root.join("seed"), b"seed\n").expect("seed"); - git(&["add", "-A"]); - git(&["commit", "--quiet", "-m", "init"]); -} - -#[test] -fn post_commit_entry_point_with_no_mutation_events_yields_empty() { - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - - let temp = tempfile::Builder::new() - .prefix("sce-post-commit-lineage-no-events-") - .tempdir() - .expect("temp dir"); - let repo_root = temp.path().join("repo"); - init_repo_with_commit(&repo_root); - - let db = RepositoryAgentTraceDb::new_at(temp.path().join("agent-trace.db")).expect("db opens"); - - let result = resolve_post_commit_mutation_ai_patch( - &repo_root, - &db, - &empty(), - &committed("file.rs", 1, 1, 1, vec![added(2, "two")]), - ); - - assert!(result.files.is_empty()); -} - -#[test] -fn post_commit_entry_point_resolves_current_worktree_and_ignores_foreign_rows() { - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use crate::services::mutation_trace::store::encode_revision; - - let temp = tempfile::Builder::new() - .prefix("sce-post-commit-lineage-current-") - .tempdir() - .expect("temp dir"); - let repo_root = temp.path().join("repo"); - init_repo_with_commit(&repo_root); - - let worktree_id = resolve_worktree_id(&repo_root).expect("worktree id"); - - let snapshot = GitSnapshotService::new(&repo_root).expect("snapshot service"); - std::fs::write(repo_root.join("file.rs"), b"one\n").expect("write"); - let before = snapshot.capture_tree().expect("capture before"); - std::fs::write(repo_root.join("file.rs"), b"one\ntwo\n").expect("write"); - let after = snapshot.capture_tree().expect("capture after"); - - std::process::Command::new("git") - .args(["add", "-A"]) - .current_dir(&repo_root) - .output() - .expect("git add"); - std::process::Command::new("git") - .args(["commit", "--quiet", "-m", "two"]) - .current_dir(&repo_root) - .output() - .expect("git commit"); - - let db = RepositoryAgentTraceDb::new_at(temp.path().join("agent-trace.db")).expect("db opens"); - for (row_worktree_id, revision, scope) in [ - (worktree_id.0.as_str(), 1u64, "scope-current"), - ("wt-foreign", 2u64, "scope-foreign"), - ] { - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ai_exclusive', ?5, 'flush', NULL, NULL)", - ( - row_worktree_id, - encode_revision(revision).as_slice(), - before.0.as_str(), - after.0.as_str(), - scope, - ), - ) - .expect("event insert"); - } - - let result = resolve_post_commit_mutation_ai_patch( - &repo_root, - &db, - &empty(), - &committed("file.rs", 1, 1, 1, vec![added(2, "two")]), - ); - - let contents: Vec = result - .files - .iter() - .flat_map(|file| file.hunks.iter()) - .flat_map(|hunk| hunk.lines.iter()) - .map(|line| line.content.clone()) - .collect(); - assert_eq!(contents, vec!["two".to_owned()]); -} diff --git a/cli/src/services/mutation_trace/runtime/protected_worktree.rs b/cli/src/services/mutation_trace/runtime/protected_worktree.rs index 114f397ba..eefd57665 100644 --- a/cli/src/services/mutation_trace/runtime/protected_worktree.rs +++ b/cli/src/services/mutation_trace/runtime/protected_worktree.rs @@ -1,11 +1,15 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use std::time::Duration; +use anyhow::Context; + use crate::services::mutation_trace::types::WorktreeId; use super::external_taint::ExternalTaintMarker; use super::git_snapshot::{resolve_git_dir, resolve_worktree_id}; -use super::worktree_lock::{acquire_inner, WorktreeLock, WorktreeLockError}; +use super::worktree_lock::{ + acquire_inner_async, WorktreeLock, WorktreeLockError, WorktreeLockLease, +}; pub const WORKTREE_LOCK_TIMEOUT: Duration = Duration::from_secs(10); @@ -24,6 +28,7 @@ pub enum ProtectedWorktreeError { source: anyhow::Error, }, CheckoutIdentity(anyhow::Error), + MarkerWorkerFailed(tokio::task::JoinError), } impl std::fmt::Display for ProtectedWorktreeError { @@ -34,6 +39,11 @@ impl std::fmt::Display for ProtectedWorktreeError { "External-taint marker {operation:?} operation failed before any \ protected runtime work began: {source}" ), + ProtectedWorktreeError::MarkerWorkerFailed(source) => write!( + f, + "External-taint marker worker failed while holding the worktree lock; \ + the marker state is unknown and no protected runtime work began: {source}" + ), ProtectedWorktreeError::LockAcquisition(source) => write!(f, "{source}"), ProtectedWorktreeError::GitDirResolution(source) | ProtectedWorktreeError::CheckoutIdentity(source) => write!(f, "{source}"), @@ -51,51 +61,72 @@ pub struct ProtectedWorktree { lock: WorktreeLock, } +async fn arm_marker_with_lock( + git_dir: PathBuf, + lock: WorktreeLock, + before_persist: F, +) -> Result<(WorktreeLock, ExternalTaintMarker, bool), ProtectedWorktreeError> +where + F: FnOnce() + Send + 'static, +{ + tokio::task::spawn_blocking(move || { + let marker = ExternalTaintMarker::new(&git_dir); + let inherited_external_taint = + marker + .exists() + .map_err(|source| ProtectedWorktreeError::ExternalTaintMarker { + operation: ExternalTaintOperation::Inspect, + source, + })?; + before_persist(); + marker + .persist() + .map_err(|source| ProtectedWorktreeError::ExternalTaintMarker { + operation: ExternalTaintOperation::Persist, + source, + })?; + Ok((lock, marker, inherited_external_taint)) + }) + .await + .map_err(ProtectedWorktreeError::MarkerWorkerFailed)? +} + impl ProtectedWorktree { - pub fn acquire(repository_root: &Path) -> Result { - Self::acquire_inner(repository_root, || {}) + pub async fn acquire(repository_root: &Path) -> Result { + Self::acquire_inner(repository_root, || {}).await } - pub(super) fn acquire_inner( + pub(super) async fn acquire_inner( repository_root: &Path, on_lock_contention: F, ) -> Result where - F: FnOnce(), + F: FnOnce() + Send + 'static, { - Self::acquire_with_timeout(repository_root, WORKTREE_LOCK_TIMEOUT, on_lock_contention) + Self::acquire_with_timeout(repository_root, WORKTREE_LOCK_TIMEOUT, on_lock_contention).await } - fn acquire_with_timeout( + pub(super) async fn acquire_with_timeout( repository_root: &Path, lock_timeout: Duration, on_lock_contention: F, ) -> Result where - F: FnOnce(), + F: FnOnce() + Send + 'static, { - let git_dir = - resolve_git_dir(repository_root).map_err(ProtectedWorktreeError::GitDirResolution)?; + let git_dir = resolve_git_dir(repository_root) + .await + .map_err(ProtectedWorktreeError::GitDirResolution)?; - let lock = acquire_inner(&git_dir, lock_timeout, on_lock_contention) + let lock = acquire_inner_async(&git_dir, lock_timeout, on_lock_contention) + .await .map_err(ProtectedWorktreeError::LockAcquisition)?; - let marker = ExternalTaintMarker::new(&git_dir); - let inherited_external_taint = - marker - .exists() - .map_err(|source| ProtectedWorktreeError::ExternalTaintMarker { - operation: ExternalTaintOperation::Inspect, - source, - })?; - marker - .persist() - .map_err(|source| ProtectedWorktreeError::ExternalTaintMarker { - operation: ExternalTaintOperation::Persist, - source, - })?; + let (lock, marker, inherited_external_taint) = + arm_marker_with_lock(git_dir, lock, || {}).await?; let worktree_id = resolve_worktree_id(repository_root) + .await .map_err(ProtectedWorktreeError::CheckoutIdentity)?; Ok(Self { @@ -116,13 +147,35 @@ impl ProtectedWorktree { self.inherited_external_taint } + #[must_use] + pub(super) fn lock_lease(&self) -> WorktreeLockLease { + self.lock.lease() + } + #[cfg(unix)] #[must_use] pub(super) fn lock_raw_fd(&self) -> std::os::unix::io::RawFd { self.lock.as_raw_fd() } - pub fn complete(self) -> anyhow::Result<()> { + pub async fn complete(self) -> anyhow::Result<()> { + self.complete_inner(|| {}).await + } + + async fn complete_inner(self, before_clear: F) -> anyhow::Result<()> + where + F: FnOnce() + Send + 'static, + { + tokio::task::spawn_blocking(move || self.complete_blocking(before_clear)) + .await + .context("External-taint marker completion worker failed")? + } + + fn complete_blocking(self, before_clear: F) -> anyhow::Result<()> + where + F: FnOnce(), + { + before_clear(); self.marker.clear() } @@ -134,53 +187,29 @@ impl ProtectedWorktree { #[cfg(test)] mod tests { - use std::path::PathBuf; + use std::path::Path; use std::process::Command; use std::sync::mpsc; - use std::thread; + use std::time::Duration; - use super::*; + use tokio::sync::oneshot; - struct TestRepo { - _temp_dir: tempfile::TempDir, - repo_root: PathBuf, - git_dir: PathBuf, - } + use super::super::external_taint::ExternalTaintMarker; + use super::super::git_snapshot::resolve_git_dir; + use super::super::worktree_lock::{acquire_inner_async, WorktreeLockError}; + use super::{arm_marker_with_lock, ProtectedWorktree}; - impl TestRepo { - fn new(label: &str) -> Self { - let temp_dir = tempfile::Builder::new() - .prefix(&format!("sce-protected-worktree-{label}-")) - .tempdir() - .expect("test temp directory should be created"); - let repo_root = temp_dir.path().join("repo"); - std::fs::create_dir_all(&repo_root).expect("repository directory should be created"); - run_git(&repo_root, &["init", "--quiet"]); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - Self { - _temp_dir: temp_dir, - repo_root, - git_dir, - } - } - - fn marker(&self) -> ExternalTaintMarker { - ExternalTaintMarker::new(&self.git_dir) - } - - fn marker_exists(&self) -> bool { - self.marker() - .exists() - .expect("marker existence should resolve") - } - } + const LOCK_CONTENDED_TIMEOUT: Duration = Duration::from_millis(300); + const LOCK_RELEASE_TIMEOUT: Duration = Duration::from_secs(10); + const WORKER_HOOK_TIMEOUT: Duration = Duration::from_secs(10); + const WORKER_PROGRESS_LIMIT: Duration = Duration::from_secs(2); - fn run_git(dir: &std::path::Path, args: &[&str]) { + fn git(cwd: &Path, args: &[&str]) { let output = Command::new("git") .args(args) - .current_dir(dir) + .current_dir(cwd) .output() - .expect("git command should spawn"); + .expect("git command should run"); assert!( output.status.success(), "git {args:?} failed: {}", @@ -188,138 +217,144 @@ mod tests { ); } - #[test] - fn acquire_arms_a_fresh_marker_and_reports_no_inherited_taint() { - let repo = TestRepo::new("fresh-marker"); + fn init_committed_repository(root: &Path) { + std::fs::create_dir_all(root).expect("repo dir"); + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "protected@example.invalid"]); + git(root, &["config", "user.name", "Protected Test"]); + git(root, &["config", "commit.gpgsign", "false"]); + std::fs::write(root.join("tracked.txt"), "tracked\n").expect("write tracked file"); + git(root, &["add", "tracked.txt"]); + git(root, &["commit", "-q", "-m", "initial"]); + } - let guard = ProtectedWorktree::acquire(&repo.repo_root) - .expect("the prefix should be establishable on a clean worktree"); + fn paused_worker_hook() -> ( + impl FnOnce() + Send + 'static, + oneshot::Receiver<()>, + mpsc::Sender<()>, + ) { + let (entered_tx, entered_rx) = oneshot::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let hook = move || { + let _ = entered_tx.send(()); + let _ = release_rx.recv_timeout(WORKER_HOOK_TIMEOUT); + }; + (hook, entered_rx, release_tx) + } - assert!( - !guard.inherited_external_taint(), - "a worktree with no marker on entry must not report inherited taint" - ); - assert!( - !guard.worktree_id().0.is_empty(), - "the guard must expose the derived WorktreeId" - ); - assert!( - repo.marker_exists(), - "acquire must arm the external-taint marker write-ahead" - ); + async fn assert_runtime_worker_progresses() { + tokio::time::timeout( + WORKER_PROGRESS_LIMIT, + tokio::spawn(tokio::time::sleep(Duration::from_millis(10))), + ) + .await + .expect("an unrelated Tokio timer should progress while marker I/O is paused") + .expect("unrelated timer task should join"); + } - guard - .complete() - .expect("completion should clear the marker"); + async fn assert_worktree_lock_contended(git_dir: &Path) { + let contended = acquire_inner_async(git_dir, LOCK_CONTENDED_TIMEOUT, || {}).await; assert!( - !repo.marker_exists(), - "an explicit completion must clear the marker it armed" + matches!(contended, Err(WorktreeLockError::TimedOut { .. })), + "the paused marker worker must still own the worktree lock, got {contended:?}" ); } - #[test] - fn acquire_reports_a_marker_inherited_from_an_earlier_invocation() { - let repo = TestRepo::new("inherited-marker"); - repo.marker() - .persist() - .expect("the earlier invocation's marker should arm"); + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn cancelled_marker_arming_keeps_the_lock_until_the_marker_is_persisted() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let marker = ExternalTaintMarker::new(&git_dir); - let guard = ProtectedWorktree::acquire(&repo.repo_root) - .expect("an inherited marker must not fail the prefix"); + let lock = acquire_inner_async(&git_dir, LOCK_RELEASE_TIMEOUT, || {}) + .await + .expect("initial worktree lock"); + let (hook, entered_rx, release_tx) = paused_worker_hook(); + let arming = tokio::spawn(arm_marker_with_lock(git_dir.clone(), lock, hook)); - assert!( - guard.inherited_external_taint(), - "a marker present on entry must be reported as inherited taint" + tokio::time::timeout(WORKER_HOOK_TIMEOUT, entered_rx) + .await + .expect("arming worker should inspect the marker") + .expect("arming hook sender should stay alive"); + assert!(!marker.exists().expect("inspect marker")); + + assert_runtime_worker_progresses().await; + + arming.abort(); + let aborted = arming.await.expect_err("arming caller should be aborted"); + assert!(aborted.is_cancelled()); + + assert_worktree_lock_contended(&git_dir).await; + + release_tx.send(()).expect("release arming worker"); + drop( + acquire_inner_async(&git_dir, LOCK_RELEASE_TIMEOUT, || {}) + .await + .expect("the arming worker should release the lock after persisting"), ); assert!( - repo.marker_exists(), - "the inherited marker must stay armed for the protected operation" + marker.exists().expect("inspect marker"), + "cancelled arming must leave the marker armed" ); + + let protected = ProtectedWorktree::acquire(&repo_root) + .await + .expect("protected worktree"); + assert!(protected.inherited_external_taint()); + protected.complete().await.expect("complete"); + assert!(!marker.exists().expect("inspect marker")); } - #[test] - fn a_dropped_guard_releases_the_lock_but_leaves_the_marker_armed() { - let repo = TestRepo::new("dropped-guard"); + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn cancelled_completion_clears_the_marker_before_a_later_operation_can_arm() { + let dir = tempfile::tempdir().expect("temp dir"); + let repo_root = dir.path().join("repo"); + init_committed_repository(&repo_root); + let git_dir = resolve_git_dir(&repo_root).await.expect("git dir"); + let marker = ExternalTaintMarker::new(&git_dir); - let guard = ProtectedWorktree::acquire(&repo.repo_root) - .expect("the prefix should be establishable on a clean worktree"); - drop(guard); + let protected = ProtectedWorktree::acquire(&repo_root) + .await + .expect("first protected worktree"); + assert!(!protected.inherited_external_taint()); + assert!(marker.exists().expect("inspect marker")); - assert!( - repo.marker_exists(), - "dropping a guard that never completed must leave the fence armed" - ); + let (hook, entered_rx, release_tx) = paused_worker_hook(); + let completion = tokio::spawn(protected.complete_inner(hook)); - ProtectedWorktree::acquire(&repo.repo_root) - .expect("a dropped guard must have released the worktree lock"); - } + tokio::time::timeout(WORKER_HOOK_TIMEOUT, entered_rx) + .await + .expect("completion worker should take ownership") + .expect("completion hook sender should stay alive"); - #[test] - fn acquire_fails_with_lock_acquisition_while_the_lock_is_still_held() { - let repo = TestRepo::new("lock-timeout"); + assert_runtime_worker_progresses().await; - let held = acquire_inner(&repo.git_dir, Duration::from_secs(5), || {}) - .expect("the test should hold the worktree lock before the guard runs"); + completion.abort(); + let aborted = completion + .await + .expect_err("completion caller should be aborted"); + assert!(aborted.is_cancelled()); - let error = ProtectedWorktree::acquire_with_timeout( - &repo.repo_root, - Duration::from_millis(250), - || {}, - ) - .expect_err("the prefix must not be establishable while the lock is held"); - assert!( - matches!( - error, - ProtectedWorktreeError::LockAcquisition(WorktreeLockError::TimedOut { .. }) - ), - "expected a LockAcquisition timeout, got {error:?}" - ); - assert!( - !repo.marker_exists(), - "a prefix that never acquired the lock must not have armed the fence" - ); + assert_worktree_lock_contended(&git_dir).await; + assert!(marker.exists().expect("inspect marker")); - drop(held); - } + release_tx.send(()).expect("release completion worker"); - #[test] - fn the_lock_is_held_for_the_whole_guard_lifetime() { - let repo = TestRepo::new("lock-lifetime"); - - let guard = ProtectedWorktree::acquire(&repo.repo_root) - .expect("the prefix should be establishable on a clean worktree"); - - let (contention_tx, contention_rx) = mpsc::channel(); - let (result_tx, result_rx) = mpsc::channel(); - let repo_root = repo.repo_root.clone(); - let worker = thread::spawn(move || { - let result = ProtectedWorktree::acquire_inner(&repo_root, move || { - contention_tx - .send(()) - .expect("contention signal channel should still be open"); - }); - result_tx - .send(()) - .expect("result signal channel should still be open"); - result - }); - - contention_rx - .recv_timeout(Duration::from_secs(5)) - .expect("a second acquirer should observe the held worktree lock"); + let next = ProtectedWorktree::acquire(&repo_root) + .await + .expect("second protected worktree"); assert!( - result_rx.recv_timeout(Duration::from_millis(300)).is_err(), - "a second acquirer must not establish the prefix while the guard is alive" + !next.inherited_external_taint(), + "the earlier completion must clear its marker before releasing the lock" + ); + assert!( + marker.exists().expect("inspect marker"), + "the later operation must keep its own armed marker" ); - drop(guard); - - result_rx - .recv_timeout(Duration::from_secs(5)) - .expect("the second acquirer should proceed once the guard is dropped"); - worker - .join() - .expect("the second acquirer thread should not panic") - .expect("the second acquirer should succeed once the lock is released"); + next.complete().await.expect("complete second operation"); + assert!(!marker.exists().expect("inspect marker")); } } diff --git a/cli/src/services/mutation_trace/runtime/ref_advisory.rs b/cli/src/services/mutation_trace/runtime/ref_advisory.rs new file mode 100644 index 000000000..b70a368ab --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_advisory.rs @@ -0,0 +1,950 @@ +use std::path::Path; +use std::time::Duration; + +use super::maintenance_state::{ + derive_git_dir, plan_advisory, read_state, state_path, write_state_atomically, + AdvisoryDecision, MaintenanceState, PersistFailure, +}; +use super::worktree_lock::{acquire_inner, WorktreeLockError}; + +pub(super) const RECONCILIATION_RECOMMENDED_MESSAGE: &str = + "Reconciliation is recommended. Run sce doctor --fix."; +const DIAGNOSTIC_PREFIX: &str = "SCE: "; +const DURABILITY_UNCERTAIN_DIAGNOSTIC: &str = "Advisory-state durability could not be confirmed"; +const STATE_WRITE_FAILED_DIAGNOSTIC: &str = "Reconciliation advisory state could not be written"; +const STATE_DURABILITY_UNCONFIRMED_DIAGNOSTIC: &str = + "Reconciliation advisory-state durability could not be confirmed"; +const STATE_UNAVAILABLE_DIAGNOSTIC: &str = "Reconciliation advisory state is unavailable"; +const CHECK_DOCTOR_HINT: &str = "Run sce doctor to check reconciliation status."; + +#[derive(Debug)] +pub(super) enum AdvisoryOutcome { + Anchored, + NoAction, + Advised, + AdvisedDurabilityUncertain { warning: PersistFailure }, + Busy, + StateWriteFailed { warning: PersistFailure }, + StateUnavailable, +} + +impl AdvisoryOutcome { + pub(super) fn recommendation(&self) -> Option<&'static str> { + match self { + AdvisoryOutcome::Advised | AdvisoryOutcome::AdvisedDurabilityUncertain { .. } => { + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + } + AdvisoryOutcome::Anchored + | AdvisoryOutcome::NoAction + | AdvisoryOutcome::Busy + | AdvisoryOutcome::StateWriteFailed { .. } + | AdvisoryOutcome::StateUnavailable => None, + } + } + + pub(super) fn persistence_warning(&self) -> Option<&PersistFailure> { + match self { + AdvisoryOutcome::AdvisedDurabilityUncertain { warning } + | AdvisoryOutcome::StateWriteFailed { warning } => Some(warning), + AdvisoryOutcome::Anchored + | AdvisoryOutcome::NoAction + | AdvisoryOutcome::Advised + | AdvisoryOutcome::Busy + | AdvisoryOutcome::StateUnavailable => None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum AdvisorySeverity { + Debug, + Warn, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AdvisoryReport { + pub(crate) outcome: &'static str, + pub(crate) severity: AdvisorySeverity, + pub(crate) recommendation: Option<&'static str>, + pub(crate) warning: Option, + pub(crate) diagnostic: Option, +} + +impl AdvisoryOutcome { + pub(crate) fn report(&self) -> AdvisoryReport { + let (outcome, severity) = match self { + AdvisoryOutcome::Anchored => ("anchored", AdvisorySeverity::Debug), + AdvisoryOutcome::NoAction => ("no_action", AdvisorySeverity::Debug), + AdvisoryOutcome::Busy => ("busy", AdvisorySeverity::Debug), + AdvisoryOutcome::Advised => ("advised", AdvisorySeverity::Warn), + AdvisoryOutcome::AdvisedDurabilityUncertain { .. } => { + ("advised_durability_uncertain", AdvisorySeverity::Warn) + } + AdvisoryOutcome::StateWriteFailed { .. } => { + ("state_write_failed", AdvisorySeverity::Warn) + } + AdvisoryOutcome::StateUnavailable => ("state_unavailable", AdvisorySeverity::Warn), + }; + AdvisoryReport { + outcome, + severity, + recommendation: self.recommendation(), + warning: self.persistence_warning().map(ToString::to_string), + diagnostic: self.diagnostic(), + } + } + + fn diagnostic(&self) -> Option { + match self { + AdvisoryOutcome::Advised => Some(format!( + "{DIAGNOSTIC_PREFIX}{RECONCILIATION_RECOMMENDED_MESSAGE}" + )), + AdvisoryOutcome::AdvisedDurabilityUncertain { warning } => Some(format!( + "{DIAGNOSTIC_PREFIX}{RECONCILIATION_RECOMMENDED_MESSAGE}\n\ + {DIAGNOSTIC_PREFIX}{DURABILITY_UNCERTAIN_DIAGNOSTIC}: {warning}." + )), + AdvisoryOutcome::StateWriteFailed { + warning: warning @ PersistFailure::NotApplied { .. }, + } => Some(format!( + "{DIAGNOSTIC_PREFIX}{STATE_WRITE_FAILED_DIAGNOSTIC}: {warning}. {CHECK_DOCTOR_HINT}" + )), + AdvisoryOutcome::StateWriteFailed { + warning: warning @ PersistFailure::DurabilityUncertain { .. }, + } => Some(format!( + "{DIAGNOSTIC_PREFIX}{STATE_DURABILITY_UNCONFIRMED_DIAGNOSTIC}: {warning}. {CHECK_DOCTOR_HINT}" + )), + AdvisoryOutcome::StateUnavailable => Some(format!( + "{DIAGNOSTIC_PREFIX}{STATE_UNAVAILABLE_DIAGNOSTIC}. {CHECK_DOCTOR_HINT}" + )), + AdvisoryOutcome::Anchored | AdvisoryOutcome::NoAction | AdvisoryOutcome::Busy => None, + } + } +} + +fn system_unix_time_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .ok() + .and_then(|elapsed| i64::try_from(elapsed.as_millis()).ok()) + .unwrap_or(0) +} + +pub(crate) fn advise_after_completed_boundary(repository_root: &Path) -> AdvisoryReport { + advise_if_due(repository_root, system_unix_time_ms).report() +} + +fn outcome_after_write( + decision: AdvisoryDecision, + write: Result<(), PersistFailure>, +) -> AdvisoryOutcome { + match (decision, write) { + (AdvisoryDecision::Anchored, Ok(())) => AdvisoryOutcome::Anchored, + (AdvisoryDecision::NoAction, Ok(())) => AdvisoryOutcome::NoAction, + (AdvisoryDecision::Advised, Ok(())) => AdvisoryOutcome::Advised, + (AdvisoryDecision::Advised, Err(warning @ PersistFailure::DurabilityUncertain { .. })) => { + AdvisoryOutcome::AdvisedDurabilityUncertain { warning } + } + (_, Err(warning)) => AdvisoryOutcome::StateWriteFailed { warning }, + } +} + +pub(super) fn advise_if_due(repository_root: &Path, now: C) -> AdvisoryOutcome +where + C: Fn() -> i64, +{ + advise_if_due_with(repository_root, now, write_state_atomically) +} + +pub(super) fn advise_if_due_with( + repository_root: &Path, + now: C, + write_state: W, +) -> AdvisoryOutcome +where + C: Fn() -> i64, + W: Fn(&Path, &MaintenanceState) -> Result<(), PersistFailure>, +{ + let Some(git_dir) = derive_git_dir(repository_root) else { + return AdvisoryOutcome::StateUnavailable; + }; + let path = state_path(&git_dir); + + let Ok(unlocked) = read_state(&path) else { + return AdvisoryOutcome::StateUnavailable; + }; + let (decision, pending) = plan_advisory(&unlocked, now()); + if pending.is_none() && decision == AdvisoryDecision::NoAction { + return AdvisoryOutcome::NoAction; + } + + let lock = match acquire_inner(&git_dir, Duration::ZERO, || {}) { + Ok(lock) => lock, + Err(WorktreeLockError::TimedOut { .. }) => return AdvisoryOutcome::Busy, + Err(_) => return AdvisoryOutcome::StateUnavailable, + }; + + let Ok(locked) = read_state(&path) else { + return AdvisoryOutcome::StateUnavailable; + }; + let now_ms = now(); + let (decision, pending) = plan_advisory(&locked, now_ms); + + let result = match pending { + Some(next) => outcome_after_write(decision, write_state(&path, &next)), + None => outcome_after_write(decision, Ok(())), + }; + drop(lock); + result +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::process::Command; + + use super::super::maintenance_state::{ + evaluate, evaluate_recommendation, read_state, record_failure, record_success, state_path, + write_state_atomically, write_state_atomically_with, AttemptOutcome, + FaultInjectingFilesystem, MaintenanceState, PersistFailure, PersistPhase, StateRead, + StoredReport, FUTURE_SKEW_TOLERANCE_MS, RECONCILIATION_ADVISORY_AFTER_MS, + }; + use super::super::worktree_lock::acquire_inner; + use super::{ + advise_if_due_with, AdvisoryOutcome, AdvisorySeverity, RECONCILIATION_RECOMMENDED_MESSAGE, + }; + + const NOW: i64 = 1_000_000_000_000; + const RECOMMENDED: &str = RECONCILIATION_RECOMMENDED_MESSAGE; + + fn init_repo(root: &Path) { + std::fs::create_dir_all(root).expect("repo dir"); + let output = Command::new("git") + .args(["init", "-q"]) + .current_dir(root) + .output() + .expect("git init"); + assert!(output.status.success()); + } + + fn real_write(path: &Path, state: &MaintenanceState) -> Result<(), PersistFailure> { + write_state_atomically(path, state) + } + + fn advise(root: &Path, now: i64) -> AdvisoryOutcome { + advise_if_due_with(root, || now, real_write) + } + + fn stored(root: &Path) -> MaintenanceState { + match read_state(&state_path(&root.join(".git"))).expect("read state") { + StateRead::Valid(state) => state, + other => panic!("expected valid state, got {other:?}"), + } + } + + fn seed(root: &Path, state: &MaintenanceState) { + let path = state_path(&root.join(".git")); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + real_write(&path, state).expect("seed state"); + } + + #[test] + fn maintenance_report_maps_every_outcome_to_severity_and_diagnostic() { + let uncertain = || PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + source: std::io::Error::other("sync failed"), + }; + let not_applied = || PersistFailure::NotApplied { + phase: PersistPhase::Rename, + source: std::io::Error::other("rename failed"), + staging_cleanup: None, + }; + let recommended = format!("SCE: {RECOMMENDED}"); + let cases = [ + (AdvisoryOutcome::Anchored, "anchored", AdvisorySeverity::Debug, None), + (AdvisoryOutcome::NoAction, "no_action", AdvisorySeverity::Debug, None), + (AdvisoryOutcome::Busy, "busy", AdvisorySeverity::Debug, None), + ( + AdvisoryOutcome::Advised, + "advised", + AdvisorySeverity::Warn, + Some(recommended.clone()), + ), + ( + AdvisoryOutcome::StateUnavailable, + "state_unavailable", + AdvisorySeverity::Warn, + Some( + "SCE: Reconciliation advisory state is unavailable. Run sce doctor to check reconciliation status." + .to_string(), + ), + ), + ]; + for (outcome, name, severity, diagnostic) in cases { + let report = outcome.report(); + assert_eq!(report.outcome, name); + assert_eq!(report.severity, severity); + assert_eq!(report.diagnostic, diagnostic, "{name}"); + assert_eq!(report.warning, None, "{name}"); + } + + let report = AdvisoryOutcome::AdvisedDurabilityUncertain { + warning: uncertain(), + } + .report(); + assert_eq!(report.outcome, "advised_durability_uncertain"); + assert_eq!(report.severity, AdvisorySeverity::Warn); + assert_eq!( + report.recommendation, + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + let warning = report.warning.expect("persistence warning"); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with(&format!( + "{recommended}\nSCE: Advisory-state durability could not be confirmed: " + ))); + assert!(diagnostic.contains(&warning)); + + let report = AdvisoryOutcome::StateWriteFailed { + warning: not_applied(), + } + .report(); + assert_eq!(report.outcome, "state_write_failed"); + assert_eq!(report.severity, AdvisorySeverity::Warn); + assert_eq!(report.recommendation, None); + let warning = report.warning.expect("persistence warning"); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with("SCE: Reconciliation advisory state could not be written: ")); + assert!(diagnostic.contains(&warning)); + assert!(!diagnostic.contains(RECONCILIATION_RECOMMENDED_MESSAGE)); + + let report = AdvisoryOutcome::StateWriteFailed { + warning: uncertain(), + } + .report(); + assert_eq!(report.outcome, "state_write_failed"); + assert_eq!(report.severity, AdvisorySeverity::Warn); + assert_eq!(report.recommendation, None); + let warning = report.warning.expect("persistence warning"); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic + .starts_with("SCE: Reconciliation advisory-state durability could not be confirmed: ")); + assert!(diagnostic.ends_with("Run sce doctor to check reconciliation status.")); + assert!(diagnostic.contains(&warning)); + assert!(!diagnostic.contains("could not be written")); + assert!(!diagnostic.contains(RECONCILIATION_RECOMMENDED_MESSAGE)); + } + + #[test] + fn maintenance_advisory_anchors_once_then_leaves_state_untouched() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Anchored)); + let path = state_path(&dir.path().join(".git")); + let first = std::fs::read(&path).expect("state bytes"); + assert_eq!(stored(dir.path()).anchor, Some(NOW)); + + assert!(matches!( + advise(dir.path(), NOW + 1000), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), first); + } + + #[test] + fn maintenance_advisory_advises_once_per_window_and_persists_before_returning() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + + let due = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + assert!(matches!(advise(dir.path(), due), AdvisoryOutcome::Advised)); + assert_eq!(stored(dir.path()).last_advised, Some(due)); + assert!(matches!( + advise(dir.path(), due + 1000), + AdvisoryOutcome::NoAction + )); + assert!(matches!( + advise(dir.path(), due + RECONCILIATION_ADVISORY_AFTER_MS), + AdvisoryOutcome::Advised + )); + } + + #[test] + fn maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let invalid = MaintenanceState { + anchor: Some(NOW + FUTURE_SKEW_TOLERANCE_MS + 1), + last_success: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + seed(dir.path(), &invalid); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Advised)); + let normalized = stored(dir.path()); + assert_eq!(normalized.last_advised, Some(NOW)); + assert_eq!(normalized.anchor, Some(NOW)); + assert_eq!(normalized.last_success, None); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_advisory_reports_busy_and_leaves_invalid_state_byte_identical() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let invalid = MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + seed(dir.path(), &invalid); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + let held = acquire_inner(&dir.path().join(".git"), std::time::Duration::ZERO, || {}) + .expect("hold lock"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Busy)); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + drop(held); + + let recommendation = evaluate_recommendation(&invalid, NOW); + assert!(recommendation.recommended); + assert!(recommendation.invalid_timestamps); + } + + fn advise_with_fault(root: &Path, now: i64, phase: PersistPhase) -> AdvisoryOutcome { + advise_if_due_with( + root, + || now, + |path, state| { + write_state_atomically_with( + &FaultInjectingFilesystem::failing_at(phase), + path, + state, + ) + }, + ) + } + + fn failed_state(now: i64) -> MaintenanceState { + record_failure( + &MaintenanceState::default(), + now, + "agent_trace_db_unavailable", + "agent trace database is missing", + ) + } + + #[test] + fn maintenance_first_failed_explicit_pass_without_anchor_is_recommended() { + let state = failed_state(NOW); + assert_eq!(state.anchor, None); + assert_eq!(state.last_success, None); + + let recommendation = evaluate_recommendation(&state, NOW); + assert!(recommendation.recommended); + assert_eq!( + recommendation.last_attempt_outcome, + Some(AttemptOutcome::Failed) + ); + assert_eq!(recommendation.consecutive_failures, 1); + assert!(!evaluate_recommendation(&MaintenanceState::default(), NOW).recommended); + } + + #[test] + fn maintenance_failure_advisory_emits_once_and_doctor_still_recommends_repair() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &failed_state(NOW)); + + let first = advise(dir.path(), NOW + 1000); + assert!(matches!(first, AdvisoryOutcome::Advised)); + assert_eq!( + first.recommendation(), + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + let persisted = stored(dir.path()); + assert_eq!(persisted.last_advised, Some(NOW + 1000)); + assert_eq!(persisted.consecutive_failures, 1); + assert_eq!(persisted.last_attempt_outcome, Some(AttemptOutcome::Failed)); + assert_eq!(persisted.last_failure, failed_state(NOW).last_failure); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + let second = advise(dir.path(), NOW + 2000); + assert!(matches!(second, AdvisoryOutcome::NoAction)); + assert_eq!(second.recommendation(), None); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + + let recommendation = evaluate_recommendation(&stored(dir.path()), NOW + 2000); + assert!(recommendation.recommended); + assert!(recommendation.last_advised_age_ms.is_some()); + } + + #[test] + fn maintenance_failure_advisory_repeats_after_the_cadence_window() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &failed_state(NOW)); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Advised)); + let later = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + assert!(matches!( + advise(dir.path(), later), + AdvisoryOutcome::Advised + )); + assert_eq!(stored(dir.path()).consecutive_failures, 1); + } + + #[test] + fn maintenance_repeated_failed_passes_keep_recommendation_active() { + let first = failed_state(NOW); + let second = record_failure(&first, NOW + 10, "pin_inventory", "inventory failed"); + let third = record_failure(&second, NOW + 20, "pin_inventory", "inventory failed"); + assert_eq!(third.consecutive_failures, 3); + assert!(evaluate_recommendation(&first, NOW + 30).recommended); + assert!(evaluate_recommendation(&second, NOW + 30).recommended); + assert!(evaluate_recommendation(&third, NOW + 30).recommended); + } + + #[test] + fn maintenance_successful_pass_clears_failure_streak_and_recommendation() { + let failed = record_failure(&failed_state(NOW), NOW + 10, "pin_inventory", "failed"); + let healed = record_success( + &failed, + NOW + 20, + StoredReport { + retained: 1, + deleted: 0, + local_required: 0, + }, + ); + assert_eq!(healed.consecutive_failures, 0); + assert_eq!(healed.last_failure, None); + assert_eq!(healed.last_attempt_outcome, Some(AttemptOutcome::Completed)); + assert!(!evaluate_recommendation(&healed, NOW + 30).recommended); + } + + #[test] + fn maintenance_failure_after_recent_success_is_recommended_and_advised_despite_older_reminder() + { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let success = record_success( + &MaintenanceState::default(), + NOW, + StoredReport { + retained: 1, + deleted: 0, + local_required: 0, + }, + ); + let reminded = MaintenanceState { + last_advised: Some(NOW + 1000), + ..success + }; + assert!(!evaluate_recommendation(&reminded, NOW + 1500).recommended); + + let failed = record_failure(&reminded, NOW + 2000, "delete_transaction", "failed"); + assert!(evaluate_recommendation(&failed, NOW + 2500).recommended); + seed(dir.path(), &failed); + + assert!(matches!( + advise(dir.path(), NOW + 2500), + AdvisoryOutcome::Advised + )); + assert!(matches!( + advise(dir.path(), NOW + 3000), + AdvisoryOutcome::NoAction + )); + } + + #[test] + fn maintenance_advisory_rename_failure_preserves_state_and_next_call_can_advise() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + let due = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + let outcome = advise_with_fault(dir.path(), due, PersistPhase::Rename); + assert!(matches!( + outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::NotApplied { + phase: PersistPhase::Rename, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + assert!(outcome.persistence_warning().is_some()); + let diagnostic = outcome.report().diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with("SCE: Reconciliation advisory state could not be written: ")); + assert!(!diagnostic.contains("durability")); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + assert!(evaluate_recommendation(&stored(dir.path()), due).recommended); + assert!(matches!(advise(dir.path(), due), AdvisoryOutcome::Advised)); + } + + fn assert_durability_uncertain_advisory(phase: PersistPhase) { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + + let due = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + let outcome = advise_with_fault(dir.path(), due, phase); + assert!(!matches!(outcome, AdvisoryOutcome::Advised)); + let AdvisoryOutcome::AdvisedDurabilityUncertain { warning } = &outcome else { + panic!("expected AdvisedDurabilityUncertain for {phase:?}, got {outcome:?}"); + }; + assert!(matches!( + warning, + PersistFailure::DurabilityUncertain { phase: reported, .. } if *reported == phase + )); + assert_eq!( + outcome.recommendation(), + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + assert!(outcome.persistence_warning().is_some()); + let report = outcome.report(); + assert_eq!( + report.recommendation, + Some(RECONCILIATION_RECOMMENDED_MESSAGE) + ); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic.starts_with(&format!( + "SCE: {RECOMMENDED}\nSCE: Advisory-state durability could not be confirmed: " + ))); + assert!(!diagnostic.contains("could not be written")); + + let visible = stored(dir.path()); + assert_eq!(visible.last_advised, Some(due)); + + assert!(matches!( + advise(dir.path(), due + 1000), + AdvisoryOutcome::NoAction + )); + let recommendation = evaluate_recommendation(&visible, due + 1000); + assert!(recommendation.recommended); + assert!(recommendation.last_advised_age_ms.is_some()); + } + + #[test] + fn maintenance_advisory_parent_directory_open_failure_is_durability_uncertain() { + assert_durability_uncertain_advisory(PersistPhase::OpenParentDirectory); + } + + #[test] + fn maintenance_advisory_parent_directory_sync_failure_is_durability_uncertain() { + assert_durability_uncertain_advisory(PersistPhase::SyncParentDirectory); + } + + #[test] + fn maintenance_advisory_durability_uncertain_failure_state_stays_recommended() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &failed_state(NOW)); + + let outcome = advise_with_fault(dir.path(), NOW + 10, PersistPhase::SyncParentDirectory); + assert!(matches!( + outcome, + AdvisoryOutcome::AdvisedDurabilityUncertain { .. } + )); + let visible = stored(dir.path()); + assert_eq!(visible.last_advised, Some(NOW + 10)); + assert_eq!(visible.consecutive_failures, 1); + assert!(evaluate_recommendation(&visible, NOW + 20).recommended); + } + + #[test] + fn maintenance_advisory_accepts_valid_timestamp_offsets_without_rewrite() { + for ahead in [0, 60_000, FUTURE_SKEW_TOLERANCE_MS] { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let state = MaintenanceState { + last_success: Some(NOW + ahead), + ..MaintenanceState::default() + }; + seed(dir.path(), &state); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + assert!( + matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction), + "offset {ahead}" + ); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + assert!( + !evaluate_recommendation(&state, NOW).recommended, + "offset {ahead}" + ); + } + } + + #[test] + fn maintenance_advisory_clock_rollback_beyond_tolerance_normalizes_while_within_it_does_not() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + advise(dir.path(), NOW); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + assert!(matches!( + advise(dir.path(), NOW - FUTURE_SKEW_TOLERANCE_MS), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + + let rolled_back = NOW - FUTURE_SKEW_TOLERANCE_MS - 1; + assert!(matches!( + advise(dir.path(), rolled_back), + AdvisoryOutcome::Advised + )); + let normalized = stored(dir.path()); + assert_within_skew_tolerance(&normalized, rolled_back); + assert_eq!(normalized.last_advised, Some(rolled_back)); + } + + #[test] + fn maintenance_advisory_reanchors_unusable_state_variants() { + let cases: [(&str, Vec); 3] = [ + ("wrong schema version", br#"{"version":9999}"#.to_vec()), + ("oversized state file", vec![b' '; 5000]), + ("corrupt JSON", b"not json".to_vec()), + ]; + for (label, bytes) in cases { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let path = state_path(&dir.path().join(".git")); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + std::fs::write(&path, &bytes).expect("unusable state"); + + assert!( + matches!(read_state(&path).expect("read"), StateRead::Unusable), + "{label}" + ); + let outcome = advise(dir.path(), NOW); + assert!( + matches!(outcome, AdvisoryOutcome::Anchored), + "{label}: {outcome:?}" + ); + assert_eq!(outcome.recommendation(), None, "{label}"); + let anchored = stored(dir.path()); + assert_eq!(anchored.anchor, Some(NOW), "{label}"); + assert!( + !evaluate_recommendation(&anchored, NOW).recommended, + "{label}" + ); + + let anchored_bytes = std::fs::read(&path).expect("state bytes"); + assert!( + matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction), + "{label}" + ); + assert_eq!( + std::fs::read(&path).expect("state bytes"), + anchored_bytes, + "{label}" + ); + } + } + + fn invalid_future_state() -> MaintenanceState { + MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + last_success: Some(NOW + FUTURE_SKEW_TOLERANCE_MS + 1), + ..MaintenanceState::default() + } + } + + fn assert_within_skew_tolerance(state: &MaintenanceState, now: i64) { + for timestamp in [ + state.anchor, + state.last_success, + state.last_attempt, + state.last_advised, + ] + .into_iter() + .flatten() + { + assert!(timestamp <= now + FUTURE_SKEW_TOLERANCE_MS); + } + } + + #[test] + fn maintenance_normalization_pre_rename_failure_keeps_previous_bytes_and_recovers_later() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &invalid_future_state()); + let path = state_path(&dir.path().join(".git")); + let before = std::fs::read(&path).expect("state bytes"); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::Rename); + + assert!(matches!( + outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::NotApplied { + phase: PersistPhase::Rename, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + assert!(evaluate_recommendation(&stored(dir.path()), NOW).invalid_timestamps); + + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Advised)); + let normalized = stored(dir.path()); + assert_within_skew_tolerance(&normalized, NOW); + assert_eq!(normalized.last_advised, Some(NOW)); + } + + #[test] + fn maintenance_normalization_post_rename_failure_is_visible_but_durability_is_uncertain() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + seed(dir.path(), &invalid_future_state()); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::SyncParentDirectory); + + assert!(matches!( + &outcome, + AdvisoryOutcome::AdvisedDurabilityUncertain { + warning: PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + } + } + )); + assert!(outcome.persistence_warning().is_some()); + assert!(outcome.report().diagnostic.is_some()); + let visible = stored(dir.path()); + assert_within_skew_tolerance(&visible, NOW); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_advisory_anchor_write_failure_reports_state_write_failed_without_advice() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::Rename); + assert!(matches!(outcome, AdvisoryOutcome::StateWriteFailed { .. })); + assert_eq!(outcome.recommendation(), None); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::Anchored)); + } + + #[test] + fn maintenance_anchor_post_rename_failure_is_state_write_failed_with_visible_anchor() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::SyncParentDirectory); + + assert!(matches!( + &outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + let report = outcome.report(); + assert!(report.warning.is_some()); + let diagnostic = report.diagnostic.expect("diagnostic"); + assert!(diagnostic + .starts_with("SCE: Reconciliation advisory-state durability could not be confirmed: ")); + assert!(!diagnostic.contains("could not be written")); + assert_eq!(stored(dir.path()).anchor, Some(NOW)); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_no_action_normalization_post_rename_failure_is_state_write_failed() { + let dir = tempfile::tempdir().expect("temp dir"); + init_repo(dir.path()); + let invalid = MaintenanceState { + anchor: Some(NOW), + last_advised: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + seed(dir.path(), &invalid); + assert!(!evaluate(&invalid, NOW).advise); + + let outcome = advise_with_fault(dir.path(), NOW, PersistPhase::SyncParentDirectory); + + assert!(matches!( + &outcome, + AdvisoryOutcome::StateWriteFailed { + warning: PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + } + } + )); + assert_eq!(outcome.recommendation(), None); + let diagnostic = outcome.report().diagnostic.expect("diagnostic"); + assert!(diagnostic.contains("durability could not be confirmed")); + let visible = stored(dir.path()); + assert_eq!(visible.anchor, Some(NOW)); + assert_eq!(visible.last_advised, None); + assert_within_skew_tolerance(&visible, NOW); + + let path = state_path(&dir.path().join(".git")); + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!(advise(dir.path(), NOW), AdvisoryOutcome::NoAction)); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + } + + #[test] + fn maintenance_advisory_derives_linked_worktree_git_dir_from_subdirectory() { + let dir = tempfile::tempdir().expect("temp dir"); + let main = dir.path().join("main"); + init_repo(&main); + for args in [ + vec!["config", "user.email", "t@example.invalid"], + vec!["config", "user.name", "T"], + vec!["commit", "-q", "--allow-empty", "-m", "init"], + vec!["worktree", "add", "-q", "../linked"], + ] { + let output = Command::new("git") + .args(&args) + .current_dir(&main) + .output() + .expect("git"); + assert!(output.status.success(), "{args:?}"); + } + let linked = dir.path().join("linked"); + let sub = linked.join("nested"); + std::fs::create_dir_all(&sub).expect("subdir"); + + let expected = Command::new("git") + .args(["rev-parse", "--absolute-git-dir"]) + .current_dir(&sub) + .output() + .expect("rev-parse"); + let expected = std::fs::canonicalize(String::from_utf8_lossy(&expected.stdout).trim()) + .expect("canonical"); + + assert_eq!(super::derive_git_dir(&sub), Some(expected.clone())); + assert!(matches!(advise(&sub, NOW), AdvisoryOutcome::Anchored)); + assert!(state_path(&expected).is_file()); + } +} diff --git a/cli/src/services/mutation_trace/runtime/ref_doctor.rs b/cli/src/services/mutation_trace/runtime/ref_doctor.rs new file mode 100644 index 000000000..752e224f5 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_doctor.rs @@ -0,0 +1,405 @@ +use std::path::Path; + +use super::maintenance_state::{ + derive_git_dir, evaluate_recommendation, read_state, state_path, system_now_ms, AttemptOutcome, + MaintenanceState, PersistFailure, PersistPhase, StateRead, +}; +use super::ref_maintenance::{reconcile_explicit, ExplicitOutcome, StatePersistWarning}; +use super::ref_reconciliation::{ReconcileError, ReconciliationReport}; +use crate::services::agent_trace_db::repository::ExistingRepositoryDbError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationCounts { + pub(crate) deleted: usize, + pub(crate) retained: usize, + pub(crate) local_required: usize, +} + +const NOT_APPLIED_MESSAGE: &str = "Maintenance-state update was not applied."; +const DURABILITY_UNCERTAIN_MESSAGE: &str = "Maintenance-state durability could not be confirmed."; +const PREVIOUS_STATE_UNREADABLE_MESSAGE: &str = + "Maintenance state could not be read; the update was not attempted."; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum StateWarning { + NotApplied { phase: &'static str, cause: String }, + DurabilityUncertain { phase: &'static str, cause: String }, + PreviousStateUnreadable { cause: String }, +} + +impl StateWarning { + pub(crate) fn kind(&self) -> &'static str { + match self { + StateWarning::NotApplied { .. } => "not_applied", + StateWarning::DurabilityUncertain { .. } => "durability_uncertain", + StateWarning::PreviousStateUnreadable { .. } => "previous_state_unreadable", + } + } + + pub(crate) fn phase(&self) -> Option<&'static str> { + match self { + StateWarning::NotApplied { phase, .. } + | StateWarning::DurabilityUncertain { phase, .. } => Some(phase), + StateWarning::PreviousStateUnreadable { .. } => None, + } + } + + pub(crate) fn message(&self) -> &'static str { + match self { + StateWarning::NotApplied { .. } => NOT_APPLIED_MESSAGE, + StateWarning::DurabilityUncertain { .. } => DURABILITY_UNCERTAIN_MESSAGE, + StateWarning::PreviousStateUnreadable { .. } => PREVIOUS_STATE_UNREADABLE_MESSAGE, + } + } + + pub(crate) fn cause(&self) -> &str { + match self { + StateWarning::NotApplied { cause, .. } + | StateWarning::DurabilityUncertain { cause, .. } + | StateWarning::PreviousStateUnreadable { cause } => cause, + } + } +} + +impl std::fmt::Display for StateWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message())?; + match self.phase() { + Some(phase) => write!(f, " Cause ({phase}): {}", self.cause()), + None => write!(f, " Cause: {}", self.cause()), + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ReconciliationFix { + Completed(ReconciliationCounts), + CompletedStatePersistFailed { + counts: ReconciliationCounts, + warning: StateWarning, + }, + Failed { + kind: &'static str, + message: String, + state_warning: Option, + }, + Busy, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReconciliationRecommendation { + pub(crate) invalid_timestamps: bool, + pub(crate) state_unusable: bool, + pub(crate) last_success_age_ms: Option, + pub(crate) last_attempt_age_ms: Option, + pub(crate) last_advised_age_ms: Option, + pub(crate) last_attempt_failed: bool, + pub(crate) consecutive_failures: u32, +} + +pub(crate) async fn run_explicit_reconciliation(repository_root: &Path) -> ReconciliationFix { + fix_from_outcome(reconcile_explicit(repository_root).await) +} + +pub(crate) fn inspect_reconciliation_recommendation( + repository_root: &Path, +) -> Option { + inspect_recommendation_at(repository_root, system_now_ms()) +} + +fn inspect_recommendation_at( + repository_root: &Path, + now: i64, +) -> Option { + let git_dir = derive_git_dir(repository_root)?; + let (state, state_unusable) = match read_state(&state_path(&git_dir)) { + Ok(StateRead::Valid(state)) => (state, false), + Ok(StateRead::Absent) => (MaintenanceState::default(), false), + Ok(StateRead::Unusable) | Err(_) => (MaintenanceState::default(), true), + }; + let recommendation = evaluate_recommendation(&state, now); + if !recommendation.recommended && !state_unusable { + return None; + } + Some(ReconciliationRecommendation { + invalid_timestamps: recommendation.invalid_timestamps, + state_unusable, + last_success_age_ms: recommendation.last_success_age_ms, + last_attempt_age_ms: recommendation.last_attempt_age_ms, + last_advised_age_ms: recommendation.last_advised_age_ms, + last_attempt_failed: recommendation.last_attempt_outcome == Some(AttemptOutcome::Failed), + consecutive_failures: recommendation.consecutive_failures, + }) +} + +fn fix_from_outcome(outcome: ExplicitOutcome) -> ReconciliationFix { + match outcome { + ExplicitOutcome::Completed(report) => ReconciliationFix::Completed(counts(report)), + ExplicitOutcome::CompletedStatePersistFailed { report, warning } => { + ReconciliationFix::CompletedStatePersistFailed { + counts: counts(report), + warning: state_warning_from(warning), + } + } + ExplicitOutcome::Failed { + error, + state_warning, + } => ReconciliationFix::Failed { + kind: failure_kind(&error), + message: error.to_string(), + state_warning: state_warning.map(state_warning_from), + }, + ExplicitOutcome::Skipped(_) => ReconciliationFix::Busy, + } +} + +fn state_warning_from(warning: StatePersistWarning) -> StateWarning { + match warning { + StatePersistWarning::PreviousStateUnreadable(error) => { + StateWarning::PreviousStateUnreadable { + cause: error.to_string(), + } + } + StatePersistWarning::WriteFailed(PersistFailure::NotApplied { phase, source, .. }) => { + StateWarning::NotApplied { + phase: phase_name(phase), + cause: source.to_string(), + } + } + StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { phase, source }) => { + StateWarning::DurabilityUncertain { + phase: phase_name(phase), + cause: source.to_string(), + } + } + } +} + +fn phase_name(phase: PersistPhase) -> &'static str { + match phase { + PersistPhase::Serialize => "serialize", + PersistPhase::CreateStaging => "create_staging", + PersistPhase::WriteStaging => "write_staging", + PersistPhase::SyncStaging => "sync_staging", + PersistPhase::Rename => "rename", + PersistPhase::OpenParentDirectory => "open_parent_directory", + PersistPhase::SyncParentDirectory => "sync_parent_directory", + } +} + +fn counts(report: ReconciliationReport) -> ReconciliationCounts { + ReconciliationCounts { + deleted: report.deleted, + retained: report.retained, + local_required: report.local_required, + } +} + +fn failure_kind(error: &ReconcileError) -> &'static str { + match error { + ReconcileError::GitDir(_) => "git_dir_unavailable", + ReconcileError::Lock(_) => "lock_unavailable", + ReconcileError::CheckoutIdentity(_) => "worktree_identity_unavailable", + ReconcileError::AgentTraceDbUnavailable(source) => { + match source.downcast_ref::() { + Some(ExistingRepositoryDbError::Missing { .. }) => "agent_trace_db_missing", + Some(ExistingRepositoryDbError::Unreadable(_)) => "agent_trace_db_unreadable", + Some(ExistingRepositoryDbError::IncompatibleSchema(_)) => { + "agent_trace_db_incompatible_schema" + } + Some(ExistingRepositoryDbError::MissingMetadata) => { + "agent_trace_db_missing_metadata" + } + Some(ExistingRepositoryDbError::RepositoryMismatch { .. }) => { + "agent_trace_db_repository_mismatch" + } + None => "agent_trace_db_unavailable", + } + } + ReconcileError::SnapshotService(_) => "snapshot_service_unavailable", + ReconcileError::PinInventory(_) => "pin_inventory_failed", + ReconcileError::MalformedPin { .. } => "malformed_ref", + ReconcileError::DurableRoots(_) => "durable_roots_unavailable", + ReconcileError::MissingRequiredPins { .. } => "missing_required_pins", + ReconcileError::DeleteTransaction(_) => "delete_transaction_failed", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::mutation_trace::runtime::maintenance_state::{ + record_failure, record_success, StoredReport, + }; + + fn write_state(git_dir: &Path, state: &MaintenanceState) { + let path = state_path(git_dir); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, serde_json::to_vec(state).unwrap()).unwrap(); + } + + fn repo() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join(".git")).unwrap(); + dir + } + + #[test] + fn recommendation_is_absent_for_fresh_and_recent_state() { + let repo = repo(); + assert_eq!(inspect_recommendation_at(repo.path(), 1_000), None); + + let state = record_success( + &MaintenanceState::default(), + 1_000, + StoredReport { + retained: 1, + deleted: 0, + local_required: 1, + }, + ); + write_state(&repo.path().join(".git"), &state); + assert_eq!(inspect_recommendation_at(repo.path(), 2_000), None); + } + + #[test] + fn recommendation_reports_failure_and_never_writes_state() { + let repo = repo(); + let git_dir = repo.path().join(".git"); + let state = record_failure(&MaintenanceState::default(), 1_000, "kind", "message"); + write_state(&git_dir, &state); + let before = std::fs::read(state_path(&git_dir)).unwrap(); + + let recommendation = inspect_recommendation_at(repo.path(), 2_000).unwrap(); + assert!(recommendation.last_attempt_failed); + assert_eq!(recommendation.consecutive_failures, 1); + assert!(!recommendation.state_unusable); + assert_eq!(std::fs::read(state_path(&git_dir)).unwrap(), before); + } + + #[test] + fn recommendation_flags_unusable_and_invalid_state_without_rewriting() { + let repo = repo(); + let git_dir = repo.path().join(".git"); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"not json").unwrap(); + + let unusable = inspect_recommendation_at(repo.path(), 2_000).unwrap(); + assert!(unusable.state_unusable); + assert_eq!(std::fs::read(&path).unwrap(), b"not json"); + + let state = MaintenanceState { + last_success: Some(i64::MAX / 2), + ..MaintenanceState::default() + }; + write_state(&git_dir, &state); + let before = std::fs::read(&path).unwrap(); + let invalid = inspect_recommendation_at(repo.path(), 2_000).unwrap(); + assert!(invalid.invalid_timestamps); + assert_eq!(std::fs::read(&path).unwrap(), before); + } + + fn report() -> ReconciliationReport { + ReconciliationReport { + local_required: 1, + retained: 2, + deleted: 3, + } + } + + fn io_error() -> std::io::Error { + std::io::Error::other("injected") + } + + #[test] + fn pre_rename_persist_failure_maps_to_not_applied() { + let fix = fix_from_outcome(ExplicitOutcome::CompletedStatePersistFailed { + report: report(), + warning: StatePersistWarning::WriteFailed(PersistFailure::NotApplied { + phase: PersistPhase::Rename, + source: io_error(), + staging_cleanup: None, + }), + }); + let ReconciliationFix::CompletedStatePersistFailed { counts, warning } = fix else { + panic!("expected persist-failed completion"); + }; + assert_eq!(counts.deleted, 3); + assert_eq!(warning.kind(), "not_applied"); + assert_eq!(warning.phase(), Some("rename")); + assert_eq!( + warning.message(), + "Maintenance-state update was not applied." + ); + } + + #[test] + fn post_rename_persist_failure_maps_to_durability_uncertain() { + let fix = fix_from_outcome(ExplicitOutcome::CompletedStatePersistFailed { + report: report(), + warning: StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + source: io_error(), + }), + }); + let ReconciliationFix::CompletedStatePersistFailed { warning, .. } = fix else { + panic!("expected persist-failed completion"); + }; + assert_eq!(warning.kind(), "durability_uncertain"); + assert_eq!(warning.phase(), Some("sync_parent_directory")); + assert_eq!( + warning.message(), + "Maintenance-state durability could not be confirmed." + ); + } + + #[test] + fn unreadable_previous_state_maps_to_its_own_warning_on_failure() { + let fix = fix_from_outcome(ExplicitOutcome::Failed { + error: ReconcileError::MissingRequiredPins { missing: vec![] }, + state_warning: Some(StatePersistWarning::PreviousStateUnreadable(io_error())), + }); + let ReconciliationFix::Failed { state_warning, .. } = fix else { + panic!("expected failure"); + }; + let warning = state_warning.expect("warning retained"); + assert_eq!(warning.kind(), "previous_state_unreadable"); + assert_eq!(warning.phase(), None); + assert!(!warning.message().contains("not recorded")); + } + + #[test] + fn outcomes_map_to_distinct_fix_results() { + let report = ReconciliationReport { + local_required: 1, + retained: 2, + deleted: 3, + }; + assert_eq!( + fix_from_outcome(ExplicitOutcome::Completed(report)), + ReconciliationFix::Completed(ReconciliationCounts { + deleted: 3, + retained: 2, + local_required: 1, + }) + ); + assert_eq!( + fix_from_outcome(ExplicitOutcome::Skipped( + super::super::ref_maintenance::SkipReason::Busy + )), + ReconciliationFix::Busy + ); + let failed = fix_from_outcome(ExplicitOutcome::Failed { + error: ReconcileError::MissingRequiredPins { missing: vec![] }, + state_warning: None, + }); + assert!(matches!( + failed, + ReconciliationFix::Failed { + kind: "missing_required_pins", + .. + } + )); + } +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance.rs new file mode 100644 index 000000000..0c5ce8d92 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance.rs @@ -0,0 +1,231 @@ +use std::path::Path; +use std::time::Duration; + +use anyhow::{Context, Result}; + +use super::git_snapshot::resolve_git_dir; +use super::maintenance_state::{ + read_state, record_failure, record_success, state_path, system_now_ms, write_state_atomically, + MaintenanceState, PersistFailure, StateRead, StoredReport, +}; +use super::ref_reconciliation::{ + reconcile_with_held_lock, ReconcileError, ReconcilePhase, ReconciliationOutcome, + ReconciliationReport, +}; +use super::worktree_lock::{acquire_inner_async, WorktreeLockError}; +use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::agent_trace_storage::{ + resolve_existing_agent_trace_storage_for_maintenance, AgentTraceStorageContext, +}; +use crate::services::config; + +#[cfg(test)] +mod tests; + +const EXPLICIT_LOCK_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SkipReason { + Busy, +} + +#[derive(Debug)] +pub(super) enum StatePersistWarning { + PreviousStateUnreadable(std::io::Error), + WriteFailed(PersistFailure), +} + +impl std::fmt::Display for StatePersistWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + StatePersistWarning::PreviousStateUnreadable(error) => { + write!(f, "maintenance state could not be read: {error}") + } + StatePersistWarning::WriteFailed(failure) => write!(f, "{failure}"), + } + } +} + +#[derive(Debug)] +pub(super) enum ExplicitOutcome { + Completed(ReconciliationReport), + CompletedStatePersistFailed { + report: ReconciliationReport, + warning: StatePersistWarning, + }, + Failed { + error: ReconcileError, + state_warning: Option, + }, + Skipped(SkipReason), +} + +pub(super) async fn reconcile_explicit(repository_root: &Path) -> ExplicitOutcome { + reconcile_explicit_with( + repository_root, + async || open_authoritative_db(repository_root).await, + system_now_ms, + write_state_atomically, + |_| {}, + EXPLICIT_LOCK_TIMEOUT, + ) + .await +} + +async fn open_authoritative_db(repository_root: &Path) -> Result { + let storage_config = config::resolve_agent_trace_storage_runtime_config(repository_root) + .context("Failed to resolve Agent Trace repository storage config.")?; + let context = AgentTraceStorageContext { + repository_root, + explicit_repository_id: storage_config.repository_id.as_deref(), + repository_remote: &storage_config.repository_remote, + }; + resolve_existing_agent_trace_storage_for_maintenance(&context) + .await + .map(|storage| storage.db) +} + +async fn reconcile_explicit_with( + repository_root: &Path, + open_db: P, + now: C, + write_state: W, + on_phase: H, + lock_timeout: Duration, +) -> ExplicitOutcome +where + P: std::ops::AsyncFnOnce() -> Result, + C: Fn() -> i64, + W: Fn(&Path, &MaintenanceState) -> std::result::Result<(), PersistFailure>, + H: FnMut(ReconcilePhase), +{ + let git_dir = match resolve_git_dir(repository_root).await { + Ok(git_dir) => git_dir, + Err(source) => { + return ExplicitOutcome::Failed { + error: ReconcileError::GitDir(source), + state_warning: None, + }; + } + }; + + let lock = match acquire_inner_async(&git_dir, lock_timeout, || {}).await { + Ok(lock) => lock, + Err(WorktreeLockError::TimedOut { .. }) => { + return ExplicitOutcome::Skipped(SkipReason::Busy); + } + Err(error) => { + return ExplicitOutcome::Failed { + error: ReconcileError::Lock(error), + state_warning: None, + }; + } + }; + + let result = reconcile_with_held_lock(repository_root, &lock, open_db, on_phase).await; + let path = state_path(&git_dir); + let now_ms = now(); + let previous = read_state(&path); + + let outcome = match result { + Ok(ReconciliationOutcome::Reconciled(report)) => { + let next = record_success( + &previous_state(&previous), + now_ms, + StoredReport { + retained: report.retained, + deleted: report.deleted, + local_required: report.local_required, + }, + ); + match persist(&previous, &write_state, &path, &next) { + Ok(()) => ExplicitOutcome::Completed(report), + Err(warning) => ExplicitOutcome::CompletedStatePersistFailed { report, warning }, + } + } + Err(error) => record_failed_pass(&previous, &write_state, &path, now_ms, error), + }; + + drop(lock); + outcome +} + +fn previous_state(previous: &std::io::Result) -> MaintenanceState { + match previous { + Ok(read) => read.clone().into_state(), + Err(_) => MaintenanceState::default(), + } +} + +fn persist( + previous: &std::io::Result, + write_state: &W, + path: &Path, + next: &MaintenanceState, +) -> std::result::Result<(), StatePersistWarning> +where + W: Fn(&Path, &MaintenanceState) -> std::result::Result<(), PersistFailure>, +{ + if let Err(error) = previous { + return Err(StatePersistWarning::PreviousStateUnreadable( + std::io::Error::new(error.kind(), error.to_string()), + )); + } + write_state(path, next).map_err(StatePersistWarning::WriteFailed) +} + +fn record_failed_pass( + previous: &std::io::Result, + write_state: &W, + path: &Path, + now_ms: i64, + error: ReconcileError, +) -> ExplicitOutcome +where + W: Fn(&Path, &MaintenanceState) -> std::result::Result<(), PersistFailure>, +{ + let next = record_failure( + &previous_state(previous), + now_ms, + failure_kind(&error), + &error.to_string(), + ); + let state_warning = persist(previous, write_state, path, &next).err(); + ExplicitOutcome::Failed { + error, + state_warning, + } +} + +fn failure_kind(error: &ReconcileError) -> &'static str { + match error { + ReconcileError::GitDir(_) => "git_dir", + ReconcileError::Lock(_) => "lock", + ReconcileError::CheckoutIdentity(_) => "checkout_identity", + ReconcileError::AgentTraceDbUnavailable(_) => "agent_trace_db_unavailable", + ReconcileError::SnapshotService(_) => "snapshot_service", + ReconcileError::PinInventory(_) => "pin_inventory", + ReconcileError::MalformedPin { .. } => "malformed_pin", + ReconcileError::DurableRoots(_) => "durable_roots", + ReconcileError::MissingRequiredPins { .. } => "missing_required_pins", + ReconcileError::DeleteTransaction(_) => "delete_transaction", + } +} + +#[cfg(test)] +async fn open_authoritative_db_at_state_root( + repository_root: &Path, + state_root: &Path, +) -> Result { + let context = AgentTraceStorageContext { + repository_root, + explicit_repository_id: None, + repository_remote: "origin", + }; + crate::services::agent_trace_storage::resolve_existing_agent_trace_storage_for_maintenance_at_state_root( + &context, + state_root, + ) + .await + .map(|storage| storage.db) +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs new file mode 100644 index 000000000..d19be1183 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs @@ -0,0 +1,940 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Duration; + +use super::super::git_snapshot::{resolve_git_dir, GitSnapshotService, PinnedRef}; +use super::super::maintenance_state::{ + evaluate_recommendation, read_state, state_path, write_state_atomically, + write_state_atomically_with, FaultInjectingFilesystem, MaintenanceState, PersistFailure, + PersistPhase, StateRead, +}; +use super::super::ref_reconciliation::{ + reconcile_with_held_lock, ReconcileError, ReconcilePhase, ReconciliationOutcome, + ReconciliationReport, +}; +use super::super::worktree_lock::acquire_inner_async; +use super::{ + open_authoritative_db_at_state_root, reconcile_explicit_with, ExplicitOutcome, SkipReason, + StatePersistWarning, +}; +use crate::services::agent_trace_db::repository::{ + ExistingRepositoryDbError, RepositoryAgentTraceDb, +}; +use crate::services::default_paths::agent_trace_db_path_for_repository_at; +use crate::services::mutation_trace::types::WorktreeId; +use crate::services::repository_identity::resolve::resolve_repository_identity; + +const NOW: i64 = 1_000_000_000_000; +const OTHER_REPOSITORY_ID: &str = "some-other-repository"; + +mod coordination; +mod cross_process; +mod fail_closed; +mod state_recovery; +mod support; +mod worktrees; + +struct Fixture { + dir: tempfile::TempDir, + root: PathBuf, + state_root: PathBuf, + repository_id: String, +} + +impl Fixture { + fn new(remote_url: &str) -> Self { + let dir = tempfile::tempdir().expect("temp dir"); + let root = dir.path().join("repo"); + init_repo(&root, remote_url); + let repository_id = resolve_repository_identity(&root, None, "origin") + .expect("identity") + .identity + .repository_id; + let state_root = dir.path().join("state"); + Self { + dir, + root, + state_root, + repository_id, + } + } + + fn db_path(&self) -> PathBuf { + agent_trace_db_path_for_repository_at(&self.state_root, &self.repository_id) + .expect("db path") + } + + async fn create_db(&self) { + create_db(&self.db_path(), &self.repository_id).await; + } + + async fn run(&self) -> (ExplicitOutcome, Vec) { + self.run_with(write_state_atomically).await + } + + async fn run_with(&self, write_state: W) -> (ExplicitOutcome, Vec) + where + W: Fn(&Path, &MaintenanceState) -> Result<(), PersistFailure>, + { + let mut phases = Vec::new(); + let outcome = reconcile_explicit_with( + &self.root, + async || open_authoritative_db_at_state_root(&self.root, &self.state_root).await, + || NOW, + write_state, + |phase| phases.push(phase), + Duration::from_secs(10), + ) + .await; + (outcome, phases) + } + + async fn pins(&self) -> Vec { + GitSnapshotService::new(&self.root) + .await + .expect("snapshot") + .list_pins(&WorktreeId("main".to_string())) + .await + .expect("pins") + } + + async fn stored_state(&self) -> StateRead { + let git_dir = resolve_git_dir(&self.root).await.expect("git dir"); + read_state(&state_path(&git_dir)).expect("state") + } +} + +fn git(cwd: &Path, args: &[&str]) { + let output = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git command"); + assert!(output.status.success(), "git {args:?} failed"); +} + +fn init_repo(root: &Path, remote_url: &str) { + std::fs::create_dir_all(root).expect("repo dir"); + git(root, &["init", "-q"]); + git(root, &["config", "user.email", "t@example.invalid"]); + git(root, &["config", "user.name", "T"]); + git(root, &["config", "commit.gpgsign", "false"]); + git(root, &["remote", "add", "origin", remote_url]); + std::fs::write(root.join("tracked.txt"), "tracked\n").expect("write file"); + git(root, &["add", "tracked.txt"]); + git(root, &["commit", "-q", "-m", "initial"]); +} + +async fn create_db(path: &Path, repository_id: &str) { + let db = RepositoryAgentTraceDb::new_at(path) + .await + .expect("create db"); + db.verify_or_initialize_repository_metadata(repository_id) + .await + .expect("initialize metadata"); +} + +fn ref_inventory(root: &Path) -> Vec { + let output = Command::new("git") + .args([ + "for-each-ref", + "--format=%(refname) %(objectname) %(symref)", + "refs/sce/", + ]) + .current_dir(root) + .output() + .expect("for-each-ref"); + assert!(output.status.success(), "for-each-ref failed"); + let mut lines: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_string) + .collect(); + lines.sort(); + lines +} + +async fn pin_orphan(root: &Path) -> String { + let snapshot = GitSnapshotService::new(root).await.expect("snapshot"); + let tree = snapshot.capture_tree().await.expect("tree"); + let git_dir = resolve_git_dir(root).await.expect("git dir"); + let lock = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("lock"); + snapshot + .pin_tree(lock.lease(), &WorktreeId("main".to_string()), &tree) + .await + .expect("pin"); + tree.0 +} + +async fn seed_durable_root(db_path: &Path, tree: &str) { + seed_durable_root_for(db_path, "main", tree).await; +} + +async fn seed_durable_root_for(db_path: &Path, worktree: &str, tree: &str) { + let db = RepositoryAgentTraceDb::open_without_migrations_at(db_path) + .await + .expect("writer"); + db.execute( + "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES (?1, ?2, x'0000000000000000', 0, 'healthy', 0)", + (worktree, tree), + ) + .await + .expect("seed durable root"); +} + +fn unavailable_cause(error: &ReconcileError) -> &ExistingRepositoryDbError { + let ReconcileError::AgentTraceDbUnavailable(source) = error else { + panic!("expected AgentTraceDbUnavailable, got {error:?}"); + }; + source + .downcast_ref::() + .expect("verification failure keeps its typed source") +} + +fn failing_write( + phase: PersistPhase, +) -> impl Fn(&Path, &MaintenanceState) -> Result<(), PersistFailure> { + move |path, state| { + write_state_atomically_with(&FaultInjectingFilesystem::failing_at(phase), path, state) + } +} + +fn staging_leftovers(dir: &Path) -> usize { + std::fs::read_dir(dir) + .expect("read dir") + .filter_map(Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().contains("staging")) + .count() +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_database_reconciles_orphan_pin_and_records_success() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + + let (outcome, phases) = fixture.run().await; + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 0, + deleted: 1, + } + ); + assert_eq!( + phases, + vec![ReconcilePhase::DbOpened, ReconcilePhase::PinsInventoried] + ); + assert!(fixture.pins().await.is_empty()); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.last_success, Some(NOW)); + assert_eq!(state.consecutive_failures, 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_database_of_another_repository_cannot_authorize_deletion() { + let repo_a = Fixture::new("https://example.invalid/org/repo-a.git"); + let repo_b = Fixture::new("https://example.invalid/org/repo-b.git"); + assert_ne!(repo_a.repository_id, repo_b.repository_id); + repo_b.create_db().await; + pin_orphan(&repo_a.root).await; + + let (outcome, phases) = repo_a.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + assert!(phases.is_empty()); + assert_eq!(repo_a.pins().await.len(), 1); + + create_db(&repo_a.db_path(), &repo_b.repository_id).await; + let (outcome, phases) = repo_a.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + let ExistingRepositoryDbError::RepositoryMismatch { stored, resolved } = + unavailable_cause(&error) + else { + panic!("expected repository mismatch"); + }; + assert_eq!(stored, &repo_b.repository_id); + assert_eq!(resolved, &repo_a.repository_id); + assert!(phases.is_empty()); + assert_eq!(repo_a.pins().await.len(), 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_metadata_or_schema_fails_before_pin_inventory() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + RepositoryAgentTraceDb::new_at(fixture.db_path()) + .await + .expect("migrated db without metadata"); + let (outcome, phases) = fixture.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::MissingMetadata + )); + assert!(phases.is_empty()); + assert_eq!(fixture.pins().await.len(), 1); + + std::fs::remove_file(fixture.db_path()).expect("remove db"); + drop( + RepositoryAgentTraceDb::open_without_migrations_at(fixture.db_path()) + .await + .expect("unmigrated db"), + ); + let (outcome, phases) = fixture.run().await; + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::IncompatibleSchema(_) + )); + assert!(phases.is_empty()); + assert_eq!(fixture.pins().await.len(), 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_database_creates_nothing_and_deletes_nothing() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + let (outcome, phases) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + assert!(phases.is_empty()); + assert!(!fixture.state_root.exists()); + assert_eq!(fixture.pins().await.len(), 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_first_failed_pass_without_anchor_is_recommended_and_advised_once() { + use super::super::ref_advisory::{advise_if_due_with, AdvisoryOutcome}; + + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.anchor, None); + assert_eq!(state.last_success, None); + let now = state.last_attempt.expect("attempt time"); + assert!(evaluate_recommendation(&state, now).recommended); + + let first = advise_if_due_with(&fixture.root, || now, write_state_atomically); + assert!(matches!(first, AdvisoryOutcome::Advised)); + let second = advise_if_due_with(&fixture.root, || now + 1000, write_state_atomically); + assert!(matches!(second, AdvisoryOutcome::NoAction)); + + let StateRead::Valid(after) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(after.consecutive_failures, 1); + assert_eq!(after.last_failure, state.last_failure); + assert!(evaluate_recommendation(&after, now + 1000).recommended); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_failure_streak_counts_but_lock_contention_never_does() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + + for expected in 1..=2 { + let (outcome, _) = fixture.run().await; + assert!(matches!( + outcome, + ExplicitOutcome::Failed { + state_warning: None, + .. + } + )); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.consecutive_failures, expected); + assert_eq!(state.last_success, None); + } + + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let before = std::fs::read(state_path(&git_dir)).expect("state bytes"); + let held = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("hold lock"); + let mut phases = Vec::new(); + let outcome = reconcile_explicit_with( + &fixture.root, + async || -> anyhow::Result { unreachable!("db must not open") }, + || NOW, + |_, _| unreachable!("state must not be written on contention"), + |phase| phases.push(phase), + Duration::ZERO, + ) + .await; + drop(held); + + assert!(matches!( + outcome, + ExplicitOutcome::Skipped(SkipReason::Busy) + )); + assert!(phases.is_empty()); + assert_eq!( + std::fs::read(state_path(&git_dir)).expect("state bytes"), + before + ); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.consecutive_failures, 2); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_database_and_failed_persistence_keep_both_errors() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + let tree = pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture.run_with(failing_write(PersistPhase::Rename)).await; + + let ExplicitOutcome::Failed { + error, + state_warning, + } = outcome + else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + let Some(StatePersistWarning::WriteFailed(PersistFailure::NotApplied { phase, .. })) = + state_warning + else { + panic!("expected retained persistence warning, got {state_warning:?}"); + }; + assert_eq!(phase, PersistPhase::Rename); + let pins = fixture.pins().await; + assert_eq!(pins.len(), 1); + assert_eq!(pins[0].tree.0, tree); + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + assert_eq!( + staging_leftovers(state_path(&git_dir).parent().expect("parent")), + 0 + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_required_pin_and_failed_persistence_keep_both_errors() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + let snapshot = GitSnapshotService::new(&fixture.root) + .await + .expect("snapshot"); + let unpinned = snapshot.capture_tree().await.expect("tree"); + seed_durable_root(&fixture.db_path(), &unpinned.0).await; + + let (outcome, phases) = fixture + .run_with(failing_write(PersistPhase::WriteStaging)) + .await; + + let ExplicitOutcome::Failed { + error, + state_warning, + } = outcome + else { + panic!("expected failure, got {outcome:?}"); + }; + let ReconcileError::MissingRequiredPins { missing } = &error else { + panic!("expected missing required pins, got {error:?}"); + }; + assert_eq!(missing, &vec![unpinned]); + assert!(matches!( + state_warning, + Some(StatePersistWarning::WriteFailed( + PersistFailure::NotApplied { + phase: PersistPhase::WriteStaging, + .. + } + )) + )); + assert_eq!( + phases, + vec![ReconcilePhase::DbOpened, ReconcilePhase::PinsInventoried] + ); + assert!(fixture.pins().await.is_empty()); + assert!(matches!(fixture.stored_state().await, StateRead::Absent)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_completed_pass_with_unconfirmed_durability_keeps_deletion() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture + .run_with(failing_write(PersistPhase::SyncParentDirectory)) + .await; + + let ExplicitOutcome::CompletedStatePersistFailed { report, warning } = outcome else { + panic!("expected persist-failed completion, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(matches!( + warning, + StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + }) + )); + assert!(fixture.pins().await.is_empty()); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("new state is visible even though durability is unconfirmed"); + }; + assert_eq!(state.last_success, Some(NOW)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + + let (outcome, _) = fixture + .run_with(failing_write(PersistPhase::CreateStaging)) + .await; + + let ExplicitOutcome::CompletedStatePersistFailed { report, warning } = outcome else { + panic!("expected persist-failed completion, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(warning.to_string().contains("previous state preserved")); + assert!(fixture.pins().await.is_empty()); + assert!(matches!(fixture.stored_state().await, StateRead::Absent)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_unreadable_previous_state_is_reported_without_overwriting() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let path = state_path(&git_dir); + std::fs::create_dir_all(&path).expect("directory in place of state file"); + + let pins_before = fixture.pins().await; + assert_eq!(pins_before.len(), 1); + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Failed { + error, + state_warning, + } = outcome + else { + panic!("expected failure, got {outcome:?}"); + }; + assert!(matches!( + unavailable_cause(&error), + ExistingRepositoryDbError::Missing { .. } + )); + assert!(matches!( + state_warning, + Some(StatePersistWarning::PreviousStateUnreadable(_)) + )); + assert!(path.is_dir()); + assert_eq!(std::fs::read_dir(&path).expect("read state dir").count(), 0); + assert_eq!(fixture.pins().await, pins_before); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_lock_held_implementation_never_reacquires_the_lock() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let lock = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("lock"); + + let outcome = tokio::time::timeout( + Duration::from_secs(30), + reconcile_with_held_lock( + &fixture.root, + &lock, + async || open_authoritative_db_at_state_root(&fixture.root, &fixture.state_root).await, + |_| {}, + ), + ) + .await + .expect("must not block on a second acquisition") + .expect("reconcile"); + drop(lock); + + let ReconciliationOutcome::Reconciled(report) = outcome; + assert_eq!(report.deleted, 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_never_creates_missing_database_or_directories() { + let dir = tempfile::tempdir().expect("temp dir"); + let missing = dir.path().join("nested").join("agent-trace.db"); + + let error = RepositoryAgentTraceDb::open_verified_existing_at(&missing, "repo") + .await + .err() + .expect("missing db fails"); + + assert!(matches!(error, ExistingRepositoryDbError::Missing { .. })); + assert!(!dir.path().join("nested").exists()); + + let missing_in_existing_dir = dir.path().join("agent-trace.db"); + let error = RepositoryAgentTraceDb::open_verified_existing_at(&missing_in_existing_dir, "repo") + .await + .err() + .expect("missing db fails"); + assert!(matches!(error, ExistingRepositoryDbError::Missing { .. })); + assert_eq!(std::fs::read_dir(dir.path()).expect("read dir").count(), 0); +} + +const DATABASE_SIDECAR_SUFFIXES: [&str; 3] = ["-wal", "-tshm", "-shm"]; + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) +} + +fn remove_database(path: &Path) { + std::fs::remove_file(path).ok(); + for suffix in DATABASE_SIDECAR_SUFFIXES { + std::fs::remove_file(with_suffix(path, suffix)).ok(); + } +} + +fn swap_in_database(source: &Path, target: &Path) { + remove_database(target); + for suffix in DATABASE_SIDECAR_SUFFIXES { + let sidecar = with_suffix(source, suffix); + if sidecar.exists() { + std::fs::rename(sidecar, with_suffix(target, suffix)).expect("move sidecar"); + } + } + std::fs::rename(source, target).expect("move database"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_rejects_database_removed_before_open() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + let existed_when_hook_ran = std::cell::Cell::new(false); + + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || { + existed_when_hook_ran.set(path.is_file()); + remove_database(&path); + }, + || {}, + ) + .await + .err() + .expect("removed db fails"); + + assert!(existed_when_hook_ran.get()); + assert!(matches!(error, ExistingRepositoryDbError::Missing { .. })); + assert!(!path.exists()); + assert_eq!(std::fs::read_dir(dir.path()).expect("read dir").count(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_rejects_database_replaced_before_open() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + let foreign = dir.path().join("foreign.db"); + create_db(&foreign, "someone-else").await; + + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&foreign, &path), + || {}, + ) + .await + .err() + .expect("replacement with another repository fails"); + assert!(matches!( + error, + ExistingRepositoryDbError::RepositoryMismatch { .. } + )); + + let empty = dir.path().join("empty.db"); + drop( + RepositoryAgentTraceDb::open_without_migrations_at(&empty) + .await + .expect("empty db"), + ); + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&empty, &path), + || {}, + ) + .await + .err() + .expect("replacement without schema fails"); + assert!(matches!( + error, + ExistingRepositoryDbError::IncompatibleSchema(_) + )); + + let unusable = dir.path().join("unusable.db"); + std::fs::write(&unusable, b"this is not a database").expect("garbage"); + let error = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&unusable, &path), + || {}, + ) + .await + .err() + .expect("replacement with garbage fails"); + assert!(matches!( + error, + ExistingRepositoryDbError::IncompatibleSchema(_) | ExistingRepositoryDbError::Unreadable(_) + )); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_accepts_a_replacement_only_if_it_verifies_itself() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + let replacement = dir.path().join("replacement.db"); + create_db(&replacement, "repo").await; + + let (_, metadata) = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || swap_in_database(&replacement, &path), + || {}, + ) + .await + .expect("a replacement that passes every check is itself authoritative"); + assert_eq!(metadata.repository_id, "repo"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_never_recreates_a_database_removed_after_open() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + + let (db, metadata) = RepositoryAgentTraceDb::open_verified_existing_at_with_hooks( + &path, + "repo", + || {}, + || remove_database(&path), + ) + .await + .expect("the opened handle is what gets verified"); + assert_eq!(metadata.repository_id, "repo"); + drop(db); + + assert!(!path.exists()); + assert_eq!(std::fs::read_dir(dir.path()).expect("read dir").count(), 0); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_is_read_only() { + const INSERT_WORKTREE: &str = "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES ('main', 'tree', x'0000000000000000', 0, 'healthy', 0)"; + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + create_db(&path, "repo").await; + + let writer = RepositoryAgentTraceDb::new_at(&path).await.expect("writer"); + writer.execute("BEGIN", ()).await.expect("begin"); + writer + .execute(INSERT_WORKTREE, ()) + .await + .expect("the statement is valid on a writable connection"); + writer.execute("ROLLBACK", ()).await.expect("rollback"); + drop(writer); + + let (db, _) = RepositoryAgentTraceDb::open_verified_existing_at(&path, "repo") + .await + .expect("verified open"); + let error = db + .execute(INSERT_WORKTREE, ()) + .await + .expect_err("verified connection must reject writes"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("database is readonly"), "{rendered}"); + drop(db); + + let inspector = RepositoryAgentTraceDb::new_at(&path) + .await + .expect("inspector"); + let rows = inspector + .query_map( + "SELECT worktree_id FROM mutation_trace_worktrees", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("rows"); + assert!(rows.is_empty()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verified_open_coexists_with_a_live_multiprocess_wal_writer() { + let dir = tempfile::tempdir().expect("temp dir"); + let path = dir.path().join("agent-trace.db"); + let writer = RepositoryAgentTraceDb::new_at(&path).await.expect("writer"); + writer + .verify_or_initialize_repository_metadata("repo") + .await + .expect("metadata"); + + let (reader, metadata) = RepositoryAgentTraceDb::open_verified_existing_at(&path, "repo") + .await + .expect("verified open alongside live writer"); + assert_eq!(metadata.repository_id, "repo"); + + writer + .execute( + "INSERT INTO mutation_trace_worktrees + (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) + VALUES ('main', 'tree-after-open', x'0000000000000000', 0, 'healthy', 0)", + (), + ) + .await + .expect("writer still writes"); + let rows = reader + .query_map( + "SELECT cursor_tree FROM mutation_trace_worktrees", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("reader sees committed writer data"); + assert_eq!(rows, vec!["tree-after-open".to_string()]); + assert!(with_suffix(&path, "-tshm").exists()); +} + +async fn pin_distinct(root: &Path, content: &str) -> String { + std::fs::write(root.join("distinct.txt"), content).expect("write distinct file"); + pin_orphan(root).await +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_retains_every_repository_wide_durable_root_and_never_repairs_other_worktrees( +) { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + let local = pin_distinct(&fixture.root, "local\n").await; + let other = pin_distinct(&fixture.root, "other\n").await; + let orphan_one = pin_distinct(&fixture.root, "orphan-one\n").await; + let orphan_two = pin_distinct(&fixture.root, "orphan-two\n").await; + let unpinned_elsewhere = "e".repeat(40); + seed_durable_root(&fixture.db_path(), &local).await; + seed_durable_root_for(&fixture.db_path(), "worktrees/other", &other).await; + seed_durable_root_for(&fixture.db_path(), "worktrees/ghost", &unpinned_elsewhere).await; + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 1, + retained: 2, + deleted: 2, + } + ); + let mut remaining: Vec = fixture + .pins() + .await + .into_iter() + .map(|pin| pin.tree.0) + .collect(); + remaining.sort(); + let mut expected = vec![local, other]; + expected.sort(); + assert_eq!(remaining, expected); + assert!(!remaining.contains(&orphan_one) && !remaining.contains(&orphan_two)); + assert!(!remaining.contains(&unpinned_elsewhere)); + + let (second, _) = fixture.run().await; + let ExplicitOutcome::Completed(second) = second else { + panic!("expected idempotent completed pass"); + }; + assert_eq!((second.retained, second.deleted), (2, 0)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_malformed_namespace_ref_deletes_nothing_and_records_failure() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + git( + &fixture.root, + &[ + "symbolic-ref", + "refs/sce/mutation-cursor/main/not-a-tree", + "HEAD", + ], + ); + let inventory_before = ref_inventory(&fixture.root); + assert_eq!(inventory_before.len(), 2); + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + let ReconcileError::MalformedPin { ref_name, .. } = &error else { + panic!("expected MalformedPin, got {error:?}"); + }; + assert_eq!(ref_name, "refs/sce/mutation-cursor/main/not-a-tree"); + assert_eq!( + ref_inventory(&fixture.root), + inventory_before, + "no ref may be deleted, replaced or retargeted" + ); + let StateRead::Valid(state) = fixture.stored_state().await else { + panic!("expected recorded state"); + }; + assert_eq!(state.consecutive_failures, 1); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/coordination.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/coordination.rs new file mode 100644 index 000000000..e95d5c313 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/coordination.rs @@ -0,0 +1,741 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use tokio::sync::mpsc::unbounded_channel; + +use super::super::super::coordinator::coordinate_inner; +use super::super::super::external_taint::ExternalTaintMarker; +use super::super::super::git_snapshot::resolve_git_dir; +use super::super::super::maintenance_state::{ + state_path, write_state_atomically, MaintenanceState, PersistPhase, + RECONCILIATION_ADVISORY_AFTER_MS, +}; +use super::super::super::protected_worktree::WORKTREE_LOCK_TIMEOUT; +use super::super::super::ref_advisory::{advise_if_due, advise_if_due_with, AdvisoryOutcome}; +use super::super::super::ref_reconciliation::{ReconcilePhase, ReconciliationReport}; +use super::super::super::worktree_lock::acquire_inner; +use super::super::ExplicitOutcome; +use super::support::{ + await_signal, close_payload, durable_snapshot, hook_ok, main_worktree, no_advice, pinned_trees, + run_hook, sce_refs, spawn_parked_pass, start_payload, writer, WATCHDOG, +}; +use super::{failing_write, pin_orphan, Fixture, NOW}; +use crate::services::hooks::mutation_scope::MutationScopePayload; +use crate::services::mutation_trace::runtime::{advise_after_completed_boundary, RuntimeBoundary}; +use crate::services::mutation_trace::types::ScopeStatus; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; +const SCOPE: &str = "scope-1"; + +async fn git_dir_of(fixture: &Fixture) -> PathBuf { + resolve_git_dir(&fixture.root).await.expect("git dir") +} + +async fn started_fixture() -> Fixture { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + hook_ok(&fixture, start_payload(SCOPE, "start-1")).await; + fixture +} + +fn state_bytes(git_dir: &Path) -> Option> { + std::fs::read(state_path(git_dir)).ok() +} + +async fn assert_coordinate_blocked_until_pass_releases(phase: ReconcilePhase) { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let baseline = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(baseline.roots.len(), 1); + let cursor_tree = baseline.worktree.cursor_tree.0.clone(); + + super::support::write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + super::support::write_file(&fixture.root, "next.txt", "next\n"); + + let mut pass = spawn_parked_pass(&fixture, phase); + await_signal(&mut pass.parked).await; + let refs_while_parked = sce_refs(&fixture.root); + assert_eq!( + pinned_trees(&fixture.root, "main"), + [cursor_tree.clone(), orphan.clone()].into_iter().collect() + ); + + let (contended_tx, mut contended_rx) = unbounded_channel::<()>(); + let db_opened = Arc::new(AtomicBool::new(false)); + let refs_at_load = Arc::new(Mutex::new(Vec::>::new())); + let db_path = fixture.db_path(); + let root = fixture.root.clone(); + + let coordinate = coordinate_inner( + &root, + &RuntimeBoundary::Flush, + async || { + db_opened.store(true, Ordering::SeqCst); + writer(&db_path).await + }, + WATCHDOG, + move || { + let _ = contended_tx.send(()); + }, + |_attempt| { + refs_at_load + .lock() + .expect("refs at load") + .push(sce_refs(&root)); + }, + |_attempt| Ok(()), + ); + + let marker = ExternalTaintMarker::new(&git_dir); + let driver = async { + await_signal(&mut contended_rx).await; + assert!( + !db_opened.load(Ordering::SeqCst), + "the coordinator must not reach its protected section while the pass holds the lock" + ); + assert!( + !marker.exists().expect("marker"), + "the coordinator must not arm its marker before it owns the lock" + ); + assert_eq!(sce_refs(&fixture.root), refs_while_parked); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + baseline + ); + pass.finish().await + }; + + let (coordinated, outcome) = tokio::join!(coordinate, driver); + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 1, + retained: 1, + deleted: 1, + } + ); + let coordinated = coordinated.expect("the coordinator completes once the pass releases"); + let new_tree = coordinated.observed_tree.0.clone(); + assert_ne!(new_tree, cursor_tree); + assert_ne!(new_tree, orphan); + + assert_eq!( + pinned_trees(&fixture.root, "main"), + [cursor_tree, new_tree.clone()].into_iter().collect(), + "the pin created after the pass released the lock must survive; only the orphan is gone" + ); + assert!(!refs_while_parked + .iter() + .any(|line| line.contains(&new_tree))); + let seen_at_load = refs_at_load.lock().expect("refs at load").clone(); + assert_new_pin_preceded_the_durable_root(&fixture, &new_tree, &seen_at_load).await; + + drop( + acquire_inner(&git_dir, Duration::ZERO, || {}) + .expect("neither operation leaves the lock held"), + ); +} + +async fn assert_new_pin_preceded_the_durable_root( + fixture: &Fixture, + new_tree: &str, + refs_at_load: &[Vec], +) { + assert_eq!(refs_at_load.len(), 1); + assert!( + refs_at_load[0].iter().any(|line| line.contains(new_tree)), + "the new pin must already exist before the durable root commit" + ); + let after = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(after.worktree.cursor_tree.0, new_tree); + assert!(after.roots.iter().any(|root| root.0 == new_tree)); + for root in &after.roots { + assert!( + pinned_trees(&fixture.root, "main").contains(&root.0), + "every durable root stays reachable" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_db_open() { + assert_coordinate_blocked_until_pass_releases(ReconcilePhase::DbOpened).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_pin_inventory() { + assert_coordinate_blocked_until_pass_releases(ReconcilePhase::PinsInventoried).await; +} + +#[derive(Clone, Copy)] +enum Boundary { + Close, + Flush, +} + +impl Boundary { + fn payload(self) -> MutationScopePayload { + match self { + Boundary::Close => close_payload(SCOPE, "close-1"), + Boundary::Flush => MutationScopePayload::Flush, + } + } + + fn scope_after_success(self) -> ScopeStatus { + match self { + Boundary::Close => ScopeStatus::Closed, + Boundary::Flush => ScopeStatus::Active, + } + } +} + +async fn assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries( + boundary: Boundary, +) { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + super::support::write_file(&fixture.root, "work.txt", "work\n"); + let before = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + let refs_before = sce_refs(&fixture.root); + let pins_before = pinned_trees(&fixture.root, "main"); + assert_eq!(before.scope, Some(ScopeStatus::Active)); + + let mut pass = spawn_parked_pass(&fixture, ReconcilePhase::DbOpened); + await_signal(&mut pass.parked).await; + + let advisories = Arc::new(AtomicU32::new(0)); + let contended = Arc::new(AtomicU32::new(0)); + let counting = |advisories: &Arc| { + let advisories = Arc::clone(advisories); + move |root: &Path| { + advisories.fetch_add(1, Ordering::SeqCst); + no_advice(root) + } + }; + let failed = run_hook( + &fixture.root, + &fixture.db_path(), + boundary.payload(), + Duration::from_millis(300), + { + let contended = Arc::clone(&contended); + move || { + contended.fetch_add(1, Ordering::SeqCst); + } + }, + || {}, + counting(&advisories), + ) + .await + .expect_err("the lock cannot be acquired while the pass holds it"); + + let message = format!("{failed:#}"); + assert!( + message.contains("before durable completion") + && message.contains("Timed out after") + && message.contains("waiting for worktree lock"), + "unexpected error: {message}" + ); + assert_eq!(contended.load(Ordering::SeqCst), 1); + assert_eq!(advisories.load(Ordering::SeqCst), 0); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + before + ); + assert_eq!(sce_refs(&fixture.root), refs_before); + assert_eq!(pinned_trees(&fixture.root, "main"), pins_before); + assert!(!marker.exists().expect("marker")); + assert_eq!(state_bytes(&git_dir), None); + + let outcome = pass.finish().await; + assert!(matches!(outcome, ExplicitOutcome::Completed(_))); + let after_pass_state = state_bytes(&git_dir); + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + boundary.payload(), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + counting(&advisories), + ) + .await + .expect("the same boundary succeeds once the maintenance pass released the lock"); + assert_eq!(output, ""); + assert_eq!(advisories.load(Ordering::SeqCst), 1); + assert_eq!(state_bytes(&git_dir), after_pass_state); + + let after = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(after.scope, Some(boundary.scope_after_success())); + assert_eq!(after.worktree.revision, before.worktree.revision + 1); + assert_ne!(after.worktree.cursor_tree, before.worktree.cursor_tree); + assert!(pinned_trees(&fixture.root, "main").contains(&after.worktree.cursor_tree.0)); + assert!(after.roots.contains(&after.worktree.cursor_tree)); + assert!(!marker.exists().expect("marker")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries() +{ + assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary::Close) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_flush_lock_timeout_under_maintenance_commits_nothing_and_retries() { + assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary::Flush) + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock() { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + super::support::write_file(&fixture.root, "work.txt", "work\n"); + let before = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + + let at_advisory = Arc::new(Mutex::new(None)); + let refs_around = Arc::new(Mutex::new(None)); + let db_path = fixture.db_path(); + let advise = { + let at_advisory = Arc::clone(&at_advisory); + let refs_around = Arc::clone(&refs_around); + let marker = marker.clone(); + let git_dir = git_dir.clone(); + move |root: &Path| { + drop( + acquire_inner(&git_dir, Duration::ZERO, || {}) + .expect("the coordinator lock is released before the advisory runs"), + ); + *at_advisory.lock().expect("snapshot slot") = Some(tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(durable_snapshot( + &db_path, + &main_worktree(), + SCOPE, + )) + })); + let marker_before = marker.exists().expect("marker"); + let refs_before = sce_refs(root); + let report = advise_after_completed_boundary(root); + *refs_around.lock().expect("refs slot") = Some((refs_before, sce_refs(root))); + assert_eq!( + marker_before, + marker.exists().expect("marker"), + "the advisory never creates, clears or modifies the marker" + ); + assert!(!marker_before); + report + } + }; + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload(SCOPE, "close-1"), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + advise, + ) + .await + .expect("hook succeeds"); + + assert_eq!(output, ""); + let committed = at_advisory + .lock() + .expect("snapshot slot") + .clone() + .expect("advisory callback ran"); + assert_eq!(committed.scope, Some(ScopeStatus::Closed)); + assert_eq!(committed.worktree.revision, before.worktree.revision + 1); + assert_ne!(committed.worktree.cursor_tree, before.worktree.cursor_tree); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + committed, + "the advisory must not change the committed result" + ); + let (refs_before, refs_after) = refs_around + .lock() + .expect("refs slot") + .clone() + .expect("refs recorded"); + assert_eq!(refs_before, refs_after, "the advisory never touches refs"); + assert!(state_bytes(&git_dir).is_some()); + assert!(!marker.exists().expect("marker")); + drop( + acquire_inner(&git_dir, Duration::ZERO, || {}) + .expect("no lock is held after the advisory returns"), + ); +} + +fn marker_fingerprint(path: &Path) -> (bool, Vec) { + let metadata = std::fs::symlink_metadata(path).expect("marker path exists"); + let mut children: Vec = if metadata.is_dir() { + std::fs::read_dir(path) + .expect("read marker dir") + .map(|entry| { + entry + .expect("entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect() + } else { + Vec::new() + }; + children.sort(); + (metadata.is_dir(), children) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate() +{ + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + let marker_path = git_dir.join("sce").join("mutation-cursor-tainted"); + let saved_path = git_dir.join("sce").join("mutation-cursor-tainted.saved"); + assert!(!marker.exists().expect("marker")); + + super::support::write_file(&fixture.root, "one.txt", "one\n"); + hook_ok( + &fixture, + super::support::start_payload("scope-2", "start-2"), + ) + .await; + let control = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(control.scope, Some(ScopeStatus::Active)); + assert!(!marker.exists().expect("marker")); + + super::support::write_file(&fixture.root, "two.txt", "two\n"); + let before_fault = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + let armed_len = Arc::new(Mutex::new(None)); + let fingerprints = Arc::new(Mutex::new(None)); + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload("scope-2", "close-2"), + WORKTREE_LOCK_TIMEOUT, + || {}, + { + let marker_path = marker_path.clone(); + let saved_path = saved_path.clone(); + let armed_len = Arc::clone(&armed_len); + move || { + *armed_len.lock().expect("len slot") = + Some(std::fs::metadata(&marker_path).expect("armed marker").len()); + std::fs::rename(&marker_path, &saved_path).expect("set armed marker aside"); + std::fs::create_dir(&marker_path).expect("block unlink with a directory"); + } + }, + { + let marker_path = marker_path.clone(); + let fingerprints = Arc::clone(&fingerprints); + move |root| { + let before = marker_fingerprint(&marker_path); + let refs_before = sce_refs(root); + let report = advise_after_completed_boundary(root); + let after = marker_fingerprint(&marker_path); + assert_eq!(refs_before, sce_refs(root)); + *fingerprints.lock().expect("fingerprint slot") = Some((before, after)); + report + } + }, + ) + .await + .expect("MarkerClearAfterCommit is a durable success for the hook"); + assert_eq!(output, ""); + + let (fingerprint_before, fingerprint_after) = fingerprints + .lock() + .expect("fingerprint slot") + .clone() + .expect("advisory ran after the failed marker clear"); + assert_eq!(fingerprint_before, fingerprint_after); + assert!(fingerprint_before.0, "the injected fault is still in place"); + + let after_fault = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!( + after_fault.worktree.revision, + before_fault.worktree.revision + 1 + ); + assert_eq!(after_fault.scope, Some(ScopeStatus::Active)); + + std::fs::remove_dir(&marker_path).expect("remove injected fault"); + std::fs::rename(&saved_path, &marker_path).expect("restore the armed marker"); + assert!(marker.exists().expect("marker")); + assert_eq!( + std::fs::metadata(&marker_path) + .expect("restored marker") + .len(), + armed_len + .lock() + .expect("len slot") + .expect("length recorded") + ); + + super::support::write_file(&fixture.root, "three.txt", "three\n"); + hook_ok(&fixture, close_payload("scope-2", "close-3")).await; + + let recovered = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!( + recovered.scope, + Some(ScopeStatus::Abandoned), + "inherited taint conservatively abandons the live scope" + ); + assert!(recovered.worktree.revision >= after_fault.worktree.revision + 2); + assert!(!recovered.worktree.tainted); + assert!(!recovered.worktree.needs_rebaseline); + assert!(pinned_trees(&fixture.root, "main").contains(&recovered.worktree.cursor_tree.0)); + assert!(!marker.exists().expect("marker")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_advisory_after_a_real_commit_is_busy_when_a_pass_wins_the_handoff() { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + super::support::write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + super::support::write_file(&fixture.root, "work.txt", "work\n"); + + let (handoff_tx, mut handoff_rx) = unbounded_channel::<()>(); + let (holding_tx, holding_rx) = std::sync::mpsc::channel::<()>(); + let (advisory_done_tx, mut advisory_done_rx) = unbounded_channel::<()>(); + + let orchestrator = { + let root = fixture.root.clone(); + let state_root = fixture.state_root.clone(); + tokio::spawn(async move { + await_signal(&mut handoff_rx).await; + let mut pass = spawn_parked_pass_for(&root, &state_root); + await_signal(&mut pass.parked).await; + holding_tx.send(()).expect("resume the advisory"); + await_signal(&mut advisory_done_rx).await; + pass.finish().await + }) + }; + + let observed = Arc::new(Mutex::new(None)); + let advise = { + let observed = Arc::clone(&observed); + let git_dir = git_dir.clone(); + move |root: &Path| { + handoff_tx.send(()).expect("announce the handoff point"); + holding_rx + .recv_timeout(WATCHDOG) + .expect("the pass takes the lock"); + let state_before = state_bytes(&git_dir); + let refs_before = sce_refs(root); + let report = advise_after_completed_boundary(root); + *observed.lock().expect("observed slot") = Some(( + state_before, + state_bytes(&git_dir), + refs_before, + sce_refs(root), + report.outcome, + )); + advisory_done_tx.send(()).expect("advisory finished"); + report + } + }; + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload(SCOPE, "close-1"), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + advise, + ) + .await + .expect("the committed boundary still reports success"); + assert_eq!(output, ""); + + let (state_before, state_after, refs_before, refs_after, outcome) = observed + .lock() + .expect("observed slot") + .clone() + .expect("advisory ran"); + assert_eq!(outcome, "busy"); + assert_eq!(state_before, state_after); + assert_eq!(refs_before, refs_after); + + let pass_outcome = orchestrator.await.expect("orchestrator joins"); + let ExplicitOutcome::Completed(report) = pass_outcome else { + panic!("expected completed pass, got {pass_outcome:?}"); + }; + assert_eq!(report.deleted, 1); + let committed = durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await; + assert_eq!(committed.scope, Some(ScopeStatus::Closed)); + let pins = pinned_trees(&fixture.root, "main"); + assert!(!pins.contains(&orphan)); + assert!(pins.contains(&committed.worktree.cursor_tree.0)); +} + +fn spawn_parked_pass_for(root: &Path, state_root: &Path) -> super::support::ParkedPass { + super::support::spawn_parked_pass_at(root, state_root, ReconcilePhase::DbOpened) +} + +async fn advisory_from_independent_task(root: &Path, now: i64) -> AdvisoryOutcome { + let root = root.to_path_buf(); + tokio::task::spawn_blocking(move || advise_if_due(&root, || now)) + .await + .expect("advisory task joins") +} + +async fn assert_independent_advisory_is_busy_while_parked(phase: ReconcilePhase) { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + let orphan = pin_orphan(&fixture.root).await; + let git_dir = git_dir_of(&fixture).await; + + let mut pass = spawn_parked_pass(&fixture, phase); + await_signal(&mut pass.parked).await; + let refs_while_parked = sce_refs(&fixture.root); + let state_while_parked = state_bytes(&git_dir); + + let advisory = + tokio::time::timeout(WATCHDOG, advisory_from_independent_task(&fixture.root, NOW)) + .await + .expect("the advisory must not wait for the pass"); + assert!(matches!(advisory, AdvisoryOutcome::Busy)); + assert_eq!(state_bytes(&git_dir), state_while_parked); + assert_eq!(sce_refs(&fixture.root), refs_while_parked); + assert!(pinned_trees(&fixture.root, "main").contains(&orphan)); + + let outcome = pass.finish().await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(pinned_trees(&fixture.root, "main").is_empty()); + + let recorded = state_bytes(&git_dir).expect("pass recorded state"); + assert!(matches!( + advisory_from_independent_task(&fixture.root, NOW).await, + AdvisoryOutcome::NoAction + )); + assert_eq!(state_bytes(&git_dir), Some(recorded)); + assert!(matches!( + advisory_from_independent_task(&fixture.root, NOW + RECONCILIATION_ADVISORY_AFTER_MS + 1) + .await, + AdvisoryOutcome::Advised + )); + let stored: MaintenanceState = + match super::super::super::maintenance_state::read_state(&state_path(&git_dir)) + .expect("state") + { + super::super::super::maintenance_state::StateRead::Valid(state) => state, + other => panic!("expected valid state, got {other:?}"), + }; + assert_eq!(stored.last_success, Some(NOW)); + assert_eq!( + stored.last_advised, + Some(NOW + RECONCILIATION_ADVISORY_AFTER_MS + 1) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_independent_advisory_task_is_busy_while_a_pass_is_parked_after_db_open() +{ + assert_independent_advisory_is_busy_while_parked(ReconcilePhase::DbOpened).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_independent_advisory_task_is_busy_while_a_pass_is_parked_after_pin_inventory( +) { + assert_independent_advisory_is_busy_while_parked(ReconcilePhase::PinsInventoried).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_advisory_failures_after_a_real_commit_never_change_the_hook_result() { + for (case, expected) in [ + ("write_failed", "state_write_failed"), + ("corrupt_state", "anchored"), + ("unavailable_state", "state_unavailable"), + ] { + let fixture = started_fixture().await; + let git_dir = git_dir_of(&fixture).await; + let marker = ExternalTaintMarker::new(&git_dir); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + match case { + "corrupt_state" => std::fs::write(&path, b"not json").expect("corrupt state"), + "unavailable_state" => std::fs::create_dir(&path).expect("directory as state"), + _ => {} + } + super::support::write_file(&fixture.root, "work.txt", "work\n"); + + let observed = Arc::new(Mutex::new(None)); + let db_path = fixture.db_path(); + let advise = { + let observed = Arc::clone(&observed); + let marker = marker.clone(); + move |root: &Path| { + let committed = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(durable_snapshot( + &db_path, + &main_worktree(), + SCOPE, + )) + }); + let refs_before = sce_refs(root); + let marker_before = marker.exists().expect("marker"); + let outcome = if case == "write_failed" { + advise_if_due_with(root, || NOW, failing_write(PersistPhase::Rename)) + } else { + advise_if_due_with(root, || NOW, write_state_atomically) + }; + let report = outcome.report(); + *observed.lock().expect("observed slot") = Some(( + committed, + refs_before == sce_refs(root), + marker_before == marker.exists().expect("marker"), + report.outcome, + )); + report + } + }; + + let output = run_hook( + &fixture.root, + &fixture.db_path(), + close_payload(SCOPE, "close-1"), + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + advise, + ) + .await + .unwrap_or_else(|error| panic!("{case}: hook must succeed, got {error:#}")); + + assert_eq!(output, "", "{case}"); + let (committed, refs_unchanged, marker_unchanged, outcome) = observed + .lock() + .expect("observed slot") + .clone() + .expect("advisory ran"); + assert_eq!(outcome, expected, "{case}"); + assert!(refs_unchanged, "{case}"); + assert!(marker_unchanged, "{case}"); + assert_eq!(committed.scope, Some(ScopeStatus::Closed), "{case}"); + assert_eq!( + durable_snapshot(&fixture.db_path(), &main_worktree(), SCOPE).await, + committed, + "{case}" + ); + assert!(!marker.exists().expect("marker"), "{case}"); + } +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/cross_process.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/cross_process.rs new file mode 100644 index 000000000..58b5cb461 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/cross_process.rs @@ -0,0 +1,166 @@ +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::mpsc; + +use super::super::super::maintenance_state::{ + read_state, state_path, write_state_atomically, MaintenanceState, StateRead, + RECONCILIATION_ADVISORY_AFTER_MS, +}; +use super::super::super::ref_advisory::{advise_if_due, advise_if_due_with}; +use super::support::WATCHDOG; +use super::{Fixture, NOW}; + +const ROLE_ENV: &str = "SCE_CROSS_PROCESS_ADVISORY_ROLE"; +const ROOT_ENV: &str = "SCE_CROSS_PROCESS_ADVISORY_ROOT"; +const NOW_ENV: &str = "SCE_CROSS_PROCESS_ADVISORY_NOW"; +const ROLE_PARK: &str = "park"; +const ROLE_PROBE: &str = "probe"; +const PARKED_LINE: &str = "PARKED"; +const OUTCOME_PREFIX: &str = "OUTCOME:"; +const CHILD_TEST: &str = "ref_reconciliation_cross_process_advisory_child"; + +fn child_test_path() -> String { + let module = module_path!(); + let (_, relative) = module + .split_once("::") + .expect("crate-qualified module path"); + format!("{relative}::{CHILD_TEST}") +} + +fn child_command(role: &str, root: &Path) -> Command { + let mut command = Command::new(std::env::current_exe().expect("current exe")); + command + .args(["--exact", &child_test_path(), "--nocapture"]) + .env(ROLE_ENV, role) + .env(ROOT_ENV, root) + .env(NOW_ENV, NOW.to_string()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + command +} + +fn outcome_of(stdout: &str) -> String { + stdout + .lines() + .find_map(|line| line.strip_prefix(OUTCOME_PREFIX)) + .unwrap_or_else(|| panic!("child reported no outcome:\n{stdout}")) + .to_string() +} + +fn run_probe(root: &Path) -> String { + let output = child_command(ROLE_PROBE, root) + .stdin(Stdio::null()) + .output() + .expect("probe child runs"); + assert!( + output.status.success(), + "probe child failed:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + outcome_of(&String::from_utf8_lossy(&output.stdout)) +} + +#[test] +fn ref_reconciliation_cross_process_advisory_child() { + let Ok(role) = std::env::var(ROLE_ENV) else { + return; + }; + let root = PathBuf::from(std::env::var(ROOT_ENV).expect("root")); + let now: i64 = std::env::var(NOW_ENV) + .expect("now") + .parse() + .expect("numeric now"); + let outcome = match role.as_str() { + ROLE_PARK => advise_if_due_with( + &root, + || now, + |path, state| { + println!("{PARKED_LINE}"); + std::io::stdout().flush().expect("flush stdout"); + let mut release = String::new(); + std::io::stdin() + .read_line(&mut release) + .expect("wait for release"); + write_state_atomically(path, state) + }, + ), + ROLE_PROBE => advise_if_due(&root, || now), + other => panic!("unknown role {other}"), + }; + let name = format!("{outcome:?}"); + let name = name + .split([' ', '{']) + .next() + .unwrap_or_default() + .to_string(); + println!("{OUTCOME_PREFIX}{name}"); +} + +#[test] +fn ref_reconciliation_advisory_lock_serializes_independent_processes() { + let fixture = Fixture::new("https://example.invalid/org/repo-a.git"); + let git_dir = fixture.root.join(".git"); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().expect("runtime dir")).expect("runtime dir"); + let due = MaintenanceState { + anchor: Some(NOW - RECONCILIATION_ADVISORY_AFTER_MS - 1), + ..MaintenanceState::default() + }; + write_state_atomically(&path, &due).expect("seed due state"); + let seeded = std::fs::read(&path).expect("seeded bytes"); + + let mut parked = child_command(ROLE_PARK, &fixture.root) + .stdin(Stdio::piped()) + .spawn() + .expect("park child spawns"); + let mut stdout = BufReader::new(parked.stdout.take().expect("park stdout")); + let (line_tx, line_rx) = mpsc::channel::(); + let reader = std::thread::spawn(move || { + let mut lines = Vec::new(); + let mut line = String::new(); + while stdout.read_line(&mut line).expect("read child stdout") > 0 { + let trimmed = line.trim_end().to_string(); + if trimmed == PARKED_LINE { + let _ = line_tx.send(trimmed.clone()); + } + lines.push(trimmed); + line.clear(); + } + lines + }); + assert_eq!( + line_rx.recv_timeout(WATCHDOG).expect("child parks"), + PARKED_LINE + ); + + assert_eq!(run_probe(&fixture.root), "Busy"); + assert_eq!( + std::fs::read(&path).expect("state while parked"), + seeded, + "a second process must not write while the first holds the lock" + ); + + let mut stdin = parked.stdin.take().expect("park stdin"); + stdin + .write_all(b"release\n") + .expect("release the parked child"); + drop(stdin); + let status = parked.wait().expect("park child exits"); + assert!(status.success()); + let lines = reader.join().expect("reader joins"); + assert_eq!( + outcome_of(&lines.join("\n")), + "Advised", + "the lock holder recorded the one advice" + ); + + assert_eq!(run_probe(&fixture.root), "NoAction"); + + let StateRead::Valid(stored) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_eq!(stored.last_advised, Some(NOW)); + assert_eq!(stored.anchor, due.anchor); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/fail_closed.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/fail_closed.rs new file mode 100644 index 000000000..d8183fb6c --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/fail_closed.rs @@ -0,0 +1,99 @@ +use std::time::Duration; + +use super::super::super::external_taint::ExternalTaintMarker; +use super::super::super::git_snapshot::resolve_git_dir; +use super::super::super::maintenance_state::{read_state, state_path, StateRead}; +use super::super::super::ref_reconciliation::{ReconcileError, ReconcilePhase}; +use super::super::reconcile_explicit_with; +use super::super::{ExplicitOutcome, SkipReason}; +use super::support::{await_signal, pinned_trees, spawn_parked_pass, write_file}; +use super::{pin_orphan, seed_durable_root, Fixture, NOW}; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_local_pin_fails_closed_while_other_orphans_exist() { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + write_file(&fixture.root, "required.txt", "required\n"); + let snapshot = super::super::super::git_snapshot::GitSnapshotService::new(&fixture.root) + .await + .expect("snapshot"); + let unpinned = snapshot.capture_tree().await.expect("tree"); + seed_durable_root(&fixture.db_path(), &unpinned.0).await; + let refs_before = pinned_trees(&fixture.root, "main"); + assert_eq!(refs_before, [orphan].into_iter().collect()); + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("expected failure, got {outcome:?}"); + }; + let ReconcileError::MissingRequiredPins { missing } = error else { + panic!("expected missing required pins, got {error:?}"); + }; + assert_eq!(missing, vec![unpinned]); + assert_eq!( + pinned_trees(&fixture.root, "main"), + refs_before, + "the eligible orphan is not deleted when a required pin is missing" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_database_failure_never_arms_the_external_taint_marker() { + let fixture = Fixture::new(REMOTE); + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let marker = ExternalTaintMarker::new(&git_dir); + + let (outcome, _) = fixture.run().await; + + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + assert!(!marker.exists().expect("marker")); + assert_eq!(pinned_trees(&fixture.root, "main").len(), 1); + let StateRead::Valid(state) = read_state(&state_path(&git_dir)).expect("state") else { + panic!("expected recorded failure"); + }; + assert_eq!(state.consecutive_failures, 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn ref_reconciliation_second_explicit_pass_is_busy_and_records_exactly_one_pass() { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + + let mut first = spawn_parked_pass(&fixture, ReconcilePhase::PinsInventoried); + await_signal(&mut first.parked).await; + + let second = reconcile_explicit_with( + &fixture.root, + async || -> anyhow::Result<_> { unreachable!("a busy pass never opens the database") }, + || NOW, + |_, _| unreachable!("a busy pass never writes state"), + |_| {}, + Duration::ZERO, + ) + .await; + assert!(matches!(second, ExplicitOutcome::Skipped(SkipReason::Busy))); + assert!(matches!( + read_state(&state_path(&git_dir)).expect("state"), + StateRead::Absent + )); + + let outcome = first.finish().await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1); + let StateRead::Valid(state) = read_state(&state_path(&git_dir)).expect("state") else { + panic!("expected recorded success"); + }; + assert_eq!(state.last_success, Some(NOW)); + assert_eq!(state.consecutive_failures, 0); + assert!(pinned_trees(&fixture.root, "main").is_empty()); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/state_recovery.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/state_recovery.rs new file mode 100644 index 000000000..9a8e0fb6f --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/state_recovery.rs @@ -0,0 +1,413 @@ +use std::path::Path; +use std::time::Duration; + +use super::super::super::external_taint::ExternalTaintMarker; +use super::super::super::git_snapshot::resolve_git_dir; +use super::super::super::maintenance_state::{ + evaluate_recommendation, read_state, state_path, write_state_atomically, MaintenanceState, + PersistFailure, PersistPhase, StateRead, FUTURE_SKEW_TOLERANCE_MS, + RECONCILIATION_ADVISORY_AFTER_MS, +}; +use super::super::super::ref_advisory::{advise_if_due, AdvisoryOutcome}; +use super::super::super::worktree_lock::acquire_inner_async; +use super::super::{reconcile_explicit_with, ExplicitOutcome, SkipReason, StatePersistWarning}; +use super::support::pinned_trees; +use super::{ + failing_write, pin_orphan, remove_database, unavailable_cause, Fixture, NOW, + OTHER_REPOSITORY_ID, +}; +use crate::services::agent_trace_db::repository::{ + ExistingRepositoryDbError, RepositoryAgentTraceDb, +}; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; + +async fn seed_state(fixture: &Fixture, state: &MaintenanceState) -> std::path::PathBuf { + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let path = state_path(&git_dir); + std::fs::create_dir_all(path.parent().expect("parent")).expect("runtime dir"); + write_state_atomically(&path, state).expect("seed state"); + path +} + +fn invalid_state() -> MaintenanceState { + MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + last_success: Some(NOW + FUTURE_SKEW_TOLERANCE_MS + 1), + last_advised: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + } +} + +fn assert_within_tolerance(state: &MaintenanceState) { + for timestamp in [ + state.anchor, + state.last_success, + state.last_attempt, + state.last_advised, + ] + .into_iter() + .flatten() + { + assert!(timestamp <= NOW + FUTURE_SKEW_TOLERANCE_MS); + } +} + +async fn logical_dump(path: &Path) -> Vec { + let db = RepositoryAgentTraceDb::open_without_migrations_at(path) + .await + .expect("inspection open"); + let tables = db + .query_map( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("tables"); + let mut dump = Vec::new(); + for table in &tables { + let columns = db + .query_map( + &format!("SELECT name FROM pragma_table_info('{table}') ORDER BY cid"), + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("columns"); + let projection = columns + .iter() + .map(|column| format!("quote(\"{column}\")")) + .collect::>() + .join(" || '|' || "); + let rows = db + .query_map( + &format!("SELECT {projection} FROM \"{table}\" ORDER BY rowid"), + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("rows"); + dump.push(format!("{table}:{}", rows.join(";"))); + } + if tables.iter().any(|table| table == "__sce_migrations") { + let ids = db + .query_map( + "SELECT CAST(id AS TEXT) FROM __sce_migrations ORDER BY id", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("migrations"); + dump.push(format!("migrations:{}", ids.join(","))); + } + if tables.iter().any(|table| table == "repository_metadata") { + let ids = db + .query_map( + "SELECT repository_id FROM repository_metadata ORDER BY id", + (), + |row| row.get::(0).map_err(Into::into), + ) + .await + .expect("metadata"); + dump.push(format!("metadata:{}", ids.join(","))); + } + dump +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_verification_failures_leave_the_logical_database_unchanged() { + for case in ["incompatible", "missing_metadata", "mismatch"] { + let fixture = Fixture::new(REMOTE); + match case { + "incompatible" => drop( + RepositoryAgentTraceDb::open_without_migrations_at(fixture.db_path()) + .await + .expect("unmigrated db"), + ), + "missing_metadata" => drop( + RepositoryAgentTraceDb::new_at(fixture.db_path()) + .await + .expect("migrated db"), + ), + _ => super::create_db(&fixture.db_path(), OTHER_REPOSITORY_ID).await, + } + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let before = logical_dump(&fixture.db_path()).await; + if case == "mismatch" { + assert!(before.iter().any(|line| line.contains(OTHER_REPOSITORY_ID))); + } + + let (outcome, phases) = fixture.run().await; + + let ExplicitOutcome::Failed { error, .. } = outcome else { + panic!("{case}: expected failure, got {outcome:?}"); + }; + match (case, unavailable_cause(&error)) { + ("incompatible", ExistingRepositoryDbError::IncompatibleSchema(_)) + | ("missing_metadata", ExistingRepositoryDbError::MissingMetadata) + | ("mismatch", ExistingRepositoryDbError::RepositoryMismatch { .. }) => {} + (_, other) => panic!("{case}: unexpected cause {other:?}"), + } + assert!(phases.is_empty(), "{case}"); + assert_eq!(logical_dump(&fixture.db_path()).await, before, "{case}"); + assert_eq!(pinned_trees(&fixture.root, "main").len(), 1, "{case}"); + assert!(!ExternalTaintMarker::new(&git_dir).exists().expect("marker")); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_busy_explicit_pass_leaves_invalid_state_byte_identical() { + let fixture = Fixture::new(REMOTE); + let path = seed_state(&fixture, &invalid_state()).await; + let before = std::fs::read(&path).expect("state bytes"); + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let held = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("hold lock"); + + let outcome = reconcile_explicit_with( + &fixture.root, + async || -> anyhow::Result { unreachable!("db must not open") }, + || NOW, + |_, _| unreachable!("state must not be written"), + |_| {}, + Duration::ZERO, + ) + .await; + drop(held); + + assert!(matches!( + outcome, + ExplicitOutcome::Skipped(SkipReason::Busy) + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + let StateRead::Valid(state) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert!(evaluate_recommendation(&state, NOW).invalid_timestamps); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_successful_pass_normalizes_invalid_state_and_a_later_read_does_not_rewrite( +) { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let near_future = NOW + FUTURE_SKEW_TOLERANCE_MS; + let seeded = MaintenanceState { + last_advised: Some(near_future), + ..invalid_state() + }; + let path = seed_state(&fixture, &seeded).await; + + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Completed(_))); + + let StateRead::Valid(normalized) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_within_tolerance(&normalized); + assert_eq!(normalized.last_success, Some(NOW)); + assert_eq!(normalized.last_advised, Some(near_future)); + let bytes = std::fs::read(&path).expect("state bytes"); + + assert!(matches!( + advise_if_due(&fixture.root, || NOW), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); + assert!(!evaluate_recommendation(&normalized, NOW).invalid_timestamps); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_failed_state_write_during_normalization_has_two_distinct_legs() { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let path = seed_state(&fixture, &invalid_state()).await; + let before = std::fs::read(&path).expect("state bytes"); + + let (pre_rename, _) = fixture.run_with(failing_write(PersistPhase::Rename)).await; + let ExplicitOutcome::CompletedStatePersistFailed { report, warning } = pre_rename else { + panic!("expected persist-failed completion, got {pre_rename:?}"); + }; + assert_eq!(report.deleted, 1); + assert!(matches!( + warning, + StatePersistWarning::WriteFailed(PersistFailure::NotApplied { + phase: PersistPhase::Rename, + .. + }) + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + + pin_orphan(&fixture.root).await; + let (post_rename, _) = fixture + .run_with(failing_write(PersistPhase::SyncParentDirectory)) + .await; + let ExplicitOutcome::CompletedStatePersistFailed { warning, .. } = post_rename else { + panic!("expected persist-failed completion, got {post_rename:?}"); + }; + assert!(matches!( + warning, + StatePersistWarning::WriteFailed(PersistFailure::DurabilityUncertain { + phase: PersistPhase::SyncParentDirectory, + .. + }) + )); + let StateRead::Valid(visible) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_within_tolerance(&visible); + assert_eq!(visible.last_success, Some(NOW)); + + let bytes = std::fs::read(&path).expect("state bytes"); + assert!(matches!( + advise_if_due(&fixture.root, || NOW), + AdvisoryOutcome::NoAction + )); + assert_eq!(std::fs::read(&path).expect("state bytes"), bytes); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_explicit_success_resets_the_streak_and_failure_keeps_last_success() { + let fixture = Fixture::new(REMOTE); + pin_orphan(&fixture.root).await; + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + let path = state_path(&git_dir); + let stored = || match read_state(&path).expect("state") { + StateRead::Valid(state) => state, + other => panic!("expected valid state, got {other:?}"), + }; + + for expected in 1..=2 { + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + assert_eq!(stored().consecutive_failures, expected); + } + + fixture.create_db().await; + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Completed(_))); + let healed = stored(); + assert_eq!(healed.consecutive_failures, 0); + assert_eq!(healed.last_failure, None); + assert_eq!(healed.last_success, Some(NOW)); + + remove_database(&fixture.db_path()); + let (outcome, _) = fixture.run().await; + assert!(matches!(outcome, ExplicitOutcome::Failed { .. })); + let failed = stored(); + assert_eq!(failed.consecutive_failures, 1); + assert_eq!(failed.last_success, Some(NOW)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_explicit_pass_runs_and_rewrites_valid_state_from_every_stored_condition( +) { + let aged = MaintenanceState { + anchor: Some(NOW - 3 * RECONCILIATION_ADVISORY_AFTER_MS), + last_advised: Some(NOW - 2 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + for case in ["aged", "invalid", "corrupt", "oversized"] { + let fixture = Fixture::new(REMOTE); + fixture.create_db().await; + pin_orphan(&fixture.root).await; + let path = match case { + "aged" => seed_state(&fixture, &aged).await, + "invalid" => seed_state(&fixture, &invalid_state()).await, + _ => { + let path = seed_state(&fixture, &MaintenanceState::default()).await; + let junk = if case == "corrupt" { + b"not json".to_vec() + } else { + vec![b' '; 5000] + }; + std::fs::write(&path, junk).expect("unusable state"); + path + } + }; + + let (outcome, _) = fixture.run().await; + + let ExplicitOutcome::Completed(report) = outcome else { + panic!("{case}: expected completed pass, got {outcome:?}"); + }; + assert_eq!(report.deleted, 1, "{case}"); + let StateRead::Valid(rewritten) = read_state(&path).expect("state") else { + panic!("{case}: state must be rewritten valid"); + }; + assert_eq!(rewritten.last_success, Some(NOW), "{case}"); + assert_within_tolerance(&rewritten); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_unnormalized_invalid_state_stays_eligible_until_a_successful_advisory() +{ + let fixture = Fixture::new(REMOTE); + let invalid = MaintenanceState { + anchor: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + let path = seed_state(&fixture, &invalid).await; + let before = std::fs::read(&path).expect("state bytes"); + let git_dir = resolve_git_dir(&fixture.root).await.expect("git dir"); + + for step in 0..3 { + let now = NOW + step * RECONCILIATION_ADVISORY_AFTER_MS / 2; + let recommendation = evaluate_recommendation(&invalid, now); + assert!(recommendation.recommended); + assert!(recommendation.invalid_timestamps); + } + let held = acquire_inner_async(&git_dir, Duration::from_secs(10), || {}) + .await + .expect("hold lock"); + for step in 0..3 { + assert!(matches!( + advise_if_due(&fixture.root, || NOW + + step * RECONCILIATION_ADVISORY_AFTER_MS), + AdvisoryOutcome::Busy + )); + } + drop(held); + assert_eq!(std::fs::read(&path).expect("state bytes"), before); + + let later = NOW + RECONCILIATION_ADVISORY_AFTER_MS; + assert!(matches!( + advise_if_due(&fixture.root, || later), + AdvisoryOutcome::Advised + )); + let StateRead::Valid(state) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_eq!(state.last_advised, Some(later)); + assert_eq!(state.anchor, Some(later)); + assert!(matches!( + advise_if_due(&fixture.root, || later), + AdvisoryOutcome::NoAction + )); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_invalid_last_advised_does_not_suppress_a_due_advisory() { + let fixture = Fixture::new(REMOTE); + let seeded = MaintenanceState { + last_success: Some(NOW - RECONCILIATION_ADVISORY_AFTER_MS - 1), + last_advised: Some(NOW + 10 * RECONCILIATION_ADVISORY_AFTER_MS), + ..MaintenanceState::default() + }; + let path = seed_state(&fixture, &seeded).await; + + assert!(matches!( + advise_if_due(&fixture.root, || NOW), + AdvisoryOutcome::Advised + )); + let StateRead::Valid(state) = read_state(&path).expect("state") else { + panic!("expected valid state"); + }; + assert_eq!(state.last_advised, Some(NOW)); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/support.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/support.rs new file mode 100644 index 000000000..0af626163 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/support.rs @@ -0,0 +1,263 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::mpsc; +use std::time::Duration; + +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; + +use super::super::super::coordinator::{coordinate_inner, CoordinateOutcome}; +use super::super::super::maintenance_state::write_state_atomically; +use super::super::super::protected_worktree::WORKTREE_LOCK_TIMEOUT; +use super::super::super::ref_reconciliation::ReconcilePhase; +use super::super::{open_authoritative_db_at_state_root, reconcile_explicit_with, ExplicitOutcome}; +use super::{Fixture, NOW}; +use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; +use crate::services::hooks::mutation_scope::{drive_mutation_scope, MutationScopePayload}; +use crate::services::mutation_trace::runtime::{ + AbandonScopeError, AbandonScopeOutcome, AdvisoryReport, RuntimeBoundary, +}; +use crate::services::mutation_trace::store::MutationTraceStore; +use crate::services::mutation_trace::types::{ + ActorKind, ScopeId, ScopeStatus, TreeId, WorktreeId, WorktreeState, +}; +use crate::services::observability::traits::NoopLogger; + +pub(super) const WATCHDOG: Duration = Duration::from_mins(1); + +pub(super) fn main_worktree() -> WorktreeId { + WorktreeId("main".to_string()) +} + +pub(super) fn start_payload(scope: &str, event: &str) -> MutationScopePayload { + MutationScopePayload::Start { + scope_id: scope.to_string(), + event_id: event.to_string(), + actor_kind: ActorKind::ClaudeCode, + provenance: None, + } +} + +pub(super) fn close_payload(scope: &str, event: &str) -> MutationScopePayload { + MutationScopePayload::Close { + scope_id: scope.to_string(), + event_id: event.to_string(), + actor_kind: ActorKind::ClaudeCode, + } +} + +pub(super) fn write_file(root: &Path, name: &str, content: &str) { + std::fs::write(root.join(name), content).expect("write working tree file"); +} + +pub(super) fn sce_refs(root: &Path) -> Vec { + let output = Command::new("git") + .args([ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/sce/", + ]) + .current_dir(root) + .output() + .expect("for-each-ref"); + assert!(output.status.success(), "for-each-ref failed"); + let mut lines: Vec = String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_string) + .collect(); + lines.sort(); + lines +} + +pub(super) fn pinned_trees(root: &Path, worktree: &str) -> BTreeSet { + let prefix = format!("refs/sce/mutation-cursor/{worktree}/"); + sce_refs(root) + .into_iter() + .filter_map(|line| { + let (name, tree) = line.split_once(' ')?; + name.strip_prefix(&prefix) + .filter(|rest| !rest.contains('/')) + .map(|_| tree.to_string()) + }) + .collect() +} + +pub(super) async fn writer(db_path: &Path) -> anyhow::Result { + RepositoryAgentTraceDb::open_without_migrations_at(db_path).await +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct DurableSnapshot { + pub worktree: WorktreeState, + pub scope: Option, + pub roots: BTreeSet, +} + +pub(super) async fn durable_snapshot( + db_path: &Path, + worktree: &WorktreeId, + scope: &str, +) -> DurableSnapshot { + let mut db = writer(db_path).await.expect("inspection db"); + let store = MutationTraceStore::new(&mut db); + let projection = store + .load_worktree(worktree, None, None) + .await + .expect("load worktree") + .expect("worktree row"); + let scope = store + .load_scope(&ScopeId(scope.to_string())) + .await + .expect("load scope") + .map(|state| state.status); + let roots = store.load_all_tree_roots().await.expect("roots"); + DurableSnapshot { + worktree: projection.worktree_state, + scope, + roots, + } +} + +pub(super) async fn run_hook( + root: &Path, + db_path: &Path, + payload: MutationScopePayload, + lock_timeout: Duration, + on_contention: C, + in_open_db: O, + advise: V, +) -> anyhow::Result +where + C: FnOnce() + Send + 'static, + O: FnOnce(), + V: FnOnce(&Path) -> AdvisoryReport, +{ + drive_mutation_scope( + root, + payload, + None::<&NoopLogger>, + &mut Vec::new(), + async |root: &Path, boundary: &RuntimeBoundary| { + coordinate_inner( + root, + boundary, + async || { + in_open_db(); + writer(db_path).await + }, + lock_timeout, + on_contention, + |_attempt| {}, + |_attempt| Ok(()), + ) + .await + }, + async |_root: &Path, _scope: &ScopeId| -> Result { + unreachable!("abandon is not driven by these tests") + }, + advise, + ) + .await +} + +pub(super) fn no_advice(_root: &Path) -> AdvisoryReport { + AdvisoryReport { + outcome: "no_action", + severity: crate::services::mutation_trace::runtime::AdvisorySeverity::Debug, + recommendation: None, + warning: None, + diagnostic: None, + } +} + +pub(super) async fn hook_ok(fixture: &Fixture, payload: MutationScopePayload) -> String { + run_hook( + &fixture.root, + &fixture.db_path(), + payload, + WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + no_advice, + ) + .await + .expect("hook boundary completes") +} + +pub(super) async fn coordinate_flush(root: &Path, db_path: &Path) -> CoordinateOutcome { + coordinate_inner( + root, + &RuntimeBoundary::Flush, + async || writer(db_path).await, + WORKTREE_LOCK_TIMEOUT, + || {}, + |_attempt| {}, + |_attempt| Ok(()), + ) + .await + .expect("flush completes") +} + +pub(super) struct ParkedPass { + outcome: tokio::sync::oneshot::Receiver, + pub parked: UnboundedReceiver<()>, + release: mpsc::Sender<()>, +} + +impl ParkedPass { + pub(super) async fn finish(self) -> ExplicitOutcome { + self.release.send(()).expect("release the pass"); + tokio::time::timeout(WATCHDOG, self.outcome) + .await + .expect("pass watchdog") + .expect("pass thread reports its outcome") + } +} + +pub(super) fn spawn_parked_pass(fixture: &Fixture, target: ReconcilePhase) -> ParkedPass { + spawn_parked_pass_at(&fixture.root, &fixture.state_root, target) +} + +pub(super) fn spawn_parked_pass_at( + root: &Path, + state_root: &Path, + target: ReconcilePhase, +) -> ParkedPass { + let root: PathBuf = root.to_path_buf(); + let state_root: PathBuf = state_root.to_path_buf(); + let (parked_tx, parked) = unbounded_channel(); + let (release, release_rx) = mpsc::channel::<()>(); + let (outcome_tx, outcome) = tokio::sync::oneshot::channel(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("pass runtime"); + let result = runtime.block_on(reconcile_explicit_with( + &root, + async || open_authoritative_db_at_state_root(&root, &state_root).await, + || NOW, + write_state_atomically, + move |phase| { + if phase == target { + let _ = parked_tx.send(()); + let _ = release_rx.recv_timeout(WATCHDOG); + } + }, + Duration::from_secs(10), + )); + let _ = outcome_tx.send(result); + }); + ParkedPass { + outcome, + parked, + release, + } +} + +pub(super) async fn await_signal(receiver: &mut UnboundedReceiver<()>) { + tokio::time::timeout(WATCHDOG, receiver.recv()) + .await + .expect("watchdog expired while waiting for a synchronization signal") + .expect("signal sender dropped before signalling"); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/worktrees.rs b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/worktrees.rs new file mode 100644 index 000000000..70064c806 --- /dev/null +++ b/cli/src/services/mutation_trace/runtime/ref_maintenance/tests/worktrees.rs @@ -0,0 +1,263 @@ +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use super::super::super::git_snapshot::{resolve_worktree_id, GitSnapshotService}; +use super::super::super::maintenance_state::write_state_atomically; +use super::super::super::ref_reconciliation::{ReconcileError, ReconciliationReport}; +use super::super::{open_authoritative_db_at_state_root, reconcile_explicit_with, ExplicitOutcome}; +use super::support::{ + coordinate_flush, hook_ok, pinned_trees, sce_refs, start_payload, write_file, writer, +}; +use super::{git, pin_orphan, Fixture, NOW}; +use crate::services::mutation_trace::store::MutationTraceStore; +use crate::services::mutation_trace::types::WorktreeId; +use crate::services::repository_identity::resolve::resolve_repository_identity; + +const REMOTE: &str = "https://example.invalid/org/repo-a.git"; + +struct Linked { + fixture: Fixture, + root: PathBuf, + id: String, +} + +impl Linked { + async fn new() -> Self { + let fixture = Fixture::new(REMOTE); + let root = fixture.dir.path().join("linked"); + git( + &fixture.root, + &[ + "worktree", + "add", + "-q", + "-b", + "linked-branch", + root.to_str().expect("utf-8 path"), + ], + ); + let main_identity = resolve_repository_identity(&fixture.root, None, "origin") + .expect("main identity") + .identity + .repository_id; + let linked_identity = resolve_repository_identity(&root, None, "origin") + .expect("linked identity") + .identity + .repository_id; + assert_eq!(main_identity, linked_identity); + assert_eq!( + resolve_worktree_id(&fixture.root).await.expect("main id"), + WorktreeId("main".to_string()) + ); + let id = resolve_worktree_id(&root).await.expect("linked id").0; + assert!(id.starts_with("worktrees/")); + fixture.create_db().await; + Self { fixture, root, id } + } + + async fn run(&self, root: &Path) -> ExplicitOutcome { + reconcile_explicit_with( + root, + async || open_authoritative_db_at_state_root(root, &self.fixture.state_root).await, + || NOW, + write_state_atomically, + |_| {}, + Duration::from_secs(10), + ) + .await + } + + async fn linked_hook( + &self, + payload: crate::services::hooks::mutation_scope::MutationScopePayload, + ) { + super::support::run_hook( + &self.root, + &self.fixture.db_path(), + payload, + super::super::super::protected_worktree::WORKTREE_LOCK_TIMEOUT, + || {}, + || {}, + super::support::no_advice, + ) + .await + .expect("linked hook completes"); + } + + fn linked_pins(&self) -> BTreeSet { + pinned_trees(&self.root, &self.id) + } + + fn main_pins(&self) -> BTreeSet { + pinned_trees(&self.fixture.root, "main") + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_historical_roots_of_a_real_coordinator_sequence_stay_protected() { + let linked = Linked::new().await; + let fixture = &linked.fixture; + hook_ok(fixture, start_payload("scope-1", "start-1")).await; + let db_path = fixture.db_path(); + + let mut trees = BTreeSet::new(); + let mut store_db = writer(&db_path).await.expect("db"); + let cursor = MutationTraceStore::new(&mut store_db) + .load_worktree(&WorktreeId("main".to_string()), None, None) + .await + .expect("load") + .expect("row") + .worktree_state + .cursor_tree + .0; + trees.insert(cursor); + for (name, content) in [("b.txt", "b\n"), ("c.txt", "c\n"), ("d.txt", "d\n")] { + write_file(&fixture.root, name, content); + let outcome = coordinate_flush(&fixture.root, &db_path).await; + trees.insert(outcome.observed_tree.0); + } + assert_eq!(trees.len(), 4); + assert_eq!(linked.main_pins(), trees); + + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + assert!(!trees.contains(&orphan)); + + let outcome = linked.run(&fixture.root).await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 4, + retained: 4, + deleted: 1, + } + ); + assert_eq!( + linked.main_pins(), + trees, + "historical A, B, C and D are all retained although only D is the cursor" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_pin_of_one_real_worktree_is_retained_while_another_needs_the_tree() { + let linked = Linked::new().await; + let fixture = &linked.fixture; + + write_file(&fixture.root, "shared.txt", "shared\n"); + let shared = pin_orphan(&fixture.root).await; + write_file(&linked.root, "shared.txt", "shared\n"); + linked + .linked_hook(start_payload("linked-scope", "linked-start")) + .await; + assert_eq!(linked.linked_pins(), [shared.clone()].into_iter().collect()); + + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + assert_eq!( + linked.main_pins(), + [shared.clone(), orphan.clone()].into_iter().collect() + ); + + let outcome = linked.run(&fixture.root).await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 1, + deleted: 1, + } + ); + assert_eq!( + linked.main_pins(), + [shared.clone()].into_iter().collect(), + "main's pin is the same tree the linked worktree durably requires" + ); + assert_eq!(linked.linked_pins(), [shared].into_iter().collect()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn ref_reconciliation_missing_pin_in_another_real_worktree_neither_fails_nor_is_repaired() { + let linked = Linked::new().await; + let fixture = &linked.fixture; + let db_path = fixture.db_path(); + + write_file(&fixture.root, "shared.txt", "shared\n"); + let shared = pin_orphan(&fixture.root).await; + write_file(&linked.root, "shared.txt", "shared\n"); + linked + .linked_hook(start_payload("linked-scope", "linked-start")) + .await; + write_file(&linked.root, "more.txt", "more\n"); + let flushed = coordinate_flush(&linked.root, &db_path).await; + let later = flushed.observed_tree.0; + assert_eq!( + linked.linked_pins(), + [shared.clone(), later.clone()].into_iter().collect() + ); + + let linked_ref = format!("refs/sce/mutation-cursor/{}/{shared}", linked.id); + git(&linked.root, &["update-ref", "-d", &linked_ref]); + assert_eq!(linked.linked_pins(), [later.clone()].into_iter().collect()); + + write_file(&fixture.root, "orphan.txt", "orphan\n"); + let orphan = pin_orphan(&fixture.root).await; + let refs_of_linked_before: Vec = sce_refs(&fixture.root) + .into_iter() + .filter(|line| line.contains(&linked.id)) + .collect(); + + let outcome = linked.run(&fixture.root).await; + let ExplicitOutcome::Completed(report) = outcome else { + panic!("expected completed pass, got {outcome:?}"); + }; + assert_eq!( + report, + ReconciliationReport { + local_required: 0, + retained: 1, + deleted: 1, + } + ); + assert!(!linked.main_pins().contains(&orphan)); + assert_eq!( + linked.main_pins(), + [shared.clone()].into_iter().collect(), + "the remaining protection for the other worktree's tree is retained" + ); + let refs_of_linked_after: Vec = sce_refs(&fixture.root) + .into_iter() + .filter(|line| line.contains(&linked.id)) + .collect(); + assert_eq!(refs_of_linked_before, refs_of_linked_after); + assert_eq!( + linked.linked_pins(), + [later].into_iter().collect(), + "reconciling main never recreates the other worktree's missing pin" + ); + + let degraded = linked.run(&linked.root).await; + let ExplicitOutcome::Failed { error, .. } = degraded else { + panic!("the degraded worktree fails closed in its own pass, got {degraded:?}"); + }; + let ReconcileError::MissingRequiredPins { missing } = error else { + panic!("expected missing required pins, got {error:?}"); + }; + assert_eq!(missing.len(), 1); + assert_eq!(missing[0].0, shared); + assert!(GitSnapshotService::new(&linked.root) + .await + .expect("snapshot") + .list_pins(&WorktreeId(linked.id.clone())) + .await + .expect("pins") + .iter() + .all(|pin| pin.tree.0 != shared)); +} diff --git a/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs b/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs index a6aa48efd..50c7f63f7 100644 --- a/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs +++ b/cli/src/services/mutation_trace/runtime/ref_reconciliation.rs @@ -1,6 +1,5 @@ use std::collections::BTreeSet; use std::path::Path; -use std::time::Duration; use anyhow::Result; @@ -9,28 +8,9 @@ use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::mutation_trace::store::MutationTraceStore; use crate::services::mutation_trace::types::TreeId; -use super::git_snapshot::{resolve_git_dir, GitSnapshotService, PinInventoryError, PinnedRef}; -use super::worktree_lock::{acquire_inner, WorktreeLockError}; +use super::git_snapshot::{GitSnapshotService, PinInventoryError, PinnedRef}; +use super::worktree_lock::{WorktreeLock, WorktreeLockError}; -const RECONCILIATION_LOCK_TIMEOUT: Duration = Duration::from_secs(10); - -/// Outcome counts of one successful reconciliation pass. -/// -/// - `local_required` — the target worktree's own durable-root count -/// (`load_tree_roots(W).len()`), the left side of the local consistency -/// invariant. -/// - `retained` — `actual_W.len() - deleted`: pins left in place, whether -/// because the target worktree still needs their tree or because another -/// worktree in the repository durably does. -/// - `deleted` — pins actually removed (inventoried under `W`'s prefix, tree -/// absent from the repository-wide durable root set). -/// -/// `retained == local_required` is **not** an invariant — a pin retained only -/// because another worktree durably needs its tree counts toward `retained` -/// but not `local_required`. For `ReconciliationOutcome::Reconciled(report)` -/// the only relation that holds is `report.local_required <= report.retained`. -/// `ReconciliationOutcome::SkippedNoCheckoutIdentity` carries no report, so no -/// report invariant applies to it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ReconciliationReport { pub local_required: usize, @@ -38,50 +18,22 @@ pub struct ReconciliationReport { pub deleted: usize, } -/// Outcome of one reconciliation pass: a real pass that ran, carrying its -/// [`ReconciliationReport`], versus a skip because no current checkout identity -/// could be derived (an `Ok`, never an `Err`). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ReconciliationOutcome { Reconciled(ReconciliationReport), - SkippedNoCheckoutIdentity, } -/// Why a reconciliation pass could not complete. One variant per fallible step, -/// no `Other` catch-all — mirroring `CoordinateError`'s convention. Every -/// non-`Ok` outcome leaves the SCE ref namespace in a consistent state: either -/// untouched, or (only on `Ok`) with exactly the stale refs gone. #[derive(Debug)] pub enum ReconcileError { - /// `resolve_git_dir` failed. GitDir(anyhow::Error), - /// The worktree's `WorktreeLock` could not be acquired (timeout or I/O). Lock(WorktreeLockError), - /// Git topology could not be resolved into a worktree identity. CheckoutIdentity(anyhow::Error), - /// The caller-supplied `open_db` provider returned `Err`. This is a - /// reconciliation maintenance error only: it never arms - /// `ExternalTaintMarker` and never becomes - /// `CoordinateError::AgentTraceDbUnavailable`, because no mutation boundary - /// is being coordinated. AgentTraceDbUnavailable(anyhow::Error), - /// `GitSnapshotService::new` failed. SnapshotService(anyhow::Error), - /// `git for-each-ref` itself failed to execute or exited non-zero - /// (`PinInventoryError::Git`). PinInventory(anyhow::Error), - /// A ref inside the SCE mutation-cursor namespace is not shaped like a - /// `pin_tree` output (`PinInventoryError::MalformedRef`). Reconciliation - /// deletes nothing. MalformedPin { ref_name: String, reason: String }, - /// `load_tree_roots` / `load_all_tree_roots` failed (DB query error, - /// migration `003` absent, ...). DurableRoots(anyhow::Error), - /// A durable root of the **target** worktree has no live pin — the local - /// consistency invariant is violated. Fail closed: nothing is deleted. MissingRequiredPins { missing: Vec }, - /// The atomic `delete_pins` transaction failed (including a ref that - /// changed since inventory). Nothing is deleted. DeleteTransaction(anyhow::Error), } @@ -115,77 +67,52 @@ impl std::fmt::Display for ReconcileError { impl std::error::Error for ReconcileError {} -/// Reconcile one worktree's SCE snapshot pins: remove orphan / unreferenced -/// pins while retaining every tree any current or historical durable -/// mutation-cursor state in the repository still references. -/// -/// Module-private to `runtime`, exactly like `coordinate` — never re-exported -/// outside mutation-trace `runtime`. It is a one-line delegation to -/// [`reconcile_worktree_inner`] with a no-op lock-contention closure. -pub fn reconcile_worktree

( - repository_root: &Path, - open_db: P, -) -> std::result::Result -where - P: FnOnce() -> Result, -{ - reconcile_worktree_inner(repository_root, open_db, || {}) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ReconcilePhase { + DbOpened, + PinsInventoried, } -/// Body of [`reconcile_worktree`] with a deterministic test seam: -/// `on_lock_contention` fires once the moment the pass first observes the -/// `WorktreeLock` is already held by another owner (see -/// [`super::worktree_lock::acquire_inner`]). `pub(super)` keeps it reachable -/// from `runtime` and `runtime::tests` but invisible outside `runtime`. -/// -/// Every fallible step below runs entirely while holding the worktree's -/// `WorktreeLock`, which is the same lock file `coordinate()` holds across -/// `pin -> recovery -> prepare -> CAS -> marker clear -> return` — the mutual -/// exclusion that makes the pin -> DB-CAS race structurally impossible. -pub(super) fn reconcile_worktree_inner( +pub(super) async fn reconcile_with_held_lock( repository_root: &Path, + lock: &WorktreeLock, open_db: P, - on_lock_contention: F, + mut on_phase: H, ) -> std::result::Result where - P: FnOnce() -> Result, - F: FnOnce(), + P: std::ops::AsyncFnOnce() -> Result, + H: FnMut(ReconcilePhase), { - let git_dir = resolve_git_dir(repository_root).map_err(ReconcileError::GitDir)?; + let worktree_id = resolve_worktree_id(repository_root) + .await + .map_err(ReconcileError::CheckoutIdentity)?; - let _lock = acquire_inner(&git_dir, RECONCILIATION_LOCK_TIMEOUT, on_lock_contention) - .map_err(ReconcileError::Lock)?; + let mut db = open_db() + .await + .map_err(ReconcileError::AgentTraceDbUnavailable)?; + on_phase(ReconcilePhase::DbOpened); - // The lock is held from here until this function returns. Worktree identity - // comes directly from Git topology and never creates or reads SCE identity - // metadata. - let worktree_id = - resolve_worktree_id(repository_root).map_err(ReconcileError::CheckoutIdentity)?; + let snapshot = GitSnapshotService::new(repository_root) + .await + .map_err(ReconcileError::SnapshotService)?; - let db = open_db().map_err(ReconcileError::AgentTraceDbUnavailable)?; - - let snapshot = - GitSnapshotService::new(repository_root).map_err(ReconcileError::SnapshotService)?; - - // Inventory the worktree's pins first, so every durable-root read that - // follows is compared against a fixed observation of the namespace. let actual = snapshot .list_pins(&worktree_id) + .await .map_err(|error| match error { PinInventoryError::Git(source) => ReconcileError::PinInventory(source), PinInventoryError::MalformedRef { ref_name, reason } => { ReconcileError::MalformedPin { ref_name, reason } } })?; + on_phase(ReconcilePhase::PinsInventoried); let pinned_trees: BTreeSet = actual.iter().map(|pin| pin.tree.clone()).collect(); - let store = MutationTraceStore::new(&db); + let store = MutationTraceStore::new(&mut db); - // Local consistency invariant (a strictly per-worktree check): every tree - // the target worktree's own durable evidence references must still have a - // live pin, or the pass fails closed and deletes nothing. let required_local = store .load_tree_roots(&worktree_id) + .await .map_err(ReconcileError::DurableRoots)?; let missing_local: Vec = required_local.difference(&pinned_trees).cloned().collect(); if !missing_local.is_empty() { @@ -194,12 +121,9 @@ where }); } - // Deletion safety invariant: an owned ref is removed only when its tree is - // outside the durable root set of **every** worktree in the repository — - // linked worktrees share one object database, so an A-owned ref may be the - // last SCE ref protecting a tree that only worktree B durably requires. let required_repository = store .load_all_tree_roots() + .await .map_err(ReconcileError::DurableRoots)?; let stale: Vec = actual .iter() @@ -209,7 +133,8 @@ where if !stale.is_empty() { snapshot - .delete_pins(&stale) + .delete_pins(lock.lease(), &stale) + .await .map_err(ReconcileError::DeleteTransaction)?; } @@ -219,474 +144,3 @@ where deleted: stale.len(), })) } - -#[cfg(test)] -mod tests { - use std::path::{Path, PathBuf}; - use std::process::Command; - - use super::super::git_snapshot::resolve_worktree_id; - use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; - use crate::services::mutation_trace::store::encode_revision; - use crate::services::mutation_trace::types::WorktreeId; - - use super::*; - - const NAMESPACE: &str = "refs/sce/mutation-cursor"; - - struct Fixture { - _temp_dir: tempfile::TempDir, - repo_root: PathBuf, - db_path: PathBuf, - worktree_id: WorktreeId, - } - - fn expect_reconciled( - outcome: std::result::Result, - ) -> ReconciliationReport { - match outcome.expect("reconciliation should succeed") { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - } - } - - fn git(dir: &Path, args: &[&str]) -> String { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git command should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).expect("git output should be valid UTF-8") - } - - fn fixture(label: &str) -> Fixture { - let temp_dir = tempfile::Builder::new() - .prefix(&format!("sce-ref-reconciliation-{label}-")) - .tempdir() - .expect("test temp directory should be created"); - - let repo_root = temp_dir.path().join("repo"); - std::fs::create_dir_all(&repo_root).expect("repo root should be created"); - git(&repo_root, &["init", "--quiet"]); - git(&repo_root, &["config", "user.email", "test@example.com"]); - git(&repo_root, &["config", "user.name", "Test"]); - git( - &repo_root, - &["commit", "--allow-empty", "--quiet", "-m", "init"], - ); - - let worktree_id = resolve_worktree_id(&repo_root).expect("worktree id should resolve"); - - // The DB lives beside the worktree, never inside it, so it can never - // perturb a captured tree. - let db_path = temp_dir.path().join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path).expect("repository schema DB should open"); - - Fixture { - _temp_dir: temp_dir, - repo_root, - db_path, - worktree_id, - } - } - - impl Fixture { - fn open_db(&self) -> Result { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) - } - - fn db(&self) -> RepositoryAgentTraceDb { - self.open_db().expect("repository DB should reopen") - } - - fn snapshot(&self) -> GitSnapshotService { - GitSnapshotService::new(&self.repo_root).expect("snapshot service should construct") - } - - /// Write `contents` to `file` in the worktree and capture the resulting - /// tree, so successive distinct contents yield distinct tree SHAs. - fn capture_after_writing(&self, file: &str, contents: &str) -> TreeId { - std::fs::write(self.repo_root.join(file), contents) - .expect("worktree file should write"); - self.snapshot() - .capture_tree() - .expect("capture should succeed") - } - - fn pin(&self, tree: &TreeId) { - self.snapshot() - .pin_tree(&self.worktree_id, tree) - .expect("pin should succeed"); - } - - fn pin_for(&self, worktree_id: &WorktreeId, tree: &TreeId) { - self.snapshot() - .pin_tree(worktree_id, tree) - .expect("pin should succeed"); - } - - fn reconcile(&self) -> std::result::Result { - reconcile_worktree(&self.repo_root, || self.open_db()) - } - - fn owned_ref(&self, tree: &TreeId) -> String { - format!("{NAMESPACE}/{}/{}", self.worktree_id.0, tree.0) - } - - fn ref_exists(&self, ref_name: &str) -> bool { - Command::new("git") - .args(["show-ref", "--verify", "--quiet", ref_name]) - .current_dir(&self.repo_root) - .status() - .expect("git show-ref should spawn") - .success() - } - - fn ref_representation(&self, ref_name: &str) -> String { - git( - &self.repo_root, - &[ - "for-each-ref", - "--format=%(refname)%00%(objectname)%00%(objecttype)%00%(symref)", - ref_name, - ], - ) - } - - fn object_type(&self, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", sha]) - .current_dir(&self.repo_root) - .output() - .expect("git cat-file should spawn"); - if output.status.success() { - Some( - String::from_utf8(output.stdout) - .expect("git cat-file output should be UTF-8") - .trim() - .to_string(), - ) - } else { - None - } - } - } - - fn seed_worktree_cursor( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - cursor_tree: &str, - ) { - db.execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES (?1, ?2, ?3, 0, 'healthy', 0)", - ( - worktree_id, - cursor_tree, - encode_revision(revision).as_slice(), - ), - ) - .expect("worktree row insert should succeed"); - } - - fn seed_event( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - before_tree: &str, - after_tree: &str, - ) { - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, - boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", - ( - worktree_id, - encode_revision(revision).as_slice(), - before_tree, - after_tree, - ), - ) - .expect("event row insert should succeed"); - } - - #[test] - fn orphan_pin_with_a_worktree_row_is_deleted() { - let fx = fixture("orphan-with-row"); - let cursor = fx.capture_after_writing("a.txt", "cursor\n"); - let orphan = fx.capture_after_writing("a.txt", "orphan\n"); - fx.pin(&cursor); - fx.pin(&orphan); - seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 1, &cursor.0); - - let report = expect_reconciled(fx.reconcile()); - - assert_eq!( - report, - ReconciliationReport { - local_required: 1, - retained: 1, - deleted: 1, - } - ); - assert!(fx.ref_exists(&fx.owned_ref(&cursor)), "cursor pin retained"); - assert!(!fx.ref_exists(&fx.owned_ref(&orphan)), "orphan pin deleted"); - } - - #[test] - fn orphan_pin_with_no_worktree_row_is_deleted() { - let fx = fixture("orphan-no-row"); - let orphan = fx.capture_after_writing("a.txt", "orphan\n"); - fx.pin(&orphan); - - let report = expect_reconciled(fx.reconcile()); - - assert_eq!( - report, - ReconciliationReport { - local_required: 0, - retained: 0, - deleted: 1, - } - ); - assert!(!fx.ref_exists(&fx.owned_ref(&orphan)), "orphan pin deleted"); - } - - #[test] - fn current_cursor_pin_is_retained_without_a_referencing_event() { - let fx = fixture("cursor-no-event"); - let cursor = fx.capture_after_writing("a.txt", "cursor\n"); - fx.pin(&cursor); - seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 3, &cursor.0); - - let report = expect_reconciled(fx.reconcile()); - - assert_eq!( - report, - ReconciliationReport { - local_required: 1, - retained: 1, - deleted: 0, - } - ); - assert!( - fx.ref_exists(&fx.owned_ref(&cursor)), - "current cursor pin retained" - ); - } - - #[test] - fn historical_event_before_and_after_pins_are_retained_after_the_cursor_advances() { - let fx = fixture("historical-retention"); - let tree_a = fx.capture_after_writing("a.txt", "A\n"); - let tree_b = fx.capture_after_writing("a.txt", "B\n"); - let tree_c = fx.capture_after_writing("a.txt", "C\n"); - let tree_d = fx.capture_after_writing("a.txt", "D\n"); - for tree in [&tree_a, &tree_b, &tree_c, &tree_d] { - fx.pin(tree); - } - - let db = fx.db(); - seed_worktree_cursor(&db, &fx.worktree_id.0, 3, &tree_d.0); - seed_event(&db, &fx.worktree_id.0, 1, &tree_a.0, &tree_b.0); - seed_event(&db, &fx.worktree_id.0, 2, &tree_b.0, &tree_c.0); - seed_event(&db, &fx.worktree_id.0, 3, &tree_c.0, &tree_d.0); - - let report = expect_reconciled(fx.reconcile()); - - assert_eq!( - report, - ReconciliationReport { - local_required: 4, - retained: 4, - deleted: 0, - } - ); - for tree in [&tree_a, &tree_b, &tree_c, &tree_d] { - assert!( - fx.ref_exists(&fx.owned_ref(tree)), - "historical tree {} pin retained", - tree.0 - ); - } - } - - #[test] - fn a_pin_another_worktree_durably_requires_is_retained() { - let fx = fixture("cross-worktree-retention"); - let shared = fx.capture_after_writing("a.txt", "shared\n"); - let orphan = fx.capture_after_writing("a.txt", "orphan\n"); - fx.pin(&shared); - fx.pin(&orphan); - - // Another worktree in the same repository durably references `shared`; - // this worktree does not. - seed_worktree_cursor(&fx.db(), "other-worktree", 1, &shared.0); - - let report = expect_reconciled(fx.reconcile()); - - assert_eq!( - report, - ReconciliationReport { - local_required: 0, - retained: 1, - deleted: 1, - }, - "a pin another worktree durably needs is retained even though it is \ - not a local root; retained exceeds local_required" - ); - assert!( - fx.ref_exists(&fx.owned_ref(&shared)), - "the repository-wide durable tree pin is retained" - ); - assert!( - !fx.ref_exists(&fx.owned_ref(&orphan)), - "the orphan pin is deleted" - ); - assert_eq!( - fx.object_type(&shared.0).as_deref(), - Some("tree"), - "the retained ref keeps the shared tree resolvable" - ); - } - - #[test] - fn a_missing_required_pin_fails_closed_and_deletes_nothing() { - let fx = fixture("missing-required-pin"); - let tree_a = fx.capture_after_writing("a.txt", "A\n"); - let tree_b = fx.capture_after_writing("a.txt", "B\n"); - let tree_x = fx.capture_after_writing("a.txt", "X\n"); - // Local durable roots are {A, B}; only A and X are pinned (B has no pin). - fx.pin(&tree_a); - fx.pin(&tree_x); - - let db = fx.db(); - seed_worktree_cursor(&db, &fx.worktree_id.0, 1, &tree_b.0); - seed_event(&db, &fx.worktree_id.0, 1, &tree_a.0, &tree_b.0); - - let error = fx - .reconcile() - .expect_err("reconciliation should fail closed"); - - match error { - ReconcileError::MissingRequiredPins { missing } => { - assert_eq!( - missing, - vec![tree_b.clone()], - "the missing local root is named" - ); - } - other => panic!("expected MissingRequiredPins, got {other:?}"), - } - assert!( - fx.ref_exists(&fx.owned_ref(&tree_a)), - "A's pin is left in place" - ); - assert!( - fx.ref_exists(&fx.owned_ref(&tree_x)), - "X's pin is left in place" - ); - } - - #[test] - fn a_malformed_namespace_ref_fails_closed_and_deletes_nothing() { - let fx = fixture("malformed-ref"); - let cursor = fx.capture_after_writing("a.txt", "cursor\n"); - let orphan = fx.capture_after_writing("a.txt", "orphan\n"); - fx.pin(&cursor); - fx.pin(&orphan); - seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 1, &cursor.0); - - // A symbolic ref inside the SCE mutation-cursor namespace is malformed. - let symref = format!("{NAMESPACE}/{}/symbolic", fx.worktree_id.0); - git( - &fx.repo_root, - &["symbolic-ref", &symref, &fx.owned_ref(&cursor)], - ); - - let error = fx - .reconcile() - .expect_err("reconciliation should fail closed"); - - match error { - ReconcileError::MalformedPin { ref_name, .. } => { - assert_eq!(ref_name, symref, "the malformed ref is named"); - } - other => panic!("expected MalformedPin, got {other:?}"), - } - assert!( - fx.ref_exists(&fx.owned_ref(&cursor)), - "the cursor pin is untouched" - ); - assert!( - fx.ref_exists(&fx.owned_ref(&orphan)), - "the orphan pin is untouched" - ); - } - - #[test] - fn reconciliation_is_idempotent() { - let fx = fixture("idempotent"); - let cursor = fx.capture_after_writing("a.txt", "cursor\n"); - let orphan = fx.capture_after_writing("a.txt", "orphan\n"); - fx.pin(&cursor); - fx.pin(&orphan); - seed_worktree_cursor(&fx.db(), &fx.worktree_id.0, 1, &cursor.0); - - let first = expect_reconciled(fx.reconcile()); - assert_eq!( - first, - ReconciliationReport { - local_required: 1, - retained: 1, - deleted: 1, - } - ); - - let second = expect_reconciled(fx.reconcile()); - assert_eq!( - second, - ReconciliationReport { - local_required: 1, - retained: 1, - deleted: 0, - }, - "a second pass with no intervening change deletes nothing and \ - reports identical local_required/retained counts" - ); - } - - #[test] - fn reconciliation_deletes_refs_without_reclaiming_objects() { - let fx = fixture("no-object-reclamation"); - let orphan = fx.capture_after_writing("a.txt", "orphan\n"); - fx.pin(&orphan); - - assert_eq!(fx.object_type(&orphan.0).as_deref(), Some("tree")); - - let report = expect_reconciled(fx.reconcile()); - assert_eq!(report.deleted, 1); - assert!( - !fx.ref_exists(&fx.owned_ref(&orphan)), - "the stale ref is deleted" - ); - - assert_eq!( - fx.object_type(&orphan.0).as_deref(), - Some("tree"), - "the now-unreachable object is still resolvable — reconciliation ran \ - no git gc / git prune" - ); - } -} diff --git a/cli/src/services/mutation_trace/runtime/scope_runtime.rs b/cli/src/services/mutation_trace/runtime/scope_runtime.rs index 31bd615fd..c8e220bb6 100644 --- a/cli/src/services/mutation_trace/runtime/scope_runtime.rs +++ b/cli/src/services/mutation_trace/runtime/scope_runtime.rs @@ -3,7 +3,7 @@ use std::path::Path; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::mutation_trace::protocol; use crate::services::mutation_trace::store::{CasResult, DurableTransition, MutationTraceStore}; -use crate::services::mutation_trace::types::{ScopeId, ScopeStatus, WorktreeId}; +use crate::services::mutation_trace::types::{ProtocolState, ScopeId, ScopeStatus, WorktreeId}; use super::coordinator::MAX_CAS_RETRY_ATTEMPTS; use super::protected_worktree::{ @@ -60,6 +60,10 @@ pub enum AbandonScopeError { }, MarkerClearAfterCompletion { source: anyhow::Error, + #[allow( + dead_code, + reason = "outcome payload carried but never read; suspected defect recorded in context/plans/cli-async-persistence-cleanup-pr3.md" + )] completed: Box, }, Other(anyhow::Error), @@ -110,29 +114,30 @@ impl std::fmt::Display for AbandonScopeError { impl std::error::Error for AbandonScopeError {} -pub fn abandon_scope

( +pub async fn abandon_scope

( repository_root: &Path, scope: &ScopeId, open_db: P, ) -> Result where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, { - abandon_scope_inner(repository_root, scope, open_db, |_attempt| {}) + abandon_scope_inner(repository_root, scope, open_db, |_attempt| {}).await } -pub(super) fn abandon_scope_inner( +pub(super) async fn abandon_scope_inner( repository_root: &Path, scope: &ScopeId, open_db: P, after_load: L, ) -> Result where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, L: FnMut(u32), { - let protected = - ProtectedWorktree::acquire(repository_root).map_err(protected_worktree_failure)?; + let protected = ProtectedWorktree::acquire(repository_root) + .await + .map_err(protected_worktree_failure)?; if protected.inherited_external_taint() { return Ok(AbandonScopeOutcome::RecoveryRequired { @@ -142,13 +147,13 @@ where }); } - let outcome = abandon_protected(protected.worktree_id(), scope, open_db, after_load)?; + let outcome = abandon_protected(protected.worktree_id(), scope, open_db, after_load).await?; if matches!(outcome, AbandonScopeOutcome::RecoveryRequired { .. }) { return Ok(outcome); } - match protected.complete() { + match protected.complete().await { Ok(()) => Ok(outcome), Err(source) => Err(AbandonScopeError::MarkerClearAfterCompletion { source, @@ -157,21 +162,54 @@ where } } -fn abandon_protected( +fn projected_revision_and_status( + state: &ProtocolState, + worktree_id: &WorktreeId, + scope: &ScopeId, +) -> Result<(u64, ScopeStatus), AbandonScopeError> { + let revision = state + .worktrees + .get(worktree_id) + .map(|worktree_state| worktree_state.revision) + .ok_or_else(|| { + AbandonScopeError::Other(anyhow::anyhow!( + "worktree {worktree_id:?} missing from its own loaded projection" + )) + })?; + let status = state + .scopes + .get(scope) + .map(|loaded| loaded.status) + .ok_or_else(|| { + AbandonScopeError::Other(anyhow::anyhow!( + "scope {scope:?} missing from the projection that loaded it as its \ + effective referenced scope" + )) + })?; + Ok((revision, status)) +} + +async fn abandon_protected( worktree_id: &WorktreeId, scope: &ScopeId, open_db: P, mut after_load: L, ) -> Result where - P: FnOnce() -> anyhow::Result, + P: std::ops::AsyncFnOnce() -> anyhow::Result, L: FnMut(u32), { - let db = open_db().map_err(AbandonScopeError::AgentTraceDbUnavailable)?; - let store = MutationTraceStore::new(&db); + let mut db = open_db() + .await + .map_err(AbandonScopeError::AgentTraceDbUnavailable)?; + let mut store = MutationTraceStore::new(&mut db); for attempt_index in 0..MAX_CAS_RETRY_ATTEMPTS { - let Some(scope_state) = store.load_scope(scope).map_err(AbandonScopeError::Other)? else { + let Some(scope_state) = store + .load_scope(scope) + .await + .map_err(AbandonScopeError::Other)? + else { return Ok(recovery_required( worktree_id, scope, @@ -188,6 +226,7 @@ where let Some(projection) = store .load_worktree(worktree_id, Some(scope), None) + .await .map_err(AbandonScopeError::Other)? else { return Ok(recovery_required( @@ -200,25 +239,7 @@ where after_load(attempt_index); let state = projection.into_protocol_state(); - let revision = state - .worktrees - .get(worktree_id) - .map(|worktree_state| worktree_state.revision) - .ok_or_else(|| { - AbandonScopeError::Other(anyhow::anyhow!( - "worktree {worktree_id:?} missing from its own loaded projection" - )) - })?; - let status = state - .scopes - .get(scope) - .map(|loaded| loaded.status) - .ok_or_else(|| { - AbandonScopeError::Other(anyhow::anyhow!( - "scope {scope:?} missing from the projection that loaded it as its \ - effective referenced scope" - )) - })?; + let (revision, status) = projected_revision_and_status(&state, worktree_id, scope)?; match status { ScopeStatus::NeverSeen => { @@ -252,6 +273,7 @@ where match store .commit(&transition) + .await .map_err(AbandonScopeError::Other)? { CasResult::Applied => { @@ -297,812 +319,8 @@ fn protected_worktree_failure(error: ProtectedWorktreeError) -> AbandonScopeErro ProtectedWorktreeError::ExternalTaintMarker { operation, source } => { AbandonScopeError::ExternalTaintMarker { operation, source } } - } -} - -#[cfg(test)] -mod tests { - use std::cell::Cell; - use std::path::{Path, PathBuf}; - use std::process::Command; - - use super::*; - use crate::services::mutation_trace::runtime::external_taint::ExternalTaintMarker; - use crate::services::mutation_trace::runtime::git_snapshot::{ - resolve_git_dir, resolve_worktree_id, - }; - use crate::services::mutation_trace::store::{decode_revision, encode_revision}; - use crate::services::mutation_trace::types::WorktreeState; - - struct TestScopeRepo { - _temp_dir: tempfile::TempDir, - repo_root: PathBuf, - git_dir: PathBuf, - db_path: PathBuf, - } - - impl TestScopeRepo { - fn new(label: &str) -> Self { - let temp_dir = tempfile::Builder::new() - .prefix(&format!("sce-mutation-trace-scope-runtime-{label}-")) - .tempdir() - .expect("test temp directory should be created"); - let repo_root = temp_dir.path().join("repo"); - std::fs::create_dir_all(&repo_root).expect("repository directory should be created"); - run_git(&repo_root, &["init", "--quiet"]); - let git_dir = resolve_git_dir(&repo_root).expect("git dir should resolve"); - let db_path = temp_dir.path().join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path) - .expect("the repository DB should open with schema"); - Self { - _temp_dir: temp_dir, - repo_root, - git_dir, - db_path, - } - } - - fn open_db(&self) -> anyhow::Result { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) - } - - fn db(&self) -> RepositoryAgentTraceDb { - self.open_db() - .expect("reopening the DB for assertions should succeed") - } - - fn worktree_id(&self) -> WorktreeId { - resolve_worktree_id(&self.repo_root).expect("the worktree identity should resolve") - } - - fn marker(&self) -> ExternalTaintMarker { - ExternalTaintMarker::new(&self.git_dir) - } - - fn marker_path(&self) -> PathBuf { - self.git_dir.join("sce").join("mutation-cursor-tainted") - } - - fn marker_exists(&self) -> bool { - self.marker() - .exists() - .expect("marker existence should resolve") - } - } - - fn run_git(dir: &Path, args: &[&str]) { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git command should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - - fn seed_worktree(db: &RepositoryAgentTraceDb, worktree: &WorktreeId, revision: u64) { - db.execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES (?1, 'tree-0', ?2, 0, 'healthy', 0)", - (worktree.0.as_str(), encode_revision(revision).as_slice()), - ) - .expect("worktree insert should succeed"); - } - - fn seed_scope( - db: &RepositoryAgentTraceDb, - scope: &ScopeId, - worktree: &WorktreeId, - status: ScopeStatus, - ) { - db.execute( - "INSERT INTO mutation_trace_scopes (scope_id, worktree_id, actor_kind, status) - VALUES (?1, ?2, 'claude_code', ?3)", - ( - scope.0.as_str(), - worktree.0.as_str(), - crate::services::mutation_trace::store::encode_scope_status(status), - ), - ) - .expect("scope insert should succeed"); - } - - fn bump_revision(db_path: &Path, worktree: &WorktreeId) { - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("the competing writer should open the DB"); - let current = read_worktree(&db, worktree).expect("the worktree row should exist"); - db.execute( - "UPDATE mutation_trace_worktrees SET revision = ?1 WHERE worktree_id = ?2", - ( - encode_revision(current.revision + 1).as_slice(), - worktree.0.as_str(), - ), - ) - .expect("the competing revision bump should succeed"); - } - - fn set_scope_status(db_path: &Path, scope: &ScopeId, status: ScopeStatus) { - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("the competing writer should open the DB"); - db.execute( - "UPDATE mutation_trace_scopes SET status = ?1 WHERE scope_id = ?2", - ( - crate::services::mutation_trace::store::encode_scope_status(status), - scope.0.as_str(), - ), - ) - .expect("the competing scope-status write should succeed"); - } - - fn arm_a_scope_status_write_collision(db_path: &Path) { - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("the competing writer should open the DB"); - db.execute( - "CREATE UNIQUE INDEX idx_test_one_scope_per_status - ON mutation_trace_scopes (status)", - (), - ) - .expect("the collision index should be created"); - } - - fn read_worktree(db: &RepositoryAgentTraceDb, worktree: &WorktreeId) -> Option { - let rows = db - .query_map( - "SELECT cursor_tree, revision, tainted, failure_kind, needs_rebaseline - FROM mutation_trace_worktrees WHERE worktree_id = ?1", - (worktree.0.as_str(),), - |row| { - let cursor_tree = row.get::(0)?; - let revision = row.get::>(1)?; - let tainted = row.get::(2)?; - let failure_kind = row.get::(3)?; - let needs_rebaseline = row.get::(4)?; - Ok(( - cursor_tree, - revision, - tainted, - failure_kind, - needs_rebaseline, - )) - }, - ) - .expect("the worktree read should succeed"); - - rows.into_iter().next().map( - |(cursor_tree, revision, tainted, failure_kind, needs_rebaseline)| WorktreeState { - cursor_tree: crate::services::mutation_trace::types::TreeId(cursor_tree), - revision: decode_revision(&revision).expect("the revision should decode"), - tainted: tainted != 0, - failure_kind: crate::services::mutation_trace::store::decode_failure_kind( - &failure_kind, - ) - .expect("the failure kind should decode"), - needs_rebaseline: needs_rebaseline != 0, - }, - ) - } - - fn read_scope_status(db: &RepositoryAgentTraceDb, scope: &ScopeId) -> Option { - let rows = db - .query_map( - "SELECT status FROM mutation_trace_scopes WHERE scope_id = ?1", - (scope.0.as_str(),), - |row| row.get::(0).map_err(Into::into), - ) - .expect("the scope read should succeed"); - - rows.into_iter().next().map(|status| { - crate::services::mutation_trace::store::decode_scope_status(&status) - .expect("the scope status should decode") - }) - } - - fn count_rows(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - let rows = db - .query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("the row count should succeed"); - - rows.into_iter().next().unwrap_or_default() - } - - #[test] - fn an_inherited_marker_requires_recovery_without_consulting_the_db_provider() { - let repo = TestScopeRepo::new("inherited-marker"); - let scope = ScopeId("scope-dead".to_string()); - repo.marker() - .persist() - .expect("the earlier invocation's marker should arm"); - - let provider_called = Cell::new(false); - let outcome = abandon_scope(&repo.repo_root, &scope, || { - provider_called.set(true); - Err(anyhow::anyhow!("the DB provider must never be invoked")) - }) - .expect("an inherited marker settles as a successful recovery-required outcome"); - - assert!( - matches!( - outcome, - AbandonScopeOutcome::RecoveryRequired { - reason: AbandonRecoveryReason::InheritedExternalTaint, - .. - } - ), - "expected InheritedExternalTaint, got {outcome:?}" - ); - assert!( - !provider_called.get(), - "the inherited-marker short-circuit must return before the DB provider runs" - ); - assert!( - repo.marker_exists(), - "the inherited marker must stay armed for the next invocation to recover" - ); - } - - #[test] - fn a_missing_scope_row_requires_recovery_and_leaves_the_fence_armed() { - let repo = TestScopeRepo::new("missing-scope"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-never-registered".to_string()); - seed_worktree(&repo.db(), &worktree, 3); - - let outcome = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect("a missing scope row settles as a recovery-required outcome"); - - assert!( - matches!( - outcome, - AbandonScopeOutcome::RecoveryRequired { - reason: AbandonRecoveryReason::MissingScope, - .. - } - ), - "expected MissingScope, got {outcome:?}" - ); - assert!( - repo.marker_exists(), - "a scope whose Start never committed must leave the fence armed" - ); - - let db = repo.db(); - assert_eq!( - read_worktree(&db, &worktree) - .expect("the worktree row should exist") - .revision, - 3, - "a recovery-required outcome must not advance the revision" - ); - assert_eq!(read_scope_status(&db, &scope), None); - } - - #[test] - fn a_never_seen_scope_requires_recovery_and_leaves_the_fence_armed() { - let repo = TestScopeRepo::new("never-seen-scope"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-registered-only".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 3); - seed_scope(&db, &scope, &worktree, ScopeStatus::NeverSeen); - } - - let outcome = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect("a NeverSeen scope settles as a recovery-required outcome"); - - assert!( - matches!( - outcome, - AbandonScopeOutcome::RecoveryRequired { - reason: AbandonRecoveryReason::NeverSeenScope, - .. - } - ), - "expected NeverSeenScope, got {outcome:?}" - ); - assert!( - repo.marker_exists(), - "a scope with no observed Start must leave the fence armed" - ); - - let db = repo.db(); - assert_eq!( - read_worktree(&db, &worktree) - .expect("the worktree row should exist") - .revision, - 3, - "a recovery-required outcome must not advance the revision" - ); - assert_eq!( - read_scope_status(&db, &scope), - Some(ScopeStatus::NeverSeen), - "a recovery-required outcome must not change any scope status" - ); - } - - #[test] - fn a_scope_whose_worktree_row_is_missing_requires_recovery() { - let repo = TestScopeRepo::new("missing-worktree-row"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-orphan".to_string()); - seed_scope(&repo.db(), &scope, &worktree, ScopeStatus::Active); - - let outcome = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect("a missing worktree row settles as a recovery-required outcome"); - - assert!( - matches!( - outcome, - AbandonScopeOutcome::RecoveryRequired { - reason: AbandonRecoveryReason::MissingWorktreeState, - .. - } - ), - "expected MissingWorktreeState, got {outcome:?}" - ); - assert!( - repo.marker_exists(), - "a worktree with no durable row must leave the fence armed" - ); - assert_eq!( - read_scope_status(&repo.db(), &scope), - Some(ScopeStatus::Active), - "no scope status may change when there is no worktree to transition" - ); - } - - #[test] - fn abandoning_a_live_scope_writes_only_the_scope_and_worktree_rows() { - let repo = TestScopeRepo::new("active-abandonment"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-live".to_string()); - let bystander = ScopeId("scope-bystander".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 3); - seed_scope(&db, &scope, &worktree, ScopeStatus::Active); - seed_scope(&db, &bystander, &worktree, ScopeStatus::Active); - } - - let outcome = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect("abandoning a live scope should succeed"); - - assert_eq!( - outcome, - AbandonScopeOutcome::Abandoned { - worktree_id: worktree.clone(), - scope: scope.clone(), - revision: 4, - } - ); - assert!( - !repo.marker_exists(), - "a settled abandonment must clear the marker it armed" - ); - - let db = repo.db(); - let worktree_state = read_worktree(&db, &worktree).expect("the worktree row should exist"); - assert_eq!(worktree_state.revision, 4, "the revision advances by one"); - assert!(worktree_state.needs_rebaseline); - assert_eq!( - worktree_state.cursor_tree, - crate::services::mutation_trace::types::TreeId("tree-0".to_string()), - "abandonment observes no tree, so the cursor is left where it was" - ); - assert!(!worktree_state.tainted); - assert_eq!( - worktree_state.failure_kind, - crate::services::mutation_trace::types::FailureKind::Healthy - ); - - assert_eq!( - read_scope_status(&db, &scope), - Some(ScopeStatus::Abandoned), - "the named scope is retired" - ); - assert_eq!( - read_scope_status(&db, &bystander), - Some(ScopeStatus::Active), - "abandonment retires only the scope it was given" - ); - - assert_eq!(count_rows(&db, "mutation_trace_events"), 0); - assert_eq!(count_rows(&db, "mutation_trace_event_active_scopes"), 0); - assert_eq!(count_rows(&db, "mutation_trace_processed_events"), 0); - } - - #[test] - fn a_closed_scope_settles_as_a_terminal_no_op() { - let repo = TestScopeRepo::new("closed-scope"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-closed".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 7); - seed_scope(&db, &scope, &worktree, ScopeStatus::Closed); - } - - let outcome = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect("an already-closed scope settles successfully"); - - assert_eq!( - outcome, - AbandonScopeOutcome::AlreadyTerminal { - worktree_id: worktree.clone(), - scope: scope.clone(), - status: ScopeStatus::Closed, - revision: 7, - } - ); - assert!( - !repo.marker_exists(), - "a proven-terminal no-op must clear the marker it armed" - ); - - let db = repo.db(); - assert_eq!( - read_worktree(&db, &worktree) - .expect("the worktree row should exist") - .revision, - 7, - "a terminal no-op writes nothing" - ); - assert_eq!(read_scope_status(&db, &scope), Some(ScopeStatus::Closed)); - } - - #[test] - fn an_already_abandoned_scope_settles_as_a_terminal_no_op() { - let repo = TestScopeRepo::new("abandoned-scope"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-abandoned".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 7); - seed_scope(&db, &scope, &worktree, ScopeStatus::Abandoned); - } - - let outcome = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect("an already-abandoned scope settles successfully"); - - assert_eq!( - outcome, - AbandonScopeOutcome::AlreadyTerminal { - worktree_id: worktree.clone(), - scope: scope.clone(), - status: ScopeStatus::Abandoned, - revision: 7, - } - ); - assert!( - !repo.marker_exists(), - "a proven-terminal no-op must clear the marker it armed" - ); - assert_eq!( - read_worktree(&repo.db(), &worktree) - .expect("the worktree row should exist") - .revision, - 7, - "a terminal no-op never abandons a scope a second time" - ); - } - - #[test] - fn a_scope_owned_by_another_worktree_is_rejected_without_writing() { - let repo = TestScopeRepo::new("cross-worktree"); - let worktree = repo.worktree_id(); - let other = WorktreeId("wt-other".to_string()); - let scope = ScopeId("scope-elsewhere".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 3); - seed_worktree(&db, &other, 11); - seed_scope(&db, &scope, &other, ScopeStatus::Active); - } - - let error = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect_err("a scope owned by another worktree must be rejected"); - - match &error { - AbandonScopeError::WorktreeIdentityMismatch { - scope: rejected, - scope_worktree_id, - invoking_worktree_id, - } => { - assert_eq!(rejected, &scope); - assert_eq!(scope_worktree_id, &other); - assert_eq!(invoking_worktree_id, &worktree); - } - other => panic!("expected WorktreeIdentityMismatch, got {other:?}"), - } - - let db = repo.db(); - assert_eq!( - read_worktree(&db, &worktree) - .expect("this worktree's row should exist") - .revision, - 3 - ); - assert_eq!( - read_worktree(&db, &other) - .expect("the other worktree's row should exist") - .revision, - 11 - ); - assert_eq!(read_scope_status(&db, &scope), Some(ScopeStatus::Active)); - } - - #[test] - fn a_live_scope_on_an_exhausted_revision_is_a_distinct_error() { - let repo = TestScopeRepo::new("revision-exhausted"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-live".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, u64::MAX); - seed_scope(&db, &scope, &worktree, ScopeStatus::Active); + error @ ProtectedWorktreeError::MarkerWorkerFailed(_) => { + AbandonScopeError::Other(anyhow::Error::new(error)) } - - let error = abandon_scope(&repo.repo_root, &scope, || repo.open_db()) - .expect_err("a worktree at the maximum revision cannot abandon"); - - match &error { - AbandonScopeError::RevisionExhausted { - worktree_id, - revision, - } => { - assert_eq!(worktree_id, &worktree); - assert_eq!(*revision, u64::MAX); - } - other => panic!("expected RevisionExhausted, got {other:?}"), - } - assert_eq!( - read_scope_status(&repo.db(), &scope), - Some(ScopeStatus::Active), - "the scope is still live and still needs retiring" - ); - } - - #[test] - fn a_cas_conflict_recomputes_the_abandonment_from_fresh_state() { - let repo = TestScopeRepo::new("cas-conflict-retry"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-live".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 3); - seed_scope(&db, &scope, &worktree, ScopeStatus::Active); - } - - let attempts = Cell::new(0u32); - let outcome = abandon_scope_inner( - &repo.repo_root, - &scope, - || repo.open_db(), - |attempt| { - attempts.set(attempts.get() + 1); - if attempt == 0 { - bump_revision(&repo.db_path, &worktree); - } - }, - ) - .expect("a losing CAS attempt should retry and settle"); - - assert_eq!( - attempts.get(), - 2, - "the first attempt loses the CAS and the second recomputes from fresh state" - ); - assert_eq!( - outcome, - AbandonScopeOutcome::Abandoned { - worktree_id: worktree.clone(), - scope: scope.clone(), - revision: 5, - }, - "the retry advances from the competitor's revision, not the stale one" - ); - - let db = repo.db(); - assert_eq!( - read_worktree(&db, &worktree) - .expect("the worktree row should exist") - .revision, - 5 - ); - assert_eq!(read_scope_status(&db, &scope), Some(ScopeStatus::Abandoned)); - } - - #[test] - fn a_cas_conflict_whose_competitor_ended_the_scope_settles_as_a_terminal_no_op() { - let repo = TestScopeRepo::new("cas-conflict-terminal"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-live".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 3); - seed_scope(&db, &scope, &worktree, ScopeStatus::Active); - } - - let outcome = abandon_scope_inner( - &repo.repo_root, - &scope, - || repo.open_db(), - |attempt| { - if attempt == 0 { - set_scope_status(&repo.db_path, &scope, ScopeStatus::Closed); - bump_revision(&repo.db_path, &worktree); - } - }, - ) - .expect("a competitor that ended the scope should settle the retry"); - - assert_eq!( - outcome, - AbandonScopeOutcome::AlreadyTerminal { - worktree_id: worktree.clone(), - scope: scope.clone(), - status: ScopeStatus::Closed, - revision: 4, - }, - "the retry must settle on the competitor's terminal status, not overwrite it" - ); - - let db = repo.db(); - assert_eq!( - read_scope_status(&db, &scope), - Some(ScopeStatus::Closed), - "a competitor's Close must never be overwritten by a second abandonment" - ); - assert_eq!( - read_worktree(&db, &worktree) - .expect("the worktree row should exist") - .revision, - 4 - ); - } - - #[test] - fn a_persistence_failure_rolls_back_the_whole_transition_and_leaves_the_fence_armed() { - let repo = TestScopeRepo::new("persistence-failure"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-live".to_string()); - let bystander = ScopeId("scope-already-abandoned".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 3); - seed_scope(&db, &scope, &worktree, ScopeStatus::Active); - seed_scope(&db, &bystander, &worktree, ScopeStatus::Abandoned); - } - - let error = abandon_scope_inner( - &repo.repo_root, - &scope, - || repo.open_db(), - |attempt| { - if attempt == 0 { - arm_a_scope_status_write_collision(&repo.db_path); - } - }, - ) - .expect_err("a failing durable write must fail the abandonment"); - - assert!( - matches!(error, AbandonScopeError::Other(_)), - "a persistence failure surfaces as the store-error variant, got {error:?}" - ); - assert!( - format!("{error}").contains("mutation_trace_scopes"), - "the failure must be the transaction's scope-status write — the statement that \ - runs after the worktree CAS guard — not an earlier read or a settled conflict; \ - got {error}" - ); - assert!( - repo.marker_exists(), - "a persistence failure after the fence is armed must leave it armed" - ); - - let db = repo.db(); - let worktree_state = read_worktree(&db, &worktree).expect("the worktree row should exist"); - assert_eq!( - worktree_state.revision, 3, - "the transaction's worktree CAS guard ran before the failing statement, so a \ - partially applied revision here would mean the batch is not atomic" - ); - assert!( - !worktree_state.needs_rebaseline, - "the rolled-back transition must not leave the worktree needing rebaseline" - ); - assert_eq!( - worktree_state.cursor_tree, - crate::services::mutation_trace::types::TreeId("tree-0".to_string()) - ); - assert_eq!( - read_scope_status(&db, &scope), - Some(ScopeStatus::Active), - "the target scope must not be left Abandoned by a failed transition" - ); - assert_eq!( - read_scope_status(&db, &bystander), - Some(ScopeStatus::Abandoned) - ); - - assert_eq!(count_rows(&db, "mutation_trace_events"), 0); - assert_eq!(count_rows(&db, "mutation_trace_event_active_scopes"), 0); - assert_eq!(count_rows(&db, "mutation_trace_processed_events"), 0); - } - - #[test] - fn a_db_provider_failure_leaves_the_fence_armed() { - let repo = TestScopeRepo::new("db-provider-failure"); - let scope = ScopeId("scope-live".to_string()); - - let error = abandon_scope(&repo.repo_root, &scope, || { - Err(anyhow::anyhow!("simulated Agent Trace DB open failure")) - }) - .expect_err("a DB provider that returns Err must fail abandon_scope()"); - - assert!( - matches!(error, AbandonScopeError::AgentTraceDbUnavailable(_)), - "expected AgentTraceDbUnavailable, got {error:?}" - ); - assert!( - repo.marker_exists(), - "a DB-provider failure after arming must leave the fence armed" - ); - } - - #[test] - fn a_marker_clear_failure_carries_the_already_settled_outcome() { - let repo = TestScopeRepo::new("marker-clear-failure"); - let worktree = repo.worktree_id(); - let scope = ScopeId("scope-live".to_string()); - { - let db = repo.db(); - seed_worktree(&db, &worktree, 3); - seed_scope(&db, &scope, &worktree, ScopeStatus::Active); - } - - let marker_path = repo.marker_path(); - let error = abandon_scope(&repo.repo_root, &scope, || { - std::fs::remove_file(&marker_path) - .expect("the armed marker file should be present mid-invocation"); - std::fs::create_dir_all(marker_path.join("nested")) - .expect("planting a non-empty directory at the marker path should succeed"); - repo.open_db() - }) - .expect_err("clearing a marker that is now a non-empty directory must fail"); - - let completed = match error { - AbandonScopeError::MarkerClearAfterCompletion { completed, .. } => completed, - other => panic!("expected MarkerClearAfterCompletion, got {other:?}"), - }; - assert_eq!( - *completed, - AbandonScopeOutcome::Abandoned { - worktree_id: worktree.clone(), - scope: scope.clone(), - revision: 4, - }, - "the error must carry the outcome that already settled durably" - ); - assert!( - repo.marker_exists(), - "the marker stays logically armed after a post-completion clear failure" - ); - - let db = repo.db(); - let worktree_state = read_worktree(&db, &worktree).expect("the worktree row should exist"); - assert_eq!(worktree_state.revision, 4); - assert!(worktree_state.needs_rebaseline); - assert_eq!(read_scope_status(&db, &scope), Some(ScopeStatus::Abandoned)); - - std::fs::remove_dir_all(&marker_path) - .expect("removing the planted directory should succeed"); } } diff --git a/cli/src/services/mutation_trace/runtime/tests.rs b/cli/src/services/mutation_trace/runtime/tests.rs deleted file mode 100644 index e88ececf2..000000000 --- a/cli/src/services/mutation_trace/runtime/tests.rs +++ /dev/null @@ -1,2449 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::sync::mpsc; -use std::thread; -use std::time::Duration; - -use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; -use crate::services::mutation_trace::protocol; -use crate::services::mutation_trace::store::{ - encode_revision, CasResult, DurableTransition, MutationTraceStore, -}; -use crate::services::mutation_trace::types::{ - boundary_event_key, boundary_scope, ActorKind, AttemptId, Attribution, Boundary, EventId, - FailureKind, ScopeId, ScopeStatus, -}; -use crate::services::patch::{parse_patch, ParsedPatch}; - -use super::coordinator::{coordinate, coordinate_inner, CoordinateError, RuntimeBoundary}; -use super::external_taint::ExternalTaintMarker; -use super::git_snapshot::{resolve_git_dir, resolve_worktree_id, GitSnapshotService}; -use super::mutation_attribution::resolve_bounded_mutation_attribution; -use super::ref_reconciliation::{ - reconcile_worktree, reconcile_worktree_inner, ReconcileError, ReconciliationOutcome, -}; -use super::scope_runtime::{ - abandon_scope, abandon_scope_inner, AbandonScopeError, AbandonScopeOutcome, -}; -use super::worktree_lock::{acquire_inner, WorktreeLock}; - -fn run_git(dir: &Path, args: &[&str]) -> String { - let output = Command::new("git") - .args(args) - .current_dir(dir) - .output() - .expect("git command should spawn"); - assert!( - output.status.success(), - "git {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).expect("git output should be valid UTF-8") -} - -fn init_repo(root: &Path) { - std::fs::create_dir_all(root).expect("repository directory should be created"); - run_git(root, &["init", "--quiet"]); - run_git(root, &["config", "user.email", "test@example.com"]); - run_git(root, &["config", "user.name", "Test"]); - run_git(root, &["commit", "--allow-empty", "--quiet", "-m", "init"]); -} - -struct TestRepo { - _temp_dir: tempfile::TempDir, - repo_root: PathBuf, - db_path: PathBuf, -} - -impl TestRepo { - fn new(label: &str) -> Self { - let temp_dir = tempfile::Builder::new() - .prefix(&format!("sce-mutation-trace-runtime-{label}-")) - .tempdir() - .expect("test temp directory should be created"); - let repo_root = temp_dir.path().join("repo"); - init_repo(&repo_root); - let db_path = temp_dir.path().join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path) - .expect("the repository DB should open with schema"); - Self { - _temp_dir: temp_dir, - repo_root, - db_path, - } - } - - fn open_db(&self) -> anyhow::Result { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) - } - - fn db(&self) -> RepositoryAgentTraceDb { - self.open_db() - .expect("reopening the DB for assertions should succeed") - } -} - -struct LinkedTestRepo { - _temp_dir: tempfile::TempDir, - main_root: PathBuf, - linked_root: PathBuf, - db_path: PathBuf, -} - -impl LinkedTestRepo { - fn new(label: &str) -> Self { - let temp_dir = tempfile::Builder::new() - .prefix(&format!("sce-mutation-trace-runtime-{label}-")) - .tempdir() - .expect("test temp directory should be created"); - let main_root = temp_dir.path().join("main"); - init_repo(&main_root); - let linked_root = temp_dir.path().join("linked"); - run_git( - &main_root, - &[ - "worktree", - "add", - "--quiet", - linked_root.to_str().expect("worktree path should be UTF-8"), - ], - ); - let db_path = temp_dir.path().join("agent-trace.db"); - RepositoryAgentTraceDb::new_at(&db_path) - .expect("the shared repository DB should open with schema"); - Self { - _temp_dir: temp_dir, - main_root, - linked_root, - db_path, - } - } - - fn open_db(&self) -> anyhow::Result { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&self.db_path) - } - - fn db(&self) -> RepositoryAgentTraceDb { - self.open_db() - .expect("reopening the shared DB for assertions should succeed") - } -} - -fn seed_event( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - before_tree: &str, - after_tree: &str, -) { - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, - boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", - ( - worktree_id, - encode_revision(revision).as_slice(), - before_tree, - after_tree, - ), - ) - .expect("event row insert should succeed"); -} - -fn seed_attribution_event( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - before_tree: &str, - after_tree: &str, - attribution_kind: &str, - attribution_scope_id: Option<&str>, -) { - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, - boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', ?5, ?6, 'flush', NULL, NULL)", - ( - worktree_id, - encode_revision(revision).as_slice(), - before_tree, - after_tree, - attribution_kind, - attribution_scope_id, - ), - ) - .expect("attribution event row insert should succeed"); -} - -fn ref_exists(dir: &Path, ref_name: &str) -> bool { - Command::new("git") - .args(["show-ref", "--verify", "--quiet", ref_name]) - .current_dir(dir) - .status() - .expect("git show-ref should spawn") - .success() -} - -fn row_count(db: &RepositoryAgentTraceDb, table: &str) -> i64 { - db.query_map(&format!("SELECT COUNT(*) FROM {table}"), (), |row| { - row.get::(0).map_err(Into::into) - }) - .expect("count query should succeed") - .into_iter() - .next() - .expect("count row should exist") -} - -#[test] -fn linked_worktrees_have_independent_locks_and_worktree_ids() { - let repo = LinkedTestRepo::new("linked-ids"); - - let main_git_dir = resolve_git_dir(&repo.main_root).expect("main git dir should resolve"); - let linked_git_dir = resolve_git_dir(&repo.linked_root).expect("linked git dir should resolve"); - assert_ne!( - main_git_dir, linked_git_dir, - "a linked worktree must resolve its own worktree-specific git dir, giving it a distinct lock and identity path" - ); - - let main_outcome = coordinate(&repo.main_root, &RuntimeBoundary::Flush, || repo.open_db()) - .expect("first observation on the main worktree should succeed"); - - let held = WorktreeLock::acquire(&main_git_dir, Duration::from_secs(5)) - .expect("the main worktree's runtime lock should be acquirable"); - - let linked_outcome = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, || { - repo.open_db() - }) - .expect( - "coordinate() on the linked worktree must acquire its own distinct runtime lock while the main worktree's lock is still held", - ); - - drop(held); - - assert_ne!( - main_outcome.worktree_id, linked_outcome.worktree_id, - "each linked worktree must derive a distinct WorktreeId from Git's own worktree topology" - ); - - let db_main = repo.db(); - let store = MutationTraceStore::new(&db_main); - assert!( - store - .load_worktree(&main_outcome.worktree_id, None, None) - .expect("loading the main worktree row should succeed") - .is_some(), - "the main worktree's coordinator should have persisted a row into the caller-supplied repository-scoped DB" - ); - assert!( - store - .load_worktree(&linked_outcome.worktree_id, None, None) - .expect("loading the linked worktree row should succeed") - .is_some(), - "the linked worktree's coordinator should have persisted its distinct row into the same caller-supplied DB" - ); - - let linked_snapshot = GitSnapshotService::new(&repo.linked_root) - .expect("a snapshot service should construct for the linked worktree"); - linked_snapshot - .diff_trees(&main_outcome.observed_tree, &main_outcome.observed_tree) - .expect("a tree pinned by the main worktree's coordinator must resolve through the linked worktree's git dir"); -} - -#[test] - -fn a_snapshot_failure_then_recovery_cycle_runs_through_the_public_api() { - let repo = TestRepo::new("failure-recovery"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let tmp_index_dir = git_dir.join("sce").join("tmp"); - let _ = std::fs::remove_dir_all(&tmp_index_dir); - std::fs::write(&tmp_index_dir, b"not a directory").expect( - "planting a file where the snapshot service expects its temp-index directory should succeed", - ); - - let scope = ScopeId("scope-recovery".to_string()); - let failure = coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-during-failure".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect_err("a Git snapshot failure against a materialized worktree should be reported"); - match failure { - CoordinateError::SnapshotFailure { - persisted_taint, .. - } => assert!( - persisted_taint, - "the existing worktree row must be durably tainted by the snapshot failure" - ), - other => panic!("expected CoordinateError::SnapshotFailure, got {other:?}"), - } - - { - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should still exist"); - assert!( - projection.worktree_state.tainted, - "the worktree row should be tainted after the snapshot failure" - ); - } - - std::fs::remove_file(&tmp_index_dir) - .expect("removing the planted file should let the snapshot service recreate its temp dir"); - - let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the coordinator should recover from the taint and process the boundary"); - assert_eq!( - recovered.worktree_id, worktree_id, - "recovery must operate on the same worktree identity" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should still exist"); - assert!( - !projection.worktree_state.tainted, - "taint recovery must clear the tainted flag before the triggering boundary is processed" - ); -} - -#[test] -fn a_successful_coordinate_through_the_public_api_leaves_no_external_taint_marker() { - let repo = TestRepo::new("public-success-no-marker"); - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - - let outcome = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, || repo.open_db()) - .expect("a first observation through the public entrypoint should succeed"); - assert_eq!( - outcome.revision, 0, - "a first-observation flush should not advance the revision" - ); - assert!( - !marker - .exists() - .expect("marker existence should resolve after a successful coordinate()"), - "a successful coordinate() must clear the external-taint marker it armed" - ); -} - -#[test] -#[allow(clippy::too_many_lines)] -fn a_db_open_failure_after_arming_leaves_the_marker_and_the_next_invocation_rebaselines_without_evidence( -) { - let repo = TestRepo::new("public-db-open-failure-gap"); - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let scope = ScopeId("scope-across-the-gap".to_string()); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting the scope should succeed"); - std::fs::write(repo.repo_root.join("work.txt"), b"v1").expect("the A -> B edit should write"); - let advanced = coordinate( - &repo.repo_root, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - ok_db, - ) - .expect("the advance should commit exactly one event"); - let tree_b = advanced.observed_tree.clone(); - assert!( - advanced.mutation_event.is_some(), - "the exclusive A -> B edit must land one trustworthy event before the gap" - ); - - let db_failure = coordinate( - &repo.repo_root, - &RuntimeBoundary::Flush, - || -> anyhow::Result { - Err(anyhow::anyhow!("simulated Agent Trace DB open failure")) - }, - ) - .expect_err("a failing DB provider must fail coordinate()"); - assert!( - matches!(db_failure, CoordinateError::AgentTraceDbUnavailable(_)), - "expected AgentTraceDbUnavailable, got {db_failure:?}" - ); - assert!( - marker.exists().expect("marker existence should resolve"), - "the armed marker must survive an invocation whose DB provider returned Err" - ); - std::fs::write(repo.repo_root.join("work.txt"), b"v2-during-the-gap").expect("the B -> C edit"); - - let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the follow-up invocation with a working provider should recover, then process its boundary"); - let tree_c = recovered.observed_tree.clone(); - assert_ne!( - tree_c, tree_b, - "the gap edit must have moved the observed tree" - ); - assert!( - recovered.mutation_event.is_none(), - "no mutation evidence may span the interval the DB-open failure fenced off" - ); - assert!( - !marker.exists().expect("marker existence should resolve"), - "the recovering invocation must clear the marker on success" - ); - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, Some(&scope), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - projection.worktree_state.cursor_tree, tree_c, - "recovery must rebaseline the cursor to the follow-up invocation's own observed tree" - ); - assert!(!projection.worktree_state.tainted); - assert_eq!( - projection.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "a scope live across the fenced interval must be abandoned afterward" - ); - for revision in 1..=recovered.revision { - if let Some(event) = store - .load_mutation_event(&worktree_id, revision) - .expect("loading a mutation event should succeed") - { - assert_ne!( - event.after_tree, tree_c, - "no MutationEvent may treat an interval ending at the post-gap tree as one trustworthy AI-attributable interval" - ); - } - } -} - -#[test] -fn a_stale_marker_rebaselines_to_the_current_tree_abandons_scopes_then_processes_the_boundary() { - let repo = TestRepo::new("public-stale-marker-rebaseline"); - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let tree_a = baseline.observed_tree.clone(); - let scope = ScopeId("scope-stranded".to_string()); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting scope S should succeed"); - - marker - .persist() - .expect("simulating a crashed invocation that armed but never cleared the marker"); - - std::fs::write( - repo.repo_root.join("stranded.txt"), - b"edited-while-the-process-was-gone", - ) - .expect("the A -> C edit should write"); - - let recovered = coordinate( - &repo.repo_root, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - ok_db, - ) - .expect("an inherited marker must recover, then process the triggering boundary"); - let tree_c = recovered.observed_tree.clone(); - assert_ne!( - tree_c, tree_a, - "the working tree must have moved during the gap" - ); - assert!( - recovered.mutation_event.is_none(), - "no A -> C evidence may be emitted across the fenced interval" - ); - assert!( - !marker.exists().expect("marker existence should resolve"), - "a successful recovery must clear the inherited marker" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, Some(&scope), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - projection.worktree_state.cursor_tree, tree_c, - "recovery must rebaseline the cursor to the current tree C" - ); - assert!(!projection.worktree_state.tainted); - assert_eq!( - projection.scopes.get(&scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "the scope that was live across the gap must be abandoned during recovery" - ); - - let stable = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("a plain flush after recovery should succeed"); - assert_eq!( - stable.revision, recovered.revision, - "recovery and the triggering boundary already completed in the inheriting invocation" - ); - assert!(stable.mutation_event.is_none()); -} - -#[test] -fn a_first_ever_failed_invocation_that_never_materialized_a_worktree_row_creates_no_evidence() { - let repo = TestRepo::new("public-first-ever-failure"); - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - - std::fs::write( - repo.repo_root.join("pre-existing.txt"), - b"content before any observation", - ) - .expect("a pre-existing edit should write"); - - let first = coordinate( - &repo.repo_root, - &RuntimeBoundary::Flush, - || -> anyhow::Result { - Err(anyhow::anyhow!( - "simulated Agent Trace DB open failure on the first-ever invocation" - )) - }, - ) - .expect_err("the first-ever invocation's DB provider fails"); - assert!( - matches!(first, CoordinateError::AgentTraceDbUnavailable(_)), - "expected AgentTraceDbUnavailable, got {first:?}" - ); - assert!( - marker.exists().expect("marker existence should resolve"), - "the first-ever failed invocation still leaves an armed marker" - ); - - std::fs::write( - repo.repo_root.join("during-the-gap.txt"), - b"more unknown-interval content", - ) - .expect("another unobserved edit should write"); - - let established = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, || repo.open_db()) - .expect("the first successful invocation establishes the baseline"); - assert!( - established.mutation_event.is_none(), - "a worktree with no prior durable row cannot produce evidence for the unknown interval" - ); - assert!( - !marker.exists().expect("marker existence should resolve"), - "the successful baseline must clear the inherited marker" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&established.worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should now exist"); - assert_eq!( - projection.worktree_state.cursor_tree, established.observed_tree, - "the baseline is established against the first observed tree" - ); - assert!(!projection.worktree_state.tainted); - assert_eq!(projection.worktree_state.failure_kind, FailureKind::Healthy); - for revision in 0..=established.revision { - assert!( - store - .load_mutation_event(&established.worktree_id, revision) - .expect("loading a mutation event should succeed") - .is_none(), - "no MutationEvent may exist for a worktree whose history began with an unobserved interval" - ); - } -} - -#[test] -#[allow(clippy::too_many_lines)] -fn linked_worktrees_keep_independent_external_taint_markers_over_a_shared_db() { - let repo = LinkedTestRepo::new("public-taint-linked"); - - let main_git_dir = resolve_git_dir(&repo.main_root).expect("main git dir should resolve"); - let linked_git_dir = resolve_git_dir(&repo.linked_root).expect("linked git dir should resolve"); - let main_marker = ExternalTaintMarker::new(&main_git_dir); - let linked_marker = ExternalTaintMarker::new(&linked_git_dir); - - let main_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, || repo.open_db()) - .expect("the main worktree baseline should succeed"); - let linked_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, || { - repo.open_db() - }) - .expect("the linked worktree baseline should succeed"); - assert_ne!(main_baseline.worktree_id, linked_baseline.worktree_id); - - let linked_scope = ScopeId("scope-linked".to_string()); - coordinate( - &repo.linked_root, - &RuntimeBoundary::Start { - scope: linked_scope.clone(), - event: EventId("evt-linked-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - || repo.open_db(), - ) - .expect("starting the linked worktree's scope should succeed"); - linked_marker - .persist() - .expect("arming the linked worktree's marker should succeed"); - - std::fs::write(repo.main_root.join("main-work.txt"), b"v1") - .expect("a main-worktree edit should write"); - coordinate(&repo.main_root, &RuntimeBoundary::Flush, || repo.open_db()).expect( - "the main worktree flush must succeed without inheriting the linked worktree's marker", - ); - assert!( - !main_marker - .exists() - .expect("marker existence should resolve"), - "the main worktree clears its own marker on success" - ); - assert!( - linked_marker - .exists() - .expect("marker existence should resolve"), - "the main worktree's invocation must not touch the linked worktree's independent marker" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let linked_mid = store - .load_worktree(&linked_baseline.worktree_id, Some(&linked_scope), None) - .expect("loading the linked worktree row should succeed") - .expect("the linked worktree row should exist"); - assert_eq!( - linked_mid.scopes.get(&linked_scope).map(|s| s.status), - Some(ScopeStatus::Active), - "a marker in the linked worktree must not trigger recovery of the linked worktree from the main worktree's invocation" - ); - - std::fs::write(repo.linked_root.join("linked-work.txt"), b"v1") - .expect("a linked-worktree edit should write"); - let linked_recovered = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, || { - repo.open_db() - }) - .expect("the linked worktree's own invocation recovers from its inherited marker"); - assert!( - linked_recovered.mutation_event.is_none(), - "the linked worktree's conservative recovery emits no evidence for its fenced interval" - ); - assert!( - !linked_marker - .exists() - .expect("marker existence should resolve"), - "the linked worktree clears its marker after its own successful recovery" - ); - - let linked_after = store - .load_worktree(&linked_baseline.worktree_id, Some(&linked_scope), None) - .expect("loading the linked worktree row should succeed") - .expect("the linked worktree row should exist"); - assert_eq!( - linked_after.scopes.get(&linked_scope).map(|s| s.status), - Some(ScopeStatus::Abandoned), - "the linked worktree's inherited-taint recovery abandons its own live scope" - ); -} - -#[test] -fn a_snapshot_failure_arms_the_marker_and_the_next_invocation_recovers_once() { - let repo = TestRepo::new("public-snapshot-failure-marker-recovery"); - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - assert!(!marker.exists().expect("marker existence should resolve")); - - let tmp_index_dir = git_dir.join("sce").join("tmp"); - let _ = std::fs::remove_dir_all(&tmp_index_dir); - std::fs::write(&tmp_index_dir, b"not a directory") - .expect("planting a file where the snapshot service expects its temp-index directory"); - - let scope = ScopeId("scope-during-the-failure".to_string()); - let failure = coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-during-failure".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect_err("a Git snapshot failure after marker arming should be reported"); - assert!( - matches!( - failure, - CoordinateError::SnapshotFailure { - persisted_taint: true, - .. - } - ), - "expected a SnapshotFailure that durably tainted the worktree, got {failure:?}" - ); - assert!( - marker.exists().expect("marker existence should resolve"), - "a snapshot failure after arming must leave BOTH a durable taint and the external-taint marker" - ); - - { - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should still exist"); - assert!(projection.worktree_state.tainted); - assert_eq!( - projection.worktree_state.failure_kind, - FailureKind::SnapshotFailure - ); - } - - std::fs::remove_file(&tmp_index_dir) - .expect("removing the planted file should let the snapshot service recreate its temp dir"); - std::fs::write(repo.repo_root.join("work.txt"), b"v1") - .expect("an edit before recovery should write"); - - let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the coordinator should recover from the taint and process the boundary"); - assert!( - recovered.mutation_event.is_none(), - "the conservative recovery emits no evidence for the fenced interval" - ); - assert!( - !marker.exists().expect("marker existence should resolve"), - "a successful recovery clears the marker" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should still exist"); - assert!( - !projection.worktree_state.tainted, - "recovery must clear the durable taint" - ); - assert_eq!(projection.worktree_state.failure_kind, FailureKind::Healthy); - assert_eq!( - projection.worktree_state.cursor_tree, recovered.observed_tree, - "recovery rebaselines the cursor to the recovering invocation's own observed tree" - ); - - let stable = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("a plain flush after recovery should succeed"); - assert_eq!( - stable.revision, recovered.revision, - "the single conservative recovery already completed; a follow-up flush is a no-op" - ); - assert!(stable.mutation_event.is_none()); -} - -#[test] -#[allow(clippy::too_many_lines)] -fn a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_recovery() { - let repo = TestRepo::new("public-marker-clear-failure"); - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - let marker_path = git_dir.join("sce").join("mutation-cursor-tainted"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let scope = ScopeId("scope-attributable".to_string()); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting the scope should succeed"); - std::fs::write(repo.repo_root.join("work.txt"), b"v1") - .expect("an exclusive edit before the boundary should write"); - - let clear_marker_path = marker_path.clone(); - let clear_db_path = repo.db_path.clone(); - let error = coordinate( - &repo.repo_root, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - move || { - std::fs::remove_file(&clear_marker_path) - .expect("the armed marker file should be present mid-invocation"); - std::fs::create_dir_all(clear_marker_path.join("nested")) - .expect("planting a non-empty directory at the marker path should succeed"); - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&clear_db_path) - }, - ) - .expect_err("clearing a marker that is now a non-empty directory must fail"); - - let committed = match error { - CoordinateError::MarkerClearAfterCommit { committed, .. } => committed, - other => panic!("expected MarkerClearAfterCommit, got {other:?}"), - }; - assert!( - marker.exists().expect("marker existence should resolve"), - "the marker stays logically armed after a post-commit clear failure" - ); - - let (durable_revision, durable_cursor) = { - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, Some(&scope), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - let event = store - .load_mutation_event(&worktree_id, projection.worktree_state.revision) - .expect("loading the committed mutation event should succeed") - .expect("the attributable Advance must have committed one durable event"); - assert_eq!(event.after_tree, projection.worktree_state.cursor_tree); - ( - projection.worktree_state.revision, - projection.worktree_state.cursor_tree.clone(), - ) - }; - - assert_eq!(committed.worktree_id, worktree_id); - assert_eq!(committed.revision, durable_revision); - assert_eq!(committed.observed_tree, durable_cursor); - assert!( - committed.mutation_event.is_some(), - "the committed outcome carried by the error must still expose the MutationEvent" - ); - - std::fs::remove_dir_all(&marker_path).expect("removing the planted directory should succeed"); - marker - .persist() - .expect("re-arming a plain marker file should succeed"); - std::fs::write(repo.repo_root.join("work.txt"), b"v2").expect("a later edit should write"); - - let recovered = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the later invocation recovers from the still-armed marker"); - assert!( - recovered.mutation_event.is_none(), - "the deferred conservative recovery emits no evidence for the interval it could not prove" - ); - assert!( - !marker.exists().expect("marker existence should resolve"), - "the successful later recovery finally clears the marker" - ); - assert_ne!( - recovered.observed_tree, durable_cursor, - "the later recovery rebaselines to the newer tree" - ); -} - -#[test] -fn reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it() { - let repo = TestRepo::new("reconcile-blocks-on-worktree-lock"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let baseline_tree = baseline.observed_tree.clone(); - - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let held = acquire_inner(&git_dir, Duration::from_secs(5), || {}) - .expect("the test should hold a real WorktreeLock before the worker runs"); - - let (contention_tx, contention_rx) = mpsc::channel(); - let (result_tx, result_rx) = mpsc::channel(); - let repo_root_clone = repo.repo_root.clone(); - let db_path_clone = repo.db_path.clone(); - let worker = thread::spawn(move || { - let outcome = reconcile_worktree_inner( - &repo_root_clone, - || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&db_path_clone), - move || { - contention_tx - .send(()) - .expect("contention signal channel should still be open"); - }, - ); - result_tx - .send(()) - .expect("result signal channel should still be open"); - outcome - }); - - contention_rx.recv_timeout(Duration::from_secs(5)).expect( - "reconcile_worktree_inner should reach the WorktreeLock try_lock loop and \ - observe contention while this test still holds the lock", - ); - assert!( - result_rx.recv_timeout(Duration::from_millis(300)).is_err(), - "the reconciliation pass must not complete while the worktree lock is still held" - ); - - let snapshot = GitSnapshotService::new(&repo.repo_root) - .expect("a snapshot service should construct for the worktree"); - std::fs::write(repo.repo_root.join("under-lock.txt"), b"under the lock") - .expect("an edit under the lock should write"); - let x = snapshot.capture_tree().expect("capturing X should succeed"); - snapshot - .pin_tree(&worktree_id, &x) - .expect("pinning X should succeed"); - seed_event( - &ok_db().expect("reopening the DB to seed the event should succeed"), - &worktree_id.0, - 1, - &baseline_tree.0, - &x.0, - ); - - drop(held); - - result_rx - .recv_timeout(Duration::from_secs(5)) - .expect("the same reconciliation pass should complete once the lock is released"); - let outcome = worker - .join() - .expect("the reconciliation worker thread should not panic") - .expect( - "reconciliation should succeed once it can acquire the worktree lock after release", - ); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - - assert_eq!( - report.deleted, 0, - "reconciliation must retain X: it became a durable root under the very lock it was waiting on" - ); - assert_eq!( - report.local_required, 2, - "the worktree's durable roots are exactly the baseline cursor tree and X" - ); - run_git( - &repo.repo_root, - &[ - "show-ref", - "--verify", - "--quiet", - &format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, x.0), - ], - ); -} - -#[test] -#[allow(clippy::too_many_lines)] -fn reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree() { - let repo = TestRepo::new("reconcile-blocks-until-real-cas"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let baseline_tree = baseline.observed_tree.clone(); - - std::fs::write( - repo.repo_root.join("under-real-cas.txt"), - b"observed before the real CAS", - ) - .expect("an edit before the coordinated boundary should write"); - - let (pinned_tx, pinned_rx) = mpsc::channel(); - let (release_tx, release_rx) = mpsc::channel::<()>(); - let (coordinate_done_tx, coordinate_done_rx) = mpsc::channel(); - let coord_repo_root = repo.repo_root.clone(); - let coord_db_path = repo.db_path.clone(); - let coordinator = thread::spawn(move || { - let mut paused = false; - let outcome = coordinate_inner( - &coord_repo_root, - &RuntimeBoundary::Flush, - || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&coord_db_path), - || {}, - move |_attempt| { - if !paused { - paused = true; - pinned_tx - .send(()) - .expect("the pin-done signal channel should still be open"); - release_rx - .recv() - .expect("the release channel should deliver before the real CAS"); - } - }, - |_attempt| Ok(()), - ); - coordinate_done_tx - .send(()) - .expect("the coordinate-done signal channel should still be open"); - outcome - }); - - pinned_rx.recv_timeout(Duration::from_secs(5)).expect( - "coordinate_inner should pin X, load the worktree, and pause in after_load \ - while still holding the WorktreeLock", - ); - - let (contention_tx, contention_rx) = mpsc::channel(); - let (reconcile_done_tx, reconcile_done_rx) = mpsc::channel(); - let rec_repo_root = repo.repo_root.clone(); - let rec_db_path = repo.db_path.clone(); - let reconciler = thread::spawn(move || { - let outcome = reconcile_worktree_inner( - &rec_repo_root, - || RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&rec_db_path), - move || { - contention_tx - .send(()) - .expect("the contention signal channel should still be open"); - }, - ); - reconcile_done_tx - .send(()) - .expect("the reconcile-done signal channel should still be open"); - outcome - }); - - contention_rx.recv_timeout(Duration::from_secs(5)).expect( - "reconcile_worktree_inner should observe WorktreeLock contention while the \ - real coordinate() CAS is still pending", - ); - assert!( - reconcile_done_rx - .recv_timeout(Duration::from_millis(300)) - .is_err(), - "reconciliation must not complete while coordinate() still holds the lock across pin -> CAS" - ); - - release_tx - .send(()) - .expect("releasing the coordinate worker should succeed"); - - coordinate_done_rx - .recv_timeout(Duration::from_secs(5)) - .expect("the real coordinate() invocation should complete after the CAS"); - let coordinate_outcome = coordinator - .join() - .expect("the coordinate worker thread should not panic") - .expect("the real coordinate() CAS should apply the observed drift"); - let x = coordinate_outcome.observed_tree.clone(); - assert_ne!( - x, baseline_tree, - "the coordinated Flush must observe a real drift from the baseline tree" - ); - assert!( - coordinate_outcome.mutation_event.is_some(), - "the observed drift must commit exactly one durable MutationEvent through the real CAS" - ); - - reconcile_done_rx - .recv_timeout(Duration::from_secs(5)) - .expect("reconciliation should complete once the coordinator releases the lock"); - let outcome = reconciler - .join() - .expect("the reconciliation worker thread should not panic") - .expect( - "reconciliation should succeed once it can acquire the worktree lock after release", - ); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - report.deleted, 0, - "reconciliation must retain X: it became a durable root through the real coordinate() CAS \ - under the very lock reconciliation was waiting on" - ); - assert_eq!( - report.local_required, 2, - "the worktree's durable roots are exactly the baseline cursor tree and X" - ); - - run_git( - &repo.repo_root, - &[ - "show-ref", - "--verify", - "--quiet", - &format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, x.0), - ], - ); - - let db = ok_db().expect("reopening the DB for assertions should succeed"); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - projection.worktree_state.cursor_tree, x, - "the real coordinator CAS advanced the durable cursor to X" - ); - let event = store - .load_mutation_event(&worktree_id, projection.worktree_state.revision) - .expect("loading the committed mutation event should succeed") - .expect("the observed drift must have committed one durable event"); - assert_eq!(event.before_tree, baseline_tree); - assert_eq!(event.after_tree, x); -} - -#[test] -fn a_pin_with_no_durable_root_is_reclaimed_by_a_later_reconciliation() { - let repo = TestRepo::new("orphan-reclaimed-via-public-api"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let snapshot = GitSnapshotService::new(&repo.repo_root) - .expect("a snapshot service should construct for the worktree"); - std::fs::write( - repo.repo_root.join("orphan.txt"), - b"never durably referenced", - ) - .expect("the orphan-producing edit should write"); - let x = snapshot.capture_tree().expect("capturing X should succeed"); - snapshot - .pin_tree(&worktree_id, &x) - .expect("pinning X should succeed"); - let x_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, x.0); - assert!( - ref_exists(&repo.repo_root, &x_ref), - "the orphan pin must exist before reconciliation runs" - ); - - let outcome = reconcile_worktree(&repo.repo_root, ok_db) - .expect("reconciliation should succeed through the public entrypoint"); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - - assert_eq!( - report.deleted, 1, - "the pass must delete exactly the one pin with no durable root anywhere in the repository" - ); - assert!( - !ref_exists(&repo.repo_root, &x_ref), - "X's snapshot ref must be gone after reconciliation reclaims it" - ); -} - -#[test] -fn current_cursor_pin_survives_reconciliation_without_a_referencing_event_through_the_public_api() { - let repo = TestRepo::new("cursor-retained-no-event"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let cursor_ref = format!( - "refs/sce/mutation-cursor/{}/{}", - worktree_id.0, baseline.observed_tree.0 - ); - assert!( - ref_exists(&repo.repo_root, &cursor_ref), - "the baseline flush must pin the current cursor tree" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let roots = store - .load_tree_roots(&worktree_id) - .expect("load_tree_roots should succeed"); - assert_eq!( - roots.len(), - 1, - "with no event yet, the cursor tree is the worktree's only durable root" - ); - assert!( - roots.contains(&baseline.observed_tree), - "the current cursor tree must be that sole durable root" - ); - - let outcome = reconcile_worktree(&repo.repo_root, ok_db) - .expect("reconciliation should succeed through the public entrypoint"); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - - assert_eq!( - report.deleted, 0, - "the current cursor tree is a durable root via cursor_tree alone and must be retained" - ); - assert!( - ref_exists(&repo.repo_root, &cursor_ref), - "the cursor pin must still resolve after reconciliation" - ); -} - -#[test] -fn historical_before_and_after_pins_survive_reconciliation_after_real_coordinate_transitions() { - let repo = TestRepo::new("historical-retention-real-transitions"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let tree_a = baseline.observed_tree.clone(); - - let scope = ScopeId("scope-historical-abcd".to_string()); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting the scope should succeed"); - - std::fs::write(repo.repo_root.join("work.txt"), b"v1").expect("the A -> B edit should write"); - let advance_b = coordinate( - &repo.repo_root, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance-b".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - ok_db, - ) - .expect("the A -> B advance should commit one event"); - let tree_b = advance_b.observed_tree.clone(); - - std::fs::write(repo.repo_root.join("work.txt"), b"v2").expect("the B -> C edit should write"); - let advance_c = coordinate( - &repo.repo_root, - &RuntimeBoundary::Advance { - scope: scope.clone(), - event: EventId("evt-advance-c".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - ok_db, - ) - .expect("the B -> C advance should commit one event"); - let tree_c = advance_c.observed_tree.clone(); - - std::fs::write(repo.repo_root.join("work.txt"), b"v3").expect("the C -> D edit should write"); - let advance_d = coordinate( - &repo.repo_root, - &RuntimeBoundary::Advance { - scope, - event: EventId("evt-advance-d".to_string()), - actor_kind: ActorKind::ClaudeCode, - }, - ok_db, - ) - .expect("the C -> D advance should commit one event"); - let tree_d = advance_d.observed_tree.clone(); - - assert_ne!( - tree_a, tree_b, - "the A -> B advance must observe a real drift" - ); - assert_ne!( - tree_b, tree_c, - "the B -> C advance must observe a real drift" - ); - assert_ne!( - tree_c, tree_d, - "the C -> D advance must observe a real drift" - ); - - let outcome = reconcile_worktree(&repo.repo_root, ok_db) - .expect("reconciliation should succeed through the public entrypoint"); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - report.deleted, 0, - "every historical before_tree/after_tree is a durable root: the history A -> B -> C -> D \ - must delete none of {{A, B, C, D}}" - ); - for tree in [&tree_a, &tree_b, &tree_c, &tree_d] { - let tree_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, tree.0); - assert!( - ref_exists(&repo.repo_root, &tree_ref), - "the historical pin for {tree:?} must survive reconciliation after the cursor advanced past it" - ); - } -} - -#[test] -fn reconciliation_through_the_public_api_is_idempotent() { - let repo = TestRepo::new("idempotent-via-public-api"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let snapshot = GitSnapshotService::new(&repo.repo_root) - .expect("a snapshot service should construct for the worktree"); - std::fs::write(repo.repo_root.join("orphan-1.txt"), b"orphan one") - .expect("the first orphan-producing edit should write"); - let x1 = snapshot - .capture_tree() - .expect("capturing X1 should succeed"); - snapshot - .pin_tree(&worktree_id, &x1) - .expect("pinning X1 should succeed"); - std::fs::write(repo.repo_root.join("orphan-2.txt"), b"orphan two") - .expect("the second orphan-producing edit should write"); - let x2 = snapshot - .capture_tree() - .expect("capturing X2 should succeed"); - snapshot - .pin_tree(&worktree_id, &x2) - .expect("pinning X2 should succeed"); - - let first = - match reconcile_worktree(&repo.repo_root, ok_db).expect("the first pass should succeed") { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - first.deleted, 2, - "the first pass must reclaim both orphan pins" - ); - - let second = - match reconcile_worktree(&repo.repo_root, ok_db).expect("the second pass should succeed") { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - second.deleted, 0, - "the second pass has nothing left to reclaim" - ); - assert_eq!( - first.local_required, second.local_required, - "the local durable-root count must be stable across both passes" - ); - assert_eq!( - first.retained, second.retained, - "the retained-pin count must be identical once the pass has nothing more to reclaim" - ); -} - -#[test] -fn reconcile_one_linked_worktree_leaves_the_other_worktrees_pins_and_shared_objects_intact() { - let repo = LinkedTestRepo::new("linked-isolation"); - let ok_db = || repo.open_db(); - - let a_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, ok_db) - .expect("A's baseline observation should materialize its worktree"); - let a_id = a_baseline.worktree_id.clone(); - let b_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, ok_db) - .expect("B's baseline observation should materialize its worktree"); - let b_id = b_baseline.worktree_id.clone(); - assert_ne!( - a_id, b_id, - "A and B must derive distinct checkout identities" - ); - - let b_cursor_ref = format!( - "refs/sce/mutation-cursor/{}/{}", - b_id.0, b_baseline.observed_tree.0 - ); - assert!( - ref_exists(&repo.linked_root, &b_cursor_ref), - "B's own cursor pin must exist before A reconciles" - ); - - let a_snapshot = GitSnapshotService::new(&repo.main_root) - .expect("a snapshot service should construct for A"); - std::fs::write(repo.main_root.join("a-orphan.txt"), b"A's own orphan") - .expect("A's orphan-producing edit should write"); - let a_orphan = a_snapshot - .capture_tree() - .expect("capturing A's orphan tree should succeed"); - a_snapshot - .pin_tree(&a_id, &a_orphan) - .expect("pinning A's orphan should succeed"); - let a_orphan_ref = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, a_orphan.0); - - let outcome = reconcile_worktree(&repo.main_root, ok_db) - .expect("reconciling A should succeed through the public entrypoint"); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - report.deleted, 1, - "reconcile_worktree(A) must delete only A's own orphan pin" - ); - assert!( - !ref_exists(&repo.main_root, &a_orphan_ref), - "A's orphan pin must be gone" - ); - - assert!( - ref_exists(&repo.linked_root, &b_cursor_ref), - "reconcile_worktree(A) must never enumerate or delete a refs/sce/mutation-cursor// ref" - ); - let b_snapshot = GitSnapshotService::new(&repo.linked_root) - .expect("a snapshot service should construct for B"); - b_snapshot - .diff_trees(&b_baseline.observed_tree, &b_baseline.observed_tree) - .expect("B's durable tree must still resolve in the shared object database after A's pass"); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - assert!( - store - .load_worktree(&b_id, None, None) - .expect("loading B's row should succeed") - .is_some(), - "reconcile_worktree(A) must not disturb B's durable worktree row" - ); -} - -#[test] -fn reconcile_a_retains_its_pin_when_another_worktree_durably_requires_the_same_tree() { - let repo = LinkedTestRepo::new("cross-worktree-degraded-retention"); - let ok_db = || repo.open_db(); - - let a_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, ok_db) - .expect("A's baseline observation should materialize its worktree"); - let a_id = a_baseline.worktree_id.clone(); - let b_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, ok_db) - .expect("B's baseline observation should materialize its worktree"); - let b_id = b_baseline.worktree_id.clone(); - - let b_snapshot = GitSnapshotService::new(&repo.linked_root) - .expect("a snapshot service should construct for B"); - std::fs::write( - repo.linked_root.join("shared-content.txt"), - b"identical content", - ) - .expect("B's edit toward T should write"); - let tree_t = b_snapshot - .capture_tree() - .expect("capturing T should succeed"); - let db = repo.db(); - seed_event(&db, &b_id.0, 1, &b_baseline.observed_tree.0, &tree_t.0); - let b_t_ref = format!("refs/sce/mutation-cursor/{}/{}", b_id.0, tree_t.0); - assert!( - !ref_exists(&repo.linked_root, &b_t_ref), - "B's own pin for T must be deliberately absent -- the degraded state under test" - ); - - let a_snapshot = GitSnapshotService::new(&repo.main_root) - .expect("a snapshot service should construct for A"); - std::fs::write( - repo.main_root.join("shared-content.txt"), - b"identical content", - ) - .expect("A's edit toward the byte-identical T should write"); - let a_tree_t = a_snapshot - .capture_tree() - .expect("capturing A's view of T should succeed"); - assert_eq!( - a_tree_t, tree_t, - "A and B must independently capture byte-identical tree content" - ); - a_snapshot - .pin_tree(&a_id, &a_tree_t) - .expect("A pinning T should succeed"); - let a_t_ref = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, a_tree_t.0); - - let outcome = reconcile_worktree(&repo.main_root, ok_db) - .expect("reconciling A should succeed through the public entrypoint"); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - report.deleted, 0, - "A must retain its T pin: T is a repository-wide durable root via B's historical event, \ - even though A does not durably reference T itself" - ); - assert!( - ref_exists(&repo.main_root, &a_t_ref), - "A's T pin is the last SCE ref protecting T and must survive" - ); - run_git(&repo.main_root, &["cat-file", "-t", &a_tree_t.0]); -} - -#[test] -fn missing_local_required_pin_fails_closed_and_deletes_nothing_even_when_another_worktree_pins_the_tree( -) { - let repo = LinkedTestRepo::new("missing-required-pin-fail-closed"); - let ok_db = || repo.open_db(); - - let a_baseline = coordinate(&repo.main_root, &RuntimeBoundary::Flush, ok_db) - .expect("A's baseline observation should materialize its worktree"); - let a_id = a_baseline.worktree_id.clone(); - let tree_a = a_baseline.observed_tree.clone(); - - let a_snapshot = GitSnapshotService::new(&repo.main_root) - .expect("a snapshot service should construct for A"); - std::fs::write(repo.main_root.join("required-but-unpinned.txt"), b"B") - .expect("the edit toward B should write"); - let tree_b = a_snapshot - .capture_tree() - .expect("capturing B should succeed"); - let db = repo.db(); - seed_event(&db, &a_id.0, 1, &tree_a.0, &tree_b.0); - let pin_ref_for_missing_b = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, tree_b.0); - assert!( - !ref_exists(&repo.main_root, &pin_ref_for_missing_b), - "A's pin for B must be deliberately absent" - ); - - std::fs::write(repo.main_root.join("unrelated.txt"), b"X") - .expect("the unrelated edit should write"); - let tree_x = a_snapshot - .capture_tree() - .expect("capturing X should succeed"); - a_snapshot - .pin_tree(&a_id, &tree_x) - .expect("pinning X should succeed"); - let pin_ref_for_x = format!("refs/sce/mutation-cursor/{}/{}", a_id.0, tree_x.0); - - let b_baseline = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, ok_db) - .expect("the linked worktree's baseline observation should materialize"); - let b_id = b_baseline.worktree_id.clone(); - let b_snapshot = GitSnapshotService::new(&repo.linked_root) - .expect("a snapshot service should construct for the linked worktree"); - b_snapshot - .pin_tree(&b_id, &tree_b) - .expect("the other worktree pinning B should succeed"); - - let outcome = reconcile_worktree(&repo.main_root, ok_db); - match outcome { - Err(ReconcileError::MissingRequiredPins { missing }) => { - assert_eq!( - missing, - vec![tree_b.clone()], - "the missing pin must name exactly B" - ); - } - other => panic!("expected MissingRequiredPins naming B, got {other:?}"), - } - - assert!( - ref_exists(&repo.main_root, &pin_ref_for_x), - "A's X pin must be left untouched by a fail-closed pass" - ); - assert!( - !ref_exists(&repo.main_root, &pin_ref_for_missing_b), - "A's still-absent pin for B must remain absent -- reconciliation never creates or repairs a pin" - ); -} - -#[test] -fn a_malformed_namespace_ref_fails_closed_through_the_public_entrypoint() { - let repo = TestRepo::new("malformed-ref-fail-closed"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let cursor_ref = format!( - "refs/sce/mutation-cursor/{}/{}", - worktree_id.0, baseline.observed_tree.0 - ); - - let snapshot = GitSnapshotService::new(&repo.repo_root) - .expect("a snapshot service should construct for the worktree"); - std::fs::write(repo.repo_root.join("unrelated.txt"), b"unrelated content") - .expect("the unrelated edit should write"); - let some_tree = snapshot - .capture_tree() - .expect("capturing an arbitrary tree should succeed"); - let symbolic_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, some_tree.0); - run_git( - &repo.repo_root, - &["symbolic-ref", &symbolic_ref, &cursor_ref], - ); - - let outcome = reconcile_worktree(&repo.repo_root, ok_db); - match outcome { - Err(ReconcileError::MalformedPin { ref_name, .. }) => { - assert_eq!(ref_name, symbolic_ref); - } - other => panic!("expected MalformedPin for the symbolic ref, got {other:?}"), - } - - assert!( - ref_exists(&repo.repo_root, &cursor_ref), - "the pass must fail closed before deleting anything, including the well-formed cursor pin" - ); - assert_eq!( - run_git(&repo.repo_root, &["symbolic-ref", &symbolic_ref]).trim(), - cursor_ref, - "the malformed symbolic ref itself must be left untouched" - ); -} - -#[test] -fn reconciliation_makes_no_protocol_or_marker_write() { - let repo = TestRepo::new("no-protocol-or-marker-write"); - let ok_db = || repo.open_db(); - let git_dir = resolve_git_dir(&repo.repo_root).expect("git dir should resolve"); - let marker = ExternalTaintMarker::new(&git_dir); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let snapshot = GitSnapshotService::new(&repo.repo_root) - .expect("a snapshot service should construct for the worktree"); - std::fs::write(repo.repo_root.join("orphan.txt"), b"orphan") - .expect("the orphan-producing edit should write"); - let orphan = snapshot - .capture_tree() - .expect("capturing the orphan tree should succeed"); - snapshot - .pin_tree(&worktree_id, &orphan) - .expect("pinning the orphan should succeed"); - - let tables = [ - "mutation_trace_worktrees", - "mutation_trace_scopes", - "mutation_trace_events", - "mutation_trace_processed_events", - "mutation_trace_event_active_scopes", - ]; - let db_before = repo.db(); - let counts_before: Vec = tables - .iter() - .map(|table| row_count(&db_before, table)) - .collect(); - let store_before = MutationTraceStore::new(&db_before); - let worktree_state_before = store_before - .load_worktree(&worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist") - .worktree_state; - let marker_existed_before = marker - .exists() - .expect("marker existence should resolve before the pass"); - assert!( - !marker_existed_before, - "a successful coordinate() must have already cleared the marker before reconciliation runs" - ); - - let outcome = reconcile_worktree(&repo.repo_root, ok_db) - .expect("reconciliation should succeed through the public entrypoint"); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - report.deleted, 1, - "the pass must actually mutate Git refs, not be a trivial no-op" - ); - - let db_after = repo.db(); - let counts_after: Vec = tables - .iter() - .map(|table| row_count(&db_after, table)) - .collect(); - assert_eq!( - counts_before, counts_after, - "every mutation-trace table's row count must be byte-identical after a reconciliation pass" - ); - let store_after = MutationTraceStore::new(&db_after); - let worktree_state_after = store_after - .load_worktree(&worktree_id, None, None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist") - .worktree_state; - assert_eq!( - worktree_state_before, worktree_state_after, - "cursor_tree/revision/tainted/failure_kind/needs_rebaseline must be unchanged by reconciliation" - ); - assert!( - !marker - .exists() - .expect("marker existence should resolve after the pass"), - "reconciliation must not create the external-taint marker" - ); -} - -#[test] -fn reconciliation_deletes_a_stale_ref_without_reclaiming_the_object_through_the_public_api() { - let repo = TestRepo::new("no-object-reclamation-via-public-api"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let snapshot = GitSnapshotService::new(&repo.repo_root) - .expect("a snapshot service should construct for the worktree"); - std::fs::write( - repo.repo_root - .join("only-reachable-through-a-stale-pin.txt"), - b"O", - ) - .expect("the edit toward O should write"); - let tree_o = snapshot.capture_tree().expect("capturing O should succeed"); - snapshot - .pin_tree(&worktree_id, &tree_o) - .expect("pinning O should succeed"); - let o_ref = format!("refs/sce/mutation-cursor/{}/{}", worktree_id.0, tree_o.0); - - run_git(&repo.repo_root, &["cat-file", "-t", &tree_o.0]); - - let outcome = reconcile_worktree(&repo.repo_root, ok_db) - .expect("reconciliation should succeed through the public entrypoint"); - let report = match outcome { - ReconciliationOutcome::Reconciled(report) => report, - ReconciliationOutcome::SkippedNoCheckoutIdentity => { - panic!("expected a Reconciled outcome, got SkippedNoCheckoutIdentity") - } - }; - assert_eq!( - report.deleted, 1, - "O's only SCE ref must be reclaimed as an orphan pin" - ); - assert!( - !ref_exists(&repo.repo_root, &o_ref), - "O's ref must be gone after reconciliation" - ); - - let cat_file_type = run_git(&repo.repo_root, &["cat-file", "-t", &tree_o.0]) - .trim() - .to_string(); - assert_eq!( - cat_file_type, "tree", - "O must still resolve via git cat-file -t immediately after its stale ref is deleted, \ - because reconciliation runs no git gc / git prune / git reflog expire" - ); -} -#[test] -#[allow(clippy::too_many_lines)] -fn an_abandoned_scope_rebaselines_the_successor_start_without_evidence_for_the_gap() { - let repo = TestRepo::new("public-abandon-then-successor-start"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - let tree_a = baseline.observed_tree.clone(); - - let scope_a = ScopeId("scope-dead-execution".to_string()); - let started = coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope_a.clone(), - event: EventId("evt-a-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting scope A should succeed"); - let first_gap_revision = started.revision; - - std::fs::write( - repo.repo_root.join("edited-by-a.txt"), - b"a mutation scope A made but never closed", - ) - .expect("the edit inside scope A should write"); - - let abandoned = abandon_scope(&repo.repo_root, &scope_a, ok_db) - .expect("abandoning the stale scope should succeed"); - let AbandonScopeOutcome::Abandoned { - revision: abandon_revision, - .. - } = abandoned - else { - panic!("expected a durable abandonment, got {abandoned:?}"); - }; - assert_eq!( - abandon_revision, - first_gap_revision + 1, - "abandonment advances the worktree revision by exactly one" - ); - - std::fs::write( - repo.repo_root.join("edited-after-a-died.txt"), - b"a mutation nobody observed a boundary for", - ) - .expect("the unobserved edit after abandonment should write"); - - let scope_b = ScopeId("scope-successor".to_string()); - let successor = coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope_b.clone(), - event: EventId("evt-b-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("the successor Start must rebaseline over the abandoned scope's gap"); - let tree_c = successor.observed_tree.clone(); - assert_ne!( - tree_c, tree_a, - "the working tree must have moved across the abandoned interval" - ); - assert!( - successor.mutation_event.is_none(), - "no evidence may be emitted for the A -> B interval, whose final boundary was never observed" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, Some(&scope_b), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - projection.worktree_state.cursor_tree, tree_c, - "the cursor must sit at the tree observed at Start(B), not at the pre-abandonment tree" - ); - assert!(!projection.worktree_state.needs_rebaseline); - assert!(!projection.worktree_state.tainted); - assert_eq!(projection.worktree_state.failure_kind, FailureKind::Healthy); - - let statuses = store - .load_worktree(&worktree_id, Some(&scope_a), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - statuses.scopes.get(&scope_a).map(|state| state.status), - Some(ScopeStatus::Abandoned), - "the abandoned scope must stay Abandoned across the successor's recovery" - ); - assert_eq!( - projection.scopes.get(&scope_b).map(|state| state.status), - Some(ScopeStatus::Active), - "the successor scope must be Active after its Start" - ); - - assert_eq!( - row_count(&db, "mutation_trace_events"), - 0, - "an interval bounded by an abandonment and a rebaseline can produce no MutationEvent" - ); - for revision in first_gap_revision..=successor.revision { - assert!( - store - .load_mutation_event(&worktree_id, revision) - .expect("loading a mutation event should succeed") - .is_none(), - "revision {revision} spans the unobserved A -> B interval and must carry no evidence" - ); - } -} - -#[test] -fn abandoning_a_stale_scope_leaves_an_unrelated_live_scope_active_through_the_recovery() { - let repo = TestRepo::new("public-abandon-preserves-live-scope"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let stale = ScopeId("scope-stale".to_string()); - let live = ScopeId("scope-live".to_string()); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: stale.clone(), - event: EventId("evt-stale-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting the scope that will go stale should succeed"); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: live.clone(), - event: EventId("evt-live-start".to_string()), - actor_kind: ActorKind::Codex, - provenance: None, - }, - ok_db, - ) - .expect("starting the unrelated live scope should succeed"); - - let abandoned = abandon_scope(&repo.repo_root, &stale, ok_db) - .expect("abandoning the stale scope should succeed"); - assert!( - matches!(abandoned, AbandonScopeOutcome::Abandoned { .. }), - "expected a durable abandonment, got {abandoned:?}" - ); - - std::fs::write( - repo.repo_root.join("edited-by-the-live-scope.txt"), - b"the surviving scope keeps working", - ) - .expect("the live scope's edit should write"); - - let advanced = coordinate( - &repo.repo_root, - &RuntimeBoundary::Advance { - scope: live.clone(), - event: EventId("evt-live-advance".to_string()), - actor_kind: ActorKind::Codex, - }, - ok_db, - ) - .expect("the live scope must still be able to advance after the unrelated abandonment"); - assert!( - advanced.mutation_event.is_none(), - "the needs_rebaseline recovery consumes the ambiguous interval rather than attributing it" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, Some(&live), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - projection.scopes.get(&live).map(|state| state.status), - Some(ScopeStatus::Active), - "abandoning one scope must not abandon an unrelated scope that is legitimately live" - ); - - let stale_projection = store - .load_worktree(&worktree_id, Some(&stale), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - stale_projection - .scopes - .get(&stale) - .map(|state| state.status), - Some(ScopeStatus::Abandoned), - "only the named scope may be abandoned" - ); -} - -#[test] -fn abandoning_a_scope_through_another_worktrees_checkout_is_rejected_without_writing() { - let repo = LinkedTestRepo::new("public-abandon-wrong-checkout"); - let ok_db = || repo.open_db(); - - let main = coordinate(&repo.main_root, &RuntimeBoundary::Flush, ok_db) - .expect("the main worktree should materialize"); - let linked = coordinate(&repo.linked_root, &RuntimeBoundary::Flush, ok_db) - .expect("the linked worktree should materialize"); - assert_ne!(main.worktree_id, linked.worktree_id); - - let scope = ScopeId("scope-on-main".to_string()); - let started = coordinate( - &repo.main_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-main-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting a scope on the main worktree should succeed"); - - let error = abandon_scope(&repo.linked_root, &scope, ok_db) - .expect_err("a scope may only be abandoned through its own checkout"); - match &error { - AbandonScopeError::WorktreeIdentityMismatch { - scope: rejected, - scope_worktree_id, - invoking_worktree_id, - } => { - assert_eq!(rejected, &scope); - assert_eq!(scope_worktree_id, &main.worktree_id); - assert_eq!(invoking_worktree_id, &linked.worktree_id); - } - other => panic!("expected WorktreeIdentityMismatch, got {other:?}"), - } - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let main_projection = store - .load_worktree(&main.worktree_id, Some(&scope), None) - .expect("loading the main worktree row should succeed") - .expect("the main worktree row should exist"); - assert_eq!( - main_projection.worktree_state.revision, started.revision, - "a rejected cross-checkout abandonment may not advance the target worktree's revision" - ); - assert_eq!( - main_projection.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::Active), - "the target scope must stay Active after the rejection" - ); - - let linked_projection = store - .load_worktree(&linked.worktree_id, None, None) - .expect("loading the linked worktree row should succeed") - .expect("the linked worktree row should exist"); - assert_eq!( - linked_projection.worktree_state.revision, linked.revision, - "a rejected cross-checkout abandonment may not advance the invoking worktree's revision" - ); - - let linked_git_dir = resolve_git_dir(&repo.linked_root).expect("linked git dir should resolve"); - assert!( - ExternalTaintMarker::new(&linked_git_dir) - .exists() - .expect("marker existence should resolve"), - "a failed abandonment leaves the invoking worktree's fence armed for the next recovery" - ); - let main_git_dir = resolve_git_dir(&repo.main_root).expect("main git dir should resolve"); - assert!( - !ExternalTaintMarker::new(&main_git_dir) - .exists() - .expect("marker existence should resolve"), - "the rejection touched only the invoking worktree's fence" - ); -} - -#[test] -#[allow(clippy::too_many_lines)] -fn a_real_thread_cas_race_settles_on_the_competitors_terminal_status() { - let repo = TestRepo::new("public-abandon-cas-race"); - let ok_db = || repo.open_db(); - - let baseline = coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - let worktree_id = baseline.worktree_id.clone(); - - let scope = ScopeId("scope-raced".to_string()); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: scope.clone(), - event: EventId("evt-race-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting the raced scope should succeed"); - - let (release_tx, release_rx) = mpsc::channel::<()>(); - let (closed_tx, closed_rx) = mpsc::channel::(); - let competitor_db_path = repo.db_path.clone(); - let competitor_worktree = worktree_id.clone(); - let competitor_scope = scope.clone(); - let competitor = thread::spawn(move || { - release_rx - .recv() - .expect("the abandoning thread should release the competitor"); - let db = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(&competitor_db_path) - .expect("the competing writer should open its own DB handle"); - let store = MutationTraceStore::new(&db); - let boundary = Boundary::Close { - scope: competitor_scope.clone(), - event: EventId("evt-race-close".to_string()), - }; - let scope_ref = boundary_scope(&boundary); - let event_key = boundary_event_key(&boundary); - let projection = store - .load_worktree(&competitor_worktree, scope_ref.as_ref(), event_key.as_ref()) - .expect("the competitor's load should succeed") - .expect("the worktree row should exist"); - let state = projection.into_protocol_state(); - let observed_tree = state - .worktrees - .get(&competitor_worktree) - .expect("the competitor's worktree state should be present") - .cursor_tree - .clone(); - let attempt = AttemptId("attempt-competing-close".to_string()); - let prepared = protocol::prepare(&state, attempt.clone(), boundary, observed_tree); - let outcome = protocol::commit(&prepared, &attempt); - let transition = DurableTransition::between(&state, &outcome.state, &competitor_worktree) - .expect("the competitor's transition should diff") - .expect("a Close must produce a durable transition"); - assert!( - matches!( - store - .commit(&transition) - .expect("the competitor's commit should run"), - CasResult::Applied - ), - "the competitor wins the race while the abandonment is still between load and commit" - ); - let revision = outcome - .state - .worktrees - .get(&competitor_worktree) - .expect("the competitor's committed worktree state should be present") - .revision; - closed_tx - .send(revision) - .expect("the abandoning thread should still be waiting"); - }); - - let mut release_tx = Some(release_tx); - let mut competitor_revision = None; - let settled = abandon_scope_inner(&repo.repo_root, &scope, ok_db, |attempt| { - if attempt == 0 { - release_tx - .take() - .expect("the competitor is released exactly once") - .send(()) - .expect("the competitor thread should still be listening"); - competitor_revision = Some( - closed_rx - .recv() - .expect("the competitor should report its committed revision"), - ); - } - }) - .expect("a lost CAS against a competing Close should settle, not fail"); - let competitor_revision = - competitor_revision.expect("the competitor must have run on the first attempt"); - competitor - .join() - .expect("the competing writer thread should not panic"); - - assert_eq!( - settled, - AbandonScopeOutcome::AlreadyTerminal { - worktree_id: worktree_id.clone(), - scope: scope.clone(), - status: ScopeStatus::Closed, - revision: competitor_revision, - }, - "the retry must settle on the competitor's terminal status rather than abandon again" - ); - - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let projection = store - .load_worktree(&worktree_id, Some(&scope), None) - .expect("loading the worktree row should succeed") - .expect("the worktree row should exist"); - assert_eq!( - projection.scopes.get(&scope).map(|state| state.status), - Some(ScopeStatus::Closed), - "a competitor's durable Close must never be overwritten by a second abandonment" - ); - assert_eq!( - projection.worktree_state.revision, competitor_revision, - "the settled no-op writes nothing, so the revision stays at the competitor's" - ); -} - -#[test] -fn a_relevant_event_behind_128_newer_events_is_never_loaded_or_reconstructed() { - let repo = TestRepo::new("attribution-horizon"); - let worktree_id = - resolve_worktree_id(&repo.repo_root).expect("worktree identity should resolve"); - let snapshot = - GitSnapshotService::new(&repo.repo_root).expect("a snapshot service should build"); - - std::fs::write(repo.repo_root.join("file.rs"), b"one\n").expect("the before edit should write"); - let before = snapshot - .capture_tree() - .expect("capturing the before tree should succeed"); - std::fs::write(repo.repo_root.join("file.rs"), b"one\ntwo\n") - .expect("the after edit should write"); - let after = snapshot - .capture_tree() - .expect("capturing the after tree should succeed"); - - let db = repo.db(); - seed_attribution_event( - &db, - &worktree_id.0, - 1, - &before.0, - &after.0, - "ai_exclusive", - Some("scope-behind-the-horizon"), - ); - for revision in 2..=129 { - seed_attribution_event( - &db, - &worktree_id.0, - revision, - &after.0, - &after.0, - "ineligible_unscoped", - None, - ); - } - - let store = MutationTraceStore::new(&db); - let committed = parse_patch( - "diff --git a/file.rs b/file.rs\n--- a/file.rs\n+++ b/file.rs\n@@ -1,1 +1,2 @@\n one\n+two\n", - None, - ) - .expect("the committed patch should parse"); - - let attribution = resolve_bounded_mutation_attribution( - &store, - &snapshot, - &worktree_id, - &ParsedPatch { files: Vec::new() }, - &committed, - &after, - None, - ); - - assert_eq!( - attribution.inspected_events, 128, - "the 128-event horizon caps inspection at exactly 128 events" - ); - assert_eq!( - attribution.reconstructed_events, 128, - "every inspected no-op event still reconstructs" - ); - assert_eq!( - attribution.loaded_pages, 4, - "the 32/128 constants imply exactly four pages here" - ); - assert_eq!( - attribution.loaded_rows, 128, - "the 129th row is never loaded" - ); - assert!( - attribution.barrier.is_none(), - "exhausting the horizon is not a failure barrier" - ); - assert!( - attribution - .result - .mutation_ai_patch - .files - .iter() - .all(|file| file.hunks.iter().all(|hunk| hunk.lines.is_empty())), - "the relevant event beyond the horizon never contributes AI coverage" - ); - assert!( - !attribution - .result - .unresolved_patch - .files - .iter() - .all(|file| file.hunks.iter().all(|hunk| hunk.lines.is_empty())), - "the committed line stays unresolved because its only match was never inspected" - ); -} - -fn patch_is_empty(patch: &ParsedPatch) -> bool { - patch - .files - .iter() - .all(|file| file.hunks.iter().all(|hunk| hunk.lines.is_empty())) -} - -fn drive_codex_overlap_transition( - label: &str, - closing_scope: &ScopeId, - closing_actor: ActorKind, -) -> (Attribution, ParsedPatch, ParsedPatch) { - let repo = TestRepo::new(label); - let worktree_id = - resolve_worktree_id(&repo.repo_root).expect("worktree identity should resolve"); - let snapshot = - GitSnapshotService::new(&repo.repo_root).expect("a snapshot service should build"); - let ok_db = || repo.open_db(); - - std::fs::write(repo.repo_root.join("file.rs"), b"one\n").expect("the baseline write"); - coordinate(&repo.repo_root, &RuntimeBoundary::Flush, ok_db) - .expect("the baseline observation should materialize the worktree"); - - let codex = ScopeId("codex-a".to_string()); - let claude = ScopeId("claude-c".to_string()); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: codex.clone(), - event: EventId("evt-codex-start".to_string()), - actor_kind: ActorKind::Codex, - provenance: None, - }, - ok_db, - ) - .expect("starting the Codex scope should succeed"); - coordinate( - &repo.repo_root, - &RuntimeBoundary::Start { - scope: claude.clone(), - event: EventId("evt-claude-start".to_string()), - actor_kind: ActorKind::ClaudeCode, - provenance: None, - }, - ok_db, - ) - .expect("starting the Claude scope should succeed"); - - std::fs::write(repo.repo_root.join("file.rs"), b"one\ntwo\n").expect("the mutating write"); - let outcome = coordinate( - &repo.repo_root, - &RuntimeBoundary::Close { - scope: closing_scope.clone(), - event: EventId("evt-close".to_string()), - actor_kind: closing_actor, - }, - ok_db, - ) - .expect("the closing boundary should succeed"); - - let event = outcome - .mutation_event - .expect("the observed tree change should commit a mutation event"); - assert_eq!( - event.active_scopes, - std::collections::BTreeSet::from([codex, claude]), - "the full protocol live set is still recorded on the event" - ); - - let after = snapshot - .capture_tree() - .expect("capturing the committed tree should succeed"); - let db = repo.db(); - let store = MutationTraceStore::new(&db); - let committed = parse_patch( - "diff --git a/file.rs b/file.rs\n--- a/file.rs\n+++ b/file.rs\n@@ -1,1 +1,2 @@\n one\n+two\n", - None, - ) - .expect("the committed patch should parse"); - let attribution = resolve_bounded_mutation_attribution( - &store, - &snapshot, - &worktree_id, - &ParsedPatch { files: Vec::new() }, - &committed, - &after, - None, - ); - - ( - event.attribution, - attribution.result.mutation_ai_patch, - attribution.result.resolved_non_ai_patch, - ) -} - -#[test] -fn an_unconfirmed_codex_overlap_never_reaches_the_mutation_ai_patch() { - let (attribution, ai_patch, non_ai_patch) = drive_codex_overlap_transition( - "codex-unconfirmed-lineage", - &ScopeId("claude-c".to_string()), - ActorKind::ClaudeCode, - ); - - assert_eq!(attribution, Attribution::IneligibleUnscoped); - assert_ne!(attribution, Attribution::AiContended); - assert!( - patch_is_empty(&ai_patch), - "an ambiguous transition must never become AI mutation lineage" - ); - assert!( - !patch_is_empty(&non_ai_patch), - "the line is still resolved, just not as AI" - ); -} - -#[test] -fn a_confirmed_codex_close_overlap_is_contended_and_still_not_ai_lineage() { - let (attribution, ai_patch, non_ai_patch) = drive_codex_overlap_transition( - "codex-confirmed-lineage", - &ScopeId("codex-a".to_string()), - ActorKind::Codex, - ); - - assert_eq!(attribution, Attribution::AiContended); - assert!( - patch_is_empty(&ai_patch), - "only AiExclusive becomes AI mutation lineage" - ); - assert!(!patch_is_empty(&non_ai_patch)); -} diff --git a/cli/src/services/mutation_trace/runtime/worktree_lock.rs b/cli/src/services/mutation_trace/runtime/worktree_lock.rs index 0b509772f..cd7a404f3 100644 --- a/cli/src/services/mutation_trace/runtime/worktree_lock.rs +++ b/cli/src/services/mutation_trace/runtime/worktree_lock.rs @@ -1,5 +1,7 @@ use std::fs::{File, OpenOptions, TryLockError}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{Context, Result}; @@ -11,16 +13,26 @@ const WORKTREE_LOCK_FILE: &str = "mutation-cursor.lock"; const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(100); #[derive(Debug)] -pub struct WorktreeLock { +struct WorktreeLockInner { file: File, - path: PathBuf, - unlock_on_drop: bool, + unlock_on_drop: AtomicBool, +} + +#[derive(Debug)] +pub struct WorktreeLock { + inner: Arc, +} + +#[derive(Clone, Debug)] +pub struct WorktreeLockLease { + _inner: Arc, } #[derive(Debug)] pub enum WorktreeLockError { TimedOut { path: PathBuf, timeout: Duration }, Io(anyhow::Error), + WorkerFailed(tokio::task::JoinError), } impl std::fmt::Display for WorktreeLockError { @@ -32,6 +44,9 @@ impl std::fmt::Display for WorktreeLockError { path.display() ), WorktreeLockError::Io(source) => write!(f, "{source}"), + WorktreeLockError::WorkerFailed(source) => { + write!(f, "Worktree lock acquisition worker failed: {source}") + } } } } @@ -39,11 +54,28 @@ impl std::fmt::Display for WorktreeLockError { impl std::error::Error for WorktreeLockError {} impl WorktreeLock { - pub fn acquire(git_dir: &Path, timeout: Duration) -> Result { - acquire_inner(git_dir, timeout, || {}) + pub async fn acquire_async( + git_dir: &Path, + timeout: Duration, + ) -> Result { + acquire_inner_async(git_dir, timeout, || {}).await } } +pub(super) async fn acquire_inner_async( + git_dir: &Path, + timeout: Duration, + on_contention: F, +) -> Result +where + F: FnOnce() + Send + 'static, +{ + let git_dir = git_dir.to_owned(); + tokio::task::spawn_blocking(move || acquire_inner(&git_dir, timeout, on_contention)) + .await + .map_err(WorktreeLockError::WorkerFailed)? +} + pub(super) fn acquire_inner( git_dir: &Path, timeout: Duration, @@ -82,9 +114,10 @@ where match file.try_lock() { Ok(()) => { return Ok(WorktreeLock { - file, - path: lock_path, - unlock_on_drop: true, + inner: Arc::new(WorktreeLockInner { + file, + unlock_on_drop: AtomicBool::new(true), + }), }); } Err(TryLockError::WouldBlock) => { @@ -110,14 +143,21 @@ where } impl WorktreeLock { - pub(super) fn close_without_unlock(mut self) { - self.unlock_on_drop = false; + #[must_use] + pub(super) fn lease(&self) -> WorktreeLockLease { + WorktreeLockLease { + _inner: Arc::clone(&self.inner), + } + } + + pub(super) fn close_without_unlock(self) { + self.inner.unlock_on_drop.store(false, Ordering::Release); } } -impl Drop for WorktreeLock { +impl Drop for WorktreeLockInner { fn drop(&mut self) { - if self.unlock_on_drop { + if self.unlock_on_drop.load(Ordering::Acquire) { let _ = self.file.unlock(); } } @@ -128,134 +168,6 @@ impl WorktreeLock { #[must_use] pub(crate) fn as_raw_fd(&self) -> std::os::unix::io::RawFd { use std::os::unix::io::AsRawFd; - self.file.as_raw_fd() - } -} - -#[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicU64, Ordering}; - use std::sync::mpsc; - use std::thread; - - use super::*; - - static NEXT_TEST_GIT_DIR_ID: AtomicU64 = AtomicU64::new(0); - - fn unique_test_git_dir(label: &str) -> PathBuf { - let id = NEXT_TEST_GIT_DIR_ID.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "sce-worktree-lock-{label}-{}-{id}", - std::process::id() - )) - } - - fn remove_test_git_dir(git_dir: &Path) { - let _ = std::fs::remove_dir_all(git_dir); - } - - #[test] - fn a_second_acquirer_blocks_until_the_first_releases() { - let git_dir = unique_test_git_dir("contention"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let first = WorktreeLock::acquire(&git_dir, Duration::from_secs(5)) - .expect("first acquirer should succeed immediately"); - - let (contention_tx, contention_rx) = mpsc::channel(); - let (result_tx, result_rx) = mpsc::channel(); - let git_dir_clone = git_dir.clone(); - let handle = thread::spawn(move || { - let result = acquire_inner(&git_dir_clone, Duration::from_secs(5), || { - contention_tx - .send(()) - .expect("contention signal channel should still be open"); - }); - result_tx - .send(()) - .expect("result signal channel should still be open"); - result - }); - - contention_rx - .recv_timeout(Duration::from_secs(5)) - .expect("worker should observe OS-level lock contention (TryLockError::WouldBlock)"); - - assert!( - result_rx.recv_timeout(Duration::from_millis(300)).is_err(), - "second acquirer should not succeed while the first still holds the lock" - ); - - drop(first); - - result_rx - .recv_timeout(Duration::from_secs(5)) - .expect("second acquirer should complete once the first releases the lock"); - - let result = handle - .join() - .expect("second acquirer thread should not panic"); - assert!( - result.is_ok(), - "second acquirer should succeed once the first releases the lock" - ); - - remove_test_git_dir(&git_dir); - } - - #[test] - fn distinct_worktree_paths_do_not_contend() { - let git_dir_a = unique_test_git_dir("distinct-a"); - let git_dir_b = unique_test_git_dir("distinct-b"); - std::fs::create_dir_all(&git_dir_a).expect("git dir a should be created"); - std::fs::create_dir_all(&git_dir_b).expect("git dir b should be created"); - - let lock_a = WorktreeLock::acquire(&git_dir_a, Duration::from_millis(200)) - .expect("lock on distinct path a should succeed"); - let lock_b = WorktreeLock::acquire(&git_dir_b, Duration::from_millis(200)) - .expect("lock on distinct path b should succeed independently of path a"); - - drop(lock_a); - drop(lock_b); - remove_test_git_dir(&git_dir_a); - remove_test_git_dir(&git_dir_b); - } - - #[test] - fn acquire_times_out_with_a_distinct_matchable_error_when_still_held() { - let git_dir = unique_test_git_dir("timeout"); - std::fs::create_dir_all(&git_dir).expect("git dir should be created"); - - let holder = WorktreeLock::acquire(&git_dir, Duration::from_secs(5)) - .expect("first acquirer should succeed immediately"); - - let result = WorktreeLock::acquire(&git_dir, Duration::from_millis(250)); - match result { - Err(WorktreeLockError::TimedOut { timeout, .. }) => { - assert_eq!(timeout, Duration::from_millis(250)); - } - other => panic!("expected a distinct TimedOut error, got {other:?}"), - } - - drop(holder); - remove_test_git_dir(&git_dir); - } - - #[test] - fn a_leftover_lock_file_with_no_active_os_lock_does_not_block_a_new_acquirer() { - let git_dir = unique_test_git_dir("stale-file"); - let runtime_dir = git_dir.join(SCE_RUNTIME_DIR); - std::fs::create_dir_all(&runtime_dir).expect("runtime dir should be created"); - - let lock_path = runtime_dir.join(WORKTREE_LOCK_FILE); - std::fs::write(&lock_path, b"leftover").expect("leftover lock file should be writable"); - - let result = WorktreeLock::acquire(&git_dir, Duration::from_millis(200)); - assert!( - result.is_ok(), - "a lock file with no active OS lock held against it must not block a new acquirer" - ); - - remove_test_git_dir(&git_dir); + self.inner.file.as_raw_fd() } } diff --git a/cli/src/services/mutation_trace/store.rs b/cli/src/services/mutation_trace/store.rs index 2b5ce05f7..8bdace440 100644 --- a/cli/src/services/mutation_trace/store.rs +++ b/cli/src/services/mutation_trace/store.rs @@ -194,6 +194,10 @@ pub fn encode_boundary_kind(kind: BoundaryKind) -> &'static str { } /// Decodes a [`BoundaryKind`] from `mutation_trace_events.boundary_kind`. +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] pub fn decode_boundary_kind(value: &str) -> Result { match value { "start" => Ok(BoundaryKind::Start), @@ -216,6 +220,10 @@ const SELECT_SCOPE_PROVENANCE_SQL: &str = "SELECT scope_id, session_id, model_id FROM mutation_trace_scope_provenance WHERE scope_id = ?1"; const SELECT_PROCESSED_EVENT_SQL: &str = "SELECT 1 FROM mutation_trace_processed_events WHERE scope_id = ?1 AND event_id = ?2"; +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] const SELECT_MUTATION_EVENT_SQL: &str = "SELECT before_tree, after_tree, tainted, failure_kind, attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id FROM mutation_trace_events WHERE worktree_id = ?1 AND revision = ?2"; @@ -236,21 +244,18 @@ const SELECT_LATEST_MUTATION_EVENT_REVISION_SQL: &str = "SELECT revision FROM mu WHERE worktree_id = ?1 ORDER BY revision DESC LIMIT 1"; +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] const SELECT_MUTATION_EVENT_ACTIVE_SCOPES_SQL: &str = "SELECT scope_id FROM mutation_trace_event_active_scopes WHERE worktree_id = ?1 AND revision = ?2"; -/// One worktree's complete durable tree root set — its cursor tree plus the -/// `before_tree` / `after_tree` of every historical `mutation_trace_events` -/// row — as a single `UNION` statement so the whole set is read from one -/// database snapshot, never assembled from independent `SELECT`s. const SELECT_TREE_ROOTS_BY_WORKTREE_SQL: &str = "SELECT cursor_tree AS tree FROM mutation_trace_worktrees WHERE worktree_id = ?1 UNION SELECT before_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1 UNION SELECT after_tree AS tree FROM mutation_trace_events WHERE worktree_id = ?1"; -/// The same three `TreeId` columns unioned across **every** worktree in the -/// repository, in one statement / one snapshot — the reconciler's -/// repository-wide retention set. const SELECT_ALL_TREE_ROOTS_SQL: &str = "SELECT cursor_tree AS tree FROM mutation_trace_worktrees UNION SELECT before_tree AS tree FROM mutation_trace_events @@ -523,11 +528,11 @@ pub struct ScopeProvenance { } pub struct MutationTraceStore<'a> { - db: &'a RepositoryAgentTraceDb, + db: &'a mut RepositoryAgentTraceDb, } impl<'a> MutationTraceStore<'a> { - pub fn new(db: &'a RepositoryAgentTraceDb) -> Self { + pub fn new(db: &'a mut RepositoryAgentTraceDb) -> Self { Self { db } } @@ -535,15 +540,21 @@ impl<'a> MutationTraceStore<'a> { /// healthy, not tainted, not needing rebaseline, with `cursor_tree` set to /// `initial_tree`. A no-op when the worktree row already exists — an /// existing cursor, revision, or failure state is never overwritten. - pub fn initialize_worktree(&self, worktree: &WorktreeId, initial_tree: &TreeId) -> Result<()> { - self.db.execute_idempotent_write( - INSERT_WORKTREE_IF_ABSENT_SQL, - ( - worktree.0.as_str(), - initial_tree.0.as_str(), - encode_revision(0).as_slice(), - ), - )?; + pub async fn initialize_worktree( + &self, + worktree: &WorktreeId, + initial_tree: &TreeId, + ) -> Result<()> { + self.db + .execute_idempotent_write( + INSERT_WORKTREE_IF_ABSENT_SQL, + ( + worktree.0.as_str(), + initial_tree.0.as_str(), + encode_revision(0).as_slice(), + ), + ) + .await?; Ok(()) } @@ -563,28 +574,30 @@ impl<'a> MutationTraceStore<'a> { /// existing one: an existing scope whose stored `worktree_id` has no /// worktree row is never returned as valid merely because it matches the /// arguments. - pub fn register_scope( + pub async fn register_scope( &self, scope: &ScopeId, worktree: &WorktreeId, actor_kind: ActorKind, ) -> Result { - if self.load_worktree_state(worktree)?.is_none() { + if self.load_worktree_state(worktree).await?.is_none() { bail!( "cannot register scope {scope:?}: worktree {worktree:?} has no mutation_trace_worktrees row" ); } - self.db.execute_idempotent_write( - INSERT_SCOPE_IF_ABSENT_SQL, - ( - scope.0.as_str(), - worktree.0.as_str(), - encode_actor_kind(actor_kind), - ), - )?; + self.db + .execute_idempotent_write( + INSERT_SCOPE_IF_ABSENT_SQL, + ( + scope.0.as_str(), + worktree.0.as_str(), + encode_actor_kind(actor_kind), + ), + ) + .await?; - let scope_state = self.load_scope(scope)?.ok_or_else(|| { + let scope_state = self.load_scope(scope).await?.ok_or_else(|| { anyhow::anyhow!("scope {scope:?} has no row immediately after register_scope insert") })?; @@ -626,7 +639,7 @@ impl<'a> MutationTraceStore<'a> { /// already exists; the lookup never references a `worktree_id` column, /// since `mutation_trace_processed_events` has none. This method never /// queries `mutation_trace_events`. - pub fn load_worktree( + pub async fn load_worktree( &self, worktree: &WorktreeId, scope: Option<&ScopeId>, @@ -634,15 +647,15 @@ impl<'a> MutationTraceStore<'a> { ) -> Result> { let effective_scope = effective_referenced_scope(scope, event_key)?; - let Some(worktree_state) = self.load_worktree_state(worktree)? else { + let Some(worktree_state) = self.load_worktree_state(worktree).await? else { return Ok(None); }; - let mut scopes = self.load_active_scopes(worktree)?; + let mut scopes = self.load_active_scopes(worktree).await?; if let Some(effective_scope_id) = effective_scope { if !scopes.contains_key(effective_scope_id) { - let scope_state = self.load_scope(effective_scope_id)?.ok_or_else(|| { + let scope_state = self.load_scope(effective_scope_id).await?.ok_or_else(|| { anyhow::anyhow!( "effective referenced scope {effective_scope_id:?} has no mutation_trace_scopes row" ) @@ -660,7 +673,7 @@ impl<'a> MutationTraceStore<'a> { } let processed_events = match event_key { - Some(event_key) if self.processed_event_exists(event_key)? => { + Some(event_key) if self.processed_event_exists(event_key).await? => { let mut processed_events = BTreeSet::new(); processed_events.insert(event_key.clone()); processed_events @@ -680,24 +693,33 @@ impl<'a> MutationTraceStore<'a> { /// revision)`, decoding its full `Attribution` and `Boundary`, or `None` /// when no such row exists. Never called from `load_worktree` or from /// any hook-boundary path. - pub fn load_mutation_event( + #[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" + )] + pub async fn load_mutation_event( &self, worktree: &WorktreeId, revision: u64, ) -> Result> { let revision_blob = encode_revision(revision); - let rows = self.db.query_map( - SELECT_MUTATION_EVENT_SQL, - (worktree.0.as_str(), revision_blob.as_slice()), - mutation_event_row_from_turso, - )?; + let rows = self + .db + .query_map( + SELECT_MUTATION_EVENT_SQL, + (worktree.0.as_str(), revision_blob.as_slice()), + mutation_event_row_from_turso, + ) + .await?; let Some(row) = rows.into_iter().next() else { return Ok(None); }; - let active_scopes = self.load_mutation_event_active_scopes(worktree, &revision_blob)?; + let active_scopes = self + .load_mutation_event_active_scopes(worktree, &revision_blob) + .await?; Ok(Some(MutationEvent { worktree_id: worktree.clone(), @@ -725,7 +747,7 @@ impl<'a> MutationTraceStore<'a> { /// /// This is a read-only cold path. It does not load active scopes, processed /// events, boundary data, or any timestamp column. - pub fn load_mutation_event_page( + pub async fn load_mutation_event_page( &self, worktree: &WorktreeId, revision_cursor: Option, @@ -735,88 +757,70 @@ impl<'a> MutationTraceStore<'a> { let rows = match revision_cursor { Some(cursor) => { let cursor_blob = encode_revision(cursor); - self.db.query_map( - SELECT_MUTATION_EVENT_PAGE_AFTER_SQL, - ( - worktree.0.as_str(), - cursor_blob.as_slice(), - limit_as_i64(limit), - ), - mutation_event_page_row_from_turso, - )? + self.db + .query_map( + SELECT_MUTATION_EVENT_PAGE_AFTER_SQL, + ( + worktree.0.as_str(), + cursor_blob.as_slice(), + limit_as_i64(limit), + ), + mutation_event_page_row_from_turso, + ) + .await? + } + None => { + self.db + .query_map( + SELECT_MUTATION_EVENT_PAGE_SQL, + (worktree.0.as_str(), limit_as_i64(limit)), + mutation_event_page_row_from_turso, + ) + .await? } - None => self.db.query_map( - SELECT_MUTATION_EVENT_PAGE_SQL, - (worktree.0.as_str(), limit_as_i64(limit)), - mutation_event_page_row_from_turso, - )?, }; Ok(rows) } - pub fn latest_mutation_event_revision(&self, worktree: &WorktreeId) -> Result> { - let rows = self.db.query_map( - SELECT_LATEST_MUTATION_EVENT_REVISION_SQL, - (worktree.0.as_str(),), - |row| { - let blob: Vec = row - .get(0) - .context("failed to read mutation_trace_events.revision")?; - decode_revision(&blob) - }, - )?; + pub async fn latest_mutation_event_revision( + &self, + worktree: &WorktreeId, + ) -> Result> { + let rows = self + .db + .query_map( + SELECT_LATEST_MUTATION_EVENT_REVISION_SQL, + (worktree.0.as_str(),), + |row| { + let blob: Vec = row + .get(0) + .context("failed to read mutation_trace_events.revision")?; + decode_revision(&blob) + }, + ) + .await?; Ok(rows.into_iter().next()) } - /// Reads `worktree`'s complete durable tree root set: its - /// `mutation_trace_worktrees.cursor_tree`, plus the `before_tree` and - /// `after_tree` of every `mutation_trace_events` row for `worktree`, - /// deduplicated. Returns an empty set (not an error) when `worktree` has - /// no durable row at all. - /// - /// Read-only, cold path — never called from `load_worktree` or any - /// hook-boundary path, exactly like [`MutationTraceStore::load_mutation_event`]. - /// It reads only the three `TreeId` columns above: never - /// `mutation_trace_scopes` / `mutation_trace_processed_events` / - /// `mutation_trace_event_active_scopes`, never another worktree's trees, - /// and never transient `AttemptState` / `external_taint`. - /// - /// The whole set is produced by **one** SQL statement (a `UNION` of the - /// three columns) through **one** `query_map` call, so a concurrent - /// mutation-cursor commit — which atomically moves `cursor_tree` from `T` - /// to `X` and inserts `MutationEvent { before_tree = T, after_tree = X }` - /// in the same transaction — cannot expose a torn root set that omits `T`: - /// the single statement observes either the pre-commit snapshot - /// (`cursor_tree` still contains `T`) or the post-commit snapshot - /// (`before_tree` contains `T`). - pub fn load_tree_roots(&self, worktree: &WorktreeId) -> Result> { - let rows = self.db.query_map( - SELECT_TREE_ROOTS_BY_WORKTREE_SQL, - (worktree.0.as_str(),), - tree_root_row_from_turso, - )?; + pub async fn load_tree_roots(&self, worktree: &WorktreeId) -> Result> { + let rows = self + .db + .query_map( + SELECT_TREE_ROOTS_BY_WORKTREE_SQL, + (worktree.0.as_str(),), + tree_root_row_from_turso, + ) + .await?; Ok(rows.into_iter().collect()) } - /// Reads the repository-wide durable tree root set: the union of - /// `mutation_trace_worktrees.cursor_tree`, `mutation_trace_events.before_tree`, - /// and `mutation_trace_events.after_tree` across **every** worktree, - /// deduplicated. Returns an empty set (not an error) for a repository with - /// no mutation-cursor rows. - /// - /// This is the reconciler's retention set: linked worktrees share one Git - /// object database, so a ref owned by worktree `A` may be the last SCE ref - /// protecting a tree that only worktree `B` durably requires. Read-only, - /// cold path, and — like [`MutationTraceStore::load_tree_roots`] — one SQL - /// statement through one `query_map` call, so it cannot tear across a - /// concurrent atomic `cursor T -> X` + `event T -> X` commit on another - /// worktree. - pub fn load_all_tree_roots(&self) -> Result> { + pub async fn load_all_tree_roots(&self) -> Result> { let rows = self .db - .query_map(SELECT_ALL_TREE_ROOTS_SQL, (), tree_root_row_from_turso)?; + .query_map(SELECT_ALL_TREE_ROOTS_SQL, (), tree_root_row_from_turso) + .await?; Ok(rows.into_iter().collect()) } @@ -839,38 +843,43 @@ impl<'a> MutationTraceStore<'a> { /// Comparing the two is the caller's decision, since the same row is a /// legitimate read from its owning worktree and a cross-worktree reference /// from any other. - pub fn load_scope(&self, scope_id: &ScopeId) -> Result> { - let rows = self.db.query_map( - SELECT_SCOPE_BY_ID_SQL, - (scope_id.0.as_str(),), - scope_row_from_turso, - )?; + pub async fn load_scope(&self, scope_id: &ScopeId) -> Result> { + let rows = self + .db + .query_map( + SELECT_SCOPE_BY_ID_SQL, + (scope_id.0.as_str(),), + scope_row_from_turso, + ) + .await?; Ok(rows.into_iter().next().map(|(_, scope_state)| scope_state)) } - pub fn register_scope_provenance( + pub async fn register_scope_provenance( &self, provenance: &ScopeProvenance, ) -> Result { - if self.load_scope(&provenance.scope_id)?.is_none() { + if self.load_scope(&provenance.scope_id).await?.is_none() { bail!( "cannot register provenance for scope {:?}: it has no mutation_trace_scopes row", provenance.scope_id ); } - self.db.execute_idempotent_write( - INSERT_SCOPE_PROVENANCE_IF_ABSENT_SQL, - ( - provenance.scope_id.0.as_str(), - provenance.session_id.as_str(), - provenance.model_id.as_deref(), - ), - )?; + self.db + .execute_idempotent_write( + INSERT_SCOPE_PROVENANCE_IF_ABSENT_SQL, + ( + provenance.scope_id.0.as_str(), + provenance.session_id.as_str(), + provenance.model_id.as_deref(), + ), + ) + .await?; let stored = self - .load_scope_provenance(&provenance.scope_id)? + .load_scope_provenance(&provenance.scope_id).await? .ok_or_else(|| { anyhow::anyhow!( "scope {:?} has no provenance row immediately after register_scope_provenance insert", @@ -890,64 +899,89 @@ impl<'a> MutationTraceStore<'a> { Ok(stored) } - pub fn load_scope_provenance(&self, scope_id: &ScopeId) -> Result> { - let rows = self.db.query_map( - SELECT_SCOPE_PROVENANCE_SQL, - (scope_id.0.as_str(),), - scope_provenance_row_from_turso, - )?; + pub async fn load_scope_provenance( + &self, + scope_id: &ScopeId, + ) -> Result> { + let rows = self + .db + .query_map( + SELECT_SCOPE_PROVENANCE_SQL, + (scope_id.0.as_str(),), + scope_provenance_row_from_turso, + ) + .await?; Ok(rows.into_iter().next()) } - fn load_worktree_state(&self, worktree: &WorktreeId) -> Result> { - let rows = self.db.query_map( - SELECT_WORKTREE_SQL, - (worktree.0.as_str(),), - worktree_state_row_from_turso, - )?; + async fn load_worktree_state(&self, worktree: &WorktreeId) -> Result> { + let rows = self + .db + .query_map( + SELECT_WORKTREE_SQL, + (worktree.0.as_str(),), + worktree_state_row_from_turso, + ) + .await?; Ok(rows.into_iter().next()) } - fn load_active_scopes(&self, worktree: &WorktreeId) -> Result> { - let rows = self.db.query_map( - SELECT_SCOPES_BY_WORKTREE_AND_STATUS_SQL, - ( - worktree.0.as_str(), - encode_scope_status(ScopeStatus::Active), - ), - scope_row_from_turso, - )?; + async fn load_active_scopes( + &self, + worktree: &WorktreeId, + ) -> Result> { + let rows = self + .db + .query_map( + SELECT_SCOPES_BY_WORKTREE_AND_STATUS_SQL, + ( + worktree.0.as_str(), + encode_scope_status(ScopeStatus::Active), + ), + scope_row_from_turso, + ) + .await?; Ok(rows.into_iter().collect()) } - fn processed_event_exists(&self, event_key: &EventKey) -> Result { - let rows = self.db.query_map( - SELECT_PROCESSED_EVENT_SQL, - (event_key.scope_id.0.as_str(), event_key.event_id.0.as_str()), - |row| row.get::(0).map_err(Into::into), - )?; + async fn processed_event_exists(&self, event_key: &EventKey) -> Result { + let rows = self + .db + .query_map( + SELECT_PROCESSED_EVENT_SQL, + (event_key.scope_id.0.as_str(), event_key.event_id.0.as_str()), + |row| row.get::(0).map_err(Into::into), + ) + .await?; Ok(!rows.is_empty()) } - fn load_mutation_event_active_scopes( + #[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" + )] + async fn load_mutation_event_active_scopes( &self, worktree: &WorktreeId, revision_blob: &[u8], ) -> Result> { - let rows = self.db.query_map( - SELECT_MUTATION_EVENT_ACTIVE_SCOPES_SQL, - (worktree.0.as_str(), revision_blob), - |row| row.get::(0).map(ScopeId).map_err(Into::into), - )?; + let rows = self + .db + .query_map( + SELECT_MUTATION_EVENT_ACTIVE_SCOPES_SQL, + (worktree.0.as_str(), revision_blob), + |row| row.get::(0).map(ScopeId).map_err(Into::into), + ) + .await?; Ok(rows.into_iter().collect()) } - pub fn commit(&self, transition: &DurableTransition) -> Result { + pub async fn commit(&mut self, transition: &DurableTransition) -> Result { let expected_revision_blob = encode_revision(transition.expected_revision); let next_revision_blob = encode_revision(transition.next_worktree_state.revision); @@ -1026,12 +1060,15 @@ impl<'a> MutationTraceStore<'a> { } } - let applied = self.db.execute_transactional_cas_batch( - "commit mutation-trace durable transition", - "reload the worktree and retry the transition", - &guard, - &statements, - )?; + let applied = self + .db + .execute_transactional_cas_batch( + "commit mutation-trace durable transition", + "reload the worktree and retry the transition", + &guard, + &statements, + ) + .await?; Ok(if applied { CasResult::Applied @@ -1217,6 +1254,10 @@ fn mutation_event_page_row_from_turso(row: &turso::Row) -> Result, } +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] fn mutation_event_row_from_turso(row: &turso::Row) -> Result { let before_tree: String = row .get(0) @@ -1288,6 +1333,10 @@ fn reconstruct_attribution(kind: AttributionKind, scope_id: Option) -> R } } +#[allow( + dead_code, + reason = "single-event loader has no production reader; attribution history uses load_mutation_event_page" +)] fn reconstruct_boundary( kind: BoundaryKind, worktree: &WorktreeId, @@ -1320,3436 +1369,3 @@ fn reconstruct_boundary( } } } - -#[cfg(test)] -mod tests { - use std::thread; - - use super::*; - use crate::services::agent_trace_db::repository::pre_health_invariant_fixture::seed_pre_health_invariant_fixture; - use crate::services::mutation_trace::protocol::{ - abandon, commit, database_failure, prepare, recover, taint, - }; - use crate::services::mutation_trace::types::{AttemptId, EventId, ScopeId}; - - #[test] - fn revision_round_trips_at_boundary_values() { - for revision in [0u64, 1, i64::MAX as u64, (i64::MAX as u64) + 1, u64::MAX] { - let encoded = encode_revision(revision); - assert_eq!(encoded.len(), 8); - assert_eq!(decode_revision(&encoded).unwrap(), revision); - } - } - - #[test] - fn decode_revision_rejects_wrong_length() { - assert!(decode_revision(&[0u8; 7]).is_err()); - assert!(decode_revision(&[0u8; 9]).is_err()); - } - - #[test] - fn actor_kind_round_trips_every_variant() { - for actor_kind in [ - ActorKind::ClaudeCode, - ActorKind::Codex, - ActorKind::OpenCode, - ActorKind::Pi, - ] { - let encoded = encode_actor_kind(actor_kind); - assert_eq!(decode_actor_kind(encoded).unwrap(), actor_kind); - } - } - - #[test] - fn decode_actor_kind_rejects_unknown_value() { - assert!(decode_actor_kind("unknown").is_err()); - } - - #[test] - fn failure_kind_round_trips_every_variant() { - for failure_kind in [FailureKind::Healthy, FailureKind::SnapshotFailure] { - let encoded = encode_failure_kind(failure_kind); - assert_eq!(decode_failure_kind(encoded).unwrap(), failure_kind); - } - } - - #[test] - fn decode_failure_kind_rejects_unknown_value() { - assert!(decode_failure_kind("unknown").is_err()); - } - - #[test] - fn scope_status_round_trips_every_variant() { - for status in [ - ScopeStatus::NeverSeen, - ScopeStatus::Active, - ScopeStatus::Closed, - ScopeStatus::Abandoned, - ] { - let encoded = encode_scope_status(status); - assert_eq!(decode_scope_status(encoded).unwrap(), status); - } - } - - #[test] - fn decode_scope_status_rejects_unknown_value() { - assert!(decode_scope_status("unknown").is_err()); - } - - #[test] - fn attribution_kind_round_trips_every_variant() { - let ineligible = Attribution::IneligibleUnscoped; - let exclusive = Attribution::AiExclusive(ScopeId("scope-1".to_string())); - let contended = Attribution::AiContended; - - for attribution in [&ineligible, &exclusive, &contended] { - let kind = attribution_kind(attribution); - let encoded = encode_attribution_kind(kind); - assert_eq!(decode_attribution_kind(encoded).unwrap(), kind); - } - - assert_eq!(attribution_kind(&exclusive), AttributionKind::AiExclusive); - } - - #[test] - fn decode_attribution_kind_rejects_unknown_value() { - assert!(decode_attribution_kind("unknown").is_err()); - } - - #[test] - fn boundary_kind_round_trips_every_variant() { - let start = Boundary::Start { - scope: ScopeId("scope-1".to_string()), - event: EventId("event-1".to_string()), - }; - let advance = Boundary::Advance { - scope: ScopeId("scope-1".to_string()), - event: EventId("event-2".to_string()), - }; - let close = Boundary::Close { - scope: ScopeId("scope-1".to_string()), - event: EventId("event-3".to_string()), - }; - let flush = Boundary::Flush { - worktree: crate::services::mutation_trace::types::WorktreeId("wt-1".to_string()), - }; - - for boundary in [&start, &advance, &close, &flush] { - let kind = boundary_kind(boundary); - let encoded = encode_boundary_kind(kind); - assert_eq!(decode_boundary_kind(encoded).unwrap(), kind); - } - } - - #[test] - fn decode_boundary_kind_rejects_unknown_value() { - assert!(decode_boundary_kind("unknown").is_err()); - } - - struct TestDbPath { - _temp_dir: tempfile::TempDir, - path: std::path::PathBuf, - } - - impl TestDbPath { - fn path(&self) -> &std::path::Path { - &self.path - } - } - - fn test_db_path(label: &str) -> TestDbPath { - let temp_dir = tempfile::Builder::new() - .prefix(&format!("sce-mutation-trace-store-{label}-")) - .tempdir() - .expect("test temp directory should be created"); - - let path = temp_dir.path().join("agent-trace.db"); - - TestDbPath { - _temp_dir: temp_dir, - path, - } - } - - fn insert_worktree(db: &RepositoryAgentTraceDb, worktree_id: &str, revision: u64) { - db.execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES (?1, 'tree-0', ?2, 0, 'healthy', 0)", - (worktree_id, encode_revision(revision).as_slice()), - ) - .expect("worktree insert should succeed"); - } - - fn insert_scope( - db: &RepositoryAgentTraceDb, - scope_id: &str, - worktree_id: &str, - status: ScopeStatus, - ) { - db.execute( - "INSERT INTO mutation_trace_scopes (scope_id, worktree_id, actor_kind, status) - VALUES (?1, ?2, 'claude_code', ?3)", - (scope_id, worktree_id, encode_scope_status(status)), - ) - .expect("scope insert should succeed"); - } - - fn insert_processed_event(db: &RepositoryAgentTraceDb, scope_id: &str, event_id: &str) { - db.execute( - "INSERT INTO mutation_trace_processed_events (scope_id, event_id) VALUES (?1, ?2)", - (scope_id, event_id), - ) - .expect("processed-event insert should succeed"); - } - - fn insert_active_scope( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - scope_id: &str, - ) { - db.execute( - "INSERT INTO mutation_trace_event_active_scopes (worktree_id, revision, scope_id) - VALUES (?1, ?2, ?3)", - (worktree_id, encode_revision(revision).as_slice(), scope_id), - ) - .expect("active-scope insert should succeed"); - } - - #[allow(clippy::too_many_arguments)] - fn insert_mutation_event( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - before_tree: &str, - after_tree: &str, - attribution_kind: &str, - attribution_scope_id: Option<&str>, - boundary_kind: &str, - boundary_scope_id: Option<&str>, - boundary_event_id: Option<&str>, - active_scopes: &[&str], - ) { - let revision_blob = encode_revision(revision); - - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', ?5, ?6, ?7, ?8, ?9)", - ( - worktree_id, - revision_blob.as_slice(), - before_tree, - after_tree, - attribution_kind, - attribution_scope_id, - boundary_kind, - boundary_scope_id, - boundary_event_id, - ), - ) - .expect("mutation event insert should succeed"); - - for scope_id in active_scopes { - db.execute( - "INSERT INTO mutation_trace_event_active_scopes (worktree_id, revision, scope_id) - VALUES (?1, ?2, ?3)", - (worktree_id, revision_blob.as_slice(), *scope_id), - ) - .expect("active-scope insert should succeed"); - } - } - - #[test] - fn load_worktree_returns_none_for_a_missing_worktree() { - let db_fixture = test_db_path("missing-worktree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - let projection = store - .load_worktree(&WorktreeId("wt-missing".to_string()), None, None) - .expect("load_worktree should succeed"); - assert!(projection.is_none()); - } - - #[test] - fn load_worktree_with_no_scope_or_event_key_loads_only_active_scopes() { - let db_fixture = test_db_path("case-1-active-only"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 5); - insert_scope(&db, "scope-active", "wt-1", ScopeStatus::Active); - insert_scope(&db, "scope-closed", "wt-1", ScopeStatus::Closed); - - let projection = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, None) - .expect("load_worktree should succeed") - .expect("worktree should exist"); - - assert_eq!(projection.worktree_id, WorktreeId("wt-1".to_string())); - assert_eq!(projection.worktree_state.revision, 5); - assert_eq!( - projection.scopes.keys().collect::>(), - vec![&ScopeId("scope-active".to_string())] - ); - assert!(projection.processed_events.is_empty()); - } - - #[test] - fn load_worktree_with_explicit_scope_includes_it_regardless_of_status() { - let db_fixture = test_db_path("case-2-explicit-scope"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-closed", "wt-1", ScopeStatus::Closed); - - let projection = store - .load_worktree( - &WorktreeId("wt-1".to_string()), - Some(&ScopeId("scope-closed".to_string())), - None, - ) - .expect("load_worktree should succeed") - .expect("worktree should exist"); - - assert_eq!( - projection.scopes.get(&ScopeId("scope-closed".to_string())), - Some(&ScopeState { - status: ScopeStatus::Closed, - actor_kind: ActorKind::ClaudeCode, - worktree_id: WorktreeId("wt-1".to_string()), - }) - ); - } - - #[test] - fn load_worktree_with_explicit_scope_on_another_worktree_errors() { - let db_fixture = test_db_path("case-2-wrong-worktree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_worktree(&db, "wt-2", 0); - insert_scope(&db, "scope-1", "wt-2", ScopeStatus::Active); - - let error = store - .load_worktree( - &WorktreeId("wt-1".to_string()), - Some(&ScopeId("scope-1".to_string())), - None, - ) - .expect_err("scope belonging to another worktree should error"); - assert!(error.to_string().contains("scope-1")); - } - - #[test] - fn load_worktree_with_explicit_missing_scope_errors() { - let db_fixture = test_db_path("case-2-missing-scope"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - - let error = store - .load_worktree( - &WorktreeId("wt-1".to_string()), - Some(&ScopeId("scope-missing".to_string())), - None, - ) - .expect_err("missing effective scope should error"); - assert!(error.to_string().contains("scope-missing")); - } - - #[test] - fn load_worktree_with_only_event_key_loads_its_scope_and_replay_row() { - let db_fixture = test_db_path("case-3-event-key-only"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::NeverSeen); - insert_processed_event(&db, "scope-1", "event-1"); - - let event_key = EventKey { - scope_id: ScopeId("scope-1".to_string()), - event_id: EventId("event-1".to_string()), - }; - - let projection = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, Some(&event_key)) - .expect("load_worktree should succeed") - .expect("worktree should exist"); - - assert_eq!( - projection - .scopes - .get(&ScopeId("scope-1".to_string())) - .map(|s| s.status), - Some(ScopeStatus::NeverSeen) - ); - assert_eq!( - projection.processed_events, - [event_key].into_iter().collect() - ); - } - - #[test] - fn load_worktree_with_event_key_scope_on_another_worktree_errors() { - let db_fixture = test_db_path("case-3-wrong-worktree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_worktree(&db, "wt-2", 0); - insert_scope(&db, "scope-1", "wt-2", ScopeStatus::Active); - - let event_key = EventKey { - scope_id: ScopeId("scope-1".to_string()), - event_id: EventId("event-1".to_string()), - }; - - let error = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, Some(&event_key)) - .expect_err("event_key scope on another worktree should error"); - assert!(error.to_string().contains("scope-1")); - } - - #[test] - fn load_worktree_with_event_key_missing_scope_errors() { - let db_fixture = test_db_path("case-3-missing-scope"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - - let event_key = EventKey { - scope_id: ScopeId("scope-missing".to_string()), - event_id: EventId("event-1".to_string()), - }; - - let error = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, Some(&event_key)) - .expect_err("missing event_key.scope_id should error"); - assert!(error.to_string().contains("scope-missing")); - } - - #[test] - fn load_worktree_with_event_key_missing_scope_and_orphan_replay_row_errors() { - let db_fixture = test_db_path("case-3-orphan-replay-row"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_processed_event(&db, "scope-missing", "event-1"); - - let event_key = EventKey { - scope_id: ScopeId("scope-missing".to_string()), - event_id: EventId("event-1".to_string()), - }; - - let error = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, Some(&event_key)) - .expect_err( - "an orphan processed-event row must not let a missing scope produce a projection", - ); - assert!(error.to_string().contains("scope-missing")); - } - - #[test] - fn load_worktree_with_agreeing_scope_and_event_key_loads_it_once() { - let db_fixture = test_db_path("case-4-agreeing"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Active); - - let event_key = EventKey { - scope_id: ScopeId("scope-1".to_string()), - event_id: EventId("event-1".to_string()), - }; - - let projection = store - .load_worktree( - &WorktreeId("wt-1".to_string()), - Some(&ScopeId("scope-1".to_string())), - Some(&event_key), - ) - .expect("load_worktree should succeed") - .expect("worktree should exist"); - - assert_eq!(projection.scopes.len(), 1); - assert!(projection - .scopes - .contains_key(&ScopeId("scope-1".to_string()))); - } - - #[test] - fn load_worktree_with_disagreeing_scope_and_event_key_errors_without_loading() { - let db_fixture = test_db_path("case-5-disagreeing"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-a", "wt-1", ScopeStatus::Active); - insert_scope(&db, "scope-b", "wt-1", ScopeStatus::Active); - - let event_key = EventKey { - scope_id: ScopeId("scope-b".to_string()), - event_id: EventId("event-1".to_string()), - }; - - let error = store - .load_worktree( - &WorktreeId("wt-1".to_string()), - Some(&ScopeId("scope-a".to_string())), - Some(&event_key), - ) - .expect_err("disagreeing scope/event_key.scope_id should error"); - assert!(error.to_string().contains("scope-a")); - assert!(error.to_string().contains("scope-b")); - } - - #[test] - fn load_scope_returns_the_durable_state_for_a_known_scope() { - let db_fixture = test_db_path("load-scope-known"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 7); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Closed); - insert_mutation_event( - &db, - "wt-1", - 7, - "tree-0", - "tree-1", - "ai_exclusive", - Some("scope-1"), - "close", - Some("scope-1"), - Some("event-1"), - &["scope-1"], - ); - insert_processed_event(&db, "scope-1", "event-1"); - - let scope_state = store - .load_scope(&ScopeId("scope-1".to_string())) - .expect("load_scope should succeed") - .expect("known scope should be present"); - - assert_eq!( - scope_state, - ScopeState { - status: ScopeStatus::Closed, - actor_kind: ActorKind::ClaudeCode, - worktree_id: WorktreeId("wt-1".to_string()), - } - ); - } - - #[test] - fn load_scope_returns_none_for_an_unknown_scope() { - let db_fixture = test_db_path("load-scope-unknown"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Active); - - let scope_state = store - .load_scope(&ScopeId("scope-missing".to_string())) - .expect("load_scope should succeed for an unknown scope"); - assert!(scope_state.is_none()); - } - - #[test] - fn load_scope_returns_a_scope_belonging_to_another_worktree() { - let db_fixture = test_db_path("load-scope-other-worktree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-other", "wt-2", ScopeStatus::Active); - - let scope_state = store - .load_scope(&ScopeId("scope-other".to_string())) - .expect("load_scope should not reject a scope on another worktree") - .expect("the scope row should be returned as-is"); - - assert_eq!( - scope_state, - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::ClaudeCode, - worktree_id: WorktreeId("wt-2".to_string()), - } - ); - - let error = store - .load_worktree( - &WorktreeId("wt-1".to_string()), - Some(&ScopeId("scope-other".to_string())), - None, - ) - .expect_err("load_worktree should still reject the cross-worktree scope"); - assert!(error.to_string().contains("scope-other")); - } - - #[test] - fn load_mutation_event_returns_none_when_missing() { - let db_fixture = test_db_path("cold-path-missing"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - let event = store - .load_mutation_event(&WorktreeId("wt-1".to_string()), 1) - .expect("load_mutation_event should succeed"); - assert!(event.is_none()); - } - - #[test] - fn load_mutation_event_reconstructs_ai_exclusive_start_event() { - let db_fixture = test_db_path("cold-path-ai-exclusive-start"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_mutation_event( - &db, - "wt-1", - 1, - "tree-0", - "tree-1", - "ai_exclusive", - Some("scope-1"), - "start", - Some("scope-1"), - Some("event-1"), - &["scope-1"], - ); - - let event = store - .load_mutation_event(&WorktreeId("wt-1".to_string()), 1) - .expect("load_mutation_event should succeed") - .expect("mutation event row should exist"); - - assert_eq!( - event, - MutationEvent { - worktree_id: WorktreeId("wt-1".to_string()), - revision: 1, - before_tree: TreeId("tree-0".to_string()), - after_tree: TreeId("tree-1".to_string()), - active_scopes: [ScopeId("scope-1".to_string())].into_iter().collect(), - tainted: false, - failure_kind: FailureKind::Healthy, - attribution: Attribution::AiExclusive(ScopeId("scope-1".to_string())), - boundary: Boundary::Start { - scope: ScopeId("scope-1".to_string()), - event: EventId("event-1".to_string()), - }, - } - ); - } - - #[test] - fn load_mutation_event_reconstructs_a_flush_event_with_multiple_active_scopes() { - let db_fixture = test_db_path("cold-path-flush"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_mutation_event( - &db, - "wt-1", - 3, - "tree-2", - "tree-3", - "ai_contended", - None, - "flush", - None, - None, - &["scope-1", "scope-2"], - ); - - let event = store - .load_mutation_event(&WorktreeId("wt-1".to_string()), 3) - .expect("load_mutation_event should succeed") - .expect("mutation event row should exist"); - - assert_eq!( - event, - MutationEvent { - worktree_id: WorktreeId("wt-1".to_string()), - revision: 3, - before_tree: TreeId("tree-2".to_string()), - after_tree: TreeId("tree-3".to_string()), - active_scopes: [ - ScopeId("scope-1".to_string()), - ScopeId("scope-2".to_string()) - ] - .into_iter() - .collect(), - tainted: false, - failure_kind: FailureKind::Healthy, - attribution: Attribution::AiContended, - boundary: Boundary::Flush { - worktree: WorktreeId("wt-1".to_string()), - }, - } - ); - } - - #[test] - fn into_protocol_state_carries_only_the_loaded_worktree_and_leaves_transient_fields_empty() { - let db_fixture = test_db_path("into-protocol-state"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 7); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Active); - - let projection = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, None) - .expect("load_worktree should succeed") - .expect("worktree should exist"); - - let protocol_state = projection.into_protocol_state(); - - assert_eq!(protocol_state.worktrees.len(), 1); - assert_eq!( - protocol_state - .worktrees - .get(&WorktreeId("wt-1".to_string())) - .map(|w| w.revision), - Some(7) - ); - assert!(protocol_state.attempts.is_empty()); - assert!(protocol_state.mutation_events.is_empty()); - assert!(protocol_state.external_taint.is_empty()); - } - - #[test] - fn initialize_worktree_inserts_a_fresh_healthy_cursor() { - let db_fixture = test_db_path("init-worktree-fresh"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - store - .initialize_worktree( - &WorktreeId("wt-1".to_string()), - &TreeId("tree-0".to_string()), - ) - .expect("initialize_worktree should succeed"); - - let projection = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, None) - .expect("load_worktree should succeed") - .expect("worktree should exist"); - - assert_eq!( - projection.worktree_state, - WorktreeState { - cursor_tree: TreeId("tree-0".to_string()), - revision: 0, - tainted: false, - failure_kind: FailureKind::Healthy, - needs_rebaseline: false, - } - ); - } - - #[test] - fn initialize_worktree_never_overwrites_an_existing_cursor() { - let db_fixture = test_db_path("init-worktree-idempotent"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 5); - - store - .initialize_worktree( - &WorktreeId("wt-1".to_string()), - &TreeId("tree-new".to_string()), - ) - .expect("initialize_worktree should succeed as a no-op"); - - let projection = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, None) - .expect("load_worktree should succeed") - .expect("worktree should exist"); - - assert_eq!( - projection.worktree_state.cursor_tree, - TreeId("tree-0".to_string()) - ); - assert_eq!(projection.worktree_state.revision, 5); - } - - #[test] - fn register_scope_inserts_never_seen_when_missing() { - let db_fixture = test_db_path("register-scope-fresh"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - - let scope_state = store - .register_scope( - &ScopeId("scope-1".to_string()), - &WorktreeId("wt-1".to_string()), - ActorKind::ClaudeCode, - ) - .expect("register_scope should succeed"); - - assert_eq!( - scope_state, - ScopeState { - status: ScopeStatus::NeverSeen, - actor_kind: ActorKind::ClaudeCode, - worktree_id: WorktreeId("wt-1".to_string()), - } - ); - } - - #[test] - fn register_scope_returns_existing_state_when_worktree_and_actor_match() { - let db_fixture = test_db_path("register-scope-existing-match"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Active); - - let scope_state = store - .register_scope( - &ScopeId("scope-1".to_string()), - &WorktreeId("wt-1".to_string()), - ActorKind::ClaudeCode, - ) - .expect("register_scope should succeed for a matching existing scope"); - - assert_eq!( - scope_state, - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::ClaudeCode, - worktree_id: WorktreeId("wt-1".to_string()), - } - ); - } - - #[test] - fn register_scope_errors_on_worktree_mismatch() { - let db_fixture = test_db_path("register-scope-worktree-mismatch"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_worktree(&db, "wt-2", 0); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Active); - - let error = store - .register_scope( - &ScopeId("scope-1".to_string()), - &WorktreeId("wt-2".to_string()), - ActorKind::ClaudeCode, - ) - .expect_err("a worktree mismatch on an existing scope should error"); - assert!(error.to_string().contains("scope-1")); - } - - #[test] - fn register_scope_errors_on_actor_mismatch() { - let db_fixture = test_db_path("register-scope-actor-mismatch"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Active); - - let error = store - .register_scope( - &ScopeId("scope-1".to_string()), - &WorktreeId("wt-1".to_string()), - ActorKind::Codex, - ) - .expect_err("an actor mismatch on an existing scope should error"); - assert!(error.to_string().contains("scope-1")); - } - - #[test] - fn register_scope_errors_when_worktree_does_not_exist_and_leaves_no_scope_row() { - let db_fixture = test_db_path("register-scope-missing-worktree-fresh"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - let error = store - .register_scope( - &ScopeId("scope-1".to_string()), - &WorktreeId("wt-missing".to_string()), - ActorKind::ClaudeCode, - ) - .expect_err("registering a scope against a missing worktree should error"); - assert!(error.to_string().contains("scope-1")); - assert!(error.to_string().contains("wt-missing")); - - let scope_state = store - .load_scope(&ScopeId("scope-1".to_string())) - .expect("load_scope should succeed"); - assert!( - scope_state.is_none(), - "a failed register_scope must not leave an orphan scope row" - ); - } - - #[test] - fn register_scope_errors_when_existing_scopes_worktree_row_is_missing() { - let db_fixture = test_db_path("register-scope-missing-worktree-existing"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_scope(&db, "scope-1", "wt-missing", ScopeStatus::Active); - - let error = store - .register_scope( - &ScopeId("scope-1".to_string()), - &WorktreeId("wt-missing".to_string()), - ActorKind::ClaudeCode, - ) - .expect_err( - "an existing scope whose worktree row is missing must not be accepted as valid", - ); - assert!(error.to_string().contains("scope-1")); - assert!(error.to_string().contains("wt-missing")); - } - - fn provenance(scope_id: &str, session_id: &str, model_id: Option<&str>) -> ScopeProvenance { - ScopeProvenance { - scope_id: ScopeId(scope_id.to_string()), - session_id: session_id.to_string(), - model_id: model_id.map(str::to_string), - } - } - - fn provenance_row_count(db: &RepositoryAgentTraceDb) -> i64 { - db.query_map( - "SELECT COUNT(*) FROM mutation_trace_scope_provenance", - (), - |row| row.get::(0).map_err(Into::into), - ) - .expect("provenance count query should succeed") - .into_iter() - .next() - .expect("count row should exist") - } - - fn registered_scope_store(label: &str) -> (TestDbPath, RepositoryAgentTraceDb) { - let db_fixture = test_db_path(label); - let db = - RepositoryAgentTraceDb::new_at(db_fixture.path()).expect("repository DB should open"); - insert_worktree(&db, "wt-1", 0); - insert_scope(&db, "scope-1", "wt-1", ScopeStatus::Active); - (db_fixture, db) - } - - #[test] - fn scope_provenance_loads_exactly_what_was_registered_or_none() { - let cases = [ - ( - "session and model", - Some(provenance("scope-1", "cx_session-1", Some("codex/gpt-5"))), - ), - ( - "session without a model", - Some(provenance("scope-1", "cc_session-1", None)), - ), - ("no provenance registered", None), - ]; - - for (index, (label, registered)) in cases.into_iter().enumerate() { - let (_fixture, db) = registered_scope_store(&format!("provenance-round-trip-{index}")); - let store = MutationTraceStore::new(&db); - - if let Some(incoming) = ®istered { - assert_eq!( - &store - .register_scope_provenance(incoming) - .unwrap_or_else(|error| panic!("{label}: registration failed: {error}")), - incoming, - "{label}: returned row" - ); - } - - assert_eq!( - store - .load_scope_provenance(&ScopeId("scope-1".to_string())) - .expect("load_scope_provenance should succeed"), - registered, - "{label}: loaded row" - ); - } - } - - #[test] - fn register_scope_provenance_keeps_the_first_model_observation_including_none() { - let cases: [(&str, Option<&str>, Option<&str>); 5] = [ - ("none then none", None, None), - ("none then a model", None, Some("model-a")), - ("a model then none", Some("model-a"), None), - ( - "a model then the same model", - Some("model-a"), - Some("model-a"), - ), - ( - "a model then another model", - Some("model-a"), - Some("model-b"), - ), - ]; - - for (index, (label, first_model, replay_model)) in cases.into_iter().enumerate() { - let (_fixture, db) = registered_scope_store(&format!("provenance-first-model-{index}")); - let store = MutationTraceStore::new(&db); - - let first = provenance("scope-1", "cc_session-1", first_model); - store - .register_scope_provenance(&first) - .unwrap_or_else(|error| panic!("{label}: first registration failed: {error}")); - - let replayed = store - .register_scope_provenance(&provenance("scope-1", "cc_session-1", replay_model)) - .unwrap_or_else(|error| { - panic!("{label}: a same-session replay must succeed: {error}") - }); - - assert_eq!(replayed, first, "{label}: returned row"); - assert_eq!( - store - .load_scope_provenance(&ScopeId("scope-1".to_string())) - .expect("load_scope_provenance should succeed"), - Some(first), - "{label}: stored row" - ); - } - } - - #[test] - fn register_scope_provenance_errors_on_a_session_conflict_without_rewriting_the_row() { - for (index, first_model) in [Some("model-a"), None].into_iter().enumerate() { - let (_fixture, db) = - registered_scope_store(&format!("provenance-session-conflict-{index}")); - let store = MutationTraceStore::new(&db); - - let first = provenance("scope-1", "cc_session-1", first_model); - store - .register_scope_provenance(&first) - .expect("first registration should succeed"); - - let error = store - .register_scope_provenance(&provenance("scope-1", "cc_session-2", Some("model-b"))) - .expect_err("a different session for the same scope should error"); - assert!(error.to_string().contains("scope-1")); - assert!(error.to_string().contains("cc_session-1")); - assert!(error.to_string().contains("cc_session-2")); - - assert_eq!( - store - .load_scope_provenance(&ScopeId("scope-1".to_string())) - .expect("load_scope_provenance should succeed"), - Some(first), - "a session conflict must never rewrite the stored session or model" - ); - } - } - - #[test] - fn register_scope_provenance_errors_for_an_unregistered_scope_and_creates_no_rows() { - let db_fixture = test_db_path("provenance-missing-scope"); - let db = - RepositoryAgentTraceDb::new_at(db_fixture.path()).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - let error = store - .register_scope_provenance(&provenance( - "scope-1", - "cc_session-1", - Some("claude/opus-5"), - )) - .expect_err("provenance for a scope with no mutation_trace_scopes row should error"); - assert!(error.to_string().contains("scope-1")); - - assert_eq!( - provenance_row_count(&db), - 0, - "a failed registration must not leave an orphan provenance row" - ); - assert!( - store - .load_scope(&ScopeId("scope-1".to_string())) - .expect("load_scope should succeed") - .is_none(), - "registering provenance must never create a scope implicitly" - ); - } - - fn healthy_worktree_state(revision: u64) -> WorktreeState { - WorktreeState { - cursor_tree: TreeId("tree0".to_string()), - revision, - tainted: false, - failure_kind: FailureKind::Healthy, - needs_rebaseline: false, - } - } - - fn state_with_scope( - worktree_id: &WorktreeId, - scope_id: &ScopeId, - actor_kind: ActorKind, - status: ScopeStatus, - revision: u64, - ) -> ProtocolState { - let mut state = ProtocolState::default(); - state - .worktrees - .insert(worktree_id.clone(), healthy_worktree_state(revision)); - state.scopes.insert( - scope_id.clone(), - ScopeState { - status, - actor_kind, - worktree_id: worktree_id.clone(), - }, - ); - state - } - - fn sample_mutation_event(worktree_id: &WorktreeId) -> MutationEvent { - MutationEvent { - worktree_id: worktree_id.clone(), - revision: 1, - before_tree: TreeId("tree0".to_string()), - after_tree: TreeId("tree1".to_string()), - active_scopes: BTreeSet::new(), - tainted: false, - failure_kind: FailureKind::Healthy, - attribution: Attribution::IneligibleUnscoped, - boundary: Boundary::Flush { - worktree: worktree_id.clone(), - }, - } - } - - #[test] - fn between_returns_none_for_a_database_failure_only_transition() { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - - let after = database_failure(&before, &wt); - - assert_eq!( - DurableTransition::between(&before, &after, &wt).unwrap(), - None - ); - } - - #[test] - fn between_returns_none_for_a_no_change_flush() { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - - let attempt = AttemptId("attempt0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Flush { - worktree: wt.clone(), - }, - TreeId("tree0".to_string()), - ); - let after = commit(&prepared, &attempt).state; - - assert_eq!( - DurableTransition::between(&before, &after, &wt).unwrap(), - None - ); - } - - #[test] - fn between_returns_some_with_correct_shape_for_a_start_transition() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let before = state_with_scope(&wt, &scope_id, ActorKind::Codex, ScopeStatus::NeverSeen, 0); - - let attempt = AttemptId("attempt0".to_string()); - let event_id = EventId("event0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Start { - scope: scope_id.clone(), - event: event_id.clone(), - }, - TreeId("tree1".to_string()), - ); - let after = commit(&prepared, &attempt).state; - - let transition = DurableTransition::between(&before, &after, &wt) - .unwrap() - .expect("a start transition should produce a durable transition"); - - assert_eq!(transition.worktree, wt); - assert_eq!(transition.expected_revision, 0); - assert_eq!(transition.next_worktree_state.revision, 1); - assert_eq!( - transition.next_worktree_state.cursor_tree, - TreeId("tree1".to_string()) - ); - assert_eq!( - transition.scope_status_changes.get(&scope_id), - Some(&ScopeStatus::Active) - ); - assert_eq!( - transition.new_processed_event, - Some(EventKey { - scope_id: scope_id.clone(), - event_id, - }) - ); - assert_eq!(after.mutation_events.len(), 1); - assert_eq!( - transition.new_mutation_event.as_ref(), - after.mutation_events.iter().next() - ); - } - - #[test] - fn between_returns_some_with_correct_shape_for_an_advance_transition() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let before = state_with_scope(&wt, &scope_id, ActorKind::Codex, ScopeStatus::Active, 0); - - let attempt = AttemptId("attempt0".to_string()); - let event_id = EventId("event0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Advance { - scope: scope_id.clone(), - event: event_id.clone(), - }, - TreeId("tree1".to_string()), - ); - let after = commit(&prepared, &attempt).state; - - let transition = DurableTransition::between(&before, &after, &wt) - .unwrap() - .expect("an advance transition should produce a durable transition"); - - assert!(transition.scope_status_changes.is_empty()); - assert_eq!( - transition.new_processed_event, - Some(EventKey { scope_id, event_id }) - ); - assert_eq!(after.mutation_events.len(), 1); - assert_eq!(transition.next_worktree_state.revision, 1); - } - - #[test] - fn between_returns_some_with_correct_shape_for_a_close_transition() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let before = state_with_scope(&wt, &scope_id, ActorKind::Codex, ScopeStatus::Active, 0); - - let attempt = AttemptId("attempt0".to_string()); - let event_id = EventId("event0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Close { - scope: scope_id.clone(), - event: event_id.clone(), - }, - TreeId("tree0".to_string()), - ); - let after = commit(&prepared, &attempt).state; - - let transition = DurableTransition::between(&before, &after, &wt) - .unwrap() - .expect("a close transition should produce a durable transition"); - - assert_eq!( - transition.scope_status_changes.get(&scope_id), - Some(&ScopeStatus::Closed) - ); - assert_eq!( - transition.new_processed_event, - Some(EventKey { scope_id, event_id }) - ); - assert_eq!(transition.next_worktree_state.revision, 1); - } - - #[test] - fn between_returns_some_with_correct_shape_for_a_taint_transition() { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - - let after = taint(&before, &wt); - - let transition = DurableTransition::between(&before, &after, &wt) - .unwrap() - .expect("a taint transition should produce a durable transition"); - - assert!(transition.scope_status_changes.is_empty()); - assert!(transition.new_processed_event.is_none()); - assert!(transition.new_mutation_event.is_none()); - assert!(transition.next_worktree_state.tainted); - assert_eq!( - transition.next_worktree_state.failure_kind, - FailureKind::SnapshotFailure - ); - assert_eq!(transition.next_worktree_state.revision, 1); - } - - #[test] - fn between_returns_some_with_correct_shape_for_an_abandon_transition() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let before = state_with_scope(&wt, &scope_id, ActorKind::Codex, ScopeStatus::Active, 0); - - let after = abandon(&before, &scope_id); - - let transition = DurableTransition::between(&before, &after, &wt) - .unwrap() - .expect("an abandon transition should produce a durable transition"); - - assert_eq!( - transition.scope_status_changes.get(&scope_id), - Some(&ScopeStatus::Abandoned) - ); - assert!(transition.next_worktree_state.needs_rebaseline); - assert_eq!(transition.next_worktree_state.revision, 1); - assert!(transition.new_processed_event.is_none()); - assert!(transition.new_mutation_event.is_none()); - } - - #[test] - fn between_returns_some_with_correct_shape_for_a_recover_transition() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let mut before = ProtocolState::default(); - before.worktrees.insert( - wt.clone(), - WorktreeState { - cursor_tree: TreeId("tree0".to_string()), - revision: 0, - tainted: true, - failure_kind: FailureKind::SnapshotFailure, - needs_rebaseline: false, - }, - ); - before.scopes.insert( - scope_id.clone(), - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::Codex, - worktree_id: wt.clone(), - }, - ); - - let after = recover(&before, &wt, TreeId("tree1".to_string())); - - let transition = DurableTransition::between(&before, &after, &wt) - .unwrap() - .expect("a recover transition should produce a durable transition"); - - assert_eq!( - transition.scope_status_changes.get(&scope_id), - Some(&ScopeStatus::Abandoned) - ); - assert!(!transition.next_worktree_state.tainted); - assert_eq!( - transition.next_worktree_state.failure_kind, - FailureKind::Healthy - ); - assert_eq!( - transition.next_worktree_state.cursor_tree, - TreeId("tree1".to_string()) - ); - assert_eq!(transition.next_worktree_state.revision, 1); - } - - #[test] - fn between_errors_when_a_scopes_actor_kind_changes() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let before = state_with_scope(&wt, &scope_id, ActorKind::Codex, ScopeStatus::Active, 0); - let mut after = before.clone(); - after.scopes.get_mut(&scope_id).unwrap().actor_kind = ActorKind::ClaudeCode; - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("an actor_kind change must be rejected"); - assert!(error.to_string().contains("actor_kind")); - } - - #[test] - fn between_errors_when_a_scopes_worktree_id_changes() { - let wt = WorktreeId("wt0".to_string()); - let other_wt = WorktreeId("wt1".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let before = state_with_scope(&wt, &scope_id, ActorKind::Codex, ScopeStatus::Active, 0); - let mut after = before.clone(); - after.scopes.get_mut(&scope_id).unwrap().worktree_id = other_wt; - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("a scope worktree_id change must be rejected"); - assert!(error.to_string().contains("worktree_id")); - } - - #[test] - fn between_errors_when_a_processed_event_disappears() { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - before.processed_events.insert(EventKey { - scope_id: ScopeId("scope0".to_string()), - event_id: EventId("event0".to_string()), - }); - let mut after = before.clone(); - after.processed_events.clear(); - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("a disappearing processed event must be rejected"); - assert!(error.to_string().contains("processed_events")); - } - - #[test] - fn between_errors_when_a_mutation_event_disappears() { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - before.mutation_events.insert(sample_mutation_event(&wt)); - let mut after = before.clone(); - after.mutation_events.clear(); - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("a disappearing mutation event must be rejected"); - assert!(error.to_string().contains("mutation_events")); - } - - #[test] - fn between_errors_when_a_new_mutation_events_revision_does_not_match_the_next_worktree_revision( - ) { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = 1; - let mut mismatched_event = sample_mutation_event(&wt); - mismatched_event.revision = 2; - after.mutation_events.insert(mismatched_event); - - let error = DurableTransition::between(&before, &after, &wt).expect_err( - "a mutation event revision mismatched with the next worktree revision must be rejected", - ); - assert!(error.to_string().contains("revision")); - } - - #[test] - fn between_errors_when_an_unrelated_worktree_changes() { - let wt = WorktreeId("wt0".to_string()); - let other_wt = WorktreeId("wt1".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - before - .worktrees - .insert(other_wt.clone(), healthy_worktree_state(0)); - - let mut after = before.clone(); - after.worktrees.get_mut(&other_wt).unwrap().revision = 1; - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("an unrelated worktree change must be rejected"); - assert!(error.to_string().contains("wt1")); - } - - #[test] - fn between_errors_when_the_revision_jumps_by_more_than_one() { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = 2; - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("a revision jump of more than one must be rejected"); - assert!(error.to_string().contains("revision")); - } - - #[test] - fn between_errors_when_the_revision_decreases() { - let wt = WorktreeId("wt0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(5)); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = 4; - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("a revision decrease must be rejected"); - assert!(error.to_string().contains("revision")); - } - - #[test] - fn between_errors_when_a_scope_unexpectedly_appears() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - let mut after = before.clone(); - after.scopes.insert( - scope_id, - ScopeState { - status: ScopeStatus::NeverSeen, - actor_kind: ActorKind::Codex, - worktree_id: wt.clone(), - }, - ); - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("an unexpectedly appearing scope must be rejected"); - assert!(error.to_string().contains("scope set")); - } - - #[test] - fn between_errors_when_a_scope_unexpectedly_disappears() { - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let before = state_with_scope(&wt, &scope_id, ActorKind::Codex, ScopeStatus::Active, 0); - let mut after = before.clone(); - after.scopes.remove(&scope_id); - - let error = DurableTransition::between(&before, &after, &wt) - .expect_err("an unexpectedly disappearing scope must be rejected"); - assert!(error.to_string().contains("scope set")); - } - - #[test] - fn commit_applies_a_full_transition_and_makes_every_write_visible() { - let db_fixture = test_db_path("commit-applies-full-transition"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - - let event_key = EventKey { - scope_id: scope_id.clone(), - event_id: EventId("event-1".to_string()), - }; - let mutation_event = MutationEvent { - worktree_id: wt.clone(), - revision: 1, - before_tree: TreeId("tree0".to_string()), - after_tree: TreeId("tree1".to_string()), - active_scopes: BTreeSet::from([scope_id.clone()]), - tainted: false, - failure_kind: FailureKind::Healthy, - attribution: Attribution::AiExclusive(scope_id.clone()), - boundary: Boundary::Close { - scope: scope_id.clone(), - event: EventId("event-1".to_string()), - }, - }; - - let before = state_with_scope( - &wt, - &scope_id, - ActorKind::ClaudeCode, - ScopeStatus::Active, - 0, - ); - let mut after = before.clone(); - after.worktrees.insert( - wt.clone(), - WorktreeState { - cursor_tree: TreeId("tree1".to_string()), - revision: 1, - tainted: false, - failure_kind: FailureKind::Healthy, - needs_rebaseline: false, - }, - ); - after.scopes.insert( - scope_id.clone(), - ScopeState { - status: ScopeStatus::Closed, - actor_kind: ActorKind::ClaudeCode, - worktree_id: wt.clone(), - }, - ); - after.processed_events.insert(event_key.clone()); - after.mutation_events.insert(mutation_event.clone()); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a transition should exist for this change"); - - let result = store.commit(&transition).expect("commit should succeed"); - assert_eq!(result, CasResult::Applied); - - let worktree_state = store - .load_worktree_state(&wt) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert_eq!(worktree_state, transition.next_worktree_state); - - let scope_state = store - .load_scope(&scope_id) - .expect("scope read should succeed") - .expect("scope row should exist"); - assert_eq!(scope_state.status, ScopeStatus::Closed); - - assert!(store - .processed_event_exists(&event_key) - .expect("processed-event read should succeed")); - - let reloaded_event = store - .load_mutation_event(&wt, 1) - .expect("mutation-event read should succeed") - .expect("mutation-event row should exist"); - assert_eq!(reloaded_event, mutation_event); - } - - #[test] - fn commit_returns_conflict_and_writes_nothing_when_the_worktree_revision_has_moved_on() { - let db_fixture = test_db_path("commit-conflict"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - insert_worktree(&db, &wt.0, 5); - - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = 1; - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a transition should exist for this change"); - - let result = store.commit(&transition).expect("commit should succeed"); - assert_eq!(result, CasResult::Conflict); - - let worktree_state = store - .load_worktree_state(&wt) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert_eq!(worktree_state.revision, 5); - } - - #[test] - fn commit_propagates_a_deterministic_failure_without_reporting_conflict() { - let db_fixture = test_db_path("commit-deterministic-failure"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - insert_worktree(&db, &wt.0, 0); - insert_processed_event(&db, "scope0", "event-1"); - - let before = state_with_scope( - &wt, - &scope_id, - ActorKind::ClaudeCode, - ScopeStatus::Active, - 0, - ); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = 1; - after.processed_events.insert(EventKey { - scope_id: scope_id.clone(), - event_id: EventId("event-1".to_string()), - }); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a transition should exist for this change"); - - let error = store - .commit(&transition) - .expect_err("a duplicate processed-event insert should fail deterministically"); - assert!(error.to_string().contains("execute failed")); - - let worktree_state = store - .load_worktree_state(&wt) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert_eq!( - worktree_state.revision, 0, - "the guard's own revision advance must roll back together with the failed insert" - ); - } - - struct RaceEvidence { - scope: ScopeId, - event_key: EventKey, - mutation_event: MutationEvent, - } - - fn race_evidence(worktree: &WorktreeId, scope: ScopeId, label: &str) -> RaceEvidence { - let event_key = EventKey { - scope_id: scope.clone(), - event_id: EventId(format!("event-{label}")), - }; - let mutation_event = MutationEvent { - worktree_id: worktree.clone(), - revision: 1, - before_tree: TreeId("tree0".to_string()), - after_tree: TreeId(format!("tree-after-{label}")), - active_scopes: BTreeSet::from([scope.clone()]), - tainted: false, - failure_kind: FailureKind::Healthy, - attribution: Attribution::AiExclusive(scope.clone()), - boundary: Boundary::Close { - scope: scope.clone(), - event: event_key.event_id.clone(), - }, - }; - RaceEvidence { - scope, - event_key, - mutation_event, - } - } - - fn closing_transition( - before: &ProtocolState, - worktree: &WorktreeId, - evidence: &RaceEvidence, - ) -> DurableTransition { - let mut after = before.clone(); - after.worktrees.get_mut(worktree).unwrap().revision = evidence.mutation_event.revision; - after.scopes.get_mut(&evidence.scope).unwrap().status = ScopeStatus::Closed; - after.processed_events.insert(evidence.event_key.clone()); - after - .mutation_events - .insert(evidence.mutation_event.clone()); - DurableTransition::between(before, &after, worktree) - .expect("between should succeed") - .expect("a transition should exist for this change") - } - - fn assert_race_winner_state( - store: &MutationTraceStore, - worktree: &WorktreeId, - persisted_event: &MutationEvent, - writer_a: &RaceEvidence, - writer_b: &RaceEvidence, - ) { - let (winner, loser) = if persisted_event == &writer_a.mutation_event { - (writer_a, writer_b) - } else if persisted_event == &writer_b.mutation_event { - (writer_b, writer_a) - } else { - panic!( - "persisted mutation event matches neither writer's expected event: \ - {persisted_event:?}" - ); - }; - - let winning_scope_state = store - .load_scope(&winner.scope) - .expect("winning scope read should succeed") - .expect("winning scope row should exist"); - assert_eq!( - winning_scope_state.status, - ScopeStatus::Closed, - "the winning transition's scope-status change must be durable" - ); - - let losing_scope_state = store - .load_scope(&loser.scope) - .expect("losing scope read should succeed") - .expect("losing scope row should exist"); - assert_eq!( - losing_scope_state.status, - ScopeStatus::Active, - "the losing transition's scope-status change must not have applied" - ); - - assert!( - store - .processed_event_exists(&winner.event_key) - .expect("winning processed-event read should succeed"), - "the winning transition's processed EventKey must exist" - ); - assert!( - !store - .processed_event_exists(&loser.event_key) - .expect("losing processed-event read should succeed"), - "the losing transition's processed EventKey must not exist" - ); - - assert_eq!( - persisted_event, &winner.mutation_event, - "the persisted mutation event must equal the winning transition's expected event \ - field-for-field, with no evidence from the losing transition mixed in" - ); - - let persisted_active_scopes = store - .load_mutation_event_active_scopes( - worktree, - encode_revision(winner.mutation_event.revision).as_slice(), - ) - .expect("active-scope read should succeed"); - assert_eq!( - &persisted_active_scopes, &winner.mutation_event.active_scopes, - "persisted active scopes must equal exactly the winning transition's active_scopes, \ - with no loser-only active-scope rows present" - ); - } - - #[test] - fn commit_from_two_independent_connections_races_and_only_one_applies() { - let db_fixture = test_db_path("commit-two-writer-race"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_a = ScopeId("scope-a".to_string()); - let scope_b = ScopeId("scope-b".to_string()); - - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_a.0, &wt.0, ScopeStatus::Active); - insert_scope(&db, &scope_b.0, &wt.0, ScopeStatus::Active); - } - - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(0)); - before.scopes.insert( - scope_a.clone(), - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::ClaudeCode, - worktree_id: wt.clone(), - }, - ); - before.scopes.insert( - scope_b.clone(), - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::ClaudeCode, - worktree_id: wt.clone(), - }, - ); - - let writer_a = race_evidence(&wt, scope_a, "a"); - let writer_b = race_evidence(&wt, scope_b, "b"); - let transition_a = closing_transition(&before, &wt, &writer_a); - let transition_b = closing_transition(&before, &wt, &writer_b); - - let db_a = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("writer A handle should open"); - let db_b = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("writer B handle should open"); - - let handle_a = thread::spawn(move || MutationTraceStore::new(&db_a).commit(&transition_a)); - let handle_b = thread::spawn(move || MutationTraceStore::new(&db_b).commit(&transition_b)); - - let result_a = handle_a - .join() - .expect("writer A thread should not panic") - .expect("writer A commit should not error"); - let result_b = handle_b - .join() - .expect("writer B thread should not panic") - .expect("writer B commit should not error"); - let results = [result_a, result_b]; - - assert_eq!( - results.iter().filter(|r| **r == CasResult::Applied).count(), - 1, - "exactly one writer should apply from the same starting revision: {results:?}" - ); - assert_eq!( - results - .iter() - .filter(|r| **r == CasResult::Conflict) - .count(), - 1, - "exactly one writer should conflict from the same starting revision: {results:?}" - ); - - let db_reopened = RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("reopened handle should open"); - let store = MutationTraceStore::new(&db_reopened); - - let worktree_state = store - .load_worktree_state(&wt) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert_eq!( - worktree_state.revision, 1, - "revision should have advanced exactly once, not once per writer" - ); - - let persisted_event = store - .load_mutation_event(&wt, 1) - .expect("mutation-event read should succeed") - .expect("exactly one writer's mutation event should be visible at the new revision"); - - assert_race_winner_state(&store, &wt, &persisted_event, &writer_a, &writer_b); - } - - fn assert_atomic_rollback_state( - store: &MutationTraceStore, - worktree: &WorktreeId, - scope_id: &ScopeId, - rolled_back_active_scope: &ScopeId, - surviving_active_scope: &ScopeId, - ) { - let worktree_state = store - .load_worktree_state(worktree) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert_eq!(worktree_state.revision, 0, "revision must roll back"); - assert_eq!( - worktree_state.cursor_tree, - TreeId("tree-0".to_string()), - "cursor_tree must roll back" - ); - assert_eq!( - worktree_state.failure_kind, - FailureKind::Healthy, - "failure_kind must roll back" - ); - assert!( - !worktree_state.needs_rebaseline, - "needs_rebaseline must roll back" - ); - - let scope_state = store - .load_scope(scope_id) - .expect("scope read should succeed") - .expect("scope row should exist"); - assert_eq!( - scope_state.status, - ScopeStatus::Active, - "scope status must roll back even though its UPDATE ran before the failure" - ); - - assert!( - !store - .processed_event_exists(&EventKey { - scope_id: scope_id.clone(), - event_id: EventId("event-1".to_string()), - }) - .expect("processed-event read should succeed"), - "the processed event must not exist even though its INSERT ran before the failure" - ); - - assert!( - store - .load_mutation_event(worktree, 1) - .expect("mutation-event read should succeed") - .is_none(), - "the mutation event must not exist even though its INSERT ran before the failure" - ); - - let active_scopes = store - .load_mutation_event_active_scopes(worktree, encode_revision(1).as_slice()) - .expect("active-scope read should succeed"); - assert!( - !active_scopes.contains(rolled_back_active_scope), - "the active-scope INSERT that was ordered before the colliding one ran but must \ - have rolled back with everything else in the transaction" - ); - assert_eq!( - active_scopes, - BTreeSet::from([surviving_active_scope.clone()]), - "only the pre-seeded row, which predates this transaction and was never part of \ - it, should remain" - ); - } - - #[test] - fn commit_rolls_back_every_write_kind_together_on_a_deterministic_failure() { - let db_fixture = test_db_path("commit-atomic-rollback"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let scope_a = ScopeId("scope-a".to_string()); - let scope_z = ScopeId("scope-z".to_string()); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - insert_active_scope(&db, &wt.0, 1, &scope_z.0); - - let before = state_with_scope( - &wt, - &scope_id, - ActorKind::ClaudeCode, - ScopeStatus::Active, - 0, - ); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = 1; - after.scopes.get_mut(&scope_id).unwrap().status = ScopeStatus::Closed; - after.processed_events.insert(EventKey { - scope_id: scope_id.clone(), - event_id: EventId("event-1".to_string()), - }); - after.mutation_events.insert(MutationEvent { - worktree_id: wt.clone(), - revision: 1, - before_tree: TreeId("tree0".to_string()), - after_tree: TreeId("tree1".to_string()), - active_scopes: BTreeSet::from([scope_a.clone(), scope_z.clone()]), - tainted: false, - failure_kind: FailureKind::Healthy, - attribution: Attribution::AiExclusive(scope_id.clone()), - boundary: Boundary::Close { - scope: scope_id.clone(), - event: EventId("event-1".to_string()), - }, - }); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a transition should exist for this change"); - - let error = store.commit(&transition).expect_err( - "the pre-seeded (wt0, revision=1, scope-z) active-scope row should collide with \ - the second active-scope insert, after every earlier write kind already succeeded", - ); - assert!(error.to_string().contains("execute failed")); - - assert_atomic_rollback_state(&store, &wt, &scope_id, &scope_a, &scope_z); - } - - #[test] - fn commit_round_trips_u64_max_through_the_real_database() { - let db_fixture = test_db_path("commit-u64-max"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - insert_worktree(&db, &wt.0, u64::MAX - 1); - - let mut before = ProtocolState::default(); - before - .worktrees - .insert(wt.clone(), healthy_worktree_state(u64::MAX - 1)); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = u64::MAX; - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a transition should exist for this change"); - - let result = store.commit(&transition).expect("commit should succeed"); - assert_eq!(result, CasResult::Applied); - - let worktree_state = store - .load_worktree_state(&wt) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert_eq!(worktree_state.revision, u64::MAX); - } - - #[test] - fn commit_rejects_a_replayed_event_key_via_the_processed_event_uniqueness_constraint() { - let db_fixture = test_db_path("commit-replay-uniqueness"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - insert_processed_event(&db, "scope0", "event-1"); - - let before = state_with_scope( - &wt, - &scope_id, - ActorKind::ClaudeCode, - ScopeStatus::Active, - 0, - ); - let mut after = before.clone(); - after.worktrees.get_mut(&wt).unwrap().revision = 1; - after.processed_events.insert(EventKey { - scope_id: scope_id.clone(), - event_id: EventId("event-1".to_string()), - }); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a transition should exist for this change"); - - let error = store - .commit(&transition) - .expect_err("a replayed (scope_id, event_id) must be rejected, not silently applied"); - assert!(error.to_string().contains("execute failed")); - - let worktree_state = store - .load_worktree_state(&wt) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert_eq!( - worktree_state.revision, 0, - "the whole transaction must roll back on a replay rejection" - ); - } - - #[test] - fn commit_of_strong_recovery_abandons_every_live_scope_on_the_worktree() { - let db_fixture = test_db_path("commit-strong-recovery"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - let scope_a = ScopeId("scope-a".to_string()); - let scope_b = ScopeId("scope-b".to_string()); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_a.0, &wt.0, ScopeStatus::Active); - insert_scope(&db, &scope_b.0, &wt.0, ScopeStatus::Active); - - let mut before = ProtocolState::default(); - before.worktrees.insert( - wt.clone(), - WorktreeState { - cursor_tree: TreeId("tree0".to_string()), - revision: 0, - tainted: true, - failure_kind: FailureKind::SnapshotFailure, - needs_rebaseline: false, - }, - ); - before.scopes.insert( - scope_a.clone(), - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::ClaudeCode, - worktree_id: wt.clone(), - }, - ); - before.scopes.insert( - scope_b.clone(), - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::ClaudeCode, - worktree_id: wt.clone(), - }, - ); - - let after = recover(&before, &wt, TreeId("tree1".to_string())); - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("strong recovery should produce a durable transition"); - - let result = store.commit(&transition).expect("commit should succeed"); - assert_eq!(result, CasResult::Applied); - - for scope_id in [&scope_a, &scope_b] { - let scope_state = store - .load_scope(scope_id) - .expect("scope read should succeed") - .expect("scope row should exist"); - assert_eq!(scope_state.status, ScopeStatus::Abandoned); - } - } - - #[test] - fn commit_of_needs_only_recovery_leaves_live_scopes_active() { - let db_fixture = test_db_path("commit-needs-only-recovery"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - - let mut before = ProtocolState::default(); - before.worktrees.insert( - wt.clone(), - WorktreeState { - cursor_tree: TreeId("tree0".to_string()), - revision: 0, - tainted: false, - failure_kind: FailureKind::Healthy, - needs_rebaseline: true, - }, - ); - before.scopes.insert( - scope_id.clone(), - ScopeState { - status: ScopeStatus::Active, - actor_kind: ActorKind::ClaudeCode, - worktree_id: wt.clone(), - }, - ); - - let after = recover(&before, &wt, TreeId("tree1".to_string())); - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("needs-only recovery should produce a durable transition"); - - let result = store.commit(&transition).expect("commit should succeed"); - assert_eq!(result, CasResult::Applied); - - let worktree_state = store - .load_worktree_state(&wt) - .expect("worktree read should succeed") - .expect("worktree row should exist"); - assert!(!worktree_state.needs_rebaseline); - - let scope_state = store - .load_scope(&scope_id) - .expect("scope read should succeed") - .expect("scope row should exist"); - assert_eq!( - scope_state.status, - ScopeStatus::Active, - "a live scope must survive needs-only recovery untouched" - ); - } - - fn insert_worktree_with_state( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - tainted: bool, - failure_kind: FailureKind, - needs_rebaseline: bool, - ) { - db.execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES (?1, 'tree0', ?2, ?3, ?4, ?5)", - ( - worktree_id, - encode_revision(revision).as_slice(), - tainted, - encode_failure_kind(failure_kind), - needs_rebaseline, - ), - ) - .expect("worktree insert should succeed"); - } - - fn reopen_store(db_path: &std::path::Path) -> RepositoryAgentTraceDb { - RepositoryAgentTraceDb::open_for_hooks_without_migrations_at(db_path) - .expect("reopened handle should open") - } - - fn load_before_state( - db_path: &std::path::Path, - worktree: &WorktreeId, - scope: Option<&ScopeId>, - event_key: Option<&EventKey>, - ) -> ProtocolState { - let db = reopen_store(db_path); - MutationTraceStore::new(&db) - .load_worktree(worktree, scope, event_key) - .expect("load_worktree should succeed") - .expect("worktree projection should exist") - .into_protocol_state() - } - - fn commit_transition(db_path: &std::path::Path, transition: &DurableTransition) -> CasResult { - let db = reopen_store(db_path); - MutationTraceStore::new(&db) - .commit(transition) - .expect("commit should succeed") - } - - fn expected_projection( - after: &ProtocolState, - worktree: &WorktreeId, - scope: Option<&ScopeId>, - event_key: Option<&EventKey>, - ) -> WorktreeProjection { - let worktree_state = after - .worktrees - .get(worktree) - .expect("after should contain the worktree") - .clone(); - - let mut scopes: BTreeMap = after - .scopes - .iter() - .filter(|(_, scope_state)| { - scope_state.worktree_id == *worktree && scope_state.status == ScopeStatus::Active - }) - .map(|(scope_id, scope_state)| (scope_id.clone(), scope_state.clone())) - .collect(); - - let effective_scope = scope.or(event_key.map(|key| &key.scope_id)); - if let Some(effective_scope) = effective_scope { - if let Some(scope_state) = after.scopes.get(effective_scope) { - scopes.insert(effective_scope.clone(), scope_state.clone()); - } - } - - let mut processed_events = BTreeSet::new(); - if let Some(key) = event_key { - if after.processed_events.contains(key) { - processed_events.insert(key.clone()); - } - } - - WorktreeProjection { - worktree_id: worktree.clone(), - worktree_state, - scopes, - processed_events, - } - } - - fn assert_round_trip( - db_path: &std::path::Path, - worktree: &WorktreeId, - scope: Option<&ScopeId>, - event_key: Option<&EventKey>, - after: &ProtocolState, - ) -> WorktreeProjection { - let db = reopen_store(db_path); - let store = MutationTraceStore::new(&db); - - let reloaded = store - .load_worktree(worktree, scope, event_key) - .expect("load_worktree should succeed") - .expect("worktree projection should exist"); - - assert_eq!( - reloaded, - expected_projection(after, worktree, scope, event_key) - ); - - for expected_event in &after.mutation_events { - let reloaded_event = store - .load_mutation_event(worktree, expected_event.revision) - .expect("load_mutation_event should succeed") - .expect("mutation event row should exist"); - assert_eq!(&reloaded_event, expected_event); - } - - reloaded - } - - #[test] - fn round_trip_start_persists_and_reloads_exactly_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-start"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::NeverSeen); - } - - let before = load_before_state(db_path, &wt, Some(&scope_id), None); - let attempt = AttemptId("attempt0".to_string()); - let event_id = EventId("event0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Start { - scope: scope_id.clone(), - event: event_id.clone(), - }, - TreeId("tree1".to_string()), - ); - let outcome = commit(&prepared, &attempt); - assert!( - outcome.evaluation.accepted, - "a fresh Start should be accepted" - ); - let after = outcome.state; - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a start transition should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - let event_key = EventKey { - scope_id: scope_id.clone(), - event_id, - }; - assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); - } - - #[test] - fn round_trip_advance_persists_and_reloads_exactly_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-advance"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - } - - let before = load_before_state(db_path, &wt, Some(&scope_id), None); - let attempt = AttemptId("attempt0".to_string()); - let event_id = EventId("event0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Advance { - scope: scope_id.clone(), - event: event_id.clone(), - }, - TreeId("tree1".to_string()), - ); - let outcome = commit(&prepared, &attempt); - assert!( - outcome.evaluation.accepted, - "a fresh Advance should be accepted" - ); - let after = outcome.state; - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("an advance transition should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - let event_key = EventKey { - scope_id: scope_id.clone(), - event_id, - }; - assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); - } - - #[test] - fn round_trip_close_persists_and_reloads_exactly_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-close"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - } - - let before = load_before_state(db_path, &wt, Some(&scope_id), None); - let attempt = AttemptId("attempt0".to_string()); - let event_id = EventId("event0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Close { - scope: scope_id.clone(), - event: event_id.clone(), - }, - TreeId("tree1".to_string()), - ); - let outcome = commit(&prepared, &attempt); - assert!( - outcome.evaluation.accepted, - "a fresh Close should be accepted" - ); - let after = outcome.state; - assert_eq!( - after.scopes.get(&scope_id).map(|s| s.status), - Some(ScopeStatus::Closed) - ); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a close transition should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - let event_key = EventKey { - scope_id: scope_id.clone(), - event_id, - }; - assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); - } - - #[test] - fn round_trip_flush_with_change_persists_and_reloads_exactly_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-flush-change"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - } - - let before = load_before_state(db_path, &wt, None, None); - let attempt = AttemptId("attempt0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Flush { - worktree: wt.clone(), - }, - TreeId("tree1".to_string()), - ); - let outcome = commit(&prepared, &attempt); - assert!( - outcome.evaluation.changed, - "an observed tree change should be recorded" - ); - let after = outcome.state; - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a changed flush transition should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - assert_round_trip(db_path, &wt, None, None, &after); - } - - #[test] - fn round_trip_flush_without_change_persists_nothing_new() { - let db_fixture = test_db_path("roundtrip-flush-no-change"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - } - - let before = load_before_state(db_path, &wt, None, None); - let attempt = AttemptId("attempt0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Flush { - worktree: wt.clone(), - }, - before.worktrees[&wt].cursor_tree.clone(), - ); - let outcome = commit(&prepared, &attempt); - assert!( - !outcome.evaluation.observed_change, - "flushing the same tree should observe no change" - ); - let after = outcome.state; - - assert_eq!( - DurableTransition::between(&before, &after, &wt).expect("between should succeed"), - None, - "a no-change flush must produce no durable transition to persist" - ); - - assert_round_trip(db_path, &wt, None, None, &after); - } - - #[test] - fn round_trip_taint_persists_and_reloads_exactly_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-taint"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - } - - let before = load_before_state(db_path, &wt, None, None); - let after = taint(&before, &wt); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("taint should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - assert_round_trip(db_path, &wt, None, None, &after); - } - - #[test] - fn round_trip_abandon_persists_and_reloads_exactly_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-abandon"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - } - - let before = load_before_state(db_path, &wt, Some(&scope_id), None); - let after = abandon(&before, &scope_id); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("abandon should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - assert_round_trip(db_path, &wt, Some(&scope_id), None, &after); - } - - #[test] - fn round_trip_strong_recovery_abandons_every_live_scope_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-recover-strong"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_a = ScopeId("scope-a".to_string()); - let scope_b = ScopeId("scope-b".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree_with_state(&db, &wt.0, 0, true, FailureKind::SnapshotFailure, false); - insert_scope(&db, &scope_a.0, &wt.0, ScopeStatus::Active); - insert_scope(&db, &scope_b.0, &wt.0, ScopeStatus::Active); - } - - let before = load_before_state(db_path, &wt, None, None); - let after = recover(&before, &wt, TreeId("tree1".to_string())); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("strong recovery should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - assert_round_trip(db_path, &wt, None, None, &after); - - let db = reopen_store(db_path); - let store = MutationTraceStore::new(&db); - for scope_id in [&scope_a, &scope_b] { - let scope_state = store - .load_scope(scope_id) - .expect("scope read should succeed") - .expect("scope row should exist"); - assert_eq!(scope_state.status, ScopeStatus::Abandoned); - } - } - - #[test] - fn round_trip_contended_mutation_persists_and_reloads_exactly_after_reopening_the_database() { - let db_fixture = test_db_path("roundtrip-contended"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_a = ScopeId("scope-a".to_string()); - let scope_b = ScopeId("scope-b".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_a.0, &wt.0, ScopeStatus::Active); - insert_scope(&db, &scope_b.0, &wt.0, ScopeStatus::Active); - } - - let before = load_before_state(db_path, &wt, None, None); - let attempt = AttemptId("attempt0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Flush { - worktree: wt.clone(), - }, - TreeId("tree1".to_string()), - ); - let outcome = commit(&prepared, &attempt); - assert!( - outcome.evaluation.changed, - "an observed tree change should be recorded" - ); - let after = outcome.state; - let mutation_event = after - .mutation_events - .iter() - .next() - .expect("a mutation event should have been produced"); - assert_eq!(mutation_event.attribution, Attribution::AiContended); - assert_eq!( - mutation_event.active_scopes, - BTreeSet::from([scope_a.clone(), scope_b.clone()]) - ); - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("a contended flush transition should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - let reloaded = assert_round_trip(db_path, &wt, None, None, &after); - assert_eq!( - reloaded.scopes.keys().cloned().collect::>(), - BTreeSet::from([scope_a.clone(), scope_b.clone()]), - "the reloaded live bounded projection must contain exactly the two contended scopes" - ); - for scope_id in [&scope_a, &scope_b] { - assert_eq!( - reloaded.scopes.get(scope_id).map(|s| s.status), - Some(ScopeStatus::Active) - ); - } - } - - #[test] - fn round_trip_database_failure_changes_only_non_persistent_external_taint() { - let db_fixture = test_db_path("roundtrip-database-failure"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - } - - let before = load_before_state(db_path, &wt, None, None); - let after = database_failure(&before, &wt); - - assert!(!before.external_taint.contains(&wt)); - assert!(after.external_taint.contains(&wt)); - assert_eq!(before.worktrees, after.worktrees); - assert_eq!(before.scopes, after.scopes); - assert_eq!(before.processed_events, after.processed_events); - assert_eq!( - after.worktrees[&wt].revision, - before.worktrees[&wt].revision - ); - - assert_eq!( - DurableTransition::between(&before, &after, &wt).expect("between should succeed"), - None, - "database_failure only changes external_taint, which is never durable, so no \ - DurableTransition should exist to commit" - ); - - let reloaded = assert_round_trip(db_path, &wt, None, None, &after); - assert!(reloaded.into_protocol_state().external_taint.is_empty()); - } - - #[test] - fn round_trip_a_replayed_event_key_is_rejected_and_does_not_advance_the_worktree_again() { - let db_fixture = test_db_path("roundtrip-replay"); - let db_path = db_fixture.path(); - let wt = WorktreeId("wt0".to_string()); - let scope_id = ScopeId("scope0".to_string()); - let event_key = EventKey { - scope_id: scope_id.clone(), - event_id: EventId("event0".to_string()), - }; - { - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - insert_worktree(&db, &wt.0, 0); - insert_scope(&db, &scope_id.0, &wt.0, ScopeStatus::Active); - } - - let before = load_before_state(db_path, &wt, Some(&scope_id), None); - let attempt = AttemptId("attempt0".to_string()); - let prepared = prepare( - &before, - attempt.clone(), - Boundary::Advance { - scope: scope_id.clone(), - event: event_key.event_id.clone(), - }, - TreeId("tree1".to_string()), - ); - let outcome = commit(&prepared, &attempt); - assert!( - outcome.evaluation.accepted, - "the first delivery should be accepted" - ); - let after = outcome.state; - - let transition = DurableTransition::between(&before, &after, &wt) - .expect("between should succeed") - .expect("the first delivery should produce a durable transition"); - assert_eq!(commit_transition(db_path, &transition), CasResult::Applied); - - let before_replay = load_before_state(db_path, &wt, Some(&scope_id), Some(&event_key)); - assert!(before_replay.processed_events.contains(&event_key)); - - let replay_attempt = AttemptId("attempt1".to_string()); - let replay_prepared = prepare( - &before_replay, - replay_attempt.clone(), - Boundary::Advance { - scope: scope_id.clone(), - event: event_key.event_id.clone(), - }, - TreeId("tree2".to_string()), - ); - let replay_outcome = commit(&replay_prepared, &replay_attempt); - assert!( - !replay_outcome.evaluation.accepted, - "a replayed EventKey must be rejected" - ); - let after_replay = replay_outcome.state; - - assert_eq!( - DurableTransition::between(&before_replay, &after_replay, &wt) - .expect("between should succeed"), - None, - "a rejected replay must produce no durable transition to persist" - ); - - assert_round_trip(db_path, &wt, Some(&scope_id), Some(&event_key), &after); - } - - fn insert_worktree_with_cursor( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - cursor_tree: &str, - ) { - db.execute( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES (?1, ?2, ?3, 0, 'healthy', 0)", - ( - worktree_id, - cursor_tree, - encode_revision(revision).as_slice(), - ), - ) - .expect("worktree insert should succeed"); - } - - fn insert_event_trees( - db: &RepositoryAgentTraceDb, - worktree_id: &str, - revision: u64, - before_tree: &str, - after_tree: &str, - ) { - db.execute( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", - ( - worktree_id, - encode_revision(revision).as_slice(), - before_tree, - after_tree, - ), - ) - .expect("mutation event insert should succeed"); - } - - fn tree_set(trees: [&str; N]) -> BTreeSet { - trees.into_iter().map(|t| TreeId(t.to_string())).collect() - } - - #[test] - fn load_tree_roots_returns_cursor_and_every_event_tree_deduplicated() { - let db_fixture = test_db_path("tree-roots-cursor-and-events"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-1", 2, "tree-2"); - insert_event_trees(&db, "wt-1", 1, "tree-0", "tree-1"); - insert_event_trees(&db, "wt-1", 2, "tree-1", "tree-2"); - - let roots = store - .load_tree_roots(&WorktreeId("wt-1".to_string())) - .expect("load_tree_roots should succeed"); - - assert_eq!(roots, tree_set(["tree-0", "tree-1", "tree-2"])); - } - - #[test] - fn load_tree_roots_excludes_other_worktrees_trees() { - let db_fixture = test_db_path("tree-roots-excludes-other-worktree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-1", 1, "tree-1a"); - insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-1a"); - - insert_worktree_with_cursor(&db, "wt-2", 1, "tree-1b"); - insert_event_trees(&db, "wt-2", 1, "tree-0b", "tree-1b"); - - let roots = store - .load_tree_roots(&WorktreeId("wt-1".to_string())) - .expect("load_tree_roots should succeed"); - - assert_eq!(roots, tree_set(["tree-0a", "tree-1a"])); - } - - #[test] - fn load_tree_roots_is_empty_for_an_unmaterialized_worktree() { - let db_fixture = test_db_path("tree-roots-unmaterialized-worktree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-other", 0, "tree-other"); - - let roots = store - .load_tree_roots(&WorktreeId("wt-missing".to_string())) - .expect("load_tree_roots should return Ok for a worktree with no durable row"); - - assert!(roots.is_empty()); - } - - #[test] - fn load_tree_roots_remains_worktree_scoped() { - let db_fixture = test_db_path("tree-roots-worktree-scoped"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-1", 1, "tree-1a"); - insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-1a"); - insert_worktree_with_cursor(&db, "wt-2", 2, "tree-2b"); - insert_event_trees(&db, "wt-2", 1, "tree-0b", "tree-1b"); - insert_event_trees(&db, "wt-2", 2, "tree-1b", "tree-2b"); - - assert_eq!( - store - .load_tree_roots(&WorktreeId("wt-1".to_string())) - .expect("load_tree_roots should succeed"), - tree_set(["tree-0a", "tree-1a"]), - ); - assert_eq!( - store - .load_tree_roots(&WorktreeId("wt-2".to_string())) - .expect("load_tree_roots should succeed"), - tree_set(["tree-0b", "tree-1b", "tree-2b"]), - ); - } - - #[test] - fn load_all_tree_roots_returns_every_worktree_cursor_and_event_tree_deduplicated() { - let db_fixture = test_db_path("all-tree-roots-every-worktree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-1", 1, "tree-1a"); - insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-1a"); - insert_worktree_with_cursor(&db, "wt-2", 2, "tree-2b"); - insert_event_trees(&db, "wt-2", 1, "tree-0b", "tree-1b"); - insert_event_trees(&db, "wt-2", 2, "tree-1b", "tree-2b"); - - let roots = store - .load_all_tree_roots() - .expect("load_all_tree_roots should succeed"); - - assert_eq!( - roots, - tree_set(["tree-0a", "tree-1a", "tree-0b", "tree-1b", "tree-2b",]), - ); - } - - #[test] - fn load_all_tree_roots_deduplicates_a_tree_shared_by_multiple_worktrees() { - let db_fixture = test_db_path("all-tree-roots-shared-tree"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-1", 1, "tree-shared"); - insert_event_trees(&db, "wt-1", 1, "tree-0a", "tree-shared"); - insert_worktree_with_cursor(&db, "wt-2", 1, "tree-1b"); - insert_event_trees(&db, "wt-2", 1, "tree-shared", "tree-1b"); - - let roots = store - .load_all_tree_roots() - .expect("load_all_tree_roots should succeed"); - - assert_eq!(roots, tree_set(["tree-0a", "tree-shared", "tree-1b"])); - } - - #[test] - fn load_all_tree_roots_is_empty_for_an_empty_repository() { - let db_fixture = test_db_path("all-tree-roots-empty-repository"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - let roots = store - .load_all_tree_roots() - .expect("load_all_tree_roots should return Ok for an empty repository"); - - assert!(roots.is_empty()); - } - - fn apply_atomic_cursor_advance(db: &RepositoryAgentTraceDb) { - let guard = TransactionStatement::new( - "UPDATE mutation_trace_worktrees SET cursor_tree = ?1, revision = ?2 - WHERE worktree_id = ?3 AND revision = ?4", - ( - "tree-x", - encode_revision(1).as_slice(), - "wt-b", - encode_revision(0).as_slice(), - ), - ) - .expect("guard statement should build"); - let statements = [TransactionStatement::new( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES (?1, ?2, ?3, ?4, 0, 'healthy', 'ineligible_unscoped', NULL, 'flush', NULL, NULL)", - ( - "wt-b", - encode_revision(1).as_slice(), - "tree-t", - "tree-x", - ), - ) - .expect("event statement should build")]; - - let applied = db - .execute_transactional_cas_batch( - "atomic cursor advance test", - "retry the operation", - &guard, - &statements, - ) - .expect("the atomic cursor advance should commit"); - assert!(applied, "the CAS guard should have matched revision 0"); - } - - fn select_trees(db: &RepositoryAgentTraceDb, sql: &str) -> BTreeSet { - db.query_map(sql, (), |row| { - let tree: String = row.get(0).context("failed to read a tree column")?; - Ok(TreeId(tree)) - }) - .expect("tree column select should succeed") - .into_iter() - .collect() - } - - /// State-transition coverage only: before the advance `T` is a root - /// through `cursor_tree`; after it, `T` is a root through `before_tree`. - /// This does NOT prove single-statement snapshot isolation — a torn - /// multi-read implementation would still pass this pre/post check. - /// `load_all_tree_roots_reads_every_durable_root_in_one_sql_statement` is - /// the deterministic regression for that property. - #[test] - fn load_all_tree_roots_retains_previous_cursor_after_atomic_cursor_advance() { - let db_fixture = test_db_path("all-tree-roots-retains-previous-cursor"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-b", 0, "tree-t"); - - let pre = store - .load_all_tree_roots() - .expect("load_all_tree_roots should succeed"); - assert!( - pre.contains(&TreeId("tree-t".to_string())), - "T is a durable root before the transition (via cursor_tree)" - ); - - apply_atomic_cursor_advance(&db); - - let post = store - .load_all_tree_roots() - .expect("load_all_tree_roots should succeed"); - assert!( - post.contains(&TreeId("tree-t".to_string())), - "T is still a durable root after the advance (via before_tree)" - ); - assert!( - post.contains(&TreeId("tree-x".to_string())), - "X becomes a durable root after the advance" - ); - } - - /// Deterministic regression for the actual concurrency boundary: one - /// `load_all_tree_roots()` invocation must read `cursor_tree`, - /// `before_tree`, and `after_tree` through a SINGLE SQL statement / one - /// database snapshot. If it is reimplemented as two or more independent - /// `SELECT`s unioned in Rust, an atomic `cursor T -> X` + - /// `MutationEvent { before = T, after = X }` commit interleaved between - /// those reads produces a torn set that omits `T`. The test constructs - /// that torn set explicitly (an events read, the atomic advance, then a - /// worktrees read, unioned in Rust — losing `T`) and then asserts the - /// production path issues exactly one read statement, so it can never - /// enter the interleaving and always retains `T`. - #[test] - fn load_all_tree_roots_reads_every_durable_root_in_one_sql_statement() { - let db_fixture = test_db_path("all-tree-roots-single-statement"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-b", 0, "tree-t"); - - let events_first = select_trees( - &db, - "SELECT before_tree AS tree FROM mutation_trace_events - UNION - SELECT after_tree AS tree FROM mutation_trace_events", - ); - assert!( - events_first.is_empty(), - "no event references T before the advance" - ); - - apply_atomic_cursor_advance(&db); - - let cursors_second = select_trees( - &db, - "SELECT cursor_tree AS tree FROM mutation_trace_worktrees", - ); - let torn: BTreeSet = events_first.union(&cursors_second).cloned().collect(); - assert!( - !torn.contains(&TreeId("tree-t".to_string())), - "a two-read implementation loses T across the atomic advance" - ); - - let (roots, statements_issued) = crate::services::db::count_read_statements(|| { - store - .load_all_tree_roots() - .expect("load_all_tree_roots should succeed") - }); - assert_eq!( - statements_issued, 1, - "load_all_tree_roots must read every durable-root column in one SQL statement" - ); - assert!( - roots.contains(&TreeId("tree-t".to_string())), - "the single-statement snapshot always retains T (via before_tree)" - ); - assert!(roots.contains(&TreeId("tree-x".to_string()))); - } - - /// The same single-statement / single-snapshot property, worktree-scoped: - /// one `load_tree_roots(W)` call reads W's `cursor_tree` / `before_tree` / - /// `after_tree` through one statement. A two-read reimplementation - /// (events-for-W, then cursor-for-W) would tear across an atomic cursor - /// advance in exactly the same way. - #[test] - fn load_tree_roots_reads_every_durable_root_in_one_sql_statement() { - let db_fixture = test_db_path("tree-roots-single-statement"); - let db_path = db_fixture.path(); - let db = RepositoryAgentTraceDb::new_at(db_path).expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - insert_worktree_with_cursor(&db, "wt-b", 0, "tree-t"); - - let events_first = select_trees( - &db, - "SELECT before_tree AS tree FROM mutation_trace_events WHERE worktree_id = 'wt-b' - UNION - SELECT after_tree AS tree FROM mutation_trace_events WHERE worktree_id = 'wt-b'", - ); - assert!( - events_first.is_empty(), - "no event references T before the advance" - ); - - apply_atomic_cursor_advance(&db); - - let cursors_second = select_trees( - &db, - "SELECT cursor_tree AS tree FROM mutation_trace_worktrees WHERE worktree_id = 'wt-b'", - ); - let torn: BTreeSet = events_first.union(&cursors_second).cloned().collect(); - assert!( - !torn.contains(&TreeId("tree-t".to_string())), - "a two-read implementation loses T across the atomic advance" - ); - - let (roots, statements_issued) = crate::services::db::count_read_statements(|| { - store - .load_tree_roots(&WorktreeId("wt-b".to_string())) - .expect("load_tree_roots should succeed") - }); - assert_eq!( - statements_issued, 1, - "load_tree_roots must read every durable-root column in one SQL statement" - ); - assert!( - roots.contains(&TreeId("tree-t".to_string())), - "the single-statement snapshot always retains T (via before_tree)" - ); - assert!(roots.contains(&TreeId("tree-x".to_string()))); - } - - mod mutation_attribution { - use super::*; - - #[test] - fn page_reader_orders_big_endian_revisions_and_isolates_worktrees() { - let db_fixture = test_db_path("mutation-attribution-ordering"); - let db = RepositoryAgentTraceDb::new_at(db_fixture.path()) - .expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - for revision in [1, 255, 256, u64::MAX] { - insert_mutation_event( - &db, - "wt-1", - revision, - "before", - "after", - "ai_exclusive", - Some("scope-1"), - "flush", - None, - None, - &[], - ); - } - insert_mutation_event( - &db, - "wt-2", - 999, - "foreign-before", - "foreign-after", - "ai_exclusive", - Some("foreign-scope"), - "flush", - None, - None, - &[], - ); - - let rows = store - .load_mutation_event_page(&WorktreeId("wt-1".to_string()), None, 100) - .expect("mutation event page should load"); - - assert_eq!( - rows.iter().map(|row| row.revision).collect::>(), - vec![u64::MAX, 256, 255, 1] - ); - assert_eq!(rows[0].before_tree, TreeId("before".to_string())); - assert_eq!(rows[0].after_tree, TreeId("after".to_string())); - assert_eq!(rows[0].failure_kind, FailureKind::Healthy); - assert_eq!(rows[0].attribution_kind, AttributionKind::AiExclusive); - assert_eq!( - rows[0].attribution_scope_id, - Some(ScopeId("scope-1".to_string())) - ); - } - - #[test] - fn page_reader_caps_limits_and_continues_with_an_exclusive_cursor() { - let db_fixture = test_db_path("mutation-attribution-pagination"); - let db = RepositoryAgentTraceDb::new_at(db_fixture.path()) - .expect("repository DB should open"); - let store = MutationTraceStore::new(&db); - - for revision in 1..=33 { - insert_mutation_event( - &db, - "wt-1", - revision, - "before", - "after", - "ineligible_unscoped", - None, - "flush", - None, - None, - &[], - ); - } - - let first_page = store - .load_mutation_event_page(&WorktreeId("wt-1".to_string()), None, 100) - .expect("first mutation event page should load"); - assert_eq!(first_page.len(), MUTATION_ATTRIBUTION_PAGE_SIZE); - assert_eq!(first_page.first().map(|row| row.revision), Some(33)); - assert_eq!(first_page.last().map(|row| row.revision), Some(2)); - - let cursor = first_page - .last() - .expect("the capped page should not be empty") - .revision; - let second_page = store - .load_mutation_event_page( - &WorktreeId("wt-1".to_string()), - Some(cursor), - MUTATION_ATTRIBUTION_PAGE_SIZE, - ) - .expect("second mutation event page should load"); - assert_eq!( - second_page - .iter() - .map(|row| row.revision) - .collect::>(), - vec![1] - ); - - let empty_page = store - .load_mutation_event_page(&WorktreeId("wt-1".to_string()), Some(1), 1) - .expect("page after the final cursor should load"); - assert!(empty_page.is_empty()); - } - } - - #[test] - fn validate_health_encoding_accepts_only_pairs_that_match_failure_kind() { - assert!(validate_health_encoding(false, FailureKind::Healthy).is_ok()); - assert!(validate_health_encoding(true, FailureKind::SnapshotFailure).is_ok()); - - let untainted_failure = validate_health_encoding(false, FailureKind::SnapshotFailure) - .expect_err("an untainted snapshot failure must be rejected"); - assert_eq!( - untainted_failure.to_string(), - "inconsistent mutation-trace health encoding: tainted=false with failure_kind=SnapshotFailure" - ); - - let tainted_healthy = validate_health_encoding(true, FailureKind::Healthy) - .expect_err("a tainted healthy pair must be rejected"); - assert_eq!( - tainted_healthy.to_string(), - "inconsistent mutation-trace health encoding: tainted=true with failure_kind=Healthy" - ); - } - - const INCONSISTENT_HEALTH_PAIRS: [(i64, &str); 2] = [(0, "snapshot_failure"), (1, "healthy")]; - - fn open_pre_health_invariant_db( - label: &str, - statements: &[String], - ) -> (TestDbPath, RepositoryAgentTraceDb) { - let db_fixture = test_db_path(label); - let statements: Vec<&str> = statements.iter().map(String::as_str).collect(); - seed_pre_health_invariant_fixture(db_fixture.path(), &statements); - let db = RepositoryAgentTraceDb::open_without_migrations_at(db_fixture.path()) - .expect("pre-006 fixture DB should reopen without applying 006"); - (db_fixture, db) - } - - fn inconsistent_worktree_sql(worktree_id: &str, tainted: i64, failure_kind: &str) -> String { - format!( - "INSERT INTO mutation_trace_worktrees - (worktree_id, cursor_tree, revision, tainted, failure_kind, needs_rebaseline) - VALUES ('{worktree_id}', 'tree-1', X'0000000000000001', {tainted}, '{failure_kind}', 0)" - ) - } - - fn inconsistent_event_sql(worktree_id: &str, tainted: i64, failure_kind: &str) -> String { - format!( - "INSERT INTO mutation_trace_events - (worktree_id, revision, before_tree, after_tree, tainted, failure_kind, - attribution_kind, attribution_scope_id, boundary_kind, boundary_scope_id, boundary_event_id) - VALUES ('{worktree_id}', X'0000000000000001', 'tree-0', 'tree-1', {tainted}, '{failure_kind}', - 'ineligible_unscoped', NULL, 'flush', NULL, NULL)" - ) - } - - fn assert_health_rejection(error: &anyhow::Error, table: &str) { - let rendered = format!("{error:#}"); - assert!( - rendered.contains(&format!("invalid {table}.tainted/failure_kind pair")), - "missing table context: {rendered}" - ); - assert!( - rendered.contains("inconsistent mutation-trace health encoding"), - "missing validator error: {rendered}" - ); - } - - #[test] - fn load_worktree_rejects_a_stored_health_pair_that_disagrees_with_failure_kind() { - for (tainted, failure_kind) in INCONSISTENT_HEALTH_PAIRS { - let (_db_fixture, db) = open_pre_health_invariant_db( - "inconsistent-worktree-health", - &[inconsistent_worktree_sql("wt-1", tainted, failure_kind)], - ); - let store = MutationTraceStore::new(&db); - - let error = store - .load_worktree(&WorktreeId("wt-1".to_string()), None, None) - .expect_err("an inconsistent worktree health pair must not load"); - assert_health_rejection(&error, "mutation_trace_worktrees"); - } - } - - #[test] - fn load_mutation_event_rejects_a_stored_health_pair_that_disagrees_with_failure_kind() { - for (tainted, failure_kind) in INCONSISTENT_HEALTH_PAIRS { - let (_db_fixture, db) = open_pre_health_invariant_db( - "inconsistent-event-health", - &[inconsistent_event_sql("wt-1", tainted, failure_kind)], - ); - let store = MutationTraceStore::new(&db); - - let error = store - .load_mutation_event(&WorktreeId("wt-1".to_string()), 1) - .expect_err("an inconsistent mutation-event health pair must not load"); - assert_health_rejection(&error, "mutation_trace_events"); - } - } - - #[test] - fn load_mutation_event_page_rejects_a_stored_health_pair_that_disagrees_with_failure_kind() { - for (tainted, failure_kind) in INCONSISTENT_HEALTH_PAIRS { - let (_db_fixture, db) = open_pre_health_invariant_db( - "inconsistent-event-page-health", - &[inconsistent_event_sql("wt-1", tainted, failure_kind)], - ); - let store = MutationTraceStore::new(&db); - - let error = store - .load_mutation_event_page(&WorktreeId("wt-1".to_string()), None, 1) - .expect_err("an inconsistent mutation-event health pair must not page"); - assert_health_rejection(&error, "mutation_trace_events"); - } - } -} diff --git a/cli/src/services/mutation_trace/tests.rs b/cli/src/services/mutation_trace/tests.rs index 979a5c14a..912dfadfe 100644 --- a/cli/src/services/mutation_trace/tests.rs +++ b/cli/src/services/mutation_trace/tests.rs @@ -125,20 +125,15 @@ fn prepare_and_commit( } #[test] -fn scope_status_live_and_terminal_table() { +fn scope_status_live_table() { let cases = [ - (ScopeStatus::NeverSeen, false, false), - (ScopeStatus::Active, true, false), - (ScopeStatus::Closed, false, true), - (ScopeStatus::Abandoned, false, true), + (ScopeStatus::NeverSeen, false), + (ScopeStatus::Active, true), + (ScopeStatus::Closed, false), + (ScopeStatus::Abandoned, false), ]; - for (status, expected_live, expected_terminal) in cases { + for (status, expected_live) in cases { assert_eq!(is_live(status), expected_live, "is_live({status:?})"); - assert_eq!( - is_terminal(status), - expected_terminal, - "is_terminal({status:?})" - ); } } diff --git a/cli/src/services/mutation_trace/types.rs b/cli/src/services/mutation_trace/types.rs index 30a781c5c..7ec4584e1 100644 --- a/cli/src/services/mutation_trace/types.rs +++ b/cli/src/services/mutation_trace/types.rs @@ -152,12 +152,6 @@ pub fn is_live(status: ScopeStatus) -> bool { status == ScopeStatus::Active } -/// A scope has ended and can never be reactivated. Refines `isTerminal` -/// (`spec/mutation_cursor.qnt:169-170`). -pub fn is_terminal(status: ScopeStatus) -> bool { - matches!(status, ScopeStatus::Closed | ScopeStatus::Abandoned) -} - impl ScopeState { /// The worktree this scope belongs to. Refines `scopeWorktree` /// (`spec/mutation_cursor.qnt:151-157`). @@ -175,21 +169,10 @@ impl ScopeState { self.worktree_id.clone() } - /// The harness that owns this scope. Refines `scopeActor` - /// (`spec/mutation_cursor.qnt:159-165`). - pub fn scope_actor(&self) -> ActorKind { - self.actor_kind - } - /// Refines `isLive` applied to this scope's status. pub fn is_live(&self) -> bool { is_live(self.status) } - - /// Refines `isTerminal` applied to this scope's status. - pub fn is_terminal(&self) -> bool { - is_terminal(self.status) - } } /// The worktree a boundary applies to. Refines `boundaryWorktree` diff --git a/cli/src/services/observability.rs b/cli/src/services/observability.rs index 5e6adb641..3d117b754 100644 --- a/cli/src/services/observability.rs +++ b/cli/src/services/observability.rs @@ -1,11 +1,12 @@ use std::{ cmp::Ordering, - collections::BTreeMap, + collections::hash_map::DefaultHasher, fmt::Write as FmtWrite, fs::{self, OpenOptions}, + hash::{Hash, Hasher}, io::{self, ErrorKind, Write}, path::{Path, PathBuf}, - sync::{Arc, Mutex, OnceLock}, + sync::Mutex, time::SystemTime, }; @@ -14,9 +15,7 @@ use chrono::{Local, NaiveDate, Utc}; use serde_json::json; use tracing::Level; -use crate::services::config::{ - self, LogFormat, LogLevel, ENV_LOG_DIR, ENV_LOG_FORMAT, ENV_LOG_LEVEL, -}; +use crate::services::config::{self, LogFormat, LogLevel, ENV_LOG_DIR}; use crate::services::error::CliError; use crate::services::security::redact_sensitive_text; @@ -74,34 +73,6 @@ impl Logger { }) } - #[allow(dead_code)] - fn from_env_lookup(lookup: F) -> Result - where - F: Fn(&str) -> Option, - { - let mut config = ObservabilityConfig::default(); - - if let Some(raw) = lookup(ENV_LOG_LEVEL) { - config.level = LogLevel::parse_env(&raw, ENV_LOG_LEVEL)?; - } - - if let Some(raw) = lookup(ENV_LOG_FORMAT) { - config.format = LogFormat::parse_env(&raw, ENV_LOG_FORMAT)?; - } - - let mut log_dir = None; - if let Some(raw) = lookup(ENV_LOG_DIR) { - validate_log_dir(&raw)?; - log_dir = Some(PathBuf::from(raw)); - } - - Ok(Self { - config, - log_dir, - log_file_retention_limit: config::DEFAULT_LOG_FILE_RETENTION_LIMIT, - }) - } - pub fn info( &self, event_id: &str, @@ -132,7 +103,6 @@ impl Logger { self.log_forced(LogLevel::Warn, event_id, message, fields, session_id); } - #[cfg_attr(not(test), allow(dead_code))] pub fn error( &self, event_id: &str, @@ -370,23 +340,26 @@ where .with_context(|| format!("failed to create log directory '{}'", parent.display()))?; } - let lock = file_log_lock(path); - let _guard = lock.lock().map_err(|error| { - anyhow::anyhow!("failed to lock log file '{}': {error}", path.display()) - })?; + let primary_error = { + let _guard = log_lock_stripe(path).lock().map_err(|error| { + anyhow::anyhow!("failed to lock log file '{}': {error}", path.display()) + })?; - match persist_log_line(path, redacted_line) { - Ok(write_target) => { - run_log_retention_after_creation(path, write_target, cleanup); - Ok(()) + match persist_log_line(path, redacted_line) { + Ok(write_target) => { + run_log_retention_after_creation(path, write_target, cleanup); + return Ok(()); + } + Err(primary_error) => primary_error, } - Err(primary_error) => attempt_v2_log_fallback( - path, - redacted_line, - &primary_error, - |fallback_path, line| append_log_line_once_with_cleanup(fallback_path, line, cleanup), - ), - } + }; + + attempt_v2_log_fallback( + path, + redacted_line, + &primary_error, + |fallback_path, line| append_log_line_once_with_cleanup(fallback_path, line, cleanup), + ) } fn attempt_v2_log_fallback( @@ -417,8 +390,7 @@ where .with_context(|| format!("failed to create log directory '{}'", parent.display()))?; } - let lock = file_log_lock(path); - let _guard = lock.lock().map_err(|error| { + let _guard = log_lock_stripe(path).lock().map_err(|error| { anyhow::anyhow!("failed to lock log file '{}': {error}", path.display()) })?; @@ -604,17 +576,19 @@ fn collect_managed_log_files(log_dir: &Path) -> Result<(Vec, Vec Ok((managed_files, errors)) } -fn file_log_lock(path: &Path) -> Arc> { - static FILE_LOG_LOCKS: OnceLock>>>> = OnceLock::new(); - let locks = FILE_LOG_LOCKS.get_or_init(|| Mutex::new(BTreeMap::new())); - let mut locks = locks - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - Arc::clone( - locks - .entry(path.to_path_buf()) - .or_insert_with(|| Arc::new(Mutex::new(()))), - ) +const LOG_LOCK_STRIPES: usize = 64; + +static LOG_LOCKS: [Mutex<()>; LOG_LOCK_STRIPES] = [const { Mutex::new(()) }; LOG_LOCK_STRIPES]; + +fn log_lock_stripe_index(path: &Path) -> usize { + let mut hasher = DefaultHasher::new(); + path.hash(&mut hasher); + usize::try_from(hasher.finish() % LOG_LOCK_STRIPES as u64) + .expect("log lock stripe index must fit usize") +} + +fn log_lock_stripe(path: &Path) -> &'static Mutex<()> { + &LOG_LOCKS[log_lock_stripe_index(path)] } #[cfg(unix)] @@ -737,7 +711,10 @@ mod tests { #[test] fn observability_fields_for_user_error_without_source_omit_error_source() { - let error = CliError::user(UserError::NotAuthenticated); + let error = CliError::User { + error: UserError::NotAuthenticated, + source: None, + }; let fields = cli_error_fields(&error); @@ -814,6 +791,388 @@ mod tests { assert!(!log_dir.exists()); } + const CHILD_DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); + const CHILD_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(20); + const ENV_CHILD_PATH: &str = "SCE_LOG_LOCK_CHILD_PATH"; + const SAME_STRIPE_SENTINEL: &str = "SAME_STRIPE_FALLBACK_OK"; + const POISON_SENTINEL: &str = "POISONED_STRIPE_OK"; + + fn text_logger(log_dir: &Path) -> Logger { + Logger { + config: ObservabilityConfig { + level: LogLevel::Error, + format: LogFormat::Text, + log_to_file: true, + }, + log_dir: Some(log_dir.to_path_buf()), + log_file_retention_limit: 10, + } + } + + fn read_all_log_lines(log_dir: &Path) -> Vec { + let mut lines = Vec::new(); + for entry in fs::read_dir(log_dir).unwrap() { + let path = entry.unwrap().path(); + if path + .extension() + .is_some_and(|ext| ext == LOG_FILE_EXTENSION) + { + lines.extend(fs::read_to_string(path).unwrap().lines().map(String::from)); + } + } + lines + } + + fn run_bounded_child(test_name: &str, child_path: &Path) -> std::process::Output { + use std::io::Read; + use std::process::{Command, Stdio}; + + let sandbox = tempfile::tempdir().unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", test_name, "--nocapture"]) + .current_dir(sandbox.path()) + .env(ENV_CHILD_PATH, child_path) + .env("HOME", sandbox.path()) + .env("XDG_CONFIG_HOME", sandbox.path().join("config")) + .env("XDG_STATE_HOME", sandbox.path().join("state")) + .env("XDG_CACHE_HOME", sandbox.path().join("cache")) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let mut stdout_pipe = child.stdout.take().unwrap(); + let mut stderr_pipe = child.stderr.take().unwrap(); + let stdout_reader = std::thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = stdout_pipe.read_to_end(&mut buffer); + buffer + }); + let stderr_reader = std::thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = stderr_pipe.read_to_end(&mut buffer); + buffer + }); + + let deadline = std::time::Instant::now() + CHILD_DEADLINE; + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break Some(status); + } + if std::time::Instant::now() >= deadline { + child.kill().unwrap(); + child.wait().unwrap(); + break None; + } + std::thread::sleep(CHILD_POLL_INTERVAL); + }; + + let stdout = stdout_reader.join().unwrap(); + let stderr = stderr_reader.join().unwrap(); + let Some(status) = status else { + panic!( + "child '{test_name}' exceeded {CHILD_DEADLINE:?} and was killed\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + }; + + std::process::Output { + status, + stdout, + stderr, + } + } + + fn assert_child_succeeded(output: &std::process::Output, sentinel: &str) { + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains(sentinel), + "child failed: {:?}\nstdout: {stdout}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + } + + fn blocked_primary_path(log_dir: &Path, stem: &str) -> PathBuf { + let primary = log_dir.join(format!("{stem}.log")); + fs::create_dir_all(&primary).unwrap(); + primary + } + + #[test] + fn concurrent_writers_to_one_path_produce_complete_unique_lines() { + const THREADS: usize = 8; + const LINES_PER_THREAD: usize = 50; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-concurrent.log"); + let padding = "x".repeat(512); + + std::thread::scope(|scope| { + for thread in 0..THREADS { + let path = &path; + let padding = &padding; + scope.spawn(move || { + for index in 0..LINES_PER_THREAD { + let line = format!("t{thread}-l{index}-{padding}-end"); + append_log_line_with_cleanup(path, &line, |_| Ok(())).unwrap(); + } + }); + } + }); + + let contents = fs::read_to_string(&path).unwrap(); + let mut seen = std::collections::BTreeSet::new(); + for line in contents.lines() { + assert!(line.ends_with("-end"), "interleaved line: {line}"); + assert!(seen.insert(line.to_string()), "duplicated line: {line}"); + } + assert_eq!(seen.len(), THREADS * LINES_PER_THREAD); + } + + #[test] + fn independent_logger_instances_serialize_writes_to_one_path() { + const INSTANCES: usize = 4; + const RECORDS_PER_INSTANCE: usize = 25; + + let dir = tempfile::tempdir().unwrap(); + let loggers: Vec = (0..INSTANCES).map(|_| text_logger(dir.path())).collect(); + + std::thread::scope(|scope| { + for (instance, logger) in loggers.iter().enumerate() { + scope.spawn(move || { + for index in 0..RECORDS_PER_INSTANCE { + let event_id = format!("sce.test.i{instance}.r{index}"); + logger.error(&event_id, "concurrent record", &[], Some("shared")); + } + }); + } + }); + + let lines = read_all_log_lines(dir.path()); + assert_eq!(lines.len(), INSTANCES * RECORDS_PER_INSTANCE); + for line in &lines { + assert!(line.starts_with("timestamp="), "malformed line: {line}"); + assert!( + line.ends_with("message=concurrent record"), + "malformed line: {line}" + ); + } + let unique: std::collections::BTreeSet<&String> = lines.iter().collect(); + assert_eq!(unique.len(), lines.len()); + } + + #[test] + fn stripe_selection_is_stable_bounded_and_independent_of_path_count() { + assert_eq!(LOG_LOCKS.len(), LOG_LOCK_STRIPES); + + let mut used = std::collections::BTreeSet::new(); + for index in 0..5000 { + let path = PathBuf::from(format!("/tmp/sce-distinct/sce-{index}.log")); + let stripe = log_lock_stripe_index(&path); + assert!(stripe < LOG_LOCK_STRIPES); + assert_eq!(stripe, log_lock_stripe_index(&path)); + assert!(std::ptr::eq( + log_lock_stripe(&path), + LOG_LOCKS.get(stripe).unwrap() + )); + used.insert(stripe); + } + + assert!(used.len() > 1); + assert_eq!(LOG_LOCKS.len(), LOG_LOCK_STRIPES); + } + + #[test] + fn fallback_on_a_different_stripe_writes_the_v2_file() { + let dir = tempfile::tempdir().unwrap(); + let primary = (0..1000) + .map(|index| dir.path().join(format!("sce-diff-{index}.log"))) + .find(|candidate| { + !std::ptr::eq( + log_lock_stripe(candidate), + log_lock_stripe(&v2_log_path(candidate)), + ) + }) + .unwrap(); + fs::create_dir_all(&primary).unwrap(); + + append_log_line_with_cleanup(&primary, "fallback-line", |_| Ok(())).unwrap(); + + assert_eq!( + fs::read_to_string(v2_log_path(&primary)).unwrap(), + "fallback-line\n" + ); + assert!(primary.is_dir()); + } + + #[test] + fn fallback_failure_reports_combined_primary_and_v2_error() { + let dir = tempfile::tempdir().unwrap(); + let primary = blocked_primary_path(dir.path(), "sce-both-fail"); + let fallback = v2_log_path(&primary); + fs::create_dir_all(&fallback).unwrap(); + + let error = append_log_line_with_cleanup(&primary, "line", |_| Ok(())) + .unwrap_err() + .to_string(); + + assert!( + error.starts_with(&format!( + "primary log file persistence failed for '{}': ", + primary.display() + )), + "{error}" + ); + assert!( + error.contains(&format!( + "; v2 fallback log file persistence failed for '{}': ", + fallback.display() + )), + "{error}" + ); + } + + #[test] + fn same_stripe_fallback_completes_in_bounded_child_process() { + let dir = tempfile::tempdir().unwrap(); + let primary = (0..10_000) + .map(|index| dir.path().join(format!("sce-collide-{index}.log"))) + .find(|candidate| { + std::ptr::eq( + log_lock_stripe(candidate), + log_lock_stripe(&v2_log_path(candidate)), + ) + }) + .expect("a primary path whose v2 path shares its stripe"); + fs::create_dir_all(&primary).unwrap(); + + let output = run_bounded_child( + "services::observability::tests::isolated_same_stripe_fallback", + &primary, + ); + + assert_child_succeeded(&output, SAME_STRIPE_SENTINEL); + assert_eq!( + fs::read_to_string(v2_log_path(&primary)).unwrap(), + "same-stripe-line\n" + ); + assert!(primary.is_dir()); + } + + #[test] + fn isolated_same_stripe_fallback() { + let Some(primary) = std::env::var_os(ENV_CHILD_PATH).map(PathBuf::from) else { + return; + }; + assert!(std::ptr::eq( + log_lock_stripe(&primary), + log_lock_stripe(&v2_log_path(&primary)) + )); + + append_log_line_with_cleanup(&primary, "same-stripe-line", |_| Ok(())).unwrap(); + + println!("{SAME_STRIPE_SENTINEL}"); + } + + #[test] + fn poisoned_stripe_fails_lock_without_fallback_in_child_process() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-poison.log"); + + let output = run_bounded_child( + "services::observability::tests::isolated_poisoned_stripe", + &path, + ); + + assert_child_succeeded(&output, POISON_SENTINEL); + assert!(!v2_log_path(&path).exists()); + } + + #[test] + fn isolated_poisoned_stripe() { + let Some(path) = std::env::var_os(ENV_CHILD_PATH).map(PathBuf::from) else { + return; + }; + + let poisoned = std::panic::catch_unwind(|| { + let _ = append_log_line_with_cleanup(&path, "first", |_| panic!("poison the stripe")); + }); + assert!(poisoned.is_err()); + + let error = append_log_line_with_cleanup(&path, "second", |_| Ok(())) + .unwrap_err() + .to_string(); + assert!( + error.starts_with(&format!("failed to lock log file '{}': ", path.display())), + "{error}" + ); + assert!(!v2_log_path(&path).exists()); + assert_eq!(fs::read_to_string(&path).unwrap(), "first\n"); + + println!("{POISON_SENTINEL}"); + } + + #[test] + fn retention_runs_once_for_concurrent_creation_of_one_path() { + const THREADS: usize = 8; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-retention.log"); + let cleanups = std::sync::atomic::AtomicUsize::new(0); + + std::thread::scope(|scope| { + for thread in 0..THREADS { + let path = &path; + let cleanups = &cleanups; + scope.spawn(move || { + append_log_line_with_cleanup(path, &format!("line-{thread}"), |log_dir| { + assert_eq!(log_dir, path.parent().unwrap()); + cleanups.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }) + .unwrap(); + }); + } + }); + + assert_eq!(cleanups.load(std::sync::atomic::Ordering::SeqCst), 1); + assert_eq!(fs::read_to_string(&path).unwrap().lines().count(), THREADS); + } + + #[cfg(unix)] + #[test] + fn created_log_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("sce-perms.log"); + + append_log_line_with_cleanup(&path, "line", |_| Ok(())).unwrap(); + + let mode = fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o600); + } + + #[test] + fn secrets_are_redacted_before_they_reach_the_log_file() { + let dir = tempfile::tempdir().unwrap(); + let logger = text_logger(dir.path()); + + logger.error( + "sce.test.redaction", + "request failed", + &[("password", "hunter2-super-secret")], + None, + ); + + let lines = read_all_log_lines(dir.path()); + assert_eq!(lines.len(), 1); + assert!(lines[0].contains("[REDACTED]")); + assert!(!lines[0].contains("hunter2-super-secret")); + } + #[test] fn observability_fields_for_internal_error_carry_surface_and_full_source_chain() { let source = anyhow::anyhow!("root cause").context("failed to do the thing"); diff --git a/cli/src/services/observability/traits.rs b/cli/src/services/observability/traits.rs index 330854885..a0d71d32c 100644 --- a/cli/src/services/observability/traits.rs +++ b/cli/src/services/observability/traits.rs @@ -46,7 +46,6 @@ pub trait Telemetry: Send + Sync { } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -#[allow(dead_code)] pub struct NoopLogger; impl Logger for NoopLogger { diff --git a/cli/src/services/output_format.rs b/cli/src/services/output_format.rs index 2e980c6f3..d403befcf 100644 --- a/cli/src/services/output_format.rs +++ b/cli/src/services/output_format.rs @@ -1,4 +1,3 @@ -use anyhow::{bail, Result}; use clap::ValueEnum; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] @@ -7,16 +6,3 @@ pub enum OutputFormat { Text, Json, } - -impl OutputFormat { - #[allow(dead_code)] - pub fn parse(raw: &str, help_command: &str) -> Result { - match raw { - "text" => Ok(Self::Text), - "json" => Ok(Self::Json), - _ => bail!( - "Invalid --format value '{raw}'. Valid values: text, json. Run '{help_command}' to see valid usage." - ), - } - } -} diff --git a/cli/src/services/parse/command_runtime.rs b/cli/src/services/parse/command_runtime.rs index a1b6d9a47..c40ef758c 100644 --- a/cli/src/services/parse/command_runtime.rs +++ b/cli/src/services/parse/command_runtime.rs @@ -293,13 +293,16 @@ fn convert_clap_command(command: cli_schema::Commands) -> Result convert_setup_command(services::setup::SetupCliOptions { - help: false, + target: [ + (opencode, services::setup::SetupTarget::OpenCode), + (claude, services::setup::SetupTarget::Claude), + (pi, services::setup::SetupTarget::Pi), + (codex, services::setup::SetupTarget::Codex), + (all, services::setup::SetupTarget::All), + ] + .into_iter() + .find_map(|(selected, target)| selected.then_some(target)), non_interactive, - opencode, - claude, - pi, - codex, - all, hooks, repo_path: repo, bootstrap_context, diff --git a/cli/src/services/patch.rs b/cli/src/services/patch.rs index 9e68c31ba..60572dda6 100644 --- a/cli/src/services/patch.rs +++ b/cli/src/services/patch.rs @@ -95,7 +95,6 @@ pub enum TouchedLineKind { Removed, } -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub struct ParseError { pub message: String, @@ -115,7 +114,6 @@ impl std::error::Error for ParseError {} /// payload could not be reconstructed into a valid `ParsedPatch`. Common /// causes include malformed JSON syntax, missing required fields, or type /// mismatches in the serialized structure. -#[allow(dead_code)] #[derive(Clone, Debug, Eq, PartialEq)] pub struct PatchLoadError { pub message: String, @@ -147,24 +145,6 @@ pub fn load_patch_from_json(input: &str) -> Result }) } -/// Load a `ParsedPatch` from JSON bytes previously produced by serializing -/// a `ParsedPatch`. -/// -/// This is the bytes-oriented counterpart to [`load_patch_from_json`], -/// convenient when the caller has raw bytes (for example, from a database -/// BLOB column or a file read) rather than a UTF-8 string. -/// -/// # Errors -/// -/// Returns `PatchLoadError` with an actionable message when the input is -/// not valid JSON or does not match the expected `ParsedPatch` structure. -#[allow(dead_code)] -pub fn load_patch_from_json_bytes(input: &[u8]) -> Result { - serde_json::from_slice(input).map_err(|e| PatchLoadError { - message: format!("invalid patch JSON: {e}"), - }) -} - /// Compute the touched-line intersection of two patches. /// /// Returns a `ParsedPatch` containing only the touched lines from @@ -196,7 +176,6 @@ pub fn load_patch_from_json_bytes(input: &[u8]) -> Result(full_path: &[&'a str], suffix_candidate: &[&'a s /// /// let combined = combine_patches(&[patch_a, patch_b]); /// ``` -#[allow(dead_code)] pub fn combine_patches(patches: &[ParsedPatch]) -> ParsedPatch { use std::collections::HashMap; @@ -514,7 +492,6 @@ pub fn combine_patches(patches: &[ParsedPatch]) -> ParsedPatch { /// /// Returns `ParseError` with an actionable message when the input is malformed, /// such as an invalid hunk header or a `---`/`+++` line that cannot be parsed. -#[allow(dead_code)] pub fn parse_patch(input: &str, session_id: Option<&str>) -> Result { let mut files: Vec = Vec::new(); let mut current_file: Option = None; diff --git a/cli/src/services/repository_identity/mod.rs b/cli/src/services/repository_identity/mod.rs index 405d0b875..c795169d7 100644 --- a/cli/src/services/repository_identity/mod.rs +++ b/cli/src/services/repository_identity/mod.rs @@ -26,15 +26,6 @@ pub struct RepositoryIdentity { pub repository_id: String, } -impl RepositoryIdentity { - /// Human-readable on-disk directory segment (`-`) for this - /// identity. Convenience wrapper over [`repository_dir_segment`]; the - /// authoritative identity is still [`RepositoryIdentity::repository_id`]. - pub fn dir_segment(&self) -> String { - repository_dir_segment(&self.canonical_identity) - } -} - /// Canonicalization failure. Variants carry no input fragments so /// credential-bearing URLs never leak into error output. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -101,55 +92,6 @@ pub fn derive_repository_id(canonical_identity: &str) -> String { hex_encode(&hasher.finalize()) } -/// Length in hex chars of the disambiguating short hash in a directory segment. -const DIR_SEGMENT_SHORT_HASH_LEN: usize = 4; - -/// Derives the human-readable on-disk directory segment for a canonical -/// identity as `-`, where `slug` is the lowercased canonical -/// identity with every run of non-alphanumeric characters collapsed to a -/// single `-` and leading/trailing `-` trimmed, and `short` is the first four -/// hex chars of `SHA256(canonical_identity)` computed with **no** domain -/// prefix. The short hash is deliberately distinct from [`derive_repository_id`], -/// which keeps its `sce-repository-id-v1\0` domain separation. This is a pure -/// display/layout helper: the authoritative identity remains the repository ID. -pub fn repository_dir_segment(canonical_identity: &str) -> String { - let slug = slugify(canonical_identity); - let short = derive_short_hash(canonical_identity); - if slug.is_empty() { - short - } else { - format!("{slug}-{short}") - } -} - -/// Lowercases and collapses every run of non-alphanumeric characters to a -/// single `-`, trimming leading and trailing `-`. -fn slugify(input: &str) -> String { - let mut slug = String::with_capacity(input.len()); - let mut pending_dash = false; - for ch in input.chars() { - if ch.is_ascii_alphanumeric() { - if pending_dash && !slug.is_empty() { - slug.push('-'); - } - pending_dash = false; - slug.push(ch.to_ascii_lowercase()); - } else { - pending_dash = true; - } - } - slug -} - -/// First [`DIR_SEGMENT_SHORT_HASH_LEN`] hex chars of the un-prefixed -/// `SHA256(canonical_identity)`. -fn derive_short_hash(canonical_identity: &str) -> String { - let mut hasher = Sha256::new(); - hasher.update(canonical_identity.as_bytes()); - let full = hex_encode(&hasher.finalize()); - full[..DIR_SEGMENT_SHORT_HASH_LEN].to_string() -} - /// Canonicalizes a Git remote URL to the scheme-neutral form /// `host[:port]/path` with credentials stripped, hostname lowercased, /// default ports removed, and query/fragment/trailing-slash/trailing-`.git` @@ -483,26 +425,6 @@ mod tests { ); } - fn un_prefixed_sha256_prefix(input: &str, len: usize) -> String { - let mut hasher = Sha256::new(); - hasher.update(input.as_bytes()); - hex_encode(&hasher.finalize())[..len].to_string() - } - - #[test] - fn dir_segment_matches_slug_and_un_prefixed_short_hash() { - // Slug: lowercased, non-alphanumeric runs (`.`,`/`) collapsed to `-`, - // trimmed. Short: first 4 hex of the un-prefixed SHA-256, deliberately - // distinct from the domain-prefixed repository ID. - let canonical = "github.com/crocoder-dev/shared-context-engineering"; - let short = un_prefixed_sha256_prefix(canonical, 4); - assert_eq!( - repository_dir_segment(canonical), - format!("github-com-crocoder-dev-shared-context-engineering-{short}") - ); - assert_ne!(short, derive_repository_id(canonical)[..4]); - } - #[test] fn missing_path_variants_error() { assert_eq!( diff --git a/cli/src/services/resilience.rs b/cli/src/services/resilience.rs index 2e0484d94..3fca25337 100644 --- a/cli/src/services/resilience.rs +++ b/cli/src/services/resilience.rs @@ -1,18 +1,91 @@ use std::future::Future; -use std::thread; use std::time::{Duration, Instant}; -use anyhow::{anyhow, ensure, Result}; +use std::fmt; + +use anyhow::{anyhow, Result}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum RetryPolicyError { + ZeroMaxAttempts, + ZeroTimeout, + MaxBackoffBelowInitial, +} + +impl fmt::Display for RetryPolicyError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::ZeroMaxAttempts => "Retry policy requires max_attempts >= 1", + Self::ZeroTimeout => "Retry policy requires timeout_ms >= 1", + Self::MaxBackoffBelowInitial => { + "Retry policy requires max_backoff_ms >= initial_backoff_ms" + } + }) + } +} + +impl std::error::Error for RetryPolicyError {} #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct RetryPolicy { - pub max_attempts: u32, - pub timeout_ms: u64, - pub initial_backoff_ms: u64, - pub max_backoff_ms: u64, + max_attempts: u32, + timeout_ms: u64, + initial_backoff_ms: u64, + max_backoff_ms: u64, } impl RetryPolicy { + pub const fn new( + max_attempts: u32, + timeout_ms: u64, + initial_backoff_ms: u64, + max_backoff_ms: u64, + ) -> Result { + if max_attempts == 0 { + return Err(RetryPolicyError::ZeroMaxAttempts); + } + if timeout_ms == 0 { + return Err(RetryPolicyError::ZeroTimeout); + } + if max_backoff_ms < initial_backoff_ms { + return Err(RetryPolicyError::MaxBackoffBelowInitial); + } + Ok(Self { + max_attempts, + timeout_ms, + initial_backoff_ms, + max_backoff_ms, + }) + } + + pub const fn builtin( + max_attempts: u32, + timeout_ms: u64, + initial_backoff_ms: u64, + max_backoff_ms: u64, + ) -> Self { + match Self::new(max_attempts, timeout_ms, initial_backoff_ms, max_backoff_ms) { + Ok(policy) => policy, + Err(_) => panic!("invalid built-in retry policy"), + } + } + + pub const fn max_attempts(self) -> u32 { + self.max_attempts + } + + pub const fn timeout_ms(self) -> u64 { + self.timeout_ms + } + + pub const fn initial_backoff_ms(self) -> u64 { + self.initial_backoff_ms + } + + pub const fn max_backoff_ms(self) -> u64 { + self.max_backoff_ms + } + fn timeout(self) -> Duration { Duration::from_millis(self.timeout_ms) } @@ -42,19 +115,6 @@ where Op: FnMut(u32) -> Fut, Fut: Future>, { - ensure!( - policy.max_attempts > 0, - "Retry policy requires max_attempts >= 1" - ); - ensure!( - policy.timeout_ms > 0, - "Retry policy requires timeout_ms >= 1" - ); - ensure!( - policy.max_backoff_ms >= policy.initial_backoff_ms, - "Retry policy requires max_backoff_ms >= initial_backoff_ms" - ); - let mut last_error = String::new(); for attempt in 1..=policy.max_attempts { @@ -98,34 +158,38 @@ where )) } -#[allow(dead_code)] -pub fn run_with_retry_sync( +/// One retryable attempt. Closures returning a future implement it directly; +/// operations that must lend an exclusive borrow (such as a transaction on a +/// single connection) implement it on a struct, because an `FnMut` closure +/// cannot return a future that borrows its own captured `&mut` state. +pub trait RetryOperation { + async fn run(&mut self, attempt: u32) -> Result; +} + +impl RetryOperation for F +where + F: FnMut(u32) -> Fut, + Fut: Future>, +{ + async fn run(&mut self, attempt: u32) -> Result { + self(attempt).await + } +} + +pub async fn run_with_retry_elapsed( policy: RetryPolicy, operation_name: &str, retry_hint: &str, mut operation: Op, ) -> Result where - Op: FnMut(u32) -> Result, + Op: RetryOperation, { - ensure!( - policy.max_attempts > 0, - "Retry policy requires max_attempts >= 1" - ); - ensure!( - policy.timeout_ms > 0, - "Retry policy requires timeout_ms >= 1" - ); - ensure!( - policy.max_backoff_ms >= policy.initial_backoff_ms, - "Retry policy requires max_backoff_ms >= initial_backoff_ms" - ); - let mut last_error = String::new(); for attempt in 1..=policy.max_attempts { let started_at = Instant::now(); - let outcome = operation(attempt); + let outcome = operation.run(attempt).await; match outcome { Ok(value) => return Ok(value), @@ -156,7 +220,7 @@ where error = %last_error, "Retrying operation after transient failure" ); - thread::sleep(backoff); + tokio::time::sleep(backoff).await; } Err(anyhow!( @@ -172,149 +236,30 @@ where #[cfg(test)] mod tests { - use super::*; - - #[test] - fn sync_retry_succeeds_after_transient_failures() { - let policy = RetryPolicy { - max_attempts: 3, - timeout_ms: 1_000, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }; - let mut attempts = 0; - - let result = - run_with_retry_sync(policy, "sync success", "retry the operation", |attempt| { - attempts = attempt; - if attempt < 3 { - return Err(anyhow!("transient failure {attempt}")); - } - - Ok("ok") - }); - - assert_eq!(result.unwrap(), "ok"); - assert_eq!(attempts, 3); - } - - #[test] - fn sync_retry_reports_exhausted_failures_with_guidance() { - let policy = RetryPolicy { - max_attempts: 2, - timeout_ms: 1_000, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }; - let mut attempts = 0; - - let error = - run_with_retry_sync::<(), _>(policy, "sync failure", "retry later", |attempt| { - attempts = attempt; - Err(anyhow!("nope {attempt}")) - }) - .unwrap_err(); - - assert_eq!(attempts, 2); - let message = error.to_string(); - assert!(message.contains("Operation 'sync failure' failed after 2 attempt(s)")); - assert!(message.contains("Last error: nope 2")); - assert!(message.contains("Try: retry later")); - } + use super::{RetryPolicy, RetryPolicyError}; #[test] - fn sync_retry_returns_a_slow_success_without_retrying_it() { - let policy = RetryPolicy { - max_attempts: 5, - timeout_ms: 5, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }; - let mut attempts = 0; - - let value = run_with_retry_sync( - policy, - "sync slow success", - "try again when the resource is available", - |attempt| { - attempts = attempt; - thread::sleep(Duration::from_millis(20)); - Ok("late success") - }, - ) - .expect("a completed success must never be reclassified as a timeout"); - - assert_eq!(value, "late success"); - assert_eq!(attempts, 1, "a slow success must not be retried"); - } - - #[test] - fn sync_retry_annotates_a_slow_failure_with_the_elapsed_bound() { - let policy = RetryPolicy { - max_attempts: 1, - timeout_ms: 5, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }; - - let error = run_with_retry_sync::<(), _>( - policy, - "sync slow failure", - "try again when the resource is available", - |_| { - thread::sleep(Duration::from_millis(20)); - Err(anyhow!("connection refused")) - }, - ) - .unwrap_err(); - - assert!(error - .to_string() - .contains("attempt 1 exceeded 5ms and failed: connection refused")); - } - - #[test] - fn sync_retry_does_not_re_execute_a_committed_write_after_a_slow_first_attempt() { - let policy = RetryPolicy { - max_attempts: 5, - timeout_ms: 1, - initial_backoff_ms: 0, - max_backoff_ms: 0, - }; - let mut inserted_keys: Vec = Vec::new(); - let mut calls = 0; - - let result = run_with_retry_sync(policy, "insert once", "retry later", |_| { - calls += 1; - thread::sleep(Duration::from_millis(5)); - if inserted_keys.contains(&1) { - return Err(anyhow!("UNIQUE constraint failed: keys.id")); - } - inserted_keys.push(1); - Ok(()) - }); - - assert!( - result.is_ok(), - "the committed first attempt must be returned" + fn retry_policy_new_rejects_each_invalid_invariant() { + assert_eq!( + RetryPolicy::new(0, 100, 10, 20), + Err(RetryPolicyError::ZeroMaxAttempts) + ); + assert_eq!( + RetryPolicy::new(1, 0, 10, 20), + Err(RetryPolicyError::ZeroTimeout) + ); + assert_eq!( + RetryPolicy::new(1, 100, 21, 20), + Err(RetryPolicyError::MaxBackoffBelowInitial) ); - assert_eq!(calls, 1, "the non-idempotent write must run exactly once"); - assert_eq!(inserted_keys, vec![1]); } #[test] - fn retry_policy_backoff_is_exponential_and_capped() { - let policy = RetryPolicy { - max_attempts: 5, - timeout_ms: 1_000, - initial_backoff_ms: 5, - max_backoff_ms: 12, - }; - - assert_eq!(policy.backoff_for_attempt(1), Duration::from_millis(0)); - assert_eq!(policy.backoff_for_attempt(2), Duration::from_millis(5)); - assert_eq!(policy.backoff_for_attempt(3), Duration::from_millis(10)); - assert_eq!(policy.backoff_for_attempt(4), Duration::from_millis(12)); - assert_eq!(policy.backoff_for_attempt(5), Duration::from_millis(12)); + fn retry_policy_new_accepts_boundary_values() { + let policy = RetryPolicy::new(1, 1, 0, 0).unwrap(); + assert_eq!(policy.max_attempts(), 1); + assert_eq!(policy.timeout_ms(), 1); + assert_eq!(policy.initial_backoff_ms(), 0); + assert_eq!(policy.max_backoff_ms(), 0); } } diff --git a/cli/src/services/setup/command.rs b/cli/src/services/setup/command.rs index b48d2af6e..faf0770f7 100644 --- a/cli/src/services/setup/command.rs +++ b/cli/src/services/setup/command.rs @@ -12,7 +12,7 @@ pub struct SetupCommand { } impl SetupCommand { - pub fn execute(&self, context: &C) -> Result { + pub async fn execute(&self, context: &C) -> Result { let setup_start_path = match &self.request.hooks_repo_path { Some(path) => path.clone(), None => std::env::current_dir() @@ -34,23 +34,15 @@ impl SetupCommand { mode, &setup::InquireSetupTargetPrompter, &optional_workflow_defaults, + &setup::persisted_integration_targets(&repository_root), + setup::persisted_setup_confirmations(&repository_root), ) .map_err(unexpected_failure)? { - setup::SetupDispatch::Proceed { - mode: resolved_mode, - optional_workflows, - agent_trace_auto_sync, - attribution_hooks_enabled, - } => Some(( - resolved_mode, - optional_workflows, - agent_trace_auto_sync, - attribution_hooks_enabled, - )), setup::SetupDispatch::Cancelled => { return Ok(setup::setup_cancelled_text()); } + dispatch @ setup::SetupDispatch::Proceed { .. } => Some(dispatch), } } else { None @@ -71,7 +63,7 @@ impl SetupCommand { let providers = lifecycle_providers(self.request.install_hooks); for provider in &providers { - let outcome = provider.setup(&ctx).map_err(unexpected_failure)?; + let outcome = provider.setup(&ctx).await.map_err(unexpected_failure)?; sections.extend(outcome.messages); @@ -82,20 +74,22 @@ impl SetupCommand { } } - if let Some(( - resolved_mode, - prompted_optional_workflows, + if let Some(setup::SetupDispatch::Proceed { + targets, + target_persistence, + optional_workflows: prompted_optional_workflows, agent_trace_auto_sync, attribution_hooks_enabled, - )) = setup_dispatch + }) = setup_dispatch { let optional_workflows = prompted_optional_workflows .as_deref() .or(self.request.optional_workflows.as_deref()); - let setup_message = setup::run_setup_for_mode( + let setup_message = setup::run_setup_for_targets( &repository_root, - resolved_mode, + &targets, + target_persistence, optional_workflows, agent_trace_auto_sync, attribution_hooks_enabled, diff --git a/cli/src/services/setup/install.rs b/cli/src/services/setup/install.rs index 986a4c531..ce91124ed 100644 --- a/cli/src/services/setup/install.rs +++ b/cli/src/services/setup/install.rs @@ -13,12 +13,12 @@ use crate::services::security::{ensure_directory_is_writable, redact_sensitive_t use super::config_merge; use super::hook_merge; use super::{ - classify_git_exit, cleanup_path_if_exists, concrete_targets_for, - embedded_assets_for_concrete_target, hook_install_recovery_guidance, - iter_embedded_assets_for_setup_target_with_selection, iter_required_hook_assets, - setup_install_recovery_guidance, EmbeddedAsset, GitExitKind, GitRepositoryResolutionError, - RequiredHookInstallResult, RequiredHookInstallStatus, RequiredHooksInstallOutcome, - SetupInstallOutcome, SetupInstallTargetResult, SetupTarget, + classify_git_exit, cleanup_path_if_exists, embedded_assets_for_concrete_target, + hook_install_recovery_guidance, iter_embedded_assets_for_setup_target_with_selection, + iter_required_hook_assets, setup_install_recovery_guidance, ConcreteSetupTarget, EmbeddedAsset, + GitExitKind, GitRepositoryResolutionError, RequiredHookInstallResult, + RequiredHookInstallStatus, RequiredHooksInstallOutcome, SetupInstallOutcome, + SetupInstallTargetResult, }; use crate::services::default_paths; use crate::services::default_paths::claude_asset; @@ -53,12 +53,12 @@ where pub(super) fn install_embedded_setup_assets( repository_root: &Path, - target: SetupTarget, + targets: &[ConcreteSetupTarget], selected_optional_workflows: &[String], ) -> Result { install_embedded_setup_assets_with_rename( repository_root, - target, + targets, selected_optional_workflows, |from, to| fs::rename(from, to), ) @@ -66,7 +66,7 @@ pub(super) fn install_embedded_setup_assets( pub(super) fn repair_merge_target_asset( repository_root: &Path, - target: SetupTarget, + target: ConcreteSetupTarget, relative_path: &str, ) -> Result<()> { let asset = embedded_assets_for_concrete_target(target) @@ -76,20 +76,26 @@ pub(super) fn repair_merge_target_asset( format!("No embedded asset named '{relative_path}' for target {target:?}") })?; - let install_targets = InstallTargetPaths::new(repository_root); - let destination_root = match target { - SetupTarget::OpenCode => install_targets.opencode_target_dir(), - SetupTarget::Claude => install_targets.claude_target_dir(), - SetupTarget::Pi => install_targets.pi_target_dir(), - SetupTarget::Codex => install_targets.codex_target_dir(), - SetupTarget::All => unreachable!("meta targets are expanded into concrete targets"), - }; + let destination_root = concrete_target_destination_root(repository_root, target); install_single_asset_with_rename(target, &destination_root, asset, &mut |from, to| { fs::rename(from, to) }) } +fn concrete_target_destination_root( + repository_root: &Path, + target: ConcreteSetupTarget, +) -> PathBuf { + let install_targets = InstallTargetPaths::new(repository_root); + match target { + ConcreteSetupTarget::OpenCode => install_targets.opencode_target_dir(), + ConcreteSetupTarget::Claude => install_targets.claude_target_dir(), + ConcreteSetupTarget::Pi => install_targets.pi_target_dir(), + ConcreteSetupTarget::Codex => install_targets.codex_target_dir(), + } +} + fn install_required_git_hooks_in_resolved_repository( resolved_repository_root: &Path, mut rename_fn: F, @@ -494,7 +500,7 @@ fn is_executable_file(path: &Path) -> Result { pub(super) fn install_embedded_setup_assets_with_rename( repository_root: &Path, - target: SetupTarget, + targets: &[ConcreteSetupTarget], selected_optional_workflows: &[String], mut rename_fn: F, ) -> Result @@ -505,7 +511,7 @@ where let mut target_results = Vec::new(); - for concrete_target in concrete_targets_for(target) { + for concrete_target in targets { let concrete_target = *concrete_target; let assets: Vec<&'static EmbeddedAsset> = iter_embedded_assets_for_setup_target_with_selection( @@ -527,23 +533,14 @@ where fn install_assets_for_concrete_target_with_rename( repository_root: &Path, - target: SetupTarget, + target: ConcreteSetupTarget, assets: &[&'static EmbeddedAsset], rename_fn: &mut F, ) -> Result where F: FnMut(&Path, &Path) -> io::Result<()>, { - let install_targets = InstallTargetPaths::new(repository_root); - let destination_root = match target { - SetupTarget::OpenCode => install_targets.opencode_target_dir(), - SetupTarget::Claude => install_targets.claude_target_dir(), - SetupTarget::Pi => install_targets.pi_target_dir(), - SetupTarget::Codex => install_targets.codex_target_dir(), - SetupTarget::All => { - unreachable!("meta targets are expanded into concrete targets") - } - }; + let destination_root = concrete_target_destination_root(repository_root, target); for asset in assets { install_single_asset_with_rename(target, &destination_root, asset, rename_fn)?; @@ -560,7 +557,7 @@ where fn prune_stale_assets_for_concrete_target( destination_root: &Path, - target: SetupTarget, + target: ConcreteSetupTarget, installed_assets: &[&'static EmbeddedAsset], ) -> Result<()> { let installed_paths: std::collections::HashSet<&'static str> = installed_assets @@ -604,20 +601,21 @@ fn remove_empty_ancestor_directories(destination_root: &Path, removed_file: &Pat } } -fn is_claude_settings_merge_target(target: SetupTarget, relative_path: &str) -> bool { - target == SetupTarget::Claude && relative_path == claude_asset::SETTINGS_FILE +fn is_claude_settings_merge_target(target: ConcreteSetupTarget, relative_path: &str) -> bool { + target == ConcreteSetupTarget::Claude && relative_path == claude_asset::SETTINGS_FILE } -fn is_opencode_config_merge_target(target: SetupTarget, relative_path: &str) -> bool { - target == SetupTarget::OpenCode && relative_path == default_paths::repo_file::OPENCODE_MANIFEST +fn is_opencode_config_merge_target(target: ConcreteSetupTarget, relative_path: &str) -> bool { + target == ConcreteSetupTarget::OpenCode + && relative_path == default_paths::repo_file::OPENCODE_MANIFEST } -fn is_codex_hooks_merge_target(target: SetupTarget, relative_path: &str) -> bool { - target == SetupTarget::Codex && relative_path == ".codex/hooks.json" +fn is_codex_hooks_merge_target(target: ConcreteSetupTarget, relative_path: &str) -> bool { + target == ConcreteSetupTarget::Codex && relative_path == ".codex/hooks.json" } fn install_single_asset_with_rename( - target: SetupTarget, + target: ConcreteSetupTarget, destination_root: &Path, asset: &'static EmbeddedAsset, rename_fn: &mut F, diff --git a/cli/src/services/setup/mod.rs b/cli/src/services/setup/mod.rs index feeb101ec..54c2ad2c8 100644 --- a/cli/src/services/setup/mod.rs +++ b/cli/src/services/setup/mod.rs @@ -90,6 +90,25 @@ pub enum SetupTarget { All, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConcreteSetupTarget { + OpenCode, + Claude, + Pi, + Codex, +} + +impl From for SetupTarget { + fn from(target: ConcreteSetupTarget) -> Self { + match target { + ConcreteSetupTarget::OpenCode => Self::OpenCode, + ConcreteSetupTarget::Claude => Self::Claude, + ConcreteSetupTarget::Pi => Self::Pi, + ConcreteSetupTarget::Codex => Self::Codex, + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct EmbeddedAsset { pub relative_path: &'static str, @@ -97,14 +116,6 @@ pub struct EmbeddedAsset { pub sha256: [u8; 32], } -#[allow(dead_code)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum RequiredHookAsset { - PreCommit, - CommitMsg, - PostCommit, -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct OptionalWorkflow { pub id: &'static str, @@ -121,28 +132,12 @@ pub fn iter_required_hook_assets() -> std::slice::Iter<'static, EmbeddedAsset> { HOOK_EMBEDDED_ASSETS.iter() } -#[allow(dead_code)] -pub fn get_required_hook_asset(hook: RequiredHookAsset) -> Option<&'static EmbeddedAsset> { - let hook_name = match hook { - RequiredHookAsset::PreCommit => default_paths::hook_dir::PRE_COMMIT, - RequiredHookAsset::CommitMsg => default_paths::hook_dir::COMMIT_MSG, - RequiredHookAsset::PostCommit => default_paths::hook_dir::POST_COMMIT, - }; - - HOOK_EMBEDDED_ASSETS - .iter() - .find(|asset| asset.relative_path == hook_name) -} - -fn embedded_assets_for_concrete_target(target: SetupTarget) -> &'static [EmbeddedAsset] { +fn embedded_assets_for_concrete_target(target: ConcreteSetupTarget) -> &'static [EmbeddedAsset] { match target { - SetupTarget::OpenCode => OPENCODE_EMBEDDED_ASSETS, - SetupTarget::Claude => CLAUDE_EMBEDDED_ASSETS, - SetupTarget::Pi => PI_EMBEDDED_ASSETS, - SetupTarget::Codex => CODEX_EMBEDDED_ASSETS, - SetupTarget::All => { - unreachable!("meta targets are expanded into concrete targets") - } + ConcreteSetupTarget::OpenCode => OPENCODE_EMBEDDED_ASSETS, + ConcreteSetupTarget::Claude => CLAUDE_EMBEDDED_ASSETS, + ConcreteSetupTarget::Pi => PI_EMBEDDED_ASSETS, + ConcreteSetupTarget::Codex => CODEX_EMBEDDED_ASSETS, } } @@ -152,27 +147,24 @@ struct WorkflowAssetLayout { skills_dir: &'static str, } -fn workflow_asset_layout(target: SetupTarget) -> WorkflowAssetLayout { +fn workflow_asset_layout(target: ConcreteSetupTarget) -> WorkflowAssetLayout { match target { - SetupTarget::OpenCode => WorkflowAssetLayout { + ConcreteSetupTarget::OpenCode => WorkflowAssetLayout { command_dir: Some(default_paths::opencode_asset::OPENCODE_COMMAND_DIR), skills_dir: default_paths::opencode_asset::SKILLS_DIR, }, - SetupTarget::Claude => WorkflowAssetLayout { + ConcreteSetupTarget::Claude => WorkflowAssetLayout { command_dir: Some(default_paths::claude_asset::COMMANDS_DIR), skills_dir: default_paths::claude_asset::SKILLS_DIR, }, - SetupTarget::Pi => WorkflowAssetLayout { + ConcreteSetupTarget::Pi => WorkflowAssetLayout { command_dir: Some(default_paths::pi_asset::PROMPTS_DIR), skills_dir: default_paths::pi_asset::SKILLS_DIR, }, - SetupTarget::Codex => WorkflowAssetLayout { + ConcreteSetupTarget::Codex => WorkflowAssetLayout { command_dir: None, skills_dir: default_paths::codex_asset::SKILLS_DIR, }, - SetupTarget::All => { - unreachable!("meta targets are expanded into concrete targets") - } } } @@ -190,7 +182,7 @@ fn asset_belongs_to_optional_workflow( } pub fn iter_embedded_assets_for_setup_target_with_selection( - target: SetupTarget, + target: ConcreteSetupTarget, selected_optional_workflows: &[impl AsRef], ) -> std::vec::IntoIter<&'static EmbeddedAsset> { let unselected: Vec<&'static OptionalWorkflow> = OPTIONAL_WORKFLOWS @@ -202,19 +194,15 @@ pub fn iter_embedded_assets_for_setup_target_with_selection( }) .collect(); - let mut assets: Vec<&'static EmbeddedAsset> = Vec::new(); - for concrete in concrete_targets_for(target) { - let layout = workflow_asset_layout(*concrete); - assets.extend( - embedded_assets_for_concrete_target(*concrete) - .iter() - .filter(|asset| { - !unselected.iter().any(|workflow| { - asset_belongs_to_optional_workflow(asset.relative_path, workflow, layout) - }) - }), - ); - } + let layout = workflow_asset_layout(target); + let assets: Vec<&'static EmbeddedAsset> = embedded_assets_for_concrete_target(target) + .iter() + .filter(|asset| { + !unselected.iter().any(|workflow| { + asset_belongs_to_optional_workflow(asset.relative_path, workflow, layout) + }) + }) + .collect(); assets.into_iter() } @@ -225,10 +213,17 @@ pub enum SetupMode { NonInteractive(SetupTarget), } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TargetPersistence { + Merge, + Replace, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub enum SetupDispatch { Proceed { - mode: SetupMode, + targets: Vec, + target_persistence: TargetPersistence, optional_workflows: Option>, agent_trace_auto_sync: Option, attribution_hooks_enabled: Option, @@ -237,15 +232,9 @@ pub enum SetupDispatch { } #[derive(Clone, Debug, Default, Eq, PartialEq)] -#[allow(clippy::struct_excessive_bools)] pub struct SetupCliOptions { - pub help: bool, + pub target: Option, pub non_interactive: bool, - pub opencode: bool, - pub claude: bool, - pub pi: bool, - pub codex: bool, - pub all: bool, pub hooks: bool, pub repo_path: Option, pub bootstrap_context: bool, @@ -284,11 +273,7 @@ pub fn resolve_setup_request(options: SetupCliOptions) -> Result { } let has_other_setup_options = options.non_interactive - || options.opencode - || options.claude - || options.pi - || options.codex - || options.all + || options.target.is_some() || options.hooks || options.repo_path.is_some(); if has_other_setup_options { @@ -306,41 +291,13 @@ pub fn resolve_setup_request(options: SetupCliOptions) -> Result { }); } - let mut selected_targets = Vec::new(); - - if options.opencode { - selected_targets.push(SetupTarget::OpenCode); - } - if options.claude { - selected_targets.push(SetupTarget::Claude); - } - if options.pi { - selected_targets.push(SetupTarget::Pi); - } - if options.codex { - selected_targets.push(SetupTarget::Codex); - } - if options.all { - selected_targets.push(SetupTarget::All); - } - - if selected_targets.len() > 1 { - bail!( - "Options '--opencode', '--claude', '--pi', '--codex', and '--all' are mutually exclusive. Try: choose exactly one target flag (for example 'sce setup --opencode --non-interactive') or omit all target flags for interactive mode." - ); - } - - if options.non_interactive && selected_targets.is_empty() && !options.hooks { - bail!( + let config_mode = match options.target { + Some(target) => Some(SetupMode::NonInteractive(target)), + None if options.non_interactive && !options.hooks => bail!( "Option '--non-interactive' requires a target flag. Try: 'sce setup --opencode --non-interactive', 'sce setup --claude --non-interactive', 'sce setup --pi --non-interactive', 'sce setup --codex --non-interactive', or 'sce setup --all --non-interactive'." - ); - } - - let config_mode = match selected_targets.as_slice() { - [target] => Some(SetupMode::NonInteractive(*target)), - [] if options.hooks => None, - [] => Some(SetupMode::Interactive), - _ => unreachable!("target count already validated"), + ), + None if options.hooks => None, + None => Some(SetupMode::Interactive), }; if config_mode.is_none() && optional_workflows.is_some() { @@ -395,37 +352,32 @@ fn available_optional_workflow_slugs() -> String { .join(", ") } -pub fn run_setup_for_mode( +pub fn run_setup_for_targets( repository_root: &Path, - mode: SetupMode, + targets: &[ConcreteSetupTarget], + target_persistence: TargetPersistence, optional_workflows: Option<&[String]>, agent_trace_auto_sync: Option, attribution_hooks_enabled: Option, ) -> Result { - let target = match mode { - SetupMode::Interactive => { - bail!("Interactive setup mode must be resolved before installation") - } - SetupMode::NonInteractive(target) => target, - }; - let selected_optional_workflows = match optional_workflows { Some(selection) => selection.to_vec(), None => persisted_optional_workflows(repository_root), }; let outcome = - install_embedded_setup_assets(repository_root, target, &selected_optional_workflows) + install_embedded_setup_assets(repository_root, targets, &selected_optional_workflows) .with_context(|| { format!( "Setup installation failed for {}", - setup_target_label(target) + setup_targets_label(targets) ) })?; persist_integration_targets( repository_root, - target, + targets, + target_persistence, &selected_optional_workflows, agent_trace_auto_sync, attribution_hooks_enabled, @@ -433,35 +385,75 @@ pub fn run_setup_for_mode( .with_context(|| { format!( "Setup assets were installed for {} but failed to update repo-local config", - setup_target_label(target) + setup_targets_label(targets) ) })?; Ok(format_setup_install_success_message(&outcome)) } -pub fn persisted_optional_workflows(repository_root: &Path) -> Vec { - use crate::services::config::schema::parse_file_config; - use crate::services::config::ConfigPathSource; +pub fn persisted_integration_targets(repository_root: &Path) -> Vec { + use crate::services::config::IntegrationTargetId; - let config_path = RepoPaths::new(repository_root).sce_config_file(); - - let Ok(raw) = fs::read_to_string(&config_path) else { - return Vec::new(); - }; - - let Ok(config) = - parse_file_config(&raw, &config_path, ConfigPathSource::DefaultDiscoveredLocal) + let Some(integrations) = + persisted_file_config(repository_root).and_then(|config| config.integrations) else { return Vec::new(); }; - config - .integrations + let mut targets = Vec::new(); + for target in integrations.value.target { + let concrete = match target { + IntegrationTargetId::Opencode => ConcreteSetupTarget::OpenCode, + IntegrationTargetId::Claude => ConcreteSetupTarget::Claude, + IntegrationTargetId::Pi => ConcreteSetupTarget::Pi, + IntegrationTargetId::Codex => ConcreteSetupTarget::Codex, + }; + if !targets.contains(&concrete) { + targets.push(concrete); + } + } + targets +} + +pub fn persisted_optional_workflows(repository_root: &Path) -> Vec { + persisted_file_config(repository_root) + .and_then(|config| config.integrations) .map(|integrations| integrations.value.optional_workflows) .unwrap_or_default() } +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct PersistedSetupConfirmations { + pub agent_trace_auto_sync: Option, + pub attribution_hooks_enabled: Option, +} + +pub fn persisted_setup_confirmations(repository_root: &Path) -> PersistedSetupConfirmations { + let Some(config) = persisted_file_config(repository_root) else { + return PersistedSetupConfirmations::default(); + }; + + PersistedSetupConfirmations { + agent_trace_auto_sync: config.agent_trace_auto_sync.map(|setting| setting.value), + attribution_hooks_enabled: config + .attribution_hooks_enabled + .map(|setting| setting.value), + } +} + +fn persisted_file_config( + repository_root: &Path, +) -> Option { + use crate::services::config::schema::parse_file_config; + use crate::services::config::ConfigPathSource; + + let config_path = RepoPaths::new(repository_root).sce_config_file(); + let raw = fs::read_to_string(&config_path).ok()?; + + parse_file_config(&raw, &config_path, ConfigPathSource::DefaultDiscoveredLocal).ok() +} + pub fn ensure_git_repository(directory: &Path) -> Result { install::ensure_git_repository(directory) } @@ -587,7 +579,7 @@ fn format_setup_install_success_message(outcome: &SetupInstallOutcome) -> String let selected_targets = outcome .target_results .iter() - .map(|result| setup_target_label(result.target)) + .map(|result| setup_target_label(result.target.into())) .collect::>() .join(", "); @@ -603,7 +595,7 @@ fn format_setup_install_success_message(outcome: &SetupInstallOutcome) -> String for result in &outcome.target_results { lines.push(format!( "- {}: {} {} {} '{}'", - label(&format!("{}:", setup_target_label(result.target))), + label(&format!("{}:", setup_target_label(result.target.into()))), success("installed"), value(&format!("{} file(s) to", result.installed_file_count)), value("'"), @@ -667,6 +659,14 @@ fn required_hook_status_label(status: RequiredHookInstallStatus) -> &'static str } } +fn setup_targets_label(targets: &[ConcreteSetupTarget]) -> String { + targets + .iter() + .map(|target| setup_target_label((*target).into())) + .collect::>() + .join(", ") +} + fn setup_target_label(target: SetupTarget) -> &'static str { match target { SetupTarget::OpenCode => "OpenCode", @@ -679,7 +679,7 @@ fn setup_target_label(target: SetupTarget) -> &'static str { #[derive(Clone, Debug, Eq, PartialEq)] pub struct SetupInstallTargetResult { - pub target: SetupTarget, + pub target: ConcreteSetupTarget, pub destination_root: PathBuf, pub installed_file_count: usize, } @@ -718,27 +718,27 @@ pub fn install_required_git_hooks(repository_root: &Path) -> Result Result { - install::install_embedded_setup_assets(repository_root, target, selected_optional_workflows) + install::install_embedded_setup_assets(repository_root, targets, selected_optional_workflows) } pub(crate) fn repair_merge_target_asset( repository_root: &Path, - target: SetupTarget, + target: ConcreteSetupTarget, relative_path: &str, ) -> Result<()> { install::repair_merge_target_asset(repository_root, target, relative_path) } pub(crate) fn setup_install_recovery_guidance( - target: SetupTarget, + target: ConcreteSetupTarget, destination_root: &Path, ) -> String { format!( "Setup for {} does not create backups. Recover '{}' from version control if needed.", - setup_target_label(target), + setup_target_label(target.into()), destination_root.display() ) } @@ -766,36 +766,34 @@ pub(crate) fn cleanup_path_if_exists(path: &Path) { } } -pub(crate) fn concrete_targets_for(target: SetupTarget) -> &'static [SetupTarget] { +pub(crate) fn concrete_targets_for(target: SetupTarget) -> &'static [ConcreteSetupTarget] { match target { - SetupTarget::OpenCode => &[SetupTarget::OpenCode], - SetupTarget::Claude => &[SetupTarget::Claude], - SetupTarget::Pi => &[SetupTarget::Pi], - SetupTarget::Codex => &[SetupTarget::Codex], + SetupTarget::OpenCode => &[ConcreteSetupTarget::OpenCode], + SetupTarget::Claude => &[ConcreteSetupTarget::Claude], + SetupTarget::Pi => &[ConcreteSetupTarget::Pi], + SetupTarget::Codex => &[ConcreteSetupTarget::Codex], SetupTarget::All => &[ - SetupTarget::OpenCode, - SetupTarget::Claude, - SetupTarget::Pi, - SetupTarget::Codex, + ConcreteSetupTarget::OpenCode, + ConcreteSetupTarget::Claude, + ConcreteSetupTarget::Pi, + ConcreteSetupTarget::Codex, ], } } -fn integration_target_id_str(target: SetupTarget) -> &'static str { +fn integration_target_id_str(target: ConcreteSetupTarget) -> &'static str { match target { - SetupTarget::OpenCode => "opencode", - SetupTarget::Claude => "claude", - SetupTarget::Pi => "pi", - SetupTarget::Codex => "codex", - SetupTarget::All => { - unreachable!("integration_target_id_str must not be called with meta targets") - } + ConcreteSetupTarget::OpenCode => "opencode", + ConcreteSetupTarget::Claude => "claude", + ConcreteSetupTarget::Pi => "pi", + ConcreteSetupTarget::Codex => "codex", } } pub fn persist_integration_targets( repository_root: &Path, - target: SetupTarget, + targets: &[ConcreteSetupTarget], + target_persistence: TargetPersistence, selected_optional_workflows: &[String], agent_trace_auto_sync: Option, attribution_hooks_enabled: Option, @@ -831,19 +829,21 @@ pub fn persist_integration_targets( ) })?; - let mut existing_targets: Vec = config_obj - .get("integrations") - .and_then(|i| i.get("target")) - .and_then(|t| t.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(String::from)) - .collect() - }) - .unwrap_or_default(); + let mut existing_targets: Vec = match target_persistence { + TargetPersistence::Merge => config_obj + .get("integrations") + .and_then(|i| i.get("target")) + .and_then(|t| t.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(), + TargetPersistence::Replace => Vec::new(), + }; - let new_targets = concrete_targets_for(target); - for concrete in new_targets { + for concrete in targets { let id_str = integration_target_id_str(*concrete); let id_owned = id_str.to_string(); if !existing_targets.contains(&id_owned) { @@ -906,7 +906,10 @@ pub fn persist_integration_targets( mod install; pub trait SetupTargetPrompter { - fn prompt_target(&self) -> Result; + fn prompt_targets( + &self, + defaults: &[ConcreteSetupTarget], + ) -> Result>>; fn prompt_optional_workflows(&self, defaults: &[String]) -> Result>>; fn prompt_agent_trace_auto_sync(&self) -> Result>; fn prompt_attribution_hooks_enabled(&self) -> Result>; @@ -916,8 +919,11 @@ pub trait SetupTargetPrompter { pub struct InquireSetupTargetPrompter; impl SetupTargetPrompter for InquireSetupTargetPrompter { - fn prompt_target(&self) -> Result { - prompt::prompt_target() + fn prompt_targets( + &self, + defaults: &[ConcreteSetupTarget], + ) -> Result>> { + prompt::prompt_targets(defaults) } fn prompt_optional_workflows(&self, defaults: &[String]) -> Result>> { @@ -934,13 +940,7 @@ impl SetupTargetPrompter for InquireSetupTargetPrompter { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum SetupPromptTarget { - OpenCode, - Claude, - Pi, - Codex, - All, -} +struct SetupPromptTarget(ConcreteSetupTarget); impl std::fmt::Display for SetupPromptTarget { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -952,33 +952,21 @@ fn setup_prompt_target_label(target: SetupPromptTarget) -> String { prompt::setup_prompt_target_label(target) } -#[allow(dead_code)] -fn setup_prompt_target_label_with_color_policy( - target: SetupPromptTarget, - color_enabled: bool, -) -> String { - prompt::setup_prompt_target_label_with_color_policy(target, color_enabled) -} - -#[allow(dead_code)] -fn setup_prompt_title_with_color_policy(color_enabled: bool) -> String { - prompt::setup_prompt_title_with_color_policy(color_enabled) -} - mod prompt; pub fn resolve_setup_dispatch

( mode: SetupMode, prompter: &P, optional_workflow_defaults: &[String], + configured_targets: &[ConcreteSetupTarget], + persisted_confirmations: PersistedSetupConfirmations, ) -> Result where P: SetupTargetPrompter, { match mode { SetupMode::Interactive => { - let target_dispatch = prompter.prompt_target()?; - let SetupDispatch::Proceed { mode, .. } = target_dispatch else { + let Some(targets) = prompter.prompt_targets(configured_targets)? else { return Ok(SetupDispatch::Cancelled); }; @@ -988,24 +976,36 @@ where return Ok(SetupDispatch::Cancelled); }; - let Some(agent_trace_auto_sync) = prompter.prompt_agent_trace_auto_sync()? else { + let Some(agent_trace_auto_sync) = + persisted_confirmations.agent_trace_auto_sync.map_or_else( + || prompter.prompt_agent_trace_auto_sync(), + |value| Ok(Some(value)), + )? + else { return Ok(SetupDispatch::Cancelled); }; - let Some(attribution_hooks_enabled) = prompter.prompt_attribution_hooks_enabled()? + let Some(attribution_hooks_enabled) = persisted_confirmations + .attribution_hooks_enabled + .map_or_else( + || prompter.prompt_attribution_hooks_enabled(), + |value| Ok(Some(value)), + )? else { return Ok(SetupDispatch::Cancelled); }; Ok(SetupDispatch::Proceed { - mode, + targets, + target_persistence: TargetPersistence::Replace, optional_workflows: Some(optional_workflows), agent_trace_auto_sync: Some(agent_trace_auto_sync), attribution_hooks_enabled: Some(attribution_hooks_enabled), }) } SetupMode::NonInteractive(target) => Ok(SetupDispatch::Proceed { - mode: SetupMode::NonInteractive(target), + targets: concrete_targets_for(target).to_vec(), + target_persistence: TargetPersistence::Merge, optional_workflows: None, agent_trace_auto_sync: None, attribution_hooks_enabled: None, @@ -1016,6 +1016,3 @@ where pub fn setup_cancelled_text() -> String { value("Setup cancelled. No files were changed.") } - -#[cfg(test)] -mod tests; diff --git a/cli/src/services/setup/prompt.rs b/cli/src/services/setup/prompt.rs index cf92217b4..fd867bc9c 100644 --- a/cli/src/services/setup/prompt.rs +++ b/cli/src/services/setup/prompt.rs @@ -1,41 +1,49 @@ use anyhow::{bail, Result}; -use inquire::{Confirm, InquireError, MultiSelect, Select}; +use inquire::list_option::ListOption; +use inquire::validator::Validation; +use inquire::{Confirm, InquireError, MultiSelect}; -use crate::services::style::{ - prompt_label, prompt_label_with_color_policy, prompt_value_with_color_policy, -}; +use crate::services::style::{prompt_label, prompt_value_with_color_policy}; -use super::{OptionalWorkflow, SetupDispatch, SetupMode, SetupPromptTarget, SetupTarget}; +use super::{ConcreteSetupTarget, OptionalWorkflow, SetupPromptTarget}; -fn proceed(target: SetupTarget) -> SetupDispatch { - SetupDispatch::Proceed { - mode: SetupMode::NonInteractive(target), - optional_workflows: None, - agent_trace_auto_sync: None, - attribution_hooks_enabled: None, - } -} +const SETUP_PROMPT_TARGETS: [ConcreteSetupTarget; 4] = [ + ConcreteSetupTarget::OpenCode, + ConcreteSetupTarget::Claude, + ConcreteSetupTarget::Pi, + ConcreteSetupTarget::Codex, +]; + +const SETUP_TARGET_REQUIRED_MESSAGE: &str = "Select at least one setup target."; -pub(super) fn prompt_target() -> Result { - let options = vec![ - SetupPromptTarget::OpenCode, - SetupPromptTarget::Claude, - SetupPromptTarget::Pi, - SetupPromptTarget::Codex, - SetupPromptTarget::All, - ]; +pub(super) fn prompt_targets( + defaults: &[ConcreteSetupTarget], +) -> Result>> { + let options: Vec = SETUP_PROMPT_TARGETS + .iter() + .map(|target| SetupPromptTarget(*target)) + .collect(); + let default_indices: Vec = SETUP_PROMPT_TARGETS + .iter() + .enumerate() + .filter(|(_, target)| defaults.contains(target)) + .map(|(index, _)| index) + .collect(); - let selection = Select::new(&setup_prompt_title(), options).prompt(); + let selection = MultiSelect::new(&setup_prompt_title(), options) + .with_default(&default_indices) + .with_validator(|selected: &[ListOption<&SetupPromptTarget>]| { + if selected.is_empty() { + Ok(Validation::Invalid(SETUP_TARGET_REQUIRED_MESSAGE.into())) + } else { + Ok(Validation::Valid) + } + }) + .prompt(); match selection { - Ok(SetupPromptTarget::OpenCode) => Ok(proceed(SetupTarget::OpenCode)), - Ok(SetupPromptTarget::Claude) => Ok(proceed(SetupTarget::Claude)), - Ok(SetupPromptTarget::Pi) => Ok(proceed(SetupTarget::Pi)), - Ok(SetupPromptTarget::Codex) => Ok(proceed(SetupTarget::Codex)), - Ok(SetupPromptTarget::All) => Ok(proceed(SetupTarget::All)), - Err(InquireError::OperationCanceled | InquireError::OperationInterrupted) => { - Ok(SetupDispatch::Cancelled) - } + Ok(selected) => Ok(Some(selected.into_iter().map(|row| row.0).collect())), + Err(InquireError::OperationCanceled | InquireError::OperationInterrupted) => Ok(None), Err(InquireError::NotTTY) => bail!( "Interactive setup requires a TTY. Re-run with '--non-interactive' and one of '--opencode', '--claude', '--pi', '--codex', or '--all'." ), @@ -161,7 +169,7 @@ pub(super) fn optional_workflow_row_label_with_color_policy( } pub(super) fn setup_prompt_title() -> String { - prompt_label("Select setup target") + prompt_label("Select setup targets") } pub(super) fn setup_prompt_target_label(target: SetupPromptTarget) -> String { @@ -172,18 +180,12 @@ pub(super) fn setup_prompt_target_label_with_color_policy( target: SetupPromptTarget, color_enabled: bool, ) -> String { - let label = match target { - SetupPromptTarget::OpenCode => "OpenCode", - SetupPromptTarget::Claude => "Claude", - SetupPromptTarget::Pi => "Pi", - SetupPromptTarget::Codex => "Codex", - SetupPromptTarget::All => "All (OpenCode + Claude + Pi + Codex)", + let label = match target.0 { + ConcreteSetupTarget::OpenCode => "OpenCode", + ConcreteSetupTarget::Claude => "Claude", + ConcreteSetupTarget::Pi => "Pi", + ConcreteSetupTarget::Codex => "Codex", }; prompt_value_with_color_policy(label, color_enabled) } - -#[allow(dead_code)] -pub(super) fn setup_prompt_title_with_color_policy(color_enabled: bool) -> String { - prompt_label_with_color_policy("Select setup target", color_enabled) -} diff --git a/cli/src/services/setup/tests.rs b/cli/src/services/setup/tests.rs deleted file mode 100644 index 7757ce092..000000000 --- a/cli/src/services/setup/tests.rs +++ /dev/null @@ -1,1123 +0,0 @@ -use super::*; -use std::process::Command; -use std::time::{SystemTime, UNIX_EPOCH}; - -use crate::command_surface; -use crate::services::command_registry::CommandRegistry; -use crate::services::command_registry::RuntimeCommand; -use crate::services::parse::command_runtime::parse_runtime_command; - -fn options_with(mutate: impl FnOnce(&mut SetupCliOptions)) -> SetupCliOptions { - let mut options = SetupCliOptions::default(); - mutate(&mut options); - options -} - -fn unique_temp_dir(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - let dir = std::env::temp_dir().join(format!( - "sce-setup-context-{label}-{}-{nonce}", - std::process::id() - )); - fs::create_dir_all(&dir).expect("create temp dir"); - dir -} - -fn init_git_repo(label: &str) -> PathBuf { - let repo = unique_temp_dir(label); - let output = Command::new("git") - .args(["init", "-q"]) - .current_dir(&repo) - .output() - .expect("git init should spawn"); - assert!( - output.status.success(), - "git init failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - repo -} - -fn assert_baseline_paths_exist(repo: &Path) { - let paths = RepoPaths::new(repo); - for path in [ - paths.context_overview_file(), - paths.context_architecture_file(), - paths.context_patterns_file(), - paths.context_glossary_file(), - paths.context_map_file(), - paths.context_plans_dir(), - paths.context_handovers_dir(), - paths.context_decisions_dir(), - paths.context_tmp_dir(), - paths.context_tmp_gitignore_file(), - ] { - assert!(path.exists(), "expected baseline path {}", path.display()); - } -} - -#[test] -fn repo_local_config_bootstrap_payload_uses_versioned_schema_url() { - let payload = repo_local_config_bootstrap_payload(); - - assert_eq!( - payload, - format!( - "{{\n \"$schema\": \"https://sce.crocoder.dev/v{}/config.json\",\n \"agent_trace\": {{\n \"auto_sync\": true\n }},\n \"policies\": {{\n \"attribution_hooks\": {{\n \"enabled\": true\n }}\n }}\n}}\n", - env!("CARGO_PKG_VERSION") - ) - ); -} - -#[test] -fn resolve_setup_request_accepts_pi_target() { - let request = resolve_setup_request(options_with(|options| { - options.pi = true; - options.non_interactive = true; - })) - .expect("pi target should resolve"); - - assert_eq!( - request.config_mode, - Some(SetupMode::NonInteractive(SetupTarget::Pi)) - ); - assert!(!request.context_only); -} - -#[test] -fn resolve_setup_request_accepts_codex_target() { - let request = resolve_setup_request(options_with(|options| { - options.codex = true; - options.non_interactive = true; - })) - .expect("codex target should resolve"); - - assert_eq!( - request.config_mode, - Some(SetupMode::NonInteractive(SetupTarget::Codex)) - ); - assert!(!request.context_only); -} - -#[test] -fn resolve_setup_request_accepts_all_target() { - let request = resolve_setup_request(options_with(|options| { - options.all = true; - options.non_interactive = true; - })) - .expect("all target should resolve"); - - assert_eq!( - request.config_mode, - Some(SetupMode::NonInteractive(SetupTarget::All)) - ); - assert!(!request.context_only); -} - -#[test] -fn resolve_setup_request_accepts_bootstrap_context_alone() { - let request = resolve_setup_request(options_with(|options| { - options.bootstrap_context = true; - })) - .expect("bootstrap-context alone should resolve"); - - assert!(request.context_only); - assert_eq!(request.config_mode, None); - assert!(!request.install_hooks); - assert_eq!(request.hooks_repo_path, None); -} - -#[test] -fn resolve_setup_request_rejects_bootstrap_context_with_target() { - let error = resolve_setup_request(options_with(|options| { - options.bootstrap_context = true; - options.opencode = true; - })) - .expect_err("bootstrap-context with target must be rejected"); - - assert!(error.to_string().contains("--bootstrap-context")); - assert!(error.to_string().contains("alone")); -} - -#[test] -fn resolve_setup_request_rejects_combined_target_flags() { - let error = resolve_setup_request(options_with(|options| { - options.pi = true; - options.all = true; - })) - .expect_err("combined target flags must be rejected"); - - assert!(error.to_string().contains("mutually exclusive")); -} - -#[test] -fn resolve_setup_request_non_interactive_error_lists_pi_and_all() { - let error = resolve_setup_request(options_with(|options| { - options.non_interactive = true; - })) - .expect_err("non-interactive without target must be rejected"); - - let message = error.to_string(); - assert!(message.contains("--pi")); - assert!(message.contains("--all")); -} - -#[test] -fn parser_routes_bootstrap_context_to_context_only_request() { - let registry = CommandRegistry::default(); - let command = parse_runtime_command( - [ - "sce".to_string(), - "setup".to_string(), - "--bootstrap-context".to_string(), - ], - ®istry, - None, - ) - .expect("bootstrap-context should parse"); - - match command { - RuntimeCommand::Setup(setup_command) => { - assert!(setup_command.request.context_only); - assert_eq!(setup_command.request.config_mode, None); - assert!(!setup_command.request.install_hooks); - } - _ => panic!("expected Setup command for --bootstrap-context"), - } -} - -#[test] -fn help_documents_bootstrap_context_flag() { - let top_level_help = command_surface::help_text(); - assert!( - top_level_help.contains("--bootstrap-context"), - "top-level help should document --bootstrap-context" - ); - - let registry = CommandRegistry::default(); - let command = parse_runtime_command( - ["sce".to_string(), "setup".to_string(), "--help".to_string()], - ®istry, - None, - ) - .expect("setup --help should parse"); - - match command { - RuntimeCommand::HelpText(help) => { - assert!( - help.text.contains("--bootstrap-context"), - "setup --help should document --bootstrap-context:\n{}", - help.text - ); - } - _ => panic!("expected HelpText for setup --help"), - } -} - -#[test] -fn bootstrap_context_baseline_creates_expected_paths() { - let repo = init_git_repo("create-baseline"); - let message = bootstrap_context_baseline(&repo).expect("bootstrap should create baseline"); - assert!(message.contains("Context baseline ensured.")); - assert_baseline_paths_exist(&repo); - - let paths = RepoPaths::new(&repo); - assert!(!paths.opencode_dir().exists()); - assert!(!paths.claude_dir().exists()); - assert!(!paths.pi_dir().exists()); - - let gitignore = fs::read_to_string(paths.context_tmp_gitignore_file()) - .expect("tmp gitignore should be readable"); - assert_eq!(gitignore, CONTEXT_TMP_GITIGNORE_CONTENT); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn bootstrap_context_baseline_is_additive_and_idempotent() { - let repo = init_git_repo("idempotent-baseline"); - bootstrap_context_baseline(&repo).expect("initial bootstrap"); - - let paths = RepoPaths::new(&repo); - let sentinel = "SENTINEL_OVERVIEW_CONTENT\n"; - fs::write(paths.context_overview_file(), sentinel).expect("seed overview sentinel"); - fs::write(paths.context_map_file(), "SENTINEL_CONTEXT_MAP\n") - .expect("seed context-map sentinel"); - fs::write(paths.context_tmp_gitignore_file(), "SENTINEL_GITIGNORE\n") - .expect("seed gitignore sentinel"); - - fs::remove_file(paths.context_architecture_file()).expect("remove architecture"); - fs::remove_dir_all(paths.context_plans_dir()).expect("remove plans"); - - bootstrap_context_baseline(&repo).expect("rerun bootstrap"); - - assert_eq!( - fs::read_to_string(paths.context_overview_file()).expect("read overview"), - sentinel - ); - assert_eq!( - fs::read_to_string(paths.context_map_file()).expect("read context-map"), - "SENTINEL_CONTEXT_MAP\n" - ); - assert_eq!( - fs::read_to_string(paths.context_tmp_gitignore_file()).expect("read gitignore"), - "SENTINEL_GITIGNORE\n" - ); - assert!(paths.context_architecture_file().exists()); - assert!(paths.context_plans_dir().is_dir()); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn concrete_targets_for_all_expands_to_four_targets() { - assert_eq!( - concrete_targets_for(SetupTarget::All), - &[ - SetupTarget::OpenCode, - SetupTarget::Claude, - SetupTarget::Pi, - SetupTarget::Codex - ] - ); -} - -#[test] -fn integration_target_id_str_maps_pi() { - assert_eq!(integration_target_id_str(SetupTarget::Pi), "pi"); -} - -#[test] -fn integration_target_id_str_maps_codex() { - assert_eq!(integration_target_id_str(SetupTarget::Codex), "codex"); -} - -fn every_optional_workflow() -> Vec<&'static str> { - super::OPTIONAL_WORKFLOWS - .iter() - .map(|workflow| workflow.id) - .collect() -} - -#[test] -fn iter_embedded_assets_for_all_covers_each_concrete_target() { - let selection = every_optional_workflow(); - let count = - |target| iter_embedded_assets_for_setup_target_with_selection(target, &selection).count(); - - let concrete_sum = count(SetupTarget::OpenCode) - + count(SetupTarget::Claude) - + count(SetupTarget::Pi) - + count(SetupTarget::Codex); - - assert!(count(SetupTarget::Pi) > 0); - assert!(count(SetupTarget::Codex) > 0); - assert_eq!(count(SetupTarget::All), concrete_sum); -} - -#[test] -fn embedded_build_payload_contains_generated_targets_and_static_hooks() { - let selection = every_optional_workflow(); - let contains = |target, path| { - iter_embedded_assets_for_setup_target_with_selection(target, &selection) - .any(|asset| asset.relative_path == path && !asset.bytes.is_empty()) - }; - - assert!(contains(SetupTarget::OpenCode, "command/next-task.md")); - assert!(contains( - SetupTarget::OpenCode, - "lib/bash-policy-presets.json" - )); - assert!(contains(SetupTarget::Claude, "commands/next-task.md")); - assert!(contains(SetupTarget::Pi, "prompts/next-task.md")); - assert!(contains(SetupTarget::Pi, "extensions/sce/index.ts")); - assert!(iter_required_hook_assets().all(|asset| !asset.bytes.is_empty())); -} - -#[test] -fn codex_embedded_assets_cover_both_output_roots_with_no_command_dir() { - let has = |path: &str| { - CODEX_EMBEDDED_ASSETS - .iter() - .any(|asset| asset.relative_path == path && !asset.bytes.is_empty()) - }; - - assert!(has(".agents/skills/sce-next-task/SKILL.md")); - assert!(has(".codex/hooks.json")); - assert!(has(".codex/hooks/run-sce-or-show-install-guidance.sh")); - assert!(!CODEX_EMBEDDED_ASSETS - .iter() - .any(|asset| asset.relative_path.starts_with(".agents/commands/"))); -} - -#[test] -fn install_writes_codex_assets_directly_under_repo_root() { - let repo = init_git_repo("install-codex-dual-roots"); - let selection: Vec = every_optional_workflow() - .into_iter() - .map(str::to_string) - .collect(); - - install_embedded_setup_assets(&repo, SetupTarget::Codex, &selection) - .expect("codex install should succeed"); - - assert!(repo.join(".agents/skills/sce-next-task/SKILL.md").is_file()); - assert!(repo.join(".codex/hooks.json").is_file()); - assert!(repo - .join(".codex/hooks/run-sce-or-show-install-guidance.sh") - .is_file()); - assert!(!repo.join(".codex/.agents").exists()); - assert!(!repo.join(".agents/.codex").exists()); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn install_merges_codex_hooks_and_replaces_stale_owned_handlers_idempotently() { - let repo = init_git_repo("install-merges-codex-hooks"); - let hooks_path = repo.join(".codex/hooks.json"); - fs::create_dir_all(hooks_path.parent().unwrap()).expect("create Codex directory"); - let stale_command = "bash .codex/hooks/run-sce-or-show-install-guidance.sh sce hooks codex"; - let existing = json!({ - "description": "user hooks", - "hooks": { - "UserPromptSubmit": [{"hooks": [ - {"type": "command", "command": "echo user"}, - {"type": "command", "command": stale_command} - ]}], - "SessionStart": [{"hooks": [{"type": "command", "command": "echo session"}]}] - } - }); - fs::write(&hooks_path, serde_json::to_vec(&existing).unwrap()).expect("seed hooks config"); - let selection: Vec = every_optional_workflow() - .into_iter() - .map(str::to_string) - .collect(); - - install_embedded_setup_assets(&repo, SetupTarget::Codex, &selection) - .expect("first Codex install should succeed"); - let first = fs::read(&hooks_path).expect("read merged hooks config"); - install_embedded_setup_assets(&repo, SetupTarget::Codex, &selection) - .expect("second Codex install should succeed"); - let second = fs::read(&hooks_path).expect("read merged hooks config again"); - assert_eq!(first, second); - - let merged: serde_json::Value = serde_json::from_slice(&second).unwrap(); - assert_eq!(merged["description"], "user hooks"); - assert_eq!( - merged["hooks"]["SessionStart"][0]["hooks"][0]["command"], - "echo session" - ); - assert_eq!( - merged["hooks"]["UserPromptSubmit"][0]["hooks"][0]["command"], - "echo user" - ); - assert_eq!(merged["hooks"].as_object().unwrap().len(), 8); - for event in ["Interrupt", "SubagentStop", "SessionEnd"] { - assert!( - merged["hooks"][event][0]["hooks"][0]["command"] - .as_str() - .unwrap() - .ends_with("sce hooks codex-mutation-scope"), - "{event} must route to the mutation-scope command" - ); - } - assert!(merged["hooks"]["PreToolUse"][1]["hooks"][0]["command"] - .as_str() - .unwrap() - .ends_with("sce hooks codex-mutation-scope")); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn invalid_codex_hooks_are_not_modified() { - let invalid_documents = [ - br#"{\"hooks\":{"#.to_vec(), - serde_json::to_vec(&json!({"custom": true})).unwrap(), - serde_json::to_vec(&json!({"hooks": {"Stop": [{"matcher": 42}]}})).unwrap(), - serde_json::to_vec(&json!({"hooks": {"Stop": [{"hooks": "invalid"}]}})).unwrap(), - serde_json::to_vec(&json!({"hooks": {"Stop": [{"hooks": [{"nonsense": true}]}]}})).unwrap(), - serde_json::to_vec(&json!({"hooks": {"Stop": [{"hooks": [{"type": "unknown"}]}]}})) - .unwrap(), - ]; - let selection: Vec = every_optional_workflow() - .into_iter() - .map(str::to_string) - .collect(); - - for (index, original) in invalid_documents.iter().enumerate() { - let repo = init_git_repo(&format!("install-rejects-malformed-codex-hooks-{index}")); - let hooks_path = repo.join(".codex/hooks.json"); - fs::create_dir_all(hooks_path.parent().unwrap()).expect("create Codex directory"); - fs::write(&hooks_path, original).expect("seed malformed hooks config"); - - let error = install_embedded_setup_assets(&repo, SetupTarget::Codex, &selection) - .expect_err("malformed Codex hooks should fail setup"); - assert!(error.to_string().contains(".codex/hooks.json")); - assert_eq!(fs::read(&hooks_path).unwrap(), original.as_slice()); - - let _ = fs::remove_dir_all(&repo); - } -} - -#[test] -fn install_preserves_user_owned_files_and_writes_sce_assets() { - let repo = init_git_repo("install-preserves-user-files"); - let claude_dir = default_paths::InstallTargetPaths::new(&repo).claude_target_dir(); - - fs::create_dir_all(claude_dir.join("skills/my-own-skill")).expect("create user skill dir"); - fs::create_dir_all(claude_dir.join("commands")).expect("create commands dir"); - - fs::write(claude_dir.join("MY_NOTES.md"), "top level user notes\n") - .expect("seed top-level user file"); - fs::write( - claude_dir.join("skills/my-own-skill/SKILL.md"), - "user skill content\n", - ) - .expect("seed user skill file"); - fs::write( - claude_dir.join("commands/my-command.md"), - "user command content\n", - ) - .expect("seed user command file"); - - let selection: Vec = every_optional_workflow() - .into_iter() - .map(str::to_string) - .collect(); - - install_embedded_setup_assets(&repo, SetupTarget::Claude, &selection) - .expect("install should succeed"); - - assert_eq!( - fs::read_to_string(claude_dir.join("MY_NOTES.md")).expect("read top-level user file"), - "top level user notes\n" - ); - assert_eq!( - fs::read_to_string(claude_dir.join("skills/my-own-skill/SKILL.md")) - .expect("read user skill file"), - "user skill content\n" - ); - assert_eq!( - fs::read_to_string(claude_dir.join("commands/my-command.md")) - .expect("read user command file"), - "user command content\n" - ); - - let expected_next_task_bytes = - iter_embedded_assets_for_setup_target_with_selection(SetupTarget::Claude, &selection) - .find(|asset| asset.relative_path == "commands/next-task.md") - .expect("next-task asset should be in the catalog") - .bytes; - assert_eq!( - fs::read(claude_dir.join("commands/next-task.md")).expect("read installed sce asset"), - expected_next_task_bytes - ); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn install_merges_into_existing_claude_settings_json_and_stays_idempotent() { - let repo = init_git_repo("install-merges-claude-settings"); - let claude_dir = default_paths::InstallTargetPaths::new(&repo).claude_target_dir(); - - fs::create_dir_all(&claude_dir).expect("create claude dir"); - fs::write( - claude_dir.join("settings.json"), - serde_json::to_string_pretty(&json!({ - "permissions": {"allow": ["Bash(git *)"]}, - "env": {"FOO": "bar"}, - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [{"type": "command", "command": "echo user-hook"}] - } - ] - } - })) - .expect("serialize seeded settings"), - ) - .expect("seed existing settings.json"); - - let selection: Vec = every_optional_workflow() - .into_iter() - .map(str::to_string) - .collect(); - - install_embedded_setup_assets(&repo, SetupTarget::Claude, &selection) - .expect("first install should succeed"); - - let after_first = - fs::read_to_string(claude_dir.join("settings.json")).expect("read merged settings"); - let merged: serde_json::Value = - serde_json::from_str(&after_first).expect("merged settings should be valid JSON"); - - assert_eq!(merged["permissions"]["allow"][0], "Bash(git *)"); - assert_eq!(merged["env"]["FOO"], "bar"); - let pre_tool_use = merged["hooks"]["PreToolUse"] - .as_array() - .expect("PreToolUse should be an array"); - assert!(pre_tool_use - .iter() - .any(|entry| entry["hooks"][0]["command"] == "echo user-hook")); - assert!(pre_tool_use - .iter() - .any(|entry| entry["hooks"] - .as_array() - .unwrap() - .iter() - .any(|hook| hook["command"] - .as_str() - .unwrap() - .contains("run-sce-or-show-install-guidance.sh")))); - - install_embedded_setup_assets(&repo, SetupTarget::Claude, &selection) - .expect("second install should succeed"); - - let after_second = - fs::read_to_string(claude_dir.join("settings.json")).expect("read re-merged settings"); - assert_eq!( - after_first, after_second, - "two consecutive installs should merge to byte-identical output" - ); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn install_merges_into_existing_opencode_config_json_and_stays_idempotent() { - let repo = init_git_repo("install-merges-opencode-config"); - let opencode_dir = default_paths::InstallTargetPaths::new(&repo).opencode_target_dir(); - - fs::create_dir_all(&opencode_dir).expect("create opencode dir"); - fs::write( - opencode_dir.join("opencode.json"), - serde_json::to_string_pretty(&json!({ - "model": "anthropic/claude", - "mcp": {"my-server": {"command": "my-server"}}, - "plugin": ["./plugins/my-plugin.ts", "./plugins/sce-old-feature.ts"] - })) - .expect("serialize seeded opencode config"), - ) - .expect("seed existing opencode.json"); - - let selection: Vec = every_optional_workflow() - .into_iter() - .map(str::to_string) - .collect(); - - install_embedded_setup_assets(&repo, SetupTarget::OpenCode, &selection) - .expect("first install should succeed"); - - let after_first = fs::read_to_string(opencode_dir.join("opencode.json")) - .expect("read merged opencode config"); - let merged: serde_json::Value = - serde_json::from_str(&after_first).expect("merged opencode config should be valid JSON"); - - assert_eq!(merged["model"], "anthropic/claude"); - assert_eq!(merged["mcp"]["my-server"]["command"], "my-server"); - - let plugin = merged["plugin"] - .as_array() - .expect("plugin should be an array"); - assert!(plugin.contains(&json!("./plugins/my-plugin.ts"))); - assert!(plugin.contains(&json!("./plugins/sce-bash-policy.ts"))); - assert!(plugin.contains(&json!("./plugins/sce-agent-trace.ts"))); - assert!(!plugin.contains(&json!("./plugins/sce-old-feature.ts"))); - assert_eq!( - plugin.last().and_then(serde_json::Value::as_str), - Some("./plugins/sce-mutation-scope.ts"), - "the mutation-scope plugin must be installed as the final plugin" - ); - - install_embedded_setup_assets(&repo, SetupTarget::OpenCode, &selection) - .expect("second install should succeed"); - - let after_second = fs::read_to_string(opencode_dir.join("opencode.json")) - .expect("read re-merged opencode config"); - assert_eq!( - after_first, after_second, - "two consecutive installs should merge to byte-identical output" - ); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn reinstall_with_empty_selection_prunes_deselected_workflow_without_touching_sibling_skill() { - let repo = init_git_repo("install-prunes-deselected-workflow"); - let claude_dir = default_paths::InstallTargetPaths::new(&repo).claude_target_dir(); - - let brownfield_selection = vec!["brownfield".to_string()]; - install_embedded_setup_assets(&repo, SetupTarget::Claude, &brownfield_selection) - .expect("initial install with brownfield selected should succeed"); - - let brownfield_command = claude_dir.join("commands/brownfield.md"); - let brownfield_skill_dir = claude_dir.join("skills/sce-brownfield"); - assert!( - brownfield_command.is_file(), - "brownfield command should be installed" - ); - assert!( - brownfield_skill_dir.is_dir(), - "brownfield skill dir should be installed" - ); - - fs::create_dir_all(claude_dir.join("skills/my-skill")).expect("create user skill dir"); - fs::write( - claude_dir.join("skills/my-skill/SKILL.md"), - "sibling user skill\n", - ) - .expect("seed sibling user skill file"); - - install_embedded_setup_assets(&repo, SetupTarget::Claude, &[]) - .expect("reinstall with empty selection should succeed"); - - assert!( - !brownfield_command.exists(), - "deselected workflow command should be pruned" - ); - assert!( - !brownfield_skill_dir.exists(), - "deselected workflow skill dir should be pruned entirely once empty" - ); - assert_eq!( - fs::read_to_string(claude_dir.join("skills/my-skill/SKILL.md")) - .expect("read sibling user skill file"), - "sibling user skill\n" - ); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn reinstall_with_empty_selection_keeps_pruned_skill_dir_holding_a_user_file() { - let repo = init_git_repo("install-prunes-but-keeps-user-file"); - let claude_dir = default_paths::InstallTargetPaths::new(&repo).claude_target_dir(); - - let brownfield_selection = vec!["brownfield".to_string()]; - install_embedded_setup_assets(&repo, SetupTarget::Claude, &brownfield_selection) - .expect("initial install with brownfield selected should succeed"); - - let brownfield_skill_dir = claude_dir.join("skills/sce-brownfield"); - fs::write( - brownfield_skill_dir.join("MY_OVERRIDE.md"), - "user file inside sce skill dir\n", - ) - .expect("seed user file inside sce-owned skill dir"); - - install_embedded_setup_assets(&repo, SetupTarget::Claude, &[]) - .expect("reinstall with empty selection should succeed"); - - assert!( - !brownfield_skill_dir.join("SKILL.md").exists(), - "deselected workflow skill file should be pruned" - ); - assert!( - brownfield_skill_dir.is_dir(), - "sce-owned skill dir should survive because it still holds a user file" - ); - assert_eq!( - fs::read_to_string(brownfield_skill_dir.join("MY_OVERRIDE.md")) - .expect("read user file inside pruned skill dir"), - "user file inside sce skill dir\n" - ); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn install_cleans_up_staging_and_reports_asset_path_on_rename_failure() { - let repo = init_git_repo("install-rename-failure"); - let selection: Vec = every_optional_workflow() - .into_iter() - .map(str::to_string) - .collect(); - - let claude_dir = default_paths::InstallTargetPaths::new(&repo).claude_target_dir(); - let failing_destination = claude_dir.join("commands/next-task.md"); - - fs::create_dir_all(claude_dir.join("commands")).expect("create commands dir"); - let prior_content = b"prior next-task content\n"; - fs::write(&failing_destination, prior_content).expect("seed prior next-task content"); - - let result = install::install_embedded_setup_assets_with_rename( - &repo, - SetupTarget::Claude, - &selection, - |from, to| { - if to == failing_destination { - Err(std::io::Error::other("simulated rename failure")) - } else { - fs::rename(from, to) - } - }, - ); - - let error = result.expect_err("rename failure should surface as an error"); - let message = format!("{error:#}"); - assert!( - message.contains(&failing_destination.display().to_string()), - "error should name the failing asset path: {message}" - ); - assert!( - message.contains("does not create backups"), - "error should include recovery guidance: {message}" - ); - - assert_eq!( - fs::read(&failing_destination).expect("read failing destination after rename failure"), - prior_content, - "prior content at the failing destination should survive a rename failure" - ); - - let commands_staging_dir = claude_dir.join("commands"); - if commands_staging_dir.exists() { - let leftover_staging_files = fs::read_dir(&commands_staging_dir) - .expect("read commands staging dir") - .filter_map(Result::ok) - .any(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with(".sce-setup-staging-") - }); - assert!( - !leftover_staging_files, - "staging artifact for the failed asset should be cleaned up" - ); - } - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn hook_install_leaves_prior_hook_intact_on_rename_failure() { - let repo = init_git_repo("hook-install-rename-failure"); - - let initial_outcome = - install::install_required_git_hooks(&repo).expect("initial hook install should succeed"); - let pre_commit_result = initial_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook should be installed"); - let pre_commit_path = pre_commit_result.hook_path.clone(); - - let prior_hook_bytes = b"#!/bin/sh\necho prior pre-commit\n".to_vec(); - fs::write(&pre_commit_path, &prior_hook_bytes).expect("seed prior pre-commit hook"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&pre_commit_path, fs::Permissions::from_mode(0o755)) - .expect("mark prior pre-commit hook executable"); - } - let prior_mode = fs::metadata(&pre_commit_path) - .expect("stat prior pre-commit hook") - .permissions(); - - let result = install::install_required_git_hooks_with_rename(&repo, |from, to| { - if to == pre_commit_path { - Err(std::io::Error::other("simulated rename failure")) - } else { - fs::rename(from, to) - } - }); - - let error = result.expect_err("rename failure should surface as an error"); - let message = format!("{error:#}"); - assert!( - message.contains(&pre_commit_path.display().to_string()), - "error should name the failing hook path: {message}" - ); - - assert_eq!( - fs::read(&pre_commit_path).expect("read pre-commit hook after rename failure"), - prior_hook_bytes, - "prior hook content should survive a rename failure" - ); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode_after = fs::metadata(&pre_commit_path) - .expect("stat pre-commit hook after rename failure") - .permissions(); - assert_eq!( - mode_after.mode() & 0o777, - prior_mode.mode() & 0o777, - "prior hook executable mode should survive a rename failure" - ); - } - - let hooks_staging_dir = pre_commit_path - .parent() - .expect("pre-commit hook should have a parent directory"); - let leftover_staging_files = fs::read_dir(hooks_staging_dir) - .expect("read hooks staging dir") - .filter_map(Result::ok) - .any(|entry| { - entry - .file_name() - .to_string_lossy() - .starts_with(".sce-hook-staging-") - }); - assert!( - !leftover_staging_files, - "staging artifact for the failed hook should be cleaned up" - ); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn foreign_pre_commit_hook_keeps_its_content_and_gains_the_sce_block() { - let repo = init_git_repo("hook-install-foreign-append"); - - let initial_outcome = - install::install_required_git_hooks(&repo).expect("initial hook install should succeed"); - let pre_commit_path = initial_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook should be installed") - .hook_path - .clone(); - - let foreign_bytes = b"#!/bin/sh\necho husky-style-guard\n".to_vec(); - fs::write(&pre_commit_path, &foreign_bytes).expect("seed foreign pre-commit hook"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&pre_commit_path, fs::Permissions::from_mode(0o755)) - .expect("mark foreign pre-commit hook executable"); - } - - let outcome = install::install_required_git_hooks(&repo) - .expect("hook install over a foreign hook should succeed"); - let result = outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook result should be present"); - - assert_eq!(result.status, RequiredHookInstallStatus::Updated); - assert!(!result.unreachable_block_advisory); - - let installed_bytes = fs::read(&pre_commit_path).expect("read installed pre-commit hook"); - assert!( - installed_bytes.starts_with(&foreign_bytes), - "foreign hook content should survive as an exact prefix" - ); - let installed_text = String::from_utf8(installed_bytes).expect("hook should be utf8"); - assert!(installed_text.contains(hook_merge::MANAGED_BLOCK_START)); - assert!(installed_text.contains(hook_merge::MANAGED_BLOCK_END)); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = fs::metadata(&pre_commit_path) - .expect("stat installed pre-commit hook") - .permissions() - .mode(); - assert_ne!(mode & 0o111, 0, "installed hook should remain executable"); - } - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn rerunning_hook_install_is_idempotent_for_block_only_and_foreign_plus_block_shapes() { - let repo = init_git_repo("hook-install-idempotent"); - - let first_outcome = - install::install_required_git_hooks(&repo).expect("first hook install should succeed"); - let pre_commit_result = first_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook should be installed"); - assert_eq!( - pre_commit_result.status, - RequiredHookInstallStatus::Installed - ); - - let second_outcome = - install::install_required_git_hooks(&repo).expect("second hook install should succeed"); - let second_pre_commit = second_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook result should be present"); - assert_eq!(second_pre_commit.status, RequiredHookInstallStatus::Skipped); - assert_eq!( - fs::read(&second_pre_commit.hook_path).expect("read block-only pre-commit hook"), - fs::read(&pre_commit_result.hook_path).expect("read initial pre-commit hook"), - "block-only hook bytes should be unchanged across reruns" - ); - - let commit_msg_result = first_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::COMMIT_MSG) - .expect("commit-msg hook should be installed"); - let commit_msg_path = commit_msg_result.hook_path.clone(); - let foreign_prefix = b"#!/bin/sh\necho foreign-commit-msg-guard\n".to_vec(); - fs::write(&commit_msg_path, &foreign_prefix).expect("seed foreign commit-msg hook"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&commit_msg_path, fs::Permissions::from_mode(0o755)) - .expect("mark foreign commit-msg hook executable"); - } - - let appended_outcome = install::install_required_git_hooks(&repo) - .expect("hook install appending to foreign commit-msg hook should succeed"); - let appended_result = appended_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::COMMIT_MSG) - .expect("commit-msg hook result should be present"); - assert_eq!(appended_result.status, RequiredHookInstallStatus::Updated); - let appended_bytes = fs::read(&commit_msg_path).expect("read appended commit-msg hook"); - - let rerun_outcome = install::install_required_git_hooks(&repo) - .expect("rerunning hook install over foreign-plus-block hook should succeed"); - let rerun_result = rerun_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::COMMIT_MSG) - .expect("commit-msg hook result should be present"); - assert_eq!(rerun_result.status, RequiredHookInstallStatus::Skipped); - assert_eq!( - fs::read(&commit_msg_path).expect("read commit-msg hook after rerun"), - appended_bytes, - "foreign-plus-block hook bytes should be unchanged across reruns" - ); - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn legacy_pre_marker_hook_upgrades_to_the_managed_block_form() { - let repo = init_git_repo("hook-install-legacy-upgrade"); - - let initial_outcome = - install::install_required_git_hooks(&repo).expect("initial hook install should succeed"); - let pre_commit_path = initial_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook should be installed") - .hook_path - .clone(); - let canonical_bytes = fs::read(&pre_commit_path).expect("read canonical pre-commit hook"); - - let legacy_bytes = b"#!/bin/sh\nset -eu\nif ! command -v sce >/dev/null 2>&1; then\n echo 'Install: https://sce.crocoder.dev/docs/getting-started#install-cli'\n exit 0\nfi\nexec sce hooks pre-commit \"$@\"\n".to_vec(); - fs::write(&pre_commit_path, &legacy_bytes).expect("seed legacy pre-commit hook"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&pre_commit_path, fs::Permissions::from_mode(0o755)) - .expect("mark legacy pre-commit hook executable"); - } - - let outcome = install::install_required_git_hooks(&repo) - .expect("hook install upgrading a legacy hook should succeed"); - let result = outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook result should be present"); - - assert_eq!(result.status, RequiredHookInstallStatus::Updated); - assert_eq!( - fs::read(&pre_commit_path).expect("read upgraded pre-commit hook"), - canonical_bytes, - "a legacy pre-marker hook should upgrade to the canonical marker form" - ); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = fs::metadata(&pre_commit_path) - .expect("stat upgraded pre-commit hook") - .permissions() - .mode(); - assert_ne!(mode & 0o111, 0, "upgraded hook should remain executable"); - } - - let _ = fs::remove_dir_all(&repo); -} - -#[test] -fn foreign_hook_ending_in_exec_installs_the_block_and_reports_the_advisory() { - let repo = init_git_repo("hook-install-unreachable-advisory"); - - let initial_outcome = - install::install_required_git_hooks(&repo).expect("initial hook install should succeed"); - let pre_commit_path = initial_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook should be installed") - .hook_path - .clone(); - let commit_msg_path = initial_outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::COMMIT_MSG) - .expect("commit-msg hook should be installed") - .hook_path - .clone(); - - let unreachable_foreign = b"#!/bin/sh\nexec some-other-tool \"$@\"\n".to_vec(); - fs::write(&pre_commit_path, &unreachable_foreign).expect("seed unreachable foreign hook"); - let ordinary_foreign = b"#!/bin/sh\necho foreign-commit-msg-guard\n".to_vec(); - fs::write(&commit_msg_path, &ordinary_foreign).expect("seed ordinary foreign hook"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&pre_commit_path, fs::Permissions::from_mode(0o755)) - .expect("mark unreachable foreign hook executable"); - fs::set_permissions(&commit_msg_path, fs::Permissions::from_mode(0o755)) - .expect("mark ordinary foreign hook executable"); - } - - let outcome = install::install_required_git_hooks(&repo) - .expect("hook install over foreign hooks should succeed"); - - let pre_commit_result = outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::PRE_COMMIT) - .expect("pre-commit hook result should be present"); - assert_eq!(pre_commit_result.status, RequiredHookInstallStatus::Updated); - assert!( - pre_commit_result.unreachable_block_advisory, - "a hook ending in a zero-indent exec should report the advisory" - ); - assert!( - fs::read(&pre_commit_path) - .expect("read pre-commit hook") - .starts_with(&unreachable_foreign), - "the block should still be installed even though it is unreachable" - ); - - let commit_msg_result = outcome - .hook_results - .iter() - .find(|result| result.hook_name == default_paths::hook_dir::COMMIT_MSG) - .expect("commit-msg hook result should be present"); - assert!( - !commit_msg_result.unreachable_block_advisory, - "a hook ending in an ordinary command should not report the advisory" - ); - - let _ = fs::remove_dir_all(&repo); -} diff --git a/cli/src/services/structured_patch/tests.rs b/cli/src/services/structured_patch/tests.rs index 19e0f3bdb..7ce6b3fed 100644 --- a/cli/src/services/structured_patch/tests.rs +++ b/cli/src/services/structured_patch/tests.rs @@ -129,7 +129,6 @@ fn claude_derivation_golden_tests() { "patch mismatch for scenario {name}" ); - // model_id is omitted at the structured-patch layer; hunks should not carry it. let all_hunks_empty_model_id = patch .patch .files diff --git a/cli/src/services/style.rs b/cli/src/services/style.rs index 7c1f2e8e3..329f22ffb 100644 --- a/cli/src/services/style.rs +++ b/cli/src/services/style.rs @@ -259,9 +259,7 @@ fn is_help_placeholder(token: &str) -> bool { /// Gradient color endpoints for the ASCII art banner. /// Right side: cyan (0, 255, 255), Left side: magenta (255, 0, 255). -#[allow(dead_code)] const GRADIENT_COLOR_RIGHT: (u8, u8, u8) = (0, 255, 255); -#[allow(dead_code)] const GRADIENT_COLOR_LEFT: (u8, u8, u8) = (255, 0, 255); /// Applies a right-to-left color gradient to ASCII art banner lines. @@ -270,7 +268,6 @@ const GRADIENT_COLOR_LEFT: (u8, u8, u8) = (255, 0, 255); /// column position using a cyan-to-magenta gradient (cyan on the right, /// magenta on the left). Spaces are left unstyled. /// When color is disabled, returns the plain ASCII art unchanged. -#[allow(dead_code)] #[must_use] pub fn banner_with_gradient(lines: &[&str]) -> String { banner_with_gradient_with_color_policy(lines, supports_color()) @@ -278,7 +275,7 @@ pub fn banner_with_gradient(lines: &[&str]) -> String { /// Internal variant of [`banner_with_gradient`] that accepts an explicit /// color policy flag for testability. -#[allow(dead_code, clippy::cast_precision_loss)] +#[allow(clippy::cast_precision_loss)] #[must_use] pub(crate) fn banner_with_gradient_with_color_policy( lines: &[&str], @@ -325,7 +322,7 @@ pub(crate) fn banner_with_gradient_with_color_policy( /// /// Returns `a + (b - a) * t`, clamped to the `u8` range by construction /// since `a`, `b` are `u8` and `t` is in `[0, 1]`. -#[allow(dead_code, clippy::cast_possible_truncation, clippy::cast_sign_loss)] +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] fn lerp_u8(a: u8, b: u8, t: f64) -> u8 { (f64::from(a) + (f64::from(b) - f64::from(a)) * t).round() as u8 } diff --git a/cli/src/services/sync/auto_sync.rs b/cli/src/services/sync/auto_sync.rs index 2198f3519..6b27d4e10 100644 --- a/cli/src/services/sync/auto_sync.rs +++ b/cli/src/services/sync/auto_sync.rs @@ -70,70 +70,3 @@ fn spawn_command(spec: AutoSyncCommand) -> io::Result<()> { let _child = command.spawn()?; Ok(()) } - -#[cfg(test)] -mod tests { - use std::cell::RefCell; - use std::io; - use std::path::{Path, PathBuf}; - use std::rc::Rc; - - use super::{launch_with, AutoSyncCommand, StdioMode, SYNC_ARGS}; - - #[test] - fn launch_builds_the_expected_detached_command() { - let captured = Rc::new(RefCell::new(None)); - let captured_by_spawn = Rc::clone(&captured); - - let launched = launch_with( - Path::new("/repo/root"), - || Ok(PathBuf::from("/usr/local/bin/sce")), - move |command: AutoSyncCommand| { - *captured_by_spawn.borrow_mut() = Some(command); - Ok(()) - }, - ); - - assert!(launched); - assert_eq!( - captured.borrow().clone(), - Some(AutoSyncCommand { - executable: PathBuf::from("/usr/local/bin/sce"), - args: SYNC_ARGS.iter().map(|arg| (*arg).to_string()).collect(), - current_dir: PathBuf::from("/repo/root"), - stdin: StdioMode::Null, - stdout: StdioMode::Null, - stderr: StdioMode::Null, - }) - ); - } - - #[test] - fn current_executable_failure_is_fail_open() { - let spawn_called = Rc::new(RefCell::new(false)); - let spawn_called_by_spawn = Rc::clone(&spawn_called); - - let launched = launch_with( - Path::new("/repo/root"), - || Err(io::Error::other("current executable unavailable")), - move |_| { - *spawn_called_by_spawn.borrow_mut() = true; - Ok(()) - }, - ); - - assert!(!launched); - assert!(!*spawn_called.borrow()); - } - - #[test] - fn spawn_failure_is_fail_open() { - let launched = launch_with( - Path::new("/repo/root"), - || Ok(PathBuf::from("/usr/local/bin/sce")), - |_| Err(io::Error::other("spawn unavailable")), - ); - - assert!(!launched); - } -} diff --git a/cli/src/services/sync/command.rs b/cli/src/services/sync/command.rs index 3455877fc..77adb11fb 100644 --- a/cli/src/services/sync/command.rs +++ b/cli/src/services/sync/command.rs @@ -43,15 +43,6 @@ fn classify_sync_error(err: TraceSyncError) -> CliError { } impl SyncCommand { - #[allow(dead_code)] - pub async fn execute(&self, context: &C) -> Result - where - C: ContextWithRepoRoot, - { - let mut stderr = std::io::sink(); - self.execute_with_stderr(context, &mut stderr).await - } - pub async fn execute_with_stderr( &self, context: &C, @@ -101,122 +92,3 @@ impl SyncCommand { .map_err(|error| CliError::runtime(anyhow::Error::msg(format!("{error:#}")))) } } - -#[cfg(test)] -mod tests { - use super::classify_sync_error; - use crate::services::agent_trace_sync::control_plane::ControlPlaneError; - use crate::services::agent_trace_sync::StreamSyncError; - use crate::services::error::CliError; - use crate::services::sync::sync::TraceSyncError; - - fn assert_user_not_authenticated(err: TraceSyncError) { - match classify_sync_error(err) { - CliError::User { error, source } => { - assert_eq!(error.key(), "auth.not_authenticated"); - assert!(source.is_some()); - } - other @ CliError::Internal { .. } => panic!("expected CliError::User, got {other:?}"), - } - } - - fn assert_user_error(err: TraceSyncError, expected_key: &str) { - match classify_sync_error(err) { - CliError::User { error, source } => { - assert_eq!(error.key(), expected_key); - assert!(source.is_some()); - } - other @ CliError::Internal { .. } => panic!("expected CliError::User, got {other:?}"), - } - } - - #[test] - fn initial_state_missing_credentials_classifies_as_not_authenticated() { - assert_user_not_authenticated(TraceSyncError::ControlPlane( - ControlPlaneError::MissingCredentials, - )); - } - - #[test] - fn initial_state_authentication_failed_classifies_as_not_authenticated() { - assert_user_not_authenticated(TraceSyncError::ControlPlane( - ControlPlaneError::AuthenticationFailed("token expired".to_string()), - )); - } - - #[test] - fn stream_batch_authentication_failed_classifies_as_not_authenticated() { - assert_user_not_authenticated(TraceSyncError::Stream { - stream: "prompts", - source: StreamSyncError::Terminal(ControlPlaneError::AuthenticationFailed( - "token expired".to_string(), - )), - }); - } - - #[test] - fn stream_refresh_authentication_failed_classifies_as_not_authenticated() { - assert_user_not_authenticated(TraceSyncError::Stream { - stream: "prompts", - source: StreamSyncError::Refresh(ControlPlaneError::MissingCredentials), - }); - } - - #[test] - fn other_control_plane_errors_classify_as_unexpected_failure() { - for error in [ - ControlPlaneError::Forbidden("nope".to_string()), - ControlPlaneError::BadRequest("bad".to_string()), - ControlPlaneError::Transport("down".to_string()), - ControlPlaneError::ServerError("500".to_string()), - ControlPlaneError::InvalidResponse("garbage".to_string()), - ControlPlaneError::Protocol { - status: reqwest::StatusCode::NOT_FOUND, - message: "route removed".to_string(), - }, - ] { - assert_user_error( - TraceSyncError::ControlPlane(error), - "general.unexpected_failure", - ); - } - } - - #[test] - fn credential_storage_failure_classifies_as_storage_unavailable() { - assert_user_error( - TraceSyncError::ControlPlane(ControlPlaneError::Storage("disk".to_string())), - "auth.storage_unavailable", - ); - } - - #[test] - fn runtime_failure_classifies_as_unexpected_failure() { - assert_user_error( - TraceSyncError::Runtime("local failure".to_string()), - "general.unexpected_failure", - ); - } - - #[test] - fn stream_terminal_storage_failure_classifies_as_storage_unavailable() { - assert_user_error( - TraceSyncError::Stream { - stream: "prompts", - source: StreamSyncError::Terminal(ControlPlaneError::Storage("disk".to_string())), - }, - "auth.storage_unavailable", - ); - } - - #[test] - fn stream_refresh_storage_failure_classifies_as_storage_unavailable() { - assert_user_error( - TraceSyncError::Stream { - stream: "prompts", - source: StreamSyncError::Refresh(ControlPlaneError::Storage("disk".to_string())), - }, - "auth.storage_unavailable", - ); - } -} diff --git a/cli/src/services/sync/mod.rs b/cli/src/services/sync/mod.rs index fac259368..62046d2a4 100644 --- a/cli/src/services/sync/mod.rs +++ b/cli/src/services/sync/mod.rs @@ -1,6 +1,5 @@ //! Top-level `sce sync` command and Agent Trace synchronization service. -#[allow(dead_code)] pub mod auto_sync; pub mod command; pub mod progress; diff --git a/cli/src/services/sync/progress.rs b/cli/src/services/sync/progress.rs index d30d9010c..4ca957b40 100644 --- a/cli/src/services/sync/progress.rs +++ b/cli/src/services/sync/progress.rs @@ -185,136 +185,3 @@ fn uploaded_message(uploaded: usize) -> String { let row_label = if uploaded == 1 { "row" } else { "rows" }; format!("{uploaded} {row_label} uploaded") } - -#[cfg(test)] -mod tests { - use super::*; - - #[derive(Clone, Debug, PartialEq, Eq)] - enum TestEvent { - Started, - Finished, - } - - #[derive(Default)] - struct CollectingReporter { - events: Vec, - finalized_successfully: bool, - } - - impl ProgressReporter for CollectingReporter { - fn report(&mut self, event: TestEvent) { - self.events.push(event); - } - - fn finish_successfully(&mut self) { - self.finalized_successfully = true; - } - } - - #[test] - fn collecting_reporter_observes_events_and_successful_finalization() { - let mut reporter = CollectingReporter::default(); - - reporter.report(TestEvent::Started); - reporter.report(TestEvent::Finished); - reporter.finish_successfully(); - - assert_eq!( - reporter.events, - vec![TestEvent::Started, TestEvent::Finished] - ); - assert!(reporter.finalized_successfully); - } - - #[test] - fn closure_reporter_supports_the_event_contract() { - let mut events = Vec::new(); - let mut reporter = |event: TestEvent| events.push(event); - - reporter.report(TestEvent::Started); - reporter.report(TestEvent::Finished); - reporter.finish_successfully(); - - assert_eq!(events, vec![TestEvent::Started, TestEvent::Finished]); - } - - #[test] - fn noop_reporter_ignores_events_and_finalization() { - let mut reporter = NoopProgressReporter; - reporter.report(TestEvent::Finished); - >::finish_successfully(&mut reporter); - } - - #[test] - fn progress_reporter_creates_aligned_rows_and_updates_only_the_matching_stream() { - let mut output = Vec::new(); - let mut reporter = IndicatifProgressReporter::with_policies(&mut output, false, false); - - reporter.report(SyncProgressEvent::Started { - timestamp: "2026-01-02T03:04:05Z".to_string(), - }); - reporter.report(SyncProgressEvent::BatchAccepted { - stream: "messages", - batch_rows: 500, - uploaded: 500, - cursor: 500, - }); - reporter.report(SyncProgressEvent::StreamCompleted { - stream: "parts", - uploaded: 0, - cursor: 12, - batches: 0, - }); - - let output = String::from_utf8(output).expect("progress output should be UTF-8"); - assert!(output.contains("messages ")); - assert!(output.contains("parts ")); - assert!(output.contains("agent_traces ")); - assert!(!output.contains("diff_traces")); - let first_render = output - .lines() - .take(STREAM_LABELS.len() + 1) - .map(|line| { - line.split_whitespace() - .nth(1) - .expect("progress row has a stream label") - }) - .collect::>(); - assert_eq!( - first_render, - vec!["messages", "parts", "agent_traces", "messages"] - ); - assert!(output.contains("messages 500 rows uploaded")); - assert!(output.contains("parts 0 rows uploaded")); - assert!(output.contains("✓ parts 0 rows uploaded")); - assert!(!output.contains("\x1b[")); - } - - #[test] - fn progress_reporter_finishes_rows_with_a_plain_check_when_color_is_disabled() { - let mut output = Vec::new(); - let mut reporter = IndicatifProgressReporter::with_policies(&mut output, false, false); - - reporter.report(SyncProgressEvent::StreamCompleted { - stream: "agent_traces", - uploaded: 7, - cursor: 12, - batches: 2, - }); - - let output = String::from_utf8(output).expect("progress output should be UTF-8"); - assert!(output.contains("✓ agent_traces 7 rows uploaded")); - assert!(!output.contains("\x1b[")); - } - - #[test] - fn progress_reporter_leaves_a_blank_line_after_completion() { - let mut output = Vec::new(); - let mut reporter = IndicatifProgressReporter::with_policies(&mut output, false, false); - - reporter.finish_successfully(); - - assert!(output.ends_with(b"\n\n")); - } -} diff --git a/cli/src/services/sync/render_sync.rs b/cli/src/services/sync/render_sync.rs index bc2d90347..987eb4aca 100644 --- a/cli/src/services/sync/render_sync.rs +++ b/cli/src/services/sync/render_sync.rs @@ -59,107 +59,52 @@ fn stream_json(stream: &StreamSyncReport) -> serde_json::Value { #[cfg(test)] mod tests { - use super::*; - - fn sample_report() -> AgentTraceSyncReport { - AgentTraceSyncReport { - repository_id: "repo-123".to_string(), - source_instance_id: "source-abc".to_string(), - streams: crate::services::sync::sync::StreamSyncReports { - messages: StreamSyncReport { - uploaded: 3, - initial_cursor: 10, - final_cursor: 13, - batches: 1, - }, - parts: StreamSyncReport { - uploaded: 5, - initial_cursor: 20, - final_cursor: 25, - batches: 1, - }, + use serde_json::{json, Value}; + + use super::render; + use crate::services::output_format::OutputFormat; + use crate::services::sync::sync::{AgentTraceSyncReport, StreamSyncReport, StreamSyncReports}; + + const DIFF_TRACES_CURSOR: i64 = 42; + + fn stream(uploaded: usize) -> StreamSyncReport { + StreamSyncReport { + uploaded, + initial_cursor: 0, + final_cursor: i64::try_from(uploaded).expect("small count"), + batches: usize::from(uploaded > 0), + } + } + + #[test] + fn json_report_keeps_zero_upload_diff_traces_stream() { + let report = AgentTraceSyncReport { + repository_id: "repo".to_string(), + source_instance_id: "source".to_string(), + streams: StreamSyncReports { + messages: stream(3), + parts: stream(0), diff_traces: StreamSyncReport { uploaded: 0, - initial_cursor: 7, - final_cursor: 7, + initial_cursor: DIFF_TRACES_CURSOR, + final_cursor: DIFF_TRACES_CURSOR, batches: 0, }, - agent_traces: StreamSyncReport { - uploaded: 2, - initial_cursor: 1, - final_cursor: 3, - batches: 1, - }, + agent_traces: stream(0), }, - } - } + }; - #[test] - fn json_shape_matches_contract() { - let payload = render_json(&sample_report()).expect("json render"); - let value: serde_json::Value = serde_json::from_str(&payload).expect("valid json"); - assert_eq!(value["status"], "ok"); - assert_eq!(value["command"], "sync"); - assert!(value.get("subcommand").is_none()); - assert!(value.get("repositoryId").is_none()); - assert!(value.get("sourceInstanceId").is_none()); - - let messages = &value["streams"]["messages"]; - assert_eq!(messages["uploaded"], 3); - assert_eq!(messages["initialCursor"], 10); - assert_eq!(messages["finalCursor"], 13); - assert_eq!(messages["batches"], 1); - - let diff_traces = &value["streams"]["diffTraces"]; - assert_eq!(diff_traces["uploaded"], 0); - assert_eq!(diff_traces["initialCursor"], 7); - assert_eq!(diff_traces["finalCursor"], 7); - assert_eq!(diff_traces["batches"], 0); - - let agent_traces = &value["streams"]["agentTraces"]; - assert_eq!(agent_traces["uploaded"], 2); - assert_eq!(agent_traces["initialCursor"], 1); - assert_eq!(agent_traces["finalCursor"], 3); - assert_eq!(agent_traces["batches"], 1); - - assert!(value.get("diff_traces").is_none()); - assert!(value.get("agent_traces").is_none()); - } - - fn report_with_uploads( - messages: usize, - parts: usize, - diff_traces: usize, - agent_traces: usize, - ) -> AgentTraceSyncReport { - let mut report = sample_report(); - report.streams.messages.uploaded = messages; - report.streams.parts.uploaded = parts; - report.streams.diff_traces.uploaded = diff_traces; - report.streams.agent_traces.uploaded = agent_traces; - report - } + let rendered = render(&report, OutputFormat::Json).expect("render json"); + let rendered: Value = serde_json::from_str(&rendered).expect("valid json"); - #[test] - fn text_heading_is_already_synced_when_three_active_streams_uploaded_nothing() { assert_eq!( - render_text(&report_with_uploads(0, 0, 0, 0)), - style::heading(ALREADY_SYNCED_HEADING) + rendered["streams"]["diffTraces"], + json!({ + "uploaded": 0, + "initialCursor": DIFF_TRACES_CURSOR, + "finalCursor": DIFF_TRACES_CURSOR, + "batches": 0, + }) ); - assert_eq!( - render_text(&report_with_uploads(0, 0, 1, 0)), - style::heading(ALREADY_SYNCED_HEADING) - ); - } - - #[test] - fn text_heading_is_complete_when_any_active_stream_uploaded_rows() { - for report in [ - report_with_uploads(1, 0, 0, 0), - report_with_uploads(0, 1, 0, 0), - report_with_uploads(0, 0, 0, 1), - ] { - assert_eq!(render_text(&report), style::heading(COMPLETE_HEADING)); - } } } diff --git a/cli/src/services/sync/sync.rs b/cli/src/services/sync/sync.rs index 31730a9fe..dfc1a4d6d 100644 --- a/cli/src/services/sync/sync.rs +++ b/cli/src/services/sync/sync.rs @@ -2,31 +2,30 @@ //! and [`crate::services::agent_trace_sync::control_plane`] client as-is; adds //! no local sync cursor or persisted progress of its own. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::fmt; use std::future::{poll_fn, Future}; use std::path::Path; +use std::pin::Pin; use std::rc::Rc; -use std::task::Poll; +use std::task::{Context, Poll}; use chrono::{DateTime, SecondsFormat, Utc}; use crate::services::agent_trace_db::repository::RepositoryAgentTraceDb; use crate::services::agent_trace_export::{AgentTraceExportReader, AGENT_TRACE_EXPORT_BATCH_SIZE}; -use crate::services::agent_trace_storage::{ - resolve_agent_trace_storage, AgentTraceStorageContext, ResolvedAgentTraceStorage, -}; +use crate::services::agent_trace_storage::{resolve_agent_trace_storage, AgentTraceStorageContext}; use crate::services::agent_trace_sync::control_plane::{ AgentTraceCursors, AgentTraceIngestionBatchRequest, AgentTraceIngestionBatchResponse, AgentTraceIngestionStateRequest, AuthenticatedControlPlaneClient, ControlPlaneError, - IngestionStream, + CredentialStore, IngestionStream, }; use crate::services::agent_trace_sync::{ - sync_stream, AgentTraceExportRow, BatchAttemptOutcome, StreamSyncError, SyncFuture, + sync_stream, AgentTraceExportRow, BatchAttemptOutcome, StreamSyncError, }; use crate::services::auth; use crate::services::config; -use crate::services::sync::progress::{NoopProgressReporter, ProgressReporter}; +use crate::services::sync::progress::ProgressReporter; #[derive(Clone, Debug, PartialEq, Eq)] pub struct AgentTraceSyncReport { @@ -128,7 +127,6 @@ impl TraceSyncError { /// (`ControlPlane`) or from a stream's batch/refresh path (`Stream`). /// `Runtime` never carries a `ControlPlaneError` and is never an /// authentication failure. - #[allow(dead_code)] pub fn is_authentication_failure(&self) -> bool { match self { Self::Runtime(_) => false, @@ -149,40 +147,6 @@ impl TraceSyncError { } } -struct SyncStorageGuard(Option); - -impl SyncStorageGuard { - fn storage(&self) -> &ResolvedAgentTraceStorage { - self.0 - .as_ref() - .expect("sync storage remains owned until cleanup") - } -} - -impl Drop for SyncStorageGuard { - fn drop(&mut self) { - tokio::task::block_in_place(|| drop(self.0.take())); - } -} - -#[allow(dead_code)] -pub async fn run_current_sync(repo_root: &Path) -> Result { - let mut progress = NoopProgressReporter; - run_current_sync_with_progress(repo_root, &mut progress).await -} - -/// Production entry point with an injectable progress sink. -pub async fn run_current_sync_with_progress( - repo_root: &Path, - progress: &mut S, -) -> Result -where - S: ProgressReporter, -{ - let clock = SystemSyncProgressClock; - run_current_sync_with_progress_and_clock(repo_root, progress, &clock).await -} - /// Production sync entry point with injectable progress sink and clock. pub async fn run_current_sync_with_progress_and_clock( repo_root: &Path, @@ -217,92 +181,27 @@ where explicit_repository_id: storage_config.repository_id.as_deref(), repository_remote: &storage_config.repository_remote, }; - let storage = tokio::task::block_in_place(|| resolve_agent_trace_storage(&context)) - .map_err(|error| TraceSyncError::Runtime(format!("{error:#}")))?; - let storage_guard = SyncStorageGuard(Some(storage)); - let result = async { - let storage = storage_guard.storage(); - - let auth_config = config::resolve_auth_runtime_config(repo_root) - .map_err(|error| TraceSyncError::Runtime(format!("{error:#}")))?; - let client = AuthenticatedControlPlaneClient::new( - reqwest::Client::new(), - auth_config.control_plane_base_url.value.unwrap_or_default(), - auth::WORKOS_DEFAULT_BASE_URL, - auth_config.workos_client_id.value.unwrap_or_default(), - ); - - run_sync_against_without_progress( - &storage.metadata.repository_id, - &storage.metadata.source_instance_id, - &storage.db, - &client, - progress, - ) - .await - } - .await; - drop(storage_guard); - result -} - -#[cfg(test)] -pub(crate) async fn run_sync_against( - repository_id: &str, - source_instance_id: &str, - db: &RepositoryAgentTraceDb, - client: &AuthenticatedControlPlaneClient, -) -> Result { - let mut progress = NoopProgressReporter; - run_sync_against_with_progress(repository_id, source_instance_id, db, client, &mut progress) + let storage = resolve_agent_trace_storage(&context) .await -} - -#[cfg(test)] -pub(crate) async fn run_sync_against_with_progress( - repository_id: &str, - source_instance_id: &str, - db: &RepositoryAgentTraceDb, - client: &AuthenticatedControlPlaneClient, - progress: &mut S, -) -> Result -where - S: ProgressReporter, -{ - let clock = SystemSyncProgressClock; - run_sync_against_with_progress_and_clock( - repository_id, - source_instance_id, - db, - client, + .map_err(|error| TraceSyncError::Runtime(format!("{error:#}")))?; + let auth_config = config::resolve_auth_runtime_config(repo_root) + .map_err(|error| TraceSyncError::Runtime(format!("{error:#}")))?; + let client = AuthenticatedControlPlaneClient::new( + reqwest::Client::new(), + auth_config.control_plane_base_url.value.unwrap_or_default(), + auth::WORKOS_DEFAULT_BASE_URL, + auth_config.workos_client_id.value.unwrap_or_default(), + ); + + let result = run_sync_against_without_progress( + &storage.metadata.repository_id, + &storage.metadata.source_instance_id, + &storage.db, + &client, progress, - &clock, ) - .await -} - -#[cfg(test)] -pub(crate) async fn run_sync_against_with_progress_and_clock( - repository_id: &str, - source_instance_id: &str, - db: &RepositoryAgentTraceDb, - client: &AuthenticatedControlPlaneClient, - progress: &mut S, - clock: &C, -) -> Result -where - S: ProgressReporter, - C: SyncProgressClock, -{ - progress.report(SyncProgressEvent::Started { - timestamp: timestamp(clock), - }); - let result = - run_sync_against_without_progress(repository_id, source_instance_id, db, client, progress) - .await; - progress.report(SyncProgressEvent::Finished { - timestamp: timestamp(clock), - }); + .await; + client.wait_for_pending_refresh().await; result } @@ -321,15 +220,16 @@ where run_sync_async(repository_id, source_instance_id, &reader, client, progress).await } -async fn run_sync_async<'a, S>( +async fn run_sync_async<'a, S, K>( repository_id: &'a str, source_instance_id: &'a str, reader: &'a AgentTraceExportReader<'a>, - client: &'a AuthenticatedControlPlaneClient, + client: &'a AuthenticatedControlPlaneClient, progress: &'a mut S, ) -> Result where S: ProgressReporter + 'a, + K: CredentialStore, { let state_request = AgentTraceIngestionStateRequest { repository_id: repository_id.to_string(), @@ -340,6 +240,7 @@ where .await .map_err(TraceSyncError::ControlPlane)?; let progress = Rc::new(RefCell::new(progress)); + let terminated = Cell::new(false); let diff_traces = StreamSyncReport { uploaded: 0, @@ -348,7 +249,7 @@ where batches: 0, }; - let (messages, parts, agent_traces) = try_join_three( + let (messages, parts, agent_traces) = join_three_to_completion( sync_one_stream( client, repository_id, @@ -356,9 +257,10 @@ where IngestionStream::Messages, state.cursors.messages, "messages", - |cursor, limit| reader.read_messages_after(cursor, limit), - |request| Box::pin(async move { client.ingest_messages(&request).await }), + async |cursor, limit| reader.read_messages_after(cursor, limit).await, + async |request| client.ingest_messages(&request).await, Rc::clone(&progress), + &terminated, ), sync_one_stream( client, @@ -367,9 +269,10 @@ where IngestionStream::Parts, state.cursors.parts, "parts", - |cursor, limit| reader.read_parts_after(cursor, limit), - |request| Box::pin(async move { client.ingest_parts(&request).await }), + async |cursor, limit| reader.read_parts_after(cursor, limit).await, + async |request| client.ingest_parts(&request).await, Rc::clone(&progress), + &terminated, ), sync_one_stream( client, @@ -378,9 +281,10 @@ where IngestionStream::AgentTraces, state.cursors.agent_traces, "agent_traces", - |cursor, limit| reader.read_agent_traces_after(cursor, limit), - |request| Box::pin(async move { client.ingest_agent_traces(&request).await }), + async |cursor, limit| reader.read_agent_traces_after(cursor, limit).await, + async |request| client.ingest_agent_traces(&request).await, Rc::clone(&progress), + &terminated, ), ) .await?; @@ -397,7 +301,11 @@ where }) } -async fn try_join_three(a: A, b: B, c: C) -> Result<(OA, OB, OC), E> +async fn join_three_to_completion( + a: A, + b: B, + c: C, +) -> Result<(OA, OB, OC), E> where A: Future>, B: Future>, @@ -406,48 +314,77 @@ where let mut a = Box::pin(a); let mut b = Box::pin(b); let mut c = Box::pin(c); - let mut a_output = None; - let mut b_output = None; - let mut c_output = None; + let mut a_state = JoinSlot::Running; + let mut b_state = JoinSlot::Running; + let mut c_state = JoinSlot::Running; + let mut first_error = None; poll_fn(|context| { - if a_output.is_none() { - if let Poll::Ready(result) = a.as_mut().poll(context) { - a_output = Some(result?); - } + a_state.poll(a.as_mut(), context, &mut first_error); + b_state.poll(b.as_mut(), context, &mut first_error); + c_state.poll(c.as_mut(), context, &mut first_error); + + if !(a_state.is_done() && b_state.is_done() && c_state.is_done()) { + return Poll::Pending; } - if b_output.is_none() { - if let Poll::Ready(result) = b.as_mut().poll(context) { - b_output = Some(result?); - } + if let Some(error) = first_error.take() { + return Poll::Ready(Err(error)); + } + Poll::Ready(Ok(( + a_state.take_output(), + b_state.take_output(), + c_state.take_output(), + ))) + }) + .await +} + +enum JoinSlot { + Running, + Succeeded(O), + Failed, + Taken, +} + +impl JoinSlot { + fn poll( + &mut self, + future: Pin<&mut F>, + context: &mut Context<'_>, + first_error: &mut Option, + ) where + F: Future>, + { + if !matches!(self, Self::Running) { + return; } - if c_output.is_none() { - if let Poll::Ready(result) = c.as_mut().poll(context) { - c_output = Some(result?); + match future.poll(context) { + Poll::Pending => {} + Poll::Ready(Ok(output)) => *self = Self::Succeeded(output), + Poll::Ready(Err(error)) => { + if first_error.is_none() { + *first_error = Some(error); + } + *self = Self::Failed; } } + } - match (a_output.take(), b_output.take(), c_output.take()) { - (Some(a), Some(b), Some(c)) => Poll::Ready(Ok((a, b, c))), - (a, b, c) => { - a_output = a; - b_output = b; - c_output = c; - Poll::Pending - } + fn is_done(&self) -> bool { + !matches!(self, Self::Running) + } + + fn take_output(&mut self) -> O { + match std::mem::replace(self, Self::Taken) { + Self::Succeeded(output) => output, + _ => unreachable!("join output is taken only after every future succeeded"), } - }) - .await + } } -/// Synchronizes one stream via the T04 engine. Genuine `409`/`5xx`/transport -/// ambiguity, including an undecodable successful batch body, reconciles -/// through a real `/state` refetch. A terminal control-plane failure -/// (missing/invalid auth, `400`, `403`, or a protocol mismatch such as `404`) -/// stops the stream immediately without issuing another `/state` request. -#[allow(clippy::too_many_arguments)] -async fn sync_one_stream<'a, T, ReadFn, IngestFn, S>( - client: &'a AuthenticatedControlPlaneClient, +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn sync_one_stream<'a, T, ReadFn, IngestFn, S, K>( + client: &'a AuthenticatedControlPlaneClient, repository_id: &'a str, source_instance_id: &'a str, stream: IngestionStream, @@ -456,15 +393,16 @@ async fn sync_one_stream<'a, T, ReadFn, IngestFn, S>( mut read_after: ReadFn, mut ingest: IngestFn, progress: Rc>, + terminated: &'a Cell, ) -> Result where T: AgentTraceExportRow + Clone + 'a, S: ProgressReporter + 'a, - ReadFn: FnMut(i64, usize) -> anyhow::Result> + 'a, - IngestFn: FnMut( + K: CredentialStore, + ReadFn: std::ops::AsyncFnMut(i64, usize) -> anyhow::Result> + 'a, + IngestFn: std::ops::AsyncFnMut( AgentTraceIngestionBatchRequest, - ) - -> SyncFuture<'a, Result> + ) -> Result + 'a, { let uploaded = Rc::new(RefCell::new(0usize)); @@ -472,12 +410,17 @@ where let outcome = sync_stream( initial_cursor, AGENT_TRACE_EXPORT_BATCH_SIZE, - |cursor, limit| { - let result = read_after(cursor, limit) - .map_err(|error| StreamSyncError::Read(format!("{error:#}"))); - Box::pin(std::future::ready(result)) + async |cursor, limit| { + if terminated.get() { + return Ok(Vec::new()); + } + let rows = read_after(cursor, limit).await; + if terminated.get() { + return Ok(Vec::new()); + } + rows.map_err(|error| StreamSyncError::Read(format!("{error:#}"))) }, - |cursor, rows: &[T]| { + async |cursor, rows: &[T]| { let uploaded = Rc::clone(&uploaded); let row_count = rows.len(); let last_row_id = rows @@ -493,7 +436,7 @@ where }; let ingest_future = ingest(request); let progress = Rc::clone(&progress); - Box::pin(async move { + async move { match ingest_future.await { Ok(response) => { if response.accepted == row_count && response.cursor == last_row_id { @@ -518,36 +461,43 @@ where } Err(_) => BatchAttemptOutcome::Ambiguous, } - }) + } + .await }, - || { + async || { let state_request = AgentTraceIngestionStateRequest { repository_id: repository_id.to_string(), source_instance_id: source_instance_id.to_string(), }; - Box::pin(async move { + async move { let response = client .ingestion_state(&state_request) .await .map_err(StreamSyncError::Refresh)?; Ok(cursor_for_stream(&response.cursors, stream)) - }) + } + .await }, ) .await - .map_err(|source| TraceSyncError::Stream { - stream: stream_label, - source, + .map_err(|source| { + terminated.set(true); + TraceSyncError::Stream { + stream: stream_label, + source, + } })?; - progress - .borrow_mut() - .report(SyncProgressEvent::StreamCompleted { - stream: stream_label, - uploaded: outcome.uploaded, - cursor: outcome.final_cursor, - batches: outcome.batches, - }); + if !terminated.get() { + progress + .borrow_mut() + .report(SyncProgressEvent::StreamCompleted { + stream: stream_label, + uploaded: outcome.uploaded, + cursor: outcome.final_cursor, + batches: outcome.batches, + }); + } Ok(StreamSyncReport { uploaded: outcome.uploaded, @@ -586,1024 +536,947 @@ fn cursor_for_stream(cursors: &AgentTraceCursors, stream: IngestionStream) -> i6 #[cfg(test)] mod tests { - use std::fs; - use std::path::PathBuf; + use std::cell::{Cell, RefCell}; + use std::collections::{HashMap, VecDeque}; + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::rc::Rc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread; use std::time::{Duration, SystemTime, UNIX_EPOCH}; - use chrono::{DateTime, Utc}; - use serde_json::json; + use serde_json::{json, Value}; + use tokio::sync::{oneshot, Notify}; - use super::*; - use crate::services::agent_trace_db::{ - AgentTraceInsert, DiffTraceInsert, InsertMessageInsert, InsertPartInsert, MessageRole, - PartType, PAYLOAD_TYPE_PATCH, - }; - use crate::services::agent_trace_export::JS_MAX_SAFE_INTEGER; - use crate::services::agent_trace_sync::control_plane::CredentialStore; - use crate::services::agent_trace_sync::test_http_server::{ - CannedResponse, CapturedRequest, ConcurrentBatchTestServer, TestHttpServer, + use super::{join_three_to_completion, sync_one_stream, StreamSyncReport, TraceSyncError}; + use crate::services::agent_trace_db::MessageRole; + use crate::services::agent_trace_export::AgentTraceMessageExportRow; + use crate::services::agent_trace_sync::control_plane::{ + AgentTraceIngestionBatchRequest, AgentTraceIngestionBatchResponse, + AuthenticatedControlPlaneClient, ControlPlaneError, CredentialStore, IngestionStream, }; + use crate::services::agent_trace_sync::StreamSyncError; use crate::services::auth::TokenResponse; + use crate::services::sync::progress::NoopProgressReporter; use crate::services::token_storage::StoredTokens; - struct TestDb(Option); - - fn storage_guard_fixture(sandbox: &Path) -> SyncStorageGuard { - let repo = sandbox.join("repo"); - fs::create_dir_all(&repo).unwrap(); - let output = std::process::Command::new("git") - .args(["init", "-q"]) - .current_dir(&repo) - .output() - .unwrap(); - assert!(output.status.success()); - let context = AgentTraceStorageContext { - repository_root: &repo, - explicit_repository_id: Some("sync-cleanup-boundary"), - repository_remote: "origin", + const NOT_COMPLETED_PROBE: Duration = Duration::from_millis(50); + const BACKLOG_ROWS: i64 = 5; + const TEST_BATCH_TIMEOUT: Duration = Duration::from_millis(500); + const TEST_DEADLINE: Duration = Duration::from_secs(10); + const STALL_SAFETY_BOUND: Duration = Duration::from_secs(10); + + #[tokio::test] + async fn sibling_failure_does_not_cancel_in_flight_credential_save() { + let (save_started_tx, save_started_rx) = oneshot::channel::<()>(); + let (release_tx, release_rx) = oneshot::channel::<()>(); + let failing_finished = Rc::new(Cell::new(false)); + let save_completed = Rc::new(Cell::new(false)); + let succeeding_finished = Rc::new(Cell::new(false)); + + let failing = { + let failing_finished = Rc::clone(&failing_finished); + async move { + save_started_rx.await.expect("save start signal"); + failing_finished.set(true); + Err::<(), _>("parts stream failed") + } + }; + let saving = { + let save_completed = Rc::clone(&save_completed); + async move { + save_started_tx.send(()).expect("signal save start"); + release_rx.await.expect("save release signal"); + save_completed.set(true); + Ok("messages report") + } + }; + let succeeding = { + let succeeding_finished = Rc::clone(&succeeding_finished); + async move { + succeeding_finished.set(true); + Ok("agent_traces report") + } }; - let storage = tokio::task::block_in_place(|| { - crate::services::agent_trace_storage::resolve_agent_trace_storage_at_state_root( - &context, - &sandbox.join("state"), - ) - .unwrap() - }); - SyncStorageGuard(Some(storage)) - } - #[tokio::test(flavor = "multi_thread")] - async fn storage_guard_cleans_up_on_error_before_returning() { - let sandbox = tempfile::tempdir().unwrap(); - let mut db_path = PathBuf::new(); - let result: Result<(), TraceSyncError> = async { - let guard = storage_guard_fixture(sandbox.path()); - db_path = guard.storage().db_path.clone(); - seed_one_row_per_stream(&guard.storage().db); - tokio::task::yield_now().await; - Err(TraceSyncError::Runtime( - "fixture failure after storage opened".into(), - ))? - } - .await; + let mut join = std::pin::pin!(join_three_to_completion(failing, saving, succeeding)); + + let probe = tokio::time::timeout(NOT_COMPLETED_PROBE, join.as_mut()).await; assert!( - matches!(result, Err(TraceSyncError::Runtime(reason)) if reason == "fixture failure after storage opened") + probe.is_err(), + "join completed before the in-flight save finished" ); - tokio::task::block_in_place(|| { - let db = RepositoryAgentTraceDb::open_without_migrations_at(&db_path).unwrap(); - assert_eq!(diff_trace_rows(&db).len(), 1); - }); - } + assert!(failing_finished.get()); + assert!(!save_completed.get()); - #[tokio::test(flavor = "multi_thread")] - async fn storage_guard_cleans_up_when_polled_future_is_abandoned() { - let sandbox = tempfile::tempdir().unwrap(); - let db_path = RefCell::new(None); - let mut future = Box::pin(async { - let guard = storage_guard_fixture(sandbox.path()); - seed_one_row_per_stream(&guard.storage().db); - *db_path.borrow_mut() = Some(guard.storage().db_path.clone()); - std::future::pending::<()>().await; - drop(guard); - }); - poll_fn(|cx| { - assert!(future.as_mut().poll(cx).is_pending()); - Poll::Ready(()) - }) - .await; - assert!(db_path.borrow().is_some()); - drop(future); - tokio::task::block_in_place(|| { - let db = RepositoryAgentTraceDb::open_without_migrations_at( - db_path.borrow().as_ref().unwrap(), - ) - .unwrap(); - assert_eq!(diff_trace_rows(&db).len(), 1); - }); + release_tx.send(()).expect("release save"); + let result = join.await; + + assert_eq!(result, Err("parts stream failed")); + assert!(save_completed.get()); + assert!(succeeding_finished.get()); } - impl TestDb { - fn new(path: &Path) -> Self { - Self(Some(tokio::task::block_in_place(|| { - RepositoryAgentTraceDb::new_at(path).expect("test DB should open") - }))) + fn message_row(source_row_id: i64) -> AgentTraceMessageExportRow { + AgentTraceMessageExportRow { + source_row_id, + session_id: "session".to_string(), + message_id: format!("message-{source_row_id}"), + role: MessageRole::User, + generated_at_unix_ms: source_row_id, } } - impl std::ops::Deref for TestDb { - type Target = RepositoryAgentTraceDb; + #[tokio::test] + async fn terminal_stream_failure_discards_sibling_rows_read_after_termination() { + let client = AuthenticatedControlPlaneClient::new( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + "http://127.0.0.1:1", + "http://127.0.0.1:1", + "client", + ); + let mut reporter = NoopProgressReporter; + let progress = Rc::new(RefCell::new(&mut reporter)); + let terminated = Cell::new(false); + let sibling_read_started = Cell::new(false); + let sibling_ingests = Cell::new(0usize); - fn deref(&self) -> &Self::Target { - self.0 - .as_ref() - .expect("test DB remains owned until cleanup") - } + let failing = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::Messages, + 0, + "messages", + async |cursor, _limit| Ok(vec![message_row(cursor + 1)]), + async |_request| { + while !sibling_read_started.get() { + tokio::task::yield_now().await; + } + Err(ControlPlaneError::Forbidden("denied".to_string())) + }, + Rc::clone(&progress), + &terminated, + ); + let sibling = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::Parts, + 0, + "parts", + async |cursor, _limit| { + sibling_read_started.set(true); + while !terminated.get() { + tokio::task::yield_now().await; + } + Ok(backlog_after(cursor, BACKLOG_ROWS)) + }, + async |request| { + sibling_ingests.set(sibling_ingests.get() + 1); + Ok(AgentTraceIngestionBatchResponse { + accepted: request.rows.len(), + cursor: request.rows[0].source_row_id, + }) + }, + Rc::clone(&progress), + &terminated, + ); + let idle = async { Ok::<_, TraceSyncError>(()) }; + + let result = tokio::time::timeout( + TEST_DEADLINE, + join_three_to_completion(failing, sibling, idle), + ) + .await + .expect("sync finishes within the test deadline"); + + assert!(matches!( + result, + Err(TraceSyncError::Stream { + stream: "messages", + source: StreamSyncError::Terminal(ControlPlaneError::Forbidden(_)), + }) + )); + assert!(terminated.get()); + assert_eq!(sibling_ingests.get(), 0); } - impl Drop for TestDb { - fn drop(&mut self) { - tokio::task::block_in_place(|| drop(self.0.take())); - } + fn unreachable_client() -> AuthenticatedControlPlaneClient { + AuthenticatedControlPlaneClient::new( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + "http://127.0.0.1:1", + "http://127.0.0.1:1", + "client", + ) + } + + #[tokio::test] + async fn first_terminal_error_wins_when_sibling_also_fails_and_third_stream_stops_backlog() { + let client = unreachable_client(); + let mut reporter = NoopProgressReporter; + let progress = Rc::new(RefCell::new(&mut reporter)); + let terminated = Cell::new(false); + let in_flight = Cell::new(0usize); + let second_reads = Cell::new(0usize); + let second_ingests = Cell::new(0usize); + let third_reads = Cell::new(0usize); + let third_ingests = Cell::new(0usize); + + let failing = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::Messages, + 0, + "messages", + async |cursor, _limit| Ok(vec![message_row(cursor + 1)]), + async |_request| { + while in_flight.get() < 2 { + tokio::task::yield_now().await; + } + Err(ControlPlaneError::Forbidden("denied".to_string())) + }, + Rc::clone(&progress), + &terminated, + ); + let second_failing = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::Parts, + 0, + "parts", + async |cursor, _limit| { + second_reads.set(second_reads.get() + 1); + Ok((cursor < BACKLOG_ROWS) + .then(|| message_row(cursor + 1)) + .into_iter() + .collect()) + }, + async |_request| { + second_ingests.set(second_ingests.get() + 1); + in_flight.set(in_flight.get() + 1); + while !terminated.get() { + tokio::task::yield_now().await; + } + Err(ControlPlaneError::AuthenticationFailed( + "expired".to_string(), + )) + }, + Rc::clone(&progress), + &terminated, + ); + let backlogged = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::AgentTraces, + 0, + "agent_traces", + async |cursor, _limit| { + third_reads.set(third_reads.get() + 1); + Ok((cursor < BACKLOG_ROWS) + .then(|| message_row(cursor + 1)) + .into_iter() + .collect()) + }, + async |request| { + third_ingests.set(third_ingests.get() + 1); + in_flight.set(in_flight.get() + 1); + while !terminated.get() { + tokio::task::yield_now().await; + } + Ok(AgentTraceIngestionBatchResponse { + accepted: request.rows.len(), + cursor: request.rows[0].source_row_id, + }) + }, + Rc::clone(&progress), + &terminated, + ); + + let result = join_three_to_completion(failing, second_failing, backlogged).await; + + assert!(matches!( + result, + Err(TraceSyncError::Stream { + stream: "messages", + source: StreamSyncError::Terminal(ControlPlaneError::Forbidden(_)), + }) + )); + assert_eq!(second_reads.get(), 1); + assert_eq!(second_ingests.get(), 1); + assert_eq!(third_reads.get(), 1); + assert_eq!(third_ingests.get(), 1); } - fn unique_test_db_path(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system time should be after Unix epoch") - .as_nanos(); - std::env::temp_dir() - .join(format!( - "sce-trace-sync-{label}-{}-{nonce}", - std::process::id() - )) - .join("agent-trace.db") + struct CountingEmptyCredentialStore { + loads: Arc, } - fn remove_test_db(db_path: &std::path::Path) { - if let Some(parent) = db_path.parent() { - let _ = fs::remove_dir_all(parent); + impl CredentialStore for CountingEmptyCredentialStore { + async fn load(&self) -> Result, ControlPlaneError> { + self.loads.fetch_add(1, Ordering::SeqCst); + Ok(None) + } + + async fn save(&self, _token: &TokenResponse) -> Result { + unreachable!("no refresh token is ever loaded") } } - /// Always reports a still-valid token; the sync tests exercise streaming - /// behavior, not auth refresh (covered by `control_plane`'s own tests). - struct AlwaysValidCredentialStore; + #[tokio::test] + async fn ambiguous_sibling_batch_after_termination_keeps_first_error_after_one_state_attempt() { + let loads = Arc::new(AtomicUsize::new(0)); + let client = AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + "http://127.0.0.1:1", + "http://127.0.0.1:1", + "client", + CountingEmptyCredentialStore { + loads: Arc::clone(&loads), + }, + ); + let mut reporter = NoopProgressReporter; + let progress = Rc::new(RefCell::new(&mut reporter)); + let terminated = Cell::new(false); + let sibling_ingest_started = Cell::new(false); + let sibling_reads = Cell::new(0usize); + let sibling_ingests = Cell::new(0usize); + + let failing = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::Messages, + 0, + "messages", + async |cursor, _limit| Ok(vec![message_row(cursor + 1)]), + async |_request| { + while !sibling_ingest_started.get() { + tokio::task::yield_now().await; + } + Err(ControlPlaneError::Forbidden("denied".to_string())) + }, + Rc::clone(&progress), + &terminated, + ); + let sibling = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::Parts, + 0, + "parts", + async |cursor, _limit| { + sibling_reads.set(sibling_reads.get() + 1); + Ok((cursor < BACKLOG_ROWS) + .then(|| message_row(cursor + 1)) + .into_iter() + .collect()) + }, + async |_request| { + sibling_ingests.set(sibling_ingests.get() + 1); + sibling_ingest_started.set(true); + while !terminated.get() { + tokio::task::yield_now().await; + } + Err::(ControlPlaneError::ServerError( + "unavailable".to_string(), + )) + }, + Rc::clone(&progress), + &terminated, + ); + let idle = async { Ok::<_, TraceSyncError>(()) }; + + let result = join_three_to_completion(failing, sibling, idle).await; - impl CredentialStore for AlwaysValidCredentialStore { - fn load(&self) -> Result, ControlPlaneError> { + assert!(matches!( + result, + Err(TraceSyncError::Stream { + stream: "messages", + source: StreamSyncError::Terminal(ControlPlaneError::Forbidden(_)), + }) + )); + assert_eq!(sibling_reads.get(), 1); + assert_eq!(sibling_ingests.get(), 1); + assert_eq!(loads.load(Ordering::SeqCst), 1); + } + + struct FixedCredentialStore; + + impl CredentialStore for FixedCredentialStore { + async fn load(&self) -> Result, ControlPlaneError> { Ok(Some(StoredTokens { - access_token: "valid-access-token".to_string(), + access_token: "access".to_string(), token_type: "Bearer".to_string(), - expires_in: 3_600, - refresh_token: "refresh-token".to_string(), + expires_in: 3600, + refresh_token: "refresh".to_string(), scope: None, stored_at_unix_seconds: SystemTime::now() .duration_since(UNIX_EPOCH) - .expect("system clock before unix epoch") + .expect("clock after epoch") .as_secs(), })) } - fn save(&self, _token: &TokenResponse) -> Result { - panic!("no token refresh expected in this test"); + async fn save(&self, _token: &TokenResponse) -> Result { + unreachable!("fixed credentials never refresh") } } - fn test_client(server: &TestHttpServer) -> AuthenticatedControlPlaneClient { - test_client_at(&server.base_url) - } - - fn test_client_at(base_url: &str) -> AuthenticatedControlPlaneClient { - let http = reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .build() - .expect("build test reqwest client"); - AuthenticatedControlPlaneClient::with_credential_store( - http, - base_url.to_string(), - base_url.to_string(), - "test-client-id", - Box::new(AlwaysValidCredentialStore), - ) - } - - fn seed_one_row_per_stream(db: &RepositoryAgentTraceDb) { - db.insert_messages(vec![InsertMessageInsert { - session_id: "sess-1".to_string(), - message_id: "msg-1".to_string(), - role: MessageRole::User, - generated_at_unix_ms: 1_700_000_000_000, - }]) - .expect("seed message"); - db.insert_parts(vec![InsertPartInsert { - part_type: PartType::Text, - text: "hello".to_string(), - session_id: "sess-1".to_string(), - message_id: "msg-1".to_string(), - generated_at_unix_ms: 1_700_000_000_000, - }]) - .expect("seed part"); - db.insert_diff_trace(DiffTraceInsert { - time_ms: 1_700_000_000_000, - session_id: "sess-1", - patch: "Index: a\n", - model_id: None, - tool_name: "opencode", - tool_version: None, - payload_type: PAYLOAD_TYPE_PATCH, - }) - .expect("seed diff_trace"); - db.insert_agent_trace(AgentTraceInsert { - commit_id: "abc123", - commit_time_ms: 1_700_000_000_000, - trace_json: "{\"steps\":[]}", - agent_trace_id: "trace-1", - url: "https://example.com/trace", - remote_url: "", - }) - .expect("seed agent_trace"); - } - - fn seed_messages(db: &RepositoryAgentTraceDb, count: usize) { - db.insert_messages( - (1..=count) - .map(|index| InsertMessageInsert { - session_id: "sess-progress".to_string(), - message_id: format!("msg-progress-{index}"), - role: MessageRole::User, - generated_at_unix_ms: 1_700_000_000_000 - + i64::try_from(index).expect("test message count fits in i64"), - }) - .collect(), - ) - .expect("seed progress messages"); + #[derive(Clone, Copy, Debug)] + enum BatchReply { + Accept, + Stall, + CommitThenStall, + HeadersThenStall, } - #[derive(Clone, Debug, PartialEq, Eq)] - struct DiffTraceRowSnapshot { - id: i64, - time_ms: i64, - session_id: String, - patch: String, - created_at: String, - model_id: Option, - tool_name: Option, - tool_version: Option, - payload_type: String, + #[derive(Default)] + struct ScriptedShared { + script: Mutex>, + committed: Mutex>, + batch_requests: Mutex>, + state_requests: AtomicUsize, + batch_stalled: Notify, } - fn diff_trace_rows(db: &RepositoryAgentTraceDb) -> Vec { - db.query_map( - "SELECT id, time_ms, session_id, patch, created_at, model_id, tool_name, tool_version, payload_type FROM diff_traces ORDER BY id ASC", - (), - |row| { - Ok(DiffTraceRowSnapshot { - id: row.get(0)?, - time_ms: row.get(1)?, - session_id: row.get(2)?, - patch: row.get(3)?, - created_at: row.get(4)?, - model_id: row.get(5)?, - tool_name: row.get(6)?, - tool_version: row.get(7)?, - payload_type: row.get(8)?, - }) - }, - ) - .expect("diff_traces snapshot query should succeed") + struct ScriptedControlPlane { + base_url: String, + shared: Arc, } - fn insert_diff_trace_row(db: &RepositoryAgentTraceDb, id: i64, time_ms: i64) { - db.execute( - "INSERT INTO diff_traces (id, time_ms, session_id, patch, model_id, tool_name, tool_version, payload_type) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - ( - id, - time_ms, - "sess-1", - "Index: a\n", - Option::<&str>::None, - Option::<&str>::None, - Option::<&str>::None, - PAYLOAD_TYPE_PATCH, - ), - ) - .expect("direct diff_trace insert should succeed"); - } + impl ScriptedControlPlane { + fn start(script: &[BatchReply]) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind scripted control plane"); + let base_url = format!("http://{}", listener.local_addr().expect("server addr")); + let shared = Arc::new(ScriptedShared { + script: Mutex::new(script.iter().copied().collect()), + ..ScriptedShared::default() + }); + let thread_shared = Arc::clone(&shared); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { + continue; + }; + let shared = Arc::clone(&thread_shared); + thread::spawn(move || shared.handle(stream)); + } + }); + Self { base_url, shared } + } - fn diff_traces_compatibility_report(server_cursor: i64) -> StreamSyncReport { - StreamSyncReport { - uploaded: 0, - initial_cursor: server_cursor, - final_cursor: server_cursor, - batches: 0, + fn client(&self) -> AuthenticatedControlPlaneClient { + AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + self.base_url.clone(), + "http://127.0.0.1:1", + "client", + FixedCredentialStore, + ) + .with_batch_timeout(TEST_BATCH_TIMEOUT) } - } - fn assert_state_request( - request: &CapturedRequest, - repository_id: &str, - source_instance_id: &str, - ) { - assert_eq!(request.method, "POST"); - assert_eq!(request.path, "/agent-trace/ingestion/state"); - assert_eq!( - serde_json::from_str::(&request.body) - .expect("state request body should be JSON"), - json!({ - "repositoryId": repository_id, - "sourceInstanceId": source_instance_id, - }) - ); - } + fn expected_cursors(&self, stream: &str) -> Vec { + self.shared + .batch_requests + .lock() + .expect("batch requests lock") + .iter() + .filter(|(name, _)| name == stream) + .map(|(_, cursor)| *cursor) + .collect() + } - fn state_request_count(requests: &[CapturedRequest]) -> usize { - requests - .iter() - .filter(|request| request.path == "/agent-trace/ingestion/state") - .count() + fn state_requests(&self) -> usize { + self.shared.state_requests.load(Ordering::SeqCst) + } } - fn sorted_batch_streams(requests: &[CapturedRequest]) -> Vec { - let mut streams = requests - .iter() - .filter(|request| request.path == "/agent-trace/ingestion/batch") - .map(|request| { - serde_json::from_str::(&request.body) - .expect("batch request body should be JSON")["stream"] - .as_str() - .expect("batch request should name its stream") - .to_string() - }) - .collect::>(); - streams.sort(); - streams - } + impl ScriptedShared { + fn handle(&self, mut stream: TcpStream) { + let Some((path, body)) = read_scripted_request(&stream) else { + return; + }; - fn state_response( - messages: i64, - parts: i64, - diff_traces: i64, - agent_traces: i64, - ) -> serde_json::Value { - json!({ - "cursors": { - "messages": messages, - "parts": parts, - "diffTraces": diff_traces, - "agentTraces": agent_traces, + if path.ends_with("/state") { + self.state_requests.fetch_add(1, Ordering::SeqCst); + let payload = { + let committed = self.committed.lock().expect("committed lock"); + let cursor = |name: &str| committed.get(name).copied().unwrap_or(0); + json!({"cursors": { + "messages": cursor("messages"), + "parts": cursor("parts"), + "diffTraces": 0, + "agentTraces": cursor("agent_traces"), + }}) + }; + write_scripted_response(&mut stream, 200, &payload); + return; } - }) - } - fn batch_response(cursor: i64) -> serde_json::Value { - json!({ "accepted": 1, "cursor": cursor }) - } + let stream_name = body["stream"].as_str().unwrap_or_default().to_string(); + let rows = body["rows"].as_array().cloned().unwrap_or_default(); + let last_row_id = rows + .last() + .and_then(|row| row["sourceRowId"].as_i64()) + .unwrap_or(0); + self.batch_requests + .lock() + .expect("batch requests lock") + .push(( + stream_name.clone(), + body["expectedCursor"].as_i64().unwrap_or(i64::MIN), + )); + let reply = self.script.lock().expect("script lock").pop_front(); + + match reply { + Some(BatchReply::Accept) => { + self.commit(&stream_name, last_row_id); + write_scripted_response( + &mut stream, + 200, + &json!({"accepted": rows.len(), "cursor": last_row_id}), + ); + } + Some(BatchReply::Stall) => self.stall(stream), + Some(BatchReply::CommitThenStall) => { + self.commit(&stream_name, last_row_id); + self.stall(stream); + } + Some(BatchReply::HeadersThenStall) => { + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 64\r\n\r\n{", + ); + let _ = stream.flush(); + self.stall(stream); + } + None => write_scripted_response( + &mut stream, + 500, + &json!({"error": "no scripted batch reply"}), + ), + } + } - struct FixedProgressClock { - timestamps: RefCell>>, - } + fn commit(&self, stream: &str, cursor: i64) { + self.committed + .lock() + .expect("committed lock") + .insert(stream.to_string(), cursor); + } - impl SyncProgressClock for FixedProgressClock { - fn now(&self) -> DateTime { - self.timestamps.borrow_mut().remove(0) + fn stall(&self, mut stream: TcpStream) { + self.batch_stalled.notify_one(); + let _ = stream.set_read_timeout(Some(STALL_SAFETY_BOUND)); + let mut buffer = [0u8; 64]; + while matches!(stream.read(&mut buffer), Ok(read) if read > 0) {} } } - fn fixed_progress_clock() -> FixedProgressClock { - FixedProgressClock { - timestamps: RefCell::new(vec![ - DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z") - .expect("valid start timestamp") - .with_timezone(&Utc), - DateTime::parse_from_rfc3339("2026-01-02T03:04:06Z") - .expect("valid end timestamp") - .with_timezone(&Utc), - ]), + fn read_scripted_request(stream: &TcpStream) -> Option<(String, Value)> { + let mut reader = BufReader::new(stream.try_clone().ok()?); + let mut request_line = String::new(); + reader.read_line(&mut request_line).ok()?; + let path = request_line.split_whitespace().nth(1)?.to_string(); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } } + let mut body = vec![0u8; content_length]; + reader.read_exact(&mut body).ok()?; + Some((path, serde_json::from_slice(&body).unwrap_or(Value::Null))) } - #[tokio::test(flavor = "multi_thread")] - async fn progress_events_cover_batches_empty_streams_and_fixed_order() { - let db_path = unique_test_db_path("progress-events"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-progress-events") - .expect("metadata should initialize"); - seed_messages(&db, 201); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - server.queue_response(CannedResponse::json( - 200, - &json!({ - "accepted": 100, - "cursor": 100, - }), - )); - server.queue_response(CannedResponse::json( - 200, - &json!({ - "accepted": 100, - "cursor": 200, - }), - )); - server.queue_response(CannedResponse::json( - 200, - &json!({ - "accepted": 1, - "cursor": 201, - }), - )); - let client = test_client(&server); - let events = RefCell::new(Vec::new()); - let clock = fixed_progress_clock(); - - let report = run_sync_against_with_progress_and_clock( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - &mut |event| events.borrow_mut().push(event), - &clock, - ) - .await - .expect("sync should succeed"); - - assert_eq!(report.streams.messages.uploaded, 201); - assert_eq!(report.streams.messages.batches, 3); - assert_eq!( - events.into_inner(), - vec![ - SyncProgressEvent::Started { - timestamp: "2026-01-02T03:04:05Z".to_string(), - }, - SyncProgressEvent::StreamCompleted { - stream: "parts", - uploaded: 0, - cursor: 0, - batches: 0, - }, - SyncProgressEvent::StreamCompleted { - stream: "agent_traces", - uploaded: 0, - cursor: 0, - batches: 0, - }, - SyncProgressEvent::BatchAccepted { - stream: "messages", - batch_rows: 100, - uploaded: 100, - cursor: 100, - }, - SyncProgressEvent::BatchAccepted { - stream: "messages", - batch_rows: 100, - uploaded: 200, - cursor: 200, - }, - SyncProgressEvent::BatchAccepted { - stream: "messages", - batch_rows: 1, - uploaded: 201, - cursor: 201, - }, - SyncProgressEvent::StreamCompleted { - stream: "messages", - uploaded: 201, - cursor: 201, - batches: 3, - }, - SyncProgressEvent::Finished { - timestamp: "2026-01-02T03:04:06Z".to_string(), - }, - ] + fn write_scripted_response(stream: &mut TcpStream, status: u16, payload: &Value) { + let payload = payload.to_string(); + let reason = if status == 200 { + "OK" + } else { + "Internal Server Error" + }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + payload.len() ); + let _ = stream.write_all(response.as_bytes()); + let _ = stream.flush(); + } - remove_test_db(&db_path); + fn backlog_after(cursor: i64, rows: i64) -> Vec { + (cursor < rows) + .then(|| message_row(cursor + 1)) + .into_iter() + .collect() } - #[tokio::test(flavor = "multi_thread")] - async fn full_sync_uploads_three_remote_streams_and_second_run_is_naturally_incremental() { - let db_path = unique_test_db_path("full-sync"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-full-sync") - .expect("metadata should initialize"); - seed_one_row_per_stream(&db); - let diff_traces_before = diff_trace_rows(&db); - assert_eq!(diff_traces_before.len(), 1); - let unsynced_diff_traces = diff_traces_compatibility_report(0); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - server.queue_response(CannedResponse::json(200, &batch_response(1))); - server.queue_response(CannedResponse::json(200, &batch_response(1))); - server.queue_response(CannedResponse::json(200, &batch_response(1))); - let client = test_client(&server); - - let report = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - ) - .await - .expect("first sync should succeed"); - - let first_run_requests = server.captured_requests(); - assert_eq!(first_run_requests.len(), 4); - assert_state_request( - &first_run_requests[0], - &metadata.repository_id, - &metadata.source_instance_id, - ); - assert_eq!(state_request_count(&first_run_requests), 1); - assert_eq!( - sorted_batch_streams(&first_run_requests), - vec!["agent_traces", "messages", "parts"] + async fn ingest_single_message( + client: &AuthenticatedControlPlaneClient, + ) -> Result { + let request = AgentTraceIngestionBatchRequest { + repository_id: "repo".to_string(), + source_instance_id: "source".to_string(), + stream: IngestionStream::Messages, + expected_cursor: 0, + rows: vec![message_row(1)], + }; + tokio::time::timeout(TEST_DEADLINE, client.ingest_messages(&request)) + .await + .expect("batch request finishes within the test deadline") + } + + async fn sync_messages_backlog( + client: &AuthenticatedControlPlaneClient, + rows: i64, + ) -> Result { + let mut reporter = NoopProgressReporter; + let terminated = Cell::new(false); + let sync = sync_one_stream( + client, + "repo", + "source", + IngestionStream::Messages, + 0, + "messages", + async |cursor, _limit| Ok(backlog_after(cursor, rows)), + async |request| client.ingest_messages(&request).await, + Rc::new(RefCell::new(&mut reporter)), + &terminated, ); - for stream in [ - report.streams.messages, - report.streams.parts, - report.streams.agent_traces, - ] { - assert_eq!(stream.uploaded, 1); - assert_eq!(stream.initial_cursor, 0); - assert_eq!(stream.final_cursor, 1); - assert_eq!(stream.batches, 1); - } - assert_eq!(report.streams.diff_traces, unsynced_diff_traces); - assert_eq!(diff_trace_rows(&db), diff_traces_before); + tokio::time::timeout(TEST_DEADLINE, sync) + .await + .expect("sync finishes within the test deadline") + } - server.queue_response(CannedResponse::json(200, &state_response(1, 1, 0, 1))); + #[tokio::test] + async fn stalled_batch_request_times_out_as_transport_error() { + let server = ScriptedControlPlane::start(&[BatchReply::Stall]); - let second_report = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - ) - .await - .expect("second sync should succeed"); + let error = ingest_single_message(&server.client()) + .await + .expect_err("stalled batch times out"); - let all_requests = server.captured_requests(); - assert_eq!( - all_requests.len(), - 5, - "second run should only re-check /state and re-read no already-synced rows" - ); - assert_eq!(all_requests[4].path, "/agent-trace/ingestion/state"); - assert_eq!( - sorted_batch_streams(&all_requests), - vec!["agent_traces", "messages", "parts"] - ); - assert!(!sorted_batch_streams(&all_requests).contains(&"diff_traces".to_string())); - for stream in [ - second_report.streams.messages, - second_report.streams.parts, - second_report.streams.agent_traces, - ] { - assert_eq!(stream.uploaded, 0); - assert_eq!(stream.initial_cursor, 1); - assert_eq!(stream.final_cursor, 1); - assert_eq!(stream.batches, 0); - } - assert_eq!(second_report.streams.diff_traces, unsynced_diff_traces); - assert_eq!(diff_trace_rows(&db), diff_traces_before); - - let db_file_name = db_path - .file_name() - .expect("db path has a file name") - .to_string_lossy() - .into_owned(); assert!( - !db_path - .parent() - .expect("db has parent dir") - .read_dir() - .expect("read db dir") - .filter_map(Result::ok) - .any(|entry| { - let name = entry.file_name().to_string_lossy().into_owned(); - // Turso may write WAL/SHM sidecar files alongside the DB - // itself; only reject files unrelated to that one DB, which - // would indicate a local sync cursor/DB sneaking in. - !name.starts_with(&db_file_name) - }), - "sync must not create any local cursor file/database/table on disk" + matches!(error, ControlPlaneError::Transport(_)), + "{error:?}" ); - - remove_test_db(&db_path); + assert_eq!(server.expected_cursors("messages"), [0]); } - #[tokio::test(flavor = "multi_thread")] - async fn sync_succeeds_when_local_diff_trace_row_is_rejected_by_export_reader() { - let db_path = unique_test_db_path("rejected-diff-trace"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-rejected-diff-trace") - .expect("metadata should initialize"); - seed_messages(&db, 1); - insert_diff_trace_row(&db, 1, JS_MAX_SAFE_INTEGER + 1); - let diff_traces_before = diff_trace_rows(&db); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - server.queue_response(CannedResponse::json(200, &batch_response(1))); - let client = test_client(&server); - - let report = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - ) - .await - .expect("sync should not read local diff_traces rows"); + #[tokio::test] + async fn stalled_batch_response_body_times_out_as_transport_error() { + let server = ScriptedControlPlane::start(&[BatchReply::HeadersThenStall]); - let requests = server.captured_requests(); - assert_eq!(requests.len(), 2); - assert_eq!(sorted_batch_streams(&requests), vec!["messages"]); - assert_eq!(report.streams.messages.uploaded, 1); - assert_eq!( - report.streams.diff_traces, - diff_traces_compatibility_report(0) - ); - assert_eq!(diff_trace_rows(&db), diff_traces_before); + let error = ingest_single_message(&server.client()) + .await + .expect_err("stalled batch body times out"); - remove_test_db(&db_path); + assert!( + matches!(error, ControlPlaneError::Transport(_)), + "{error:?}" + ); } - #[tokio::test(flavor = "multi_thread")] - async fn diff_traces_report_echoes_server_cursor_without_uploading_newer_local_rows() { - let db_path = unique_test_db_path("diff-traces-compat"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-diff-traces-compat") - .expect("metadata should initialize"); - seed_one_row_per_stream(&db); - insert_diff_trace_row(&db, 124, 1_700_000_000_124); - let diff_traces_before = diff_trace_rows(&db); - assert_eq!(diff_traces_before.len(), 2); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response(1, 1, 123, 1))); - let client = test_client(&server); - - let report = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - ) - .await - .expect("sync should succeed"); + #[tokio::test] + async fn timed_out_batch_committed_by_server_reconciles_without_resending() { + let server = ScriptedControlPlane::start(&[BatchReply::CommitThenStall]); - let requests = server.captured_requests(); - assert_eq!(requests.len(), 1); - assert_eq!(state_request_count(&requests), 1); - assert!(sorted_batch_streams(&requests).is_empty()); - assert_eq!( - report.streams.diff_traces, - diff_traces_compatibility_report(123) - ); - assert_eq!(diff_trace_rows(&db), diff_traces_before); + let report = sync_messages_backlog(&server.client(), 1) + .await + .expect("sync converges from /state"); - remove_test_db(&db_path); + assert_eq!(report.final_cursor, 1); + assert_eq!(report.uploaded, 0); + assert_eq!(report.batches, 0); + assert_eq!(server.expected_cursors("messages"), [0]); + assert_eq!(server.state_requests(), 1); } - #[tokio::test(flavor = "multi_thread")] - async fn concurrent_sync_overlaps_all_three_stream_batches_after_one_state_request() { - let db_path = unique_test_db_path("concurrent-overlap"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-concurrent-overlap") - .expect("metadata should initialize"); - seed_one_row_per_stream(&db); - - let server = ConcurrentBatchTestServer::start(Duration::from_millis(100)); - server.queue_state_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - for stream in ["messages", "parts", "agent_traces"] { - server.queue_batch_response(stream, 0, CannedResponse::json(200, &batch_response(1))); - } - let client = test_client_at(&server.base_url); + #[tokio::test] + async fn timed_out_batch_not_committed_by_server_is_retried() { + let server = ScriptedControlPlane::start(&[BatchReply::Stall, BatchReply::Accept]); - let report = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - ) - .await - .expect("concurrent sync should succeed"); - - assert_eq!(server.state_request_count(), 1); - assert_eq!(server.captured_requests().len(), 4); - assert_eq!(server.max_in_flight(), 3); - for stream in ["messages", "parts", "agent_traces"] { - assert_eq!(server.max_in_flight_for(stream), 1, "stream {stream}"); - assert_eq!(server.expected_cursors_for(stream), vec![0]); - } - assert_eq!(server.max_in_flight_for("diff_traces"), 0); - assert!(server.expected_cursors_for("diff_traces").is_empty()); - for stream in [ - report.streams.messages, - report.streams.parts, - report.streams.agent_traces, - ] { - assert_eq!(stream.uploaded, 1); - assert_eq!(stream.final_cursor, 1); - assert_eq!(stream.batches, 1); - } + let report = sync_messages_backlog(&server.client(), 1) + .await + .expect("sync converges after retry"); - remove_test_db(&db_path); + assert_eq!(report.final_cursor, 1); + assert_eq!(report.uploaded, 1); + assert_eq!(report.batches, 1); + assert_eq!(server.expected_cursors("messages"), [0, 0]); + assert_eq!(server.state_requests(), 1); } - #[tokio::test(flavor = "multi_thread")] - async fn concurrent_sync_keeps_batches_sequential_within_one_stream() { - let db_path = unique_test_db_path("concurrent-ordering"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-concurrent-ordering") - .expect("metadata should initialize"); - seed_messages(&db, 201); - - let server = ConcurrentBatchTestServer::start(Duration::from_millis(50)); - server.queue_state_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - server.queue_batch_response( - "messages", + #[tokio::test] + async fn terminal_failure_is_reported_after_sibling_batch_times_out() { + let server = ScriptedControlPlane::start(&[BatchReply::Stall]); + let client = server.client(); + let mut reporter = NoopProgressReporter; + let progress = Rc::new(RefCell::new(&mut reporter)); + let terminated = Cell::new(false); + let sibling_reads = Cell::new(0usize); + + let failing = sync_one_stream( + &client, + "repo", + "source", + IngestionStream::Messages, 0, - CannedResponse::json( - 200, - &json!({ - "accepted": 100, - "cursor": 100, - }), - ), - ); - server.queue_batch_response( - "messages", - 100, - CannedResponse::json( - 200, - &json!({ - "accepted": 100, - "cursor": 200, - }), - ), - ); - server.queue_batch_response( "messages", - 200, - CannedResponse::json( - 200, - &json!({ - "accepted": 1, - "cursor": 201, - }), - ), + async |cursor, _limit| Ok(vec![message_row(cursor + 1)]), + async |_request| { + server.shared.batch_stalled.notified().await; + Err(ControlPlaneError::Forbidden("denied".to_string())) + }, + Rc::clone(&progress), + &terminated, ); - let client = test_client_at(&server.base_url); - - let report = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, + let stalled = sync_one_stream( &client, - ) - .await - .expect("ordered multi-batch sync should succeed"); - - assert_eq!(server.state_request_count(), 1); - assert_eq!(server.max_in_flight(), 1); - assert_eq!(server.max_in_flight_for("messages"), 1); - assert_eq!(server.expected_cursors_for("messages"), vec![0, 100, 200]); - assert_eq!(report.streams.messages.uploaded, 201); - assert_eq!(report.streams.messages.final_cursor, 201); - assert_eq!(report.streams.messages.batches, 3); - - remove_test_db(&db_path); - } - - #[tokio::test(flavor = "multi_thread")] - async fn invalid_state_cursor_fails_before_any_batch_request() { - let db_path = unique_test_db_path("invalid-cursor"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-invalid-cursor") - .expect("metadata should initialize"); - seed_one_row_per_stream(&db); - - let server = TestHttpServer::start(); - // `agentTraces` is one past JS_MAX_SAFE_INTEGER, outside the wire - // contract's representable cursor range. - server.queue_response(CannedResponse::json( - 200, - &state_response(0, 0, 0, 9_007_199_254_740_992), - )); - let client = test_client(&server); + "repo", + "source", + IngestionStream::Parts, + 0, + "parts", + async |cursor, _limit| { + sibling_reads.set(sibling_reads.get() + 1); + Ok(backlog_after(cursor, BACKLOG_ROWS)) + }, + async |request| client.ingest_messages(&request).await, + Rc::clone(&progress), + &terminated, + ); + let idle = async { Ok::<_, TraceSyncError>(()) }; - let error = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, + let result = tokio::time::timeout( + TEST_DEADLINE, + join_three_to_completion(failing, stalled, idle), ) .await - .expect_err("out-of-range /state cursor should fail sync"); + .expect("sync finishes within the test deadline"); assert!(matches!( - error, - TraceSyncError::ControlPlane(ControlPlaneError::InvalidResponse(_)) + result, + Err(TraceSyncError::Stream { + stream: "messages", + source: StreamSyncError::Terminal(ControlPlaneError::Forbidden(_)), + }) )); - assert_eq!( - server.call_count(), - 1, - "an invalid /state cursor must fail before any /batch request is sent" - ); - - remove_test_db(&db_path); + assert_eq!(sibling_reads.get(), 1); + assert_eq!(server.expected_cursors("parts"), [0]); + assert_eq!(server.state_requests(), 1); } +} - #[tokio::test(flavor = "multi_thread")] - async fn terminal_batch_status_fails_without_state_reconciliation() { - let db_path = unique_test_db_path("terminal-batch"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-terminal-batch") - .expect("metadata should initialize"); - seed_one_row_per_stream(&db); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response(0, 1, 1, 1))); - server.queue_response(CannedResponse::json( - 404, - &json!({"message": "unknown ingestion route"}), - )); - let client = test_client(&server); +#[cfg(test)] +mod contract_tests { + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::TcpListener; + use std::sync::{Arc, Mutex}; + use std::thread; + use std::time::{SystemTime, UNIX_EPOCH}; + + use serde_json::{json, Value}; + + use super::run_sync_async; + use crate::services::agent_trace_db::transaction_tests::{message, open, part}; + use crate::services::agent_trace_db::{AgentTraceInsert, DiffTraceInsert}; + use crate::services::agent_trace_export::AgentTraceExportReader; + use crate::services::agent_trace_sync::control_plane::{ + AuthenticatedControlPlaneClient, ControlPlaneError, CredentialStore, + }; + use crate::services::auth::TokenResponse; + use crate::services::sync::progress::NoopProgressReporter; + use crate::services::token_storage::StoredTokens; - let error = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - ) - .await - .expect_err("a terminal 404 /batch response should fail the sync"); + const DIFF_TRACES_CURSOR: i64 = 7; - assert!( - matches!( - error, - TraceSyncError::Stream { - stream: "messages", - .. - } - ), - "unexpected error: {error:?}" - ); - let requests = server.captured_requests(); - assert_eq!( - requests - .iter() - .filter(|request| request.path == "/agent-trace/ingestion/state") - .count(), - 1, - "terminal /batch statuses must not trigger a /state refetch" - ); - let batch_count = requests - .iter() - .filter(|request| request.path == "/agent-trace/ingestion/batch") - .count(); - assert_eq!( - batch_count, 1, - "only `messages` should send a /batch request, and a terminal status must not resend it; observed {batch_count}" - ); + struct FixedCredentialStore; - remove_test_db(&db_path); - } + impl CredentialStore for FixedCredentialStore { + async fn load(&self) -> Result, ControlPlaneError> { + Ok(Some(StoredTokens { + access_token: "access".to_string(), + token_type: "Bearer".to_string(), + expires_in: 3600, + refresh_token: "refresh".to_string(), + scope: None, + stored_at_unix_seconds: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock after epoch") + .as_secs(), + })) + } - #[tokio::test(flavor = "multi_thread")] - async fn progress_events_end_after_terminal_failure() { - let db_path = unique_test_db_path("progress-failure"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-progress-failure") - .expect("metadata should initialize"); - seed_one_row_per_stream(&db); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - for _ in 0..3 { - server.queue_response(CannedResponse::json( - 404, - &json!({"message": "unknown ingestion route"}), - )); + async fn save(&self, _token: &TokenResponse) -> Result { + unreachable!("fixed credentials never refresh") } - let client = test_client(&server); - let events = RefCell::new(Vec::new()); - let clock = fixed_progress_clock(); - - let error = run_sync_against_with_progress_and_clock( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - &mut |event| events.borrow_mut().push(event), - &clock, - ) - .await - .expect_err("terminal batch failure should be reported"); + } - assert!(matches!(error, TraceSyncError::Stream { .. })); - assert_eq!( - events.into_inner(), - vec![ - SyncProgressEvent::Started { - timestamp: "2026-01-02T03:04:05Z".to_string(), - }, - SyncProgressEvent::Finished { - timestamp: "2026-01-02T03:04:06Z".to_string(), - }, - ] - ); - remove_test_db(&db_path); + type CapturedRequests = Arc>>; + + fn start_control_plane() -> (String, CapturedRequests) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind control plane"); + let base_url = format!("http://{}", listener.local_addr().expect("server addr")); + let requests = Arc::new(Mutex::new(Vec::new())); + let thread_requests = Arc::clone(&requests); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { + continue; + }; + let mut reader = BufReader::new(stream.try_clone().expect("clone stream")); + let mut request_line = String::new(); + let _ = reader.read_line(&mut request_line); + let path = request_line + .split_whitespace() + .nth(1) + .unwrap_or_default() + .to_string(); + let mut content_length = 0usize; + loop { + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + break; + } + let line = line.trim_end_matches(['\r', '\n']); + if line.is_empty() { + break; + } + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } + } + let mut body = vec![0u8; content_length]; + let _ = reader.read_exact(&mut body); + let body: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + + let payload = if path.ends_with("/state") { + json!({"cursors": { + "messages": 0, + "parts": 0, + "diffTraces": DIFF_TRACES_CURSOR, + "agentTraces": 0, + }}) + } else { + let rows = body["rows"].as_array().cloned().unwrap_or_default(); + let cursor = rows + .last() + .and_then(|row| row["sourceRowId"].as_i64()) + .unwrap_or(0); + json!({"accepted": rows.len(), "cursor": cursor}) + }; + thread_requests + .lock() + .expect("requests lock") + .push((path, body)); + let payload = payload.to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + payload.len() + ); + let _ = stream.write_all(response.as_bytes()); + } + }); + (base_url, requests) } - #[tokio::test(flavor = "multi_thread")] - async fn malformed_2xx_batch_response_still_reconciles_via_state() { - let db_path = unique_test_db_path("malformed-batch"); - let db = TestDb::new(&db_path); - let metadata = db - .verify_or_initialize_repository_metadata("repo-malformed-batch") - .expect("metadata should initialize"); - seed_messages(&db, 1); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - // Syntactically successful but undecodable as `AgentTraceIngestionBatchResponse`. - server.queue_response(CannedResponse::json(200, &json!({"unexpected": "shape"}))); - // Only messages has pending data in this test. The malformed successful - // batch response must cause an explicit state reconciliation before the - // same batch is retried. - server.queue_response(CannedResponse::json(200, &state_response(0, 0, 0, 0))); - server.queue_response(CannedResponse::json(200, &batch_response(1))); - let client = test_client(&server); - - let report = run_sync_against( - &metadata.repository_id, - &metadata.source_instance_id, - &db, - &client, - ) + #[tokio::test] + async fn sync_uploads_only_three_streams_and_reports_zero_upload_diff_traces() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut db = open(&dir.path().join("agent-trace.db")).await; + db.insert_conversation_text_event(message("m1"), part("m1")) + .await + .expect("insert message and part"); + db.insert_agent_trace(AgentTraceInsert { + commit_id: "commit", + commit_time_ms: 1, + trace_json: "{}", + agent_trace_id: "trace", + url: "https://example.test/trace", + remote_url: "https://example.test/repo.git", + }) .await - .expect("an undecodable 2xx /batch body should still reconcile via /state and succeed"); - - assert_eq!(server.call_count(), 4); - let requests = server.captured_requests(); - assert_eq!( - requests - .iter() - .map(|request| (request.method.as_str(), request.path.as_str())) - .collect::>(), - vec![ - ("POST", "/agent-trace/ingestion/state"), - ("POST", "/agent-trace/ingestion/batch"), - ("POST", "/agent-trace/ingestion/state"), - ("POST", "/agent-trace/ingestion/batch"), - ], - "an undecodable 2xx /batch body must reconcile via /state before retrying the same batch" + .expect("insert agent trace"); + db.insert_diff_trace(DiffTraceInsert { + time_ms: 1, + session_id: "session", + patch: "patch", + model_id: None, + tool_name: "tool", + tool_version: None, + payload_type: "patch", + }) + .await + .expect("insert diff trace"); + + let (base_url, requests) = start_control_plane(); + let client = AuthenticatedControlPlaneClient::with_credential_store( + reqwest::Client::builder() + .tls_certs_only(Vec::new()) + .build() + .expect("test http client"), + base_url, + "http://127.0.0.1:1", + "client", + FixedCredentialStore, ); - assert_eq!(report.streams.messages.uploaded, 1); - assert_eq!(report.streams.messages.final_cursor, 1); - assert_eq!(report.streams.parts.uploaded, 0); - assert_eq!(report.streams.diff_traces.uploaded, 0); - assert_eq!(report.streams.agent_traces.uploaded, 0); + let reader = AgentTraceExportReader::new(&db); + let mut reporter = NoopProgressReporter; - remove_test_db(&db_path); - } - - #[tokio::test(flavor = "multi_thread")] - async fn forbidden_state_response_fails_without_mutating_local_metadata() { - let db_path = unique_test_db_path("forbidden"); - let db = TestDb::new(&db_path); - let metadata_before = db - .verify_or_initialize_repository_metadata("repo-forbidden") - .expect("metadata should initialize"); - - let server = TestHttpServer::start(); - server.queue_response(CannedResponse::json( - 403, - &json!({"error": "not the owner"}), - )); - let client = test_client(&server); + let report = run_sync_async("repo", "source", &reader, &client, &mut reporter) + .await + .expect("sync converges"); - let error = run_sync_against( - &metadata_before.repository_id, - &metadata_before.source_instance_id, - &db, - &client, - ) - .await - .expect_err("403 should fail the sync"); - - assert!(matches!( - error, - TraceSyncError::ControlPlane(ControlPlaneError::Forbidden(_)) - )); + let requests = requests.lock().expect("requests lock").clone(); + let mut streams: Vec<&str> = requests + .iter() + .filter(|(path, _)| path.ends_with("/batch")) + .map(|(_, body)| body["stream"].as_str().expect("stream field")) + .collect(); + streams.sort_unstable(); + assert_eq!(streams, ["agent_traces", "messages", "parts"]); - let metadata_after = db - .verify_or_initialize_repository_metadata("repo-forbidden") - .expect("metadata should still verify"); - assert_eq!( - metadata_after.source_instance_id, - metadata_before.source_instance_id - ); + assert_eq!(report.streams.messages.uploaded, 1); + assert_eq!(report.streams.parts.uploaded, 1); + assert_eq!(report.streams.agent_traces.uploaded, 1); + assert_eq!(report.streams.diff_traces.uploaded, 0); + assert_eq!(report.streams.diff_traces.batches, 0); assert_eq!( - server.call_count(), - 1, - "no reconciliation call should follow a terminal /state failure" + report.streams.diff_traces.initial_cursor, + DIFF_TRACES_CURSOR ); - - remove_test_db(&db_path); + assert_eq!(report.streams.diff_traces.final_cursor, DIFF_TRACES_CURSOR); } } diff --git a/cli/src/services/token_storage.rs b/cli/src/services/token_storage.rs index ea1dd5086..8236f868f 100644 --- a/cli/src/services/token_storage.rs +++ b/cli/src/services/token_storage.rs @@ -1,6 +1,6 @@ use std::fmt; -use std::sync::OnceLock; use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::OnceCell; use serde::{Deserialize, Serialize}; @@ -13,10 +13,12 @@ const DEFAULT_TOKEN_ROW_ID: i64 = 1; /// Lazy singleton for the encrypted auth database. /// /// Stores `Result` so initialization failures are preserved across calls. -static AUTH_DB: OnceLock> = OnceLock::new(); +static AUTH_DB: OnceCell> = OnceCell::const_new(); -fn get_auth_db() -> Result<&'static AuthDb, TokenStorageError> { - let result = AUTH_DB.get_or_init(|| AuthDb::new().map_err(|e| e.to_string())); +async fn get_auth_db() -> Result<&'static AuthDb, TokenStorageError> { + let result = AUTH_DB + .get_or_init(async || AuthDb::new().await.map_err(|e| e.to_string())) + .await; match result { Ok(db) => Ok(db), Err(msg) => Err(TokenStorageError::Database(msg.clone())), @@ -70,8 +72,8 @@ impl fmt::Display for TokenStorageError { impl std::error::Error for TokenStorageError {} -pub fn save_tokens(token: &TokenResponse) -> Result { - let db = get_auth_db()?; +pub async fn save_tokens(token: &TokenResponse) -> Result { + let db = get_auth_db().await?; let stored = StoredTokens::from_token_response(token)?; let expires_in = i64::try_from(stored.expires_in).map_err(|error| { @@ -101,13 +103,14 @@ pub fn save_tokens(token: &TokenResponse) -> Result Result, TokenStorageError> { - let db = get_auth_db()?; +pub async fn load_tokens() -> Result, TokenStorageError> { + let db = get_auth_db().await?; let sql = "SELECT access_token, token_type, expires_in, refresh_token, scope, \ stored_at_unix_seconds FROM auth_credentials WHERE id = ?1"; @@ -140,19 +143,21 @@ pub fn load_tokens() -> Result, TokenStorageError> { stored_at_unix_seconds, }) }) + .await .map_err(|e| TokenStorageError::Database(e.to_string()))?; Ok(rows.into_iter().next()) } -pub fn delete_tokens() -> Result { - let db = get_auth_db()?; +pub async fn delete_tokens() -> Result { + let db = get_auth_db().await?; let affected = db .execute( "DELETE FROM auth_credentials WHERE id = ?1", (DEFAULT_TOKEN_ROW_ID,), ) + .await .map_err(|e| TokenStorageError::Database(e.to_string()))?; Ok(affected > 0) diff --git a/context/architecture.md b/context/architecture.md index cf1a7c13d..5f5b3854f 100644 --- a/context/architecture.md +++ b/context/architecture.md @@ -132,12 +132,12 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `cli/src/services/config/mod.rs` is the config service facade and `sce config` orchestration surface (`show`, `validate`, `--help`), with bare `sce config` routed by `cli/src/app.rs` to the same help payload as `sce config --help`. Focused submodules own the implementation slices: `types.rs` owns shared config/runtime primitives, `schema.rs` owns generated schema embedding plus typed file parsing, `policy.rs` owns bash-policy semantic validation plus policy-specific formatting and runtime preset-catalog access for the Rust evaluator, `resolver.rs` owns deterministic config-file discovery, file-layer merging, explicit value precedence (`flags > env > config file > defaults` where flag-backed), shared auth-key resolution, observability-runtime resolution, attribution-hooks runtime gate and `agent_trace.auto_sync` resolution, database-retry config resolution and `DATABASE_RETRY_CONFIG` `OnceLock` initialization, default-discovered invalid-file degradation, strict invalid-discovered-layer errors for Agent Trace storage, and explicit-path fatal errors for `--config` / `SCE_CONFIG_FILE`, and private `render.rs` owns `sce config show` / `sce config validate` text and JSON output construction plus rendering-specific display-value helpers. The facade preserves existing `services::config` imports for startup/auth/hooks callers while delegating command execution to resolution plus rendering submodules. - `cli/src/services/output_format.rs` defines the canonical shared CLI output-format contract (`OutputFormat`) for supporting commands, with deterministic `text|json` parsing and command-scoped actionable invalid-value guidance. - `cli/src/services/config/types.rs` is the canonical owner for the shared runtime/config primitive seam used by the CLI: `LogLevel`, `LogFormat`, `SCE_LOG_LEVEL`, `SCE_LOG_FORMAT`, `SCE_LOG_DIR`, `DEFAULT_LOG_FILE_RETENTION_LIMIT`, and the shared bool parsing helpers used by both config resolution and observability bootstrap; `cli/src/services/config/mod.rs` re-exports those primitives through the facade. -- `cli/src/services/capabilities.rs` defines the current broad CLI capability traits consumed by the borrowed, compile-time-typed `AppContext`: `FsOps` with `StdFsOps` for filesystem operations and `GitOps` with `ProcessGitOps` for git command execution plus repository-root/hooks-directory resolution. Existing service internals do not consume these traits directly yet; command execution uses narrow accessors and repo-root-scoped context derivation. +- `cli/src/services/capabilities.rs` defines the current broad CLI capability traits consumed by the borrowed, compile-time-typed `AppContext`: `FsOps` with `StdFsOps` as a method-less filesystem capability marker and `GitOps` with `ProcessGitOps` for git command execution and availability checks. Existing service internals do not consume these traits directly yet; command execution uses narrow accessors and repo-root-scoped context derivation. - `cli/src/services/lifecycle.rs` defines the current compile-safe lifecycle seam. `ServiceLifecycle` has default no-op generic `diagnose`, `fix`, and `setup` methods over `C: HasRepoRoot`, with lifecycle-owned health, fix, and setup result types so the trait contract is not publicly anchored to doctor/setup module types or the full `AppContext` shape. The same module owns the static `LifecycleProvider` enum and shared `lifecycle_providers(include_hooks)` catalog/factory, returning providers in deterministic order (config → local_db → auth_db → agent_trace_db → hooks when requested); enum dispatch calls each concrete provider through generic context methods without boxed lifecycle-provider allocation or repo-root trait-object context erasure. Hooks exposes a `HooksLifecycle` provider in `cli/src/services/hooks/lifecycle.rs` for hook rollout diagnosis/fix/setup using lifecycle-owned health records plus the canonical required-hook installer. Config exposes a `ConfigLifecycle` provider in `cli/src/services/config/lifecycle.rs` for global/repo-local config validation and repo-local `.sce/config.json` bootstrap. local_db exposes a `LocalDbLifecycle` provider in `cli/src/services/local_db/lifecycle.rs` for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup. auth_db exposes an `AuthDbLifecycle` provider in `cli/src/services/auth_db/lifecycle.rs` for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup. agent_trace_db exposes an `AgentTraceDbLifecycle` provider in `cli/src/services/agent_trace_db/lifecycle.rs` for setup-time repository-scoped Agent Trace storage initialization when a repo root is available and repository Agent Trace DB path health/fix from resolved repository identity, returning an actionable "requires a Git repository" diagnostic outside repository context (no global/checkout fallback path; the former fallback was removed by the `retire-legacy-agent-trace-db` plan). Doctor runtime aggregates the full provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor report/fix records at the orchestration boundary; setup command aggregates the shared catalog for `setup` with hooks included only when requested and adapts hook setup outcomes before rendering setup-owned messages. - Agent Trace lifecycle setup resolves `agent_trace.repository_id` / `agent_trace.repository_remote`, creates/reuses checkout identity for diagnostics, and creates or migrates the repository-scoped DB through `agent_trace_storage::resolve_agent_trace_storage(...)`; hook runtime uses the same storage identity/path resolution and no-migration open path, with missing or stale schema failing open through the existing `Run 'sce setup'.` guidance. - `cli/src/services/auth_command/mod.rs` defines the implemented auth command surface for `sce auth login|logout|whoami`, including device-flow login, stored-credential validation/renewal through login with device-flow fallback, logout, and Control Plane `/me`-backed whoami rendering in text/JSON formats; text mode uses flat `Email`, `First Name`, `Last Name`, `Role`, `Permissions`, and `Organization Name` labels, with optional names and missing role/permissions/workspace values handled deterministically. Logged-out text returns exact login guidance and renewal reports retain the `login` operation label. `cli/src/services/auth_command/command.rs` owns the `AuthCommand` payload used by the static `RuntimeCommand` enum. There is no public renewal or status subcommand. -- `cli/src/services/db/mod.rs` provides the shared generic Turso infrastructure seam: `DbSpec` supplies a service-specific name, path, ordered embedded migrations, and config-file lookup key (`db_config_key()`), while `TursoDb` owns parent-directory creation, `Builder::new_local(...)` initialization (with `experimental_multiprocess_wal(true)` for safe concurrent access), Turso connection setup, tokio current-thread runtime bridging, retry-backed blocking `execute`/`query`/`query_values`/`query_map` wrappers, and generic migration execution with per-database `__sce_migrations` metadata. `TursoDb::new()` and `EncryptedTursoDb::new()` wrap only their local open/connect block in `run_with_retry_sync` using a config-driven connection-open policy resolved from the `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults, while operation methods use a config-driven operation policy from the same source; on the Agent Trace DB only, replay-safe write units (the transactional insert-pair and CAS-batch primitives and the opt-in `execute_idempotent_write`) instead use a bounded write-contention retry layered over Turso's `busy_timeout` (see `context/sce/shared-turso-db.md`). `query_values()` returns fully fetched column names plus raw `turso::Value` rows for deterministic operator-facing rendering; `query_map()` retries the initial query and row-fetch loop, then applies caller row mapping after retry completion. Migration execution is not retried and uses batch execution so one migration file may contain multiple SQL statements while still recording one migration ID. The same module also provides `EncryptedTursoDb`, a structurally parallel encrypted adapter that resolves the encryption key through `encryption_key::get_or_create_encryption_key()`, enables Turso local encryption with strict `aegis256` cipher selection, and exposes retry-backed synchronous wrappers plus migration execution. `cli/src/services/db/encryption_key.rs` first derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text when present, otherwise falls back to keyring-backed credential-store get-or-create behavior; no plaintext auth DB fallback exists. -- `cli/src/services/local_db/mod.rs` provides the concrete local DB spec and `LocalDb` type alias over the shared generic `TursoDb` adapter. `LocalDbSpec` resolves the deterministic persistent runtime DB target through the shared default-path seam and declares no local migrations; `TursoDb` supplies retry-backed blocking `execute`/`query`, parent-directory creation, Turso connection setup, tokio current-thread runtime bridging, and generic migration execution. +- `cli/src/services/db/mod.rs` provides shared async Turso infrastructure: `DbSpec` supplies a service name, path, ordered migrations, and config key; `TursoDb` / `EncryptedTursoDb` own concrete connections but no Tokio runtime. Construction, migration, query/materialization, transaction, readiness, and checkpoint operations are awaited through `run_with_retry` on the application runtime. SQL, one-connection transaction scope, migration metadata, retry classification, WAL, encryption, and per-database policy remain adapter-owned; see `context/sce/shared-turso-db.md`. +- `cli/src/services/local_db/mod.rs` provides `LocalDb = TursoDb` with zero local migrations; inherited construction, `execute`, `query`, and `query_map` operations are async and use the shared adapter retry policies. - `cli/src/services/auth_db/mod.rs` provides the encrypted auth DB spec and `AuthDb` type alias over `EncryptedTursoDb`. `AuthDbSpec` resolves `/sce/auth.db` through the shared default-path seam and embeds ordered auth migrations. Auth DB lifecycle setup/doctor integration is wired through `AuthDbLifecycle`; auth command/token-storage reads/writes are directed through `token_storage.rs`. - `cli/src/services/agent_trace_db/mod.rs` owns the shared Agent Trace insert payloads, SQL constants, and typed row helpers (diff-trace/intersection/Agent Trace/message/part) plus `ensure_schema_ready_for_hooks()` consumed by the repository adapter. `cli/src/services/agent_trace_db/repository.rs` defines the sole `RepositoryAgentTraceDb` adapter over `TursoDb` with one fresh `agent-trace-repository/001_repository_schema.sql` baseline for `diff_traces` (including `payload_type`), `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes, and triggers, `repository_metadata` validation, no trace-table `checkout_id` columns, `agent_traces.agent_trace_id NOT NULL UNIQUE`, and `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` using the inclusive chronological parser without checkout filtering; structured-row reconstruction applies the persisted row `model_id` to every hunk and the persisted canonical `session_id` to every touched line before downstream combination and intersection. Active hook runtime, setup/lifecycle storage, and `sce sync` resolve through `agent_trace_storage` and use `RepositoryAgentTraceDb`. The checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, the 15-file `cli/migrations/agent-trace/` chain, and the former `sce trace --legacy` surface were removed by the `retire-legacy-agent-trace-db` plan. - `cli/src/services/setup/mod.rs` defines the setup command contract (`SetupMode`, `SetupTarget`, `SetupRequest`, CLI flag parser/validator), an `inquire`-backed interactive target prompter (`InquireSetupTargetPrompter`), setup dispatch outcomes (proceed/cancelled), additive durable-context bootstrap (`bootstrap_context_baseline` for standalone `--bootstrap-context` and every normal successful setup path), and compile-time embedded asset access (`EmbeddedAsset`, target-scoped iterators, required-hook asset iterators/lookups). It also owns the install-time optional-workflow seam: the `OptionalWorkflow` type plus the build-generated `OPTIONAL_WORKFLOWS` catalog, a per-target `WorkflowAssetLayout` built from the existing `default_paths` command/skill directory constants (`command`/`commands`/`prompts` plus `skills`), and `iter_embedded_assets_for_setup_target_with_selection(target, selection)`, which yields every embedded asset except the `{command_dir}/{command_slug}.md` file and `{skills_dir}/{skill_slug}/` subtree of each optional workflow the selection omits. Membership is derived from the catalog's slugs rather than an enumerated file list, so a new optional workflow needs no Rust change. This filtered iterator is the only way embedded assets are enumerated; setup installs through it and doctor inspects through it, so there is no unfiltered enumeration path that could reintroduce an unselected workflow. The non-interactive selection flows through the repeatable `--workflow ` flag into `SetupRequest.optional_workflows: Option>` (`None` meaning the flag was absent); `validate_optional_workflow_slugs` checks each slug against `OPTIONAL_WORKFLOWS` during request resolution, before any write, and `run_setup_for_mode` resolves `None` to the persisted `integrations.optional_workflows` (exported as `persisted_optional_workflows`) before installing through the filtered iterator and persisting the resolved selection. The interactive selection flows through the same seam: `SetupTargetPrompter` carries `prompt_target` plus `prompt_optional_workflows(defaults)` (returning `None` for a cancelled prompt), `SetupDispatch::Proceed { mode, optional_workflows }` carries a prompted selection alongside the resolved mode, and `resolve_setup_dispatch(mode, prompter, defaults)` runs the workflow prompt only after an interactive target prompt, mapping either cancellation to `SetupDispatch::Cancelled`. The prompt module builds its `inquire::MultiSelect` from `optional_workflow_prompt_inputs(catalog, defaults)`, which returns `None` for an empty catalog (skipping the prompt) and otherwise catalog-ordered rows plus the indices to pre-check, ignoring ids absent from the catalog. `setup/command.rs` therefore resolves the repository root before dispatch, mapping the setup domain's typed positive non-Git classification to `setup.not_git_repository` and other resolution failures to `general.unexpected_failure`, so a non-Git directory fails before any prompt without CLI-layer string matching. For repository builds, `cli/build.rs` validates the `SCE_CLI_GENERATED_INPUT_DIR` payload and canonical-input inventories, copies the payload into Cargo `OUT_DIR/pkl-generated`, stages `cli/assets/hooks/**` under `OUT_DIR/static`, requires the staged `config/optional-workflows.json`, and generates both the setup manifest and the optional-workflow catalog (`optional_workflows.rs`, rejecting a manifest whose `schemaVersion` is not 1 or whose entries lack a non-empty `id`/`title`/`description`/`commandSlug`/`skillSlug`) in `OUT_DIR`; focused internal seams separate install-flow from prompt-flow logic; `cli/src/services/setup/command.rs` owns the `SetupCommand` payload used by the static `RuntimeCommand` enum and executes against any context implementing repo-root scoping. Its install engine/orchestrator (`install_assets_for_concrete_target_with_rename`) installs each embedded asset individually: it stages the asset's content next to its final destination and swaps it into place by renaming the staging file directly over the destination — never unlinking the destination first, since `fs::rename` already replaces an existing file atomically — with deterministic recovery guidance naming the failing asset's path on swap failure (the pre-existing destination content, if any, is untouched) and no backup artifact creation; it never removes an integration target directory as a whole, so files a repository owns inside `.opencode`/`.claude`/`.pi` — including nested inside an SCE-owned subdirectory such as `skills/` or `commands/` — survive a setup run; Codex has no single target directory this way, since `CODEX_EMBEDDED_ASSETS` relative paths already carry their own `.agents/`/`.codex/` prefix and `InstallTargetPaths::codex_target_dir()` resolves to the repository root itself, but the same per-asset stage/atomic-swap and never-delete-what-it-did-not-author guarantees still apply file by file. For the three assets that are merge targets — the Claude target's `settings.json`, the OpenCode target's `opencode.json`, and Codex's user-owned `.codex/hooks.json` — the content staged is not always the embedded asset's bytes. Claude and OpenCode use `cli/src/services/setup/config_merge.rs`, while Codex uses the shared `cli/src/services/codex_hook_config.rs` service for structural validation and canonical-registration merging. For the Claude and OpenCode targets, the content staged is the result of `cli/src/services/setup/config_merge.rs::merge_or_create_claude_settings`/`merge_or_create_opencode_config(existing_bytes, generated_bytes, source_path)`: each returns the generated document verbatim when no file exists yet, otherwise parses the existing file as JSON (a parse failure is a hard error naming `source_path`, and nothing is written) and merges it with a pure per-shape function that copies `$schema` from the generated document and preserves every other key from the existing file untouched. `merge_claude_settings` replaces, per hook event key the generated document declares, only the entries whose command contains the ownership marker `run-sce-or-show-install-guidance.sh`, preserving every event key and non-SCE hook entry from the existing file. `merge_opencode_config` merges the `plugin` array as a set: existing entries whose path starts with the ownership marker `./plugins/sce-` are dropped structurally — so a stale plugin path an older or renamed catalog once installed is still recognized and pruned even after the current generated document stops declaring it — and the generated document's `plugin` entries are appended after the surviving entries. Codex's merge validates the document shape and recognizes ownership by the generated helper path plus one of two trailing command contracts — `sce hooks codex` or `sce hooks codex-mutation-scope` — attributing each handler to whichever it matches; it replaces stale or duplicate handlers with exactly one current handler per registration (the four `sce hooks codex` registrations plus six additive `sce hooks codex-mutation-scope` registrations, within that mutation-scope set, `PreToolUse` and `PostToolUse` use matcher `^(Bash|apply_patch)$`, while `Stop`, `Interrupt`, `SubagentStop`, and `SessionEnd` are appended in unmatched groups after the existing groups so an already-trusted handler keeps its `(event, matcher, group index, handler index)` identity and computed Codex trust key), touching only the matching command's handlers and preserving unrelated valid Codex fields, groups, and handlers. After that per-asset install loop, `prune_stale_assets_for_concrete_target` deletes every path the full embedded catalog for the concrete target claims but the resolved selection did not install (a deselected optional workflow, or an asset a newer catalog renamed or dropped), and `remove_empty_ancestor_directories` removes any parent directory left empty by that deletion, stopping at the target root or at a directory that still holds something such as a user file. It formats deterministic completion messaging; required-hook install orchestration (`install_required_git_hooks`, backed by the rename-injectable `install_required_git_hooks_with_rename`) is a third content-computation seam alongside the two JSON merge targets: `install_single_required_hook_with_rename` computes the bytes to stage with `cli/src/services/setup/hook_merge.rs::merge_or_create_hook(existing_bytes, canonical_bytes, hook_name)` rather than writing `hook_asset.bytes` verbatim — a foreign hook (no SCE managed block, no legacy guidance-URL marker) is kept as an exact byte prefix with the canonical block appended after it, an SCE-owned hook has only its block spliced in place or left unchanged, and a legacy pre-marker hook is replaced wholesale — then follows the same per-file stage/atomic-swap choreography as config-asset install: the staging file is renamed directly over the existing hook without unlinking it first, so a rename failure leaves the prior hook's bytes and executable bit intact, with deterministic recovery guidance on swap failure. `Installed`/`Updated`/`Skipped` are decided against the merged bytes plus the executable bit rather than the canonical asset's raw bytes, so an already-current foreign-plus-block hook reports `Skipped`; `RequiredHookInstallResult.unreachable_block_advisory` is set, and rendered as a named advisory line in setup's hook output, when an appended block follows a foreign hook's zero-indent `exec`/`exit` and so would never run. After the Git gate, setup always ensures the context baseline; context-only requests return there, while normal modes derive a repo-root-scoped context before aggregating static lifecycle provider `setup` dispatch across providers (config → local_db → auth_db → agent_trace_db → hooks when requested), so setup providers consume only repo-root access from the scoped context. @@ -152,7 +152,7 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - `context/cli/agent-trace-auto-sync.md` documents the sync-owned, one-shot post-commit launcher boundary: it reuses `sce sync`, has no daemon or local retry machinery, and fails open when child startup cannot be completed; doctor reports this capability without invoking the launcher. - `cli/src/services/sync/progress.rs` owns the sync-local, consumer-typed progress seam: generic `ProgressReporter` supports event delivery plus explicit successful finalization, closure-based collectors, and a no-op implementation alongside the fixed `indicatif` stderr presentation adapter. `cli/src/services/sync/sync.rs` owns `SyncProgressEvent` and its payload semantics for the three remotely synchronized streams (`messages`, `parts`, `agent_traces`; `diff_traces` is captured locally but not synchronized), while `sync/command.rs` selects the terminal adapter for text and the no-op reporter for JSON. There is no top-level `cli/src/services/progress/` module; sync orchestration depends only on its sync-owned contract, so terminal-library details stay at the sync presentation boundary. - `sce sync [--format text|json]` is implemented: `cli/src/services/sync/sync.rs` resolves repository-scoped Agent Trace storage, authenticates against the control plane with stored WorkOS credentials, uses the config-resolved `control_plane_base_url` with baked default `https://sce.crocoderlab.dev`, calls the ingestion `/state` endpoint once, then starts the three remote `messages`/`parts`/`agent_traces` stream state machines concurrently (`diff_traces` remains a local-only compatibility report) via `AgentTraceExportReader` and a shared per-stream reconciliation engine. Batches and cursor refreshes remain sequential within each stream, while fixed stream order is retained for final and stream-completion reporting; `cli/src/services/sync/render_sync.rs` renders the converged `AgentTraceSyncReport` as concise per-stream text or `camelCase` JSON without a nested subcommand field (see `context/cli/sync-command.md`). Local DB bootstrap and setup-time repository-scoped Agent Trace DB initialization otherwise still flow through lifecycle providers aggregated by setup, while repository-scoped DB health/repair flows through the doctor surface. The former trace database inspection and nested sync surfaces are unavailable. -- `cli/src/services/patch.rs` defines the standalone patch domain model (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation, capturing only touched lines (added/removed) plus minimal per-file/per-hunk metadata while excluding non-hunk headers and unchanged context lines. All types are `serde`-serializable/deserializable with `snake_case` JSON field naming. The module also provides `parse_patch`, a public parser function that converts raw unified-diff text (both `Index:` SVN-style and `diff --git` git-style formats) into `ParsedPatch` structs, with `ParseError` for actionable malformed-input diagnostics. Storage-agnostic JSON load helpers (`load_patch_from_json` for string input, `load_patch_from_json_bytes` for byte input) reconstruct `ParsedPatch` from serialized JSON content with `PatchLoadError` for actionable deserialization diagnostics. Its patch-set operations now include deterministic ordered combination plus target-shaped intersection that prefers exact touched-line matches and falls back to historical `kind`+`content` matching when incremental diffs and canonical post-commit diffs have drifted line numbers; `parse_patch`, `combine_patches`, and `intersect_patches` are consumed by the active post-commit hook runtime. +- `cli/src/services/patch.rs` defines the standalone patch domain model (`ParsedPatch`, `PatchFileChange`, `FileChangeKind`, `PatchHunk`, `TouchedLine`, `TouchedLineKind`) for in-memory parsed unified-diff representation, capturing only touched lines (added/removed) plus minimal per-file/per-hunk metadata while excluding non-hunk headers and unchanged context lines. All types are `serde`-serializable/deserializable with `snake_case` JSON field naming. The module also provides `parse_patch`, a public parser function that converts raw unified-diff text (both `Index:` SVN-style and `diff --git` git-style formats) into `ParsedPatch` structs, with `ParseError` for actionable malformed-input diagnostics. Storage-agnostic JSON load helper (`load_patch_from_json` for string input) reconstruct `ParsedPatch` from serialized JSON content with `PatchLoadError` for actionable deserialization diagnostics. Its patch-set operations now include deterministic ordered combination plus target-shaped intersection that prefers exact touched-line matches and falls back to historical `kind`+`content` matching when incremental diffs and canonical post-commit diffs have drifted line numbers; `parse_patch`, `combine_patches`, and `intersect_patches` are consumed by the active post-commit hook runtime. - `cli/src/services/structured_patch.rs` defines the synchronous structured editor-hook derivation seam. It derives Claude `PostToolUse` `Write` structured-update hunks, `Write` `tool_input.content` create fallback, and `Edit` structured-patch payloads into canonical `ParsedPatch` values plus Claude session/tool metadata, returning deterministic skip reasons for unsupported events/tools/payload shapes. The module is pure and side-effect-free. It is wired into `sce hooks diff-trace` for Claude payload classification at intake (T04) and into `RepositoryAgentTraceDb::recent_diff_trace_patches` for post-commit structured payload parsing dispatch at read time (T05). - `cli/src/services/` contains module boundaries for command_registry, lifecycle, auth_command, config, setup, doctor, hooks, checkout identity, bash_policy, version, completion, help, patch, sync, SCE web URL helpers, shared database infrastructure, local DB adapters, encrypted auth DB adapters, and Agent Trace DB adapters with explicit trait seams for future implementations. `cli/src/services/checkout/` owns checkout ID file infrastructure (the per-checkout Agent Trace DB opener/path helper was removed by the `retire-legacy-agent-trace-db` plan); active setup/hooks use `agent_trace_storage` to establish checkout identity as diagnostics and initialize/open the repository-scoped DB, while `sce doctor` surfaces checkout identity facts plus credential-safe repository Agent Trace DB metadata. There is no checkout-scoped discovery or former trace inspection surface; any pre-migration `agent-trace-*.db` files on disk are never touched and no longer inspectable via the CLI. `cli/src/services/bash_policy.rs` owns both the CLI-agnostic evaluator logic and the hidden `sce policy bash` command adapter used by OpenCode, Claude, and Pi callers; Codex delegates in-process to the same evaluator for its native `PreToolUse(Bash)` response. `cli/src/services/command_registry.rs` defines the static `RuntimeCommand` enum, deterministic `CommandRegistry` name catalog, and `build_default_registry()` function for command dispatch metadata. Service-owned command modules own the runtime command payload structs for help/help-text, version, completion, auth, config, setup, doctor, hooks, policy, and sync. - `cli/README.md` is the crate-local onboarding and usage source of truth for placeholder behavior, safety limitations, and roadmap mapping back to service contracts. @@ -165,34 +165,100 @@ The repository includes a new placeholder Rust binary crate at `cli/`. - The config-lib check source preserves repo-relative access to shared CLI patch fixtures: Nix copies a filtered repo-shaped source containing `config/lib/**` plus `cli/src/services/structured_patch/fixtures`, then runs Bun/Biome from `config/lib/`. - `flake.nix` exposes the **native** development package as `packages.sce` (`packages.default = packages.sce`) plus `apps.sce` and `apps.default`, all targeting `${packages.sce}/bin/sce`; this keeps repo-local and remote flake run/install flows (`nix run .`, `nix run github:crocoder-dev/shared-context-engineering`, `nix profile install github:crocoder-dev/shared-context-engineering`) aligned to the native CLI output. The static-musl release binary is a distinct output, `packages.sce-release` (`apps.sce-release`); see the native/release split note above and `context/sce/cli-release-artifact-contract.md`. - `biome.json` at the repository root is the canonical Biome configuration for the current JS tooling slice and deliberately scopes coverage to `npm/**` plus the shared `config/lib/**` plugin package root while excluding package-local `node_modules/**`; `flake.nix` exposes Biome through the default dev shell rather than through package-local installs. -- `cli/Cargo.toml` keeps crates.io publication-ready package metadata for the `shared-context-engineering` crate, and `cli/README.md` is the Cargo install surface for crates.io (`cargo install shared-context-engineering --locked`) and local checkout (`./scripts/run-cli-cargo.sh install --path cli --locked`) guidance. Direct `cargo install --git` is unsupported because it cannot invoke the repository's pre-Cargo producer. The published crate installs the `sce` binary. Tokio remains intentionally constrained (`default-features = false`) with application-level multi-thread execution, retained DB-owned current-thread runtimes and timer-backed bounded resilience wrappers for retry/timeout behavior. -- `cli/Cargo.toml` explicitly enables Tokio `rt`, `io-util`, `sync`, `time`, `macros`, and `rt-multi-thread`, retaining `default-features = false`. `main` directly awaits async `app::run` under `#[tokio::main(flavor = "multi_thread")]`; only the synchronous Turso adapters retain their DB-owned current-thread runtimes. +- `cli/Cargo.toml` keeps crates.io publication-ready package metadata for the `shared-context-engineering` crate, and `cli/README.md` is the Cargo install surface for crates.io (`cargo install shared-context-engineering --locked`) and local checkout (`./scripts/run-cli-cargo.sh install --path cli --locked`) guidance. Direct `cargo install --git` is unsupported because it cannot invoke the repository's pre-Cargo producer. The published crate installs the `sce` binary. Tokio remains intentionally constrained (`default-features = false`) with application-level multi-thread execution, async Turso operations on the application runtime and timer-backed bounded resilience wrappers for retry/timeout behavior. +- `cli/Cargo.toml` explicitly enables Tokio `rt`, `io-util`, `sync`, `time`, `macros`, and `rt-multi-thread`, retaining `default-features = false`. `main` directly awaits async `app::run` under `#[tokio::main(flavor = "multi_thread")]`; Turso adapters do not own nested runtimes. ## Application execution runtime -PR1 directly awaits static command execution under one application-level multi-thread Tokio runtime. Parsing, rendering and lifecycle providers remain synchronous; auth/sync own no command runtime. This boundary does not promise one runtime object process-wide. See the [runtime ownership decision](decisions/2026-10-07-application-owned-async-command-runtime.md) and the [command blocking-scope inventory](cli/cli-command-surface.md#application-runtime-dispatch). +The application-level multi-thread Tokio runtime owns directly awaited static +command execution through async services, lifecycle providers, hooks, sync, +export, credentials, and Turso persistence. `TursoDb` and +`EncryptedTursoDb` own no Tokio runtime and expose native async operations. +Foreground hooks await required persistence before returning; only the existing +post-commit auto-sync child remains detached. See the [directly awaited Turso +persistence decision](decisions/2026-10-07-directly-awaited-turso-persistence.md) +and [command dispatch](cli/cli-command-surface.md#application-runtime-dispatch). ```mermaid flowchart TD - A[Application Tokio multi-thread runtime] --> B[Awaited static command dispatch] - B --> C[Awaited auth, sync and control-plane HTTP] - C --> D[Awaited credential spawn_blocking] - B --> E[Direct synchronous commands] - B --> F[DB-capable Setup, Doctor and Hooks blocking scopes] - C --> G[Sync storage construction and cleanup guard blocking scopes] - D --> H[Synchronous Turso adapter with DB-owned runtime] - F --> H - G --> H + A[Application Tokio multi-thread runtime] --> B[Static RuntimeCommand dispatch] + B --> C[Async command and service operations] + C --> D[Async lifecycle and hook operations] + C --> E[Async credentials and sync/export] + D --> F[Shared async Turso adapters] + E --> F + F --> G[Turso futures] ``` -Sync retains borrowed storage across HTTP awaits and drops its owning guard before returning; guard Drop also handles error, cancellation and unwind. PR2 is future work: async Turso removes DB-owned runtimes, `build_current_thread_runtime`, `block_on_isolated` and the DB-related `block_in_place` scopes together. - -```mermaid -flowchart TD - A[PR2 future application Tokio runtime] --> B[Awaited static command dispatch] - B --> C[Async auth, sync and HTTP] - B --> D[Async Turso without DB-owned runtime or bridge] -``` +Pure parsing, rendering and unrelated service logic remain synchronous. PR2 as +implemented also moved Git snapshot subprocesses, protected-worktree marker I/O, +worktree/adapter lock acquisition, Git ref mutation and guarded-shell supervision +off the async workers (listed below) so cancellation cannot release a protected +resource early. The adapter layer owns connection and +migration policy, not an executor; retry classification, transaction SQL, +encryption, WAL, and migration boundaries remain unchanged by async execution. + +No database or Turso future runs inside `spawn_blocking`, and no nested +executor exists. Production `spawn_blocking` is limited to genuinely +synchronous external boundaries, each returning an owned result to the awaiting +caller: + +- `EncryptedTursoDb::new` resolves the encryption key through the synchronous OS + credential store (`cli/src/services/db/mod.rs`); AuthDb construction and Turso + work stay on the async side. +- `acquire_with_timeout_async` and `run_locked_blocking` + (`cli/src/services/hooks/mutation_scope_lock.rs`) run the synchronous adapter + boundary/state advisory-lock polling and the state-file transitions for the + Claude, Codex, OpenCode, and Pi mutation-scope adapters. A started transition + owns the state lock plus a clone of the boundary lease until it finishes, so a + cancelled boundary caller cannot be overtaken by a later boundary. The adapters' + `state.rs` `read_state` runs a read-only synchronous file read in a worker. +- `acquire_inner_async` (`cli/src/services/mutation_trace/runtime/worktree_lock.rs`) + runs the synchronous mutation-trace `WorktreeLock` polling loop used by + `ProtectedWorktree`, coordinate, abandon-scope, ref reconciliation, the + revision cut, and the external mutation guard. +- `arm_marker_with_lock` and `ProtectedWorktree::complete` + (`cli/src/services/mutation_trace/runtime/protected_worktree.rs`) run the + synchronous external-taint marker inspect/persist and clear `fsync` I/O. + Each worker owns the worktree lock (arming) or the whole `ProtectedWorktree` + (completion) until its marker I/O finishes. Cancelling the caller cannot + release the lock while a marker operation is still in flight. +- `run_ref_mutation_inner` + (`cli/src/services/mutation_trace/runtime/git_snapshot.rs`) runs the blocking + Git ref-mutation subprocess. The worker owns the `WorktreeLockLease`, so the + worktree lock stays held until Git exits even if the caller is cancelled. +- `TempIndexGuard::drop` (same file) removes a temporary Git index through + `Handle::try_current` + `spawn_blocking` (inline removal without a runtime). + The task is fire-and-forget and holds no lock. A killed process can leave an + orphaned `index-.lock`/temporary index file behind; this is an accepted + limitation. PR3 audited it and adds no automatic sweeper: the temporary + index has no ownership or lock record, and a file name or age alone is not + evidence of abandonment, so a sweep is deferred to a separate change that + first defines a safe ownership protocol. +- The guarded-shell supervisor + (`cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`) runs the + synchronous `Child::wait`/descriptor supervision. The worker owns the + `ProtectedWorktree` and the guarded child, ends when the child exits or + cancellation is observed, and keeps the protected lock until it returns. The + lifetime pipe is created close-on-exec atomically only where `pipe2` exists + (Linux); elsewhere it is `pipe` then `fcntl`, which leaves a small + fork/exec window that is an unresolved limitation (not fixed in PR3). The + guarded child alone clears close-on-exec on the protected descriptors in + `pre_exec`. + +Runtime shutdown waits for these workers; each is bounded by its lock timeout, +file I/O or child process. No other production `spawn_blocking` or +`block_in_place` exists, and `build_current_thread_runtime`, `block_on_isolated`, +`SyncStorageGuard`, `run_credential_operation` and the command-registry +Setup/Doctor/Hooks blocking scopes are deleted. + +Lock workers return the acquired guard. The async caller holds it across its +protected awaited work and drops it when that work ends. Lock timeouts (10 s) +and poll intervals are unchanged. A cancelled caller leaves its worker polling +only until acquisition or timeout; a guard acquired after cancellation drops +with the abandoned join result and releases the lock. + +Verified guarantees, limits and PR1–PR4 staging: [async persistence guarantees](cli/async-persistence-guarantees.md). ## Build / devShell / CI performance (flake-speedup) diff --git a/context/cli/agent-trace-storage.md b/context/cli/agent-trace-storage.md index 998696230..bbfaf237e 100644 --- a/context/cli/agent-trace-storage.md +++ b/context/cli/agent-trace-storage.md @@ -7,16 +7,16 @@ Module at `cli/src/services/agent_trace_storage/` (T04 of the `repository-scoped - `AgentTraceStorageContext { repository_root, explicit_repository_id, repository_remote }` — borrowed inputs mirroring the `agent_trace.repository_id` / `agent_trace.repository_remote` config keys; callers pass already-resolved configuration values. Active hook runtime and Agent Trace lifecycle setup/health resolve these values through the config service before constructing the storage context. - `ResolvedAgentTraceStorage { repository_identity, db_path, db, metadata }` — resolved repository identity (`ResolvedRepositoryIdentity` including source provenance), the repository-scoped DB path, the open `RepositoryAgentTraceDb`, and the typed `RepositoryMetadata { repository_id, source_instance_id }` produced by the same verification/initialization call that opened `db` (see [../sce/agent-trace-db.md](../sce/agent-trace-db.md) for the `source_instance_id` identity contract). - `resolve_agent_trace_storage(context)` — setup/lifecycle production entrypoint using the canonical state root from the default-path catalog; may run migrations, including migration `002`. -- `resolve_agent_trace_storage_at_state_root(context, state_root)` — setup/lifecycle resolution core against an explicit state root; used by tests to exercise the full path without touching the real user state directory. -- `resolve_agent_trace_storage_for_hook_runtime(context)` — hook-runtime production entrypoint using the canonical state root; never runs migrations, including migration `002`. -- `resolve_agent_trace_storage_for_hook_runtime_at_state_root(context, state_root)` — hook-runtime resolution core against an explicit state root; used by tests. +- `resolve_agent_trace_storage_for_hook_runtime(context).await` — async hook-runtime production entrypoint using the canonical state root; never runs migrations, including migration `002`. +- `resolve_agent_trace_storage_for_hook_runtime_at_state_root(context, state_root).await` — async hook-runtime resolution core against an explicit state root. ## Resolution flow 1. Agent Trace storage runtime config is resolved through the config service; invalid default-discovered config layers are skipped with the shared resolver's remaining values and validation diagnostics, while explicit `--config` / `SCE_CONFIG_FILE` failures remain fatal. Repository identity then uses `repository_identity::resolve` precedence (explicit config ID → configured remote URL, default `origin`); resolution errors carry `.sce/config.json` guidance and never echo URLs. A failed config or identity resolution creates no state directories. 2. DB path from `default_paths::agent_trace_db_path_for_repository{,_at}`, which rejects empty or path-unsafe repository IDs (separators, `.`, `..`). -3. DB open splits by caller through `agent_trace_db::repository::RepositoryAgentTraceDb`, sharing steps 1–2 through an internal `open_storage_with` helper parameterized by the DB-opener: +3. DB open splits by caller through `agent_trace_db::repository::RepositoryAgentTraceDb`, sharing steps 1–2 through an internal `open_storage_with` helper parameterized by the DB-opener; both resolver paths await async database operations: - Setup/lifecycle (`resolve_agent_trace_storage{,_at_state_root}`): a bounded concurrent-safe fast-path-then-migrate sequence — `open_without_migrations_at` + `ensure_schema_ready_for_hooks()` + `verify_or_initialize_repository_metadata(repository_id)`, falling back to migration-running `new_at` plus the same metadata validation. Directory creation rides on `TursoDb` parent-dir `create_dir_all`. If concurrent first open leaves all repository schema tables present but the one-file baseline migration record missing, the repository adapter records that missing metadata before rechecking readiness; otherwise the resolver retries the open/migrate sequence for a bounded window while SQLite/Turso locks clear. + - Maintenance (`resolve_existing_agent_trace_storage_for_maintenance` + `RepositoryAgentTraceDb::open_verified_existing_at`, used only by snapshot-ref reconciliation): a non-creating read-only Turso open (`TursoDb::open_existing_without_migrations_at`, multiprocess WAL preserved) that never creates the DB file or parent directory, never migrates, never calls the migration-metadata repair, and never runs `verify_or_initialize_repository_metadata`. It verifies schema readiness, metadata presence, `repository_id` equality, and a valid `source_instance_id` before returning, with typed `ExistingRepositoryDbError` (`Missing`, `Unreadable`, `IncompatibleSchema`, `MissingMetadata`, `RepositoryMismatch`). Plain `sce doctor` shares the open via `RepositoryAgentTraceDb::open_existing_schema_ready_at`. See [mutation-trace-ref-reconciliation.md](mutation-trace-ref-reconciliation.md). - Hook runtime (`resolve_agent_trace_storage_for_hook_runtime{,_at_state_root}`): `open_for_hooks_without_migrations_at` + `ensure_schema_ready_for_hooks()`, applying the same narrow concurrent-first-open metadata repair on failure but never falling back to `new_at`/migrations. A missing or baseline-only (pre-`002`) database fails with `ensure_schema_ready_for_hooks`'s existing `Run 'sce setup'.` guidance. Once readiness passes, `verify_or_initialize_repository_metadata(repository_id)` initializes `source_instance_id` when needed, matching setup's result for an already-set-up repository. `sce hooks` command call sites use this path via `open_agent_trace_db_for_hook_runtime` in `cli/src/services/hooks/mod.rs`. ## Never-touch on-disk boundary diff --git a/context/cli/agent-trace-sync-command.md b/context/cli/agent-trace-sync-command.md index 3ba024b10..72368b4e1 100644 --- a/context/cli/agent-trace-sync-command.md +++ b/context/cli/agent-trace-sync-command.md @@ -25,10 +25,10 @@ flowchart LR - **hooks/plugins** write local capture rows (`messages`, `parts`, `diff_traces`, `agent_traces`) into the current repository's `RepositoryAgentTraceDb` during normal Git/editor activity — this is unchanged by sync. - **`AgentTraceExportReader`** (PR #198) is the read-only local export boundary sync uses to read rows after a cursor; sync never queries the repository DB directly. The reader exposes four `read_*_after` methods; `sce sync` consumes three and never calls `read_diff_traces_after`. - **`sce sync`** resolves repository storage through `agent_trace_storage` (not the hook-runtime resolver), builds an `AuthenticatedControlPlaneClient` from stored WorkOS credentials and the resolved `control_plane_base_url`, and drives one authoritative `/state` call before starting three concurrent remote stream state machines (`messages`, `parts`, `agent_traces`). Each stream keeps its own batches and reconciliation refreshes sequential and cursor-safe; the bounded per-stream reconciliation loop remains independent. With no environment or config override, that base is `https://sce.crocoderlab.dev`; it is distinct from the `https://sce.crocoder.dev` SCE web and config-schema URL owner. -- **Credential runtime boundary:** `AuthenticatedControlPlaneClient` keeps the synchronous `CredentialStore` behind an `Arc` and runs every token-storage `load`/`save` through `tokio::task::spawn_blocking`. The underlying encrypted auth DB and Linux Secret Service/zbus APIs are blocking and may create their own Tokio runtime, so they must never execute directly inside the async control-plane request future. Token refresh and HTTP requests remain asynchronous; only credential persistence crosses the blocking boundary. The client owns a refresh single-flight guard: expired-token callers re-check credentials after acquiring it, and callers retrying the same rejected access token reuse a token saved by an earlier refresh; valid-token resolution does not acquire the guard. +- **Credential runtime boundary:** `CredentialStore` has native async `load` and `save` methods (returning `Send` futures; the trait is `Send + Sync + 'static`), and `AuthenticatedControlPlaneClient` owns a concrete generic store behind an `Arc`. `SystemCredentialStore` directly awaits async token storage on the application runtime; credential persistence uses no DB `spawn_blocking` wrapper. Token refresh and HTTP remain asynchronous. The client keeps its refresh single-flight guard: expired-token callers re-check credentials after acquiring it, callers retrying the same rejected access token reuse a token saved by an earlier refresh, and valid-token resolution does not acquire the guard. Refresh-and-save is one owned unit: the WorkOS renewal and the credential save run in a spawned task that holds the guard, so a cancelled or timed-out caller (for example the outer `/state` timeout) detaches the task instead of losing the rotated refresh token before it is persisted. `sce sync` then calls `wait_for_pending_refresh` (acquires the refresh mutex, which the task holds through the save) after the sync futures finish on both success and error, so a normal return cannot shut the runtime down under a started save; `sce auth whoami` directly awaits the task and needs no drain. Remaining limits: the renewal is bounded only by the auth module's retry/timeout policy, and a forced termination, process crash after WorkOS rotates the token but before the save completes still loses the replacement (the user re-runs `sce auth login`). - **control plane** is the sole source of cursor truth: every invocation starts from `POST /agent-trace/ingestion/state`, uploads via `POST /agent-trace/ingestion/batch`, and advances a stream's cursor only from a validated batch response (`accepted == rows.len()` and `cursor == rows.last().sourceRowId`), never by inferring `cursor + rows.len()`. -The three remote streams (`messages`, `parts`, `agent_traces`) start concurrently after the single authoritative state response; batches and cursor-refresh calls remain sequential within each stream. Accepted-batch progress and stream-completion events may arrive as their respective futures complete. Text mode creates three aligned `indicatif` rows immediately on `stderr`, before any accepted batch can be reported, each with a 15-column stream label, a steady spinner, and a `0 rows uploaded` starting message. Accepted batches update only their stream's cumulative count; each stream changes its spinner to a shared-policy styled `✓` with its final count as soon as its own future completes, including an empty stream with zero uploaded rows. After the progress rows, the final text report contains only a status heading: `Agent Trace already synced.` when the three remote streams uploaded zero rows, otherwise `Agent Trace sync complete.`. Non-TTY output uses stable aligned plain snapshots without ANSI or terminal-control sequences, and `NO_COLOR` disables completion styling. JSON mode uses a no-op progress sink and retains its JSON-only stdout contract without human progress on `stderr`. +The three remote streams (`messages`, `parts`, `agent_traces`) start concurrently after the single authoritative state response; batches and cursor-refresh calls remain sequential within each stream. Accepted-batch progress and stream-completion events may arrive as their respective futures complete. A terminal failure in any stream sets a shared flag that stops sibling streams from reading or starting further batches (the flag is rechecked after an in-flight database read, so rows read after termination are discarded); a sibling batch already in flight (including its credential save) runs to completion, aborted siblings emit no completion event, and the first observed stream error is returned. Text mode creates three aligned `indicatif` rows immediately on `stderr`, before any accepted batch can be reported, each with a 15-column stream label, a steady spinner, and a `0 rows uploaded` starting message. Accepted batches update only their stream's cumulative count; each stream changes its spinner to a shared-policy styled `✓` with its final count as soon as its own future completes, including an empty stream with zero uploaded rows. After the progress rows, the final text report contains only a status heading: `Agent Trace already synced.` when the three remote streams uploaded zero rows, otherwise `Agent Trace sync complete.`. Non-TTY output uses stable aligned plain snapshots without ANSI or terminal-control sequences, and `NO_COLOR` disables completion styling. JSON mode uses a no-op progress sink and retains its JSON-only stdout contract without human progress on `stderr`. ## Local capture streams versus remote streams diff --git a/context/cli/async-persistence-guarantees.md b/context/cli/async-persistence-guarantees.md new file mode 100644 index 000000000..f3b42eeeb --- /dev/null +++ b/context/cli/async-persistence-guarantees.md @@ -0,0 +1,67 @@ +# PR2 async persistence guarantees and staging + +Verified guarantees, known limits and PR staging for the directly awaited Turso persistence migration. The runtime architecture and retained blocking workers live in [architecture](../architecture.md#application-execution-runtime). + + +Each statement points to a test, a type-level argument or a named limitation. + +- **Exclusive transactions:** `execute_transactional_cas_batch` and + `execute_transactional_insert_pair_if_absent` take `&mut self` and use the + checked `Transaction::new`, so no other operation on the same adapter can enter + an active transaction (compile-time argument). Turso 0.8.1 rolls back lazily: + an abandoned uncommitted transaction leaves no committed partial state and the + connection stays reusable (`db::transaction_cancellation_tests`); a commit that + already completed may be durable when its caller is cancelled. Bounded + contention retry is covered by `agent_trace_db::transaction_tests`. See + [shared Turso adapter](../sce/shared-turso-db.md). +- **Credential rotation:** the replacement token from a refresh is persisted by an + owned unit that outlives caller cancellation and the outer `/state` timeout, + and the single-flight guard spans refresh and persistence. Limit: a crash after + the remote rotation and before the save, or an ambiguous remote result, can + still lose the rotated token. See [sync command](agent-trace-sync-command.md). +- **Sync termination:** a terminal stream error (for example `403`) stops sibling + streams from starting further batches while in-flight credential persistence + still completes (`terminal_stream` tests). +- **Adapter lifecycle and guard descriptors:** a started state transition keeps the + boundary lease until it ends, and guarded-shell descriptors are inherited only + by the guarded child; see [adapter lifecycle](opencode-mutation-scope-adapter-lifecycle.md) + and [external mutation guard](mutation-trace-external-mutation-guard.md). +- **Known limits:** + - Future size and `recursion_limit` (PR3 audit, Linux debug build, + `RUSTC_BOOTSTRAP=1 RUSTFLAGS=-Zprint-type-sizes`, disposable worktree): + the top-level `app::run` future is 14000 bytes and `main`'s `block_on` + future 14016; the chain `run_with_dependency_check` 13952, + `run_command_lifecycle` 13568, `execute_command_phase` 13168, + `SyncCommand::execute_with_stderr` 13072, `run_current_sync_without_progress` + 12784, `run_sync_async` 12040 and `join_three_to_completion` 11832 carries + almost all of it. The dependency's own `DatabaseSyncEngine::open_db` future + is 12704 bytes, so the size is dominated by a third-party future captured + inline and each wrapper adds roughly 50-200 bytes. No safe reduction exists + without boxing (out of scope). Building without the override fails with + "queries overflow the depth limit" (depth increased by 130 computing the + layout of `app::run`; the default limit is 128), so `#![recursion_limit = + "256"]` in `cli/src/main.rs` is kept. + - Temporary Git index: `TempIndexGuard` removes its file on completion or drop, + but a killed process can leave an orphaned `index-` file under the + runtime tmp directory. No automatic sweeper is added: there is no ownership + or lock record for the file and name or age alone is not evidence of + abandonment. A sweep needs a separate change that first defines that + protocol. + - Guarded-shell descriptor atomicity: the lifetime pipe is created + close-on-exec atomically with `pipe2` on Linux only; elsewhere `pipe` then + `fcntl` leaves a small fork/exec window. This race is an unresolved + limitation and is not fixed here. + - Blocking sites: the 14 production `spawn_blocking` sites match the + retained-worker list in [architecture](../architecture.md#application-execution-runtime); + none runs a database or Turso future. + The ordinary Rust suites deleted during the async migration are not restored; + correctness evidence is the focused regression set, the Quint MBT suite and + surviving targeted tests. + +## Staging + +- **PR1** (complete): application-owned multi-thread Tokio runtime and static async command dispatch. +- **PR2**: async Turso core and the callers it required (credentials, token storage, lifecycle, hooks, mutation trace, sync/export, setup/doctor); the Git/process/filesystem async work and cancellation protections above; transaction isolation, credential-rotation, terminal-sync, adapter-boundary and FD-inheritance fixes; removal of the DB runtime bridge and its blocking scopes. +- **PR3** (implementation complete; final `/validate` pending): post-migration cleanup and runtime audit. Verified-dead code and blanket dead-code allowances were deleted, redundant `async` wrappers, `run_with_retry_sync` and the redundant `drop(storage)` were removed, `join_three_to_completion` was retained, the `recursion_limit = "256"` override was kept, and no temporary-index sweeper was added. The final unsuppressed audit on Linux reports 32 binary-target and 27 test-target diagnostics, all covered by declaration-level `#[allow(dead_code, reason = ...)]` exceptions: the `diff_traces` compatibility surface (cites the 2026-10-01 ADR), the single-event mutation loaders with no production reader (the former ref-reconciliation exemptions were removed once `sce doctor --fix` wired the pass), cfg-gated `PlatformFamily` variants and `GuardError::UnsupportedPlatform`, and never-read `CoordinateOutcome`/`committed`/`completed` fields kept as suspected defects. Linux-only diagnostics do not prove cross-platform safety. Cosmetic and maintainability only. +- **PR4**: OpenTelemetry (not implemented in PR2). + diff --git a/context/cli/capability-traits.md b/context/cli/capability-traits.md index c919e0143..07cd66bd8 100644 --- a/context/cli/capability-traits.md +++ b/context/cli/capability-traits.md @@ -4,14 +4,12 @@ The CLI exposes broad capability traits in `cli/src/services/capabilities.rs` an ## Current contract -- `FsOps: Send + Sync` defines broad filesystem operations: `read_file`, `write_file`, `metadata`, and `exists`. -- `StdFsOps` is the production filesystem implementation and wraps `std::fs` / `Path::exists` with `anyhow` context on fallible operations. -- `GitOps: Send + Sync` defines broad git process capabilities: `run_command`, `resolve_repository_root`, `resolve_hooks_directory`, and `is_available`. +- `FsOps: Send + Sync` is a method-less filesystem capability marker; the unused read/write/metadata/exists operations were removed. +- `StdFsOps` is the production `FsOps` implementation. +- `GitOps: Send + Sync` defines broad git process capabilities: `run_command` and `is_available`. - `ProcessGitOps` is the production git implementation and shells out to `git`, returning stdout as UTF-8 text with command/directory context on failure. -- `resolve_repository_root` uses `git rev-parse --show-toplevel`. -- `resolve_hooks_directory` uses `git rev-parse --git-path hooks` and resolves relative hook paths against the provided repository root. - `AppRuntime` owns concrete `StdFsOps` and `ProcessGitOps` production dependencies alongside concrete observability dependencies; these dependencies are borrowed by `AppContext` rather than stored behind type-erased runtime containers. -- `AppContext` is generic over logger, telemetry, filesystem, and git capability implementations and borrows them from the runtime; it is passed through directly awaited static `RuntimeCommand::execute` (enum defined in `cli/src/services/command_registry.rs`) behind narrow accessor bounds. Because it borrows from `AppRuntime`, `AppContext` is command-scoped: borrowing across directly awaited execution is safe while the owner lives, but detached work cannot retain that borrowed view. No new `Send`, `Sync`, or `'static` command bounds are required. Its inherent `logger()` / `telemetry()` / `fs()` / `git()` helpers return the concrete generic references (`&L`, `&T`, `&F`, `&G`), and `HasLogger`, `HasTelemetry`, `HasFs`, and `HasGit` use associated capability types that return `&Self::{Capability}` instead of object-erased `&dyn ...` values. This is the current static-DI boundary: command and service call sites express narrow capability requirements while preserving concrete borrowed dependency types. `ContextWithRepoRoot` derives a repository-scoped context while preserving the borrowed runtime capability objects. +- `AppContext` is generic over logger, telemetry, filesystem, and git capability implementations and borrows them from the runtime; it is passed through directly awaited static `RuntimeCommand::execute` (enum defined in `cli/src/services/command_registry.rs`) behind narrow accessor bounds. Because it borrows from `AppRuntime`, `AppContext` is command-scoped: borrowing across directly awaited execution is safe while the owner lives, but detached work cannot retain that borrowed view. No new `Send`, `Sync`, or `'static` command bounds are required. Its inherent `logger()` helper returns the concrete generic reference (`&L`), and `HasLogger`, `HasTelemetry`, and `HasGit` use associated capability types that return `&Self::{Capability}`, while `HasFs` only declares its associated `Fs` type as a bound, instead of object-erased `&dyn ...` values. This is the current static-DI boundary: command and service call sites express narrow capability requirements while preserving concrete borrowed dependency types. `ContextWithRepoRoot` derives a repository-scoped context while preserving the borrowed runtime capability objects. - Current command execution bounds are capability-oriented: central dispatch requires logger access plus repo-root scoping, context-free commands accept any context, hooks require logger access, and setup/doctor require repo-root scoping. - Test-only `UnimplementedFsOps` and `UnimplementedGitOps` stubs are available under `capabilities::test_stubs` for tests that need to satisfy trait bounds before providing focused fakes. diff --git a/context/cli/cli-command-surface.md b/context/cli/cli-command-surface.md index bf9a4a377..5a65724e4 100644 --- a/context/cli/cli-command-surface.md +++ b/context/cli/cli-command-surface.md @@ -16,25 +16,29 @@ Operator onboarding currently comes from `sce --help`, command-local `--help` ou ## Application runtime dispatch -`RuntimeCommand::execute_with_stderr` directly awaits Auth and Sync. The temporary `block_in_place` scopes protect construction and destruction of legacy DB-owned runtimes; ordinary filesystem, Git, parsing and rendering work alone does not require a scope. Lifecycle providers and synchronous command APIs remain synchronous. +`RuntimeCommand::execute_with_stderr` directly awaits database-backed Auth, +Setup, Doctor, Hooks, and Sync paths under the application multi-thread Tokio +runtime. The static enum dispatches concrete service command types; async +lifecycle providers and hook handlers await the shared Turso adapters. Pure +filesystem, Git, parsing, formatting, and rendering operations remain +synchronous. No DB-related `block_in_place`, `spawn_blocking`, or storage-drop +guard remains in production. See [application execution +runtime](../architecture.md#application-execution-runtime) and the [directly +awaited Turso persistence decision](../decisions/2026-10-07-directly-awaited-turso-persistence.md). | Path | Execution and lifetime reason | | --- | --- | -| Help, HelpText, Version, Completion, Policy, Config Show/Validate | Direct synchronous execution; no runtime-owning DB resources. | -| Setup | Context-only requests execute directly. Other requests execute inside one blocking scope because LocalDb/AuthDb/AgentTraceDb lifecycle setup constructs and destroys DB adapters. | -| Doctor Diagnose/Fix | One blocking scope around execution; AgentTraceDb diagnosis opens a local DB even without repair. | -| Hooks PreCommit/PostRewrite | Direct synchronous execution; these paths do not own DB resources. | -| Hooks CommitMsg, PostCommit, DiffTrace, ConversationTrace, Codex, ClaudeModelState, MutationScope, ClaudeMutationScope, CodexMutationScope, OpenCodeMutationScope, PiMutationScope, ExternalMutationGuard | One blocking scope around each invocation, including its error cleanup; these paths can open or persist through runtime-backed DB adapters. | -| Sync storage construction | One blocking scope around `resolve_agent_trace_storage`, including partially constructed resources on failure. | -| Sync storage destruction | A concrete `SyncStorageGuard` owns storage across HTTP awaits. Its Drop takes and destroys storage inside one blocking scope on completion, error, cancellation or unwind; normal execution explicitly drops the guard before returning. | - -Auth's direct credential load/save/delete and control-plane credential load/save use awaited owned `spawn_blocking` closures. Refresh and HTTP stay async. Hooks finish foreground persistence before returning; the existing post-commit auto-sync child remains detached. See [application execution runtime](../architecture.md#application-execution-runtime), [capability borrowing](capability-traits.md), and [sync](sync-command.md). +| Help, HelpText, Version, Completion, Policy, Config Show/Validate | Direct synchronous execution; no database operations. | +| Setup and Doctor | Async lifecycle-provider dispatch awaits local, auth, and repository DB operations; pure Config/Hooks work stays synchronous within async lifecycle methods. | +| DB-backed Hooks and MutationScope | Static handler dispatch awaits required foreground persistence before returning. | +| Sync | Async repository storage, export reads, credential persistence, and HTTP remain awaited on the application runtime; ordinary ownership retains storage across HTTP calls. | +| Post-commit auto-sync | The existing one-shot sync process remains detached after local persistence succeeds. | ## Onboarding documentation - `sce --help` includes a slim top-level command list and quick-start examples for `setup`, `doctor`, and `version`; `auth` is visible while `hooks` remains implemented in code but hidden from `sce`, `sce help`, and `sce --help` for this phase. - `cli/src/cli_schema.rs` owns the real top-level command catalog metadata for clap-backed commands (purpose text plus `show_in_top_level_help`), while `command_surface::help_text()` consumes that catalog and adds the synthetic `help` row plus the ASCII banner. -- `cli/src/app.rs` now owns an explicit startup lifecycle (`perform_dependency_check` -> `build_startup_context` -> `initialize_runtime` -> `run_command_lifecycle` -> `render_run_outcome`) with command execution directly awaited under the application-level multi-thread Tokio runtime before final stream rendering. See [application execution runtime](../architecture.md#application-execution-runtime) for runtime ownership and temporary DB lifetime scopes. +- `cli/src/app.rs` now owns an explicit startup lifecycle (`perform_dependency_check` -> `build_startup_context` -> `initialize_runtime` -> `run_command_lifecycle` -> `render_run_outcome`) with command execution directly awaited under the application-level multi-thread Tokio runtime before final stream rendering. See [application execution runtime](../architecture.md#application-execution-runtime) for runtime ownership and awaited persistence. - `cli/src/app.rs` also routes clap output through an internal static `RuntimeCommand` enum defined in `cli/src/services/command_registry.rs`, so parse-time conversion and run-time command execution stay separated while avoiding boxed command trait objects. - Command-local help is available for implemented commands including bare `sce auth`, `sce auth --help`, `sce auth login --help`, `sce setup --help`, `sce doctor --help`, and `sce completion --help`; when stdout color is enabled those help payloads now reuse the shared heading/command/placeholder styling pass while non-TTY and `NO_COLOR` flows stay plain text. Human-readable stderr diagnostics and interactive setup prompt text now follow the same shared styling policy on their respective terminal streams. - Current repository verification guidance for this CLI slice uses `nix flake check` for normal repository checks and the separate `nix build .#ci-checks` for the release build (static musl on Linux, native on Darwin) plus Linux portability validation of the real release binary. Both tiers verify changes to the execution/build model. Native packaged installability uses `nix build .#sce` / `nix run .#sce -- --help`; targeted Cargo checks use `nix develop -c ./scripts/run-cli-cargo.sh ...` so generated payload preparation remains intact. @@ -135,7 +139,7 @@ An interactive `setup` run instead resolves the selection through an `inquire` m - `cli/src/services/default_paths.rs` defines the canonical per-user persisted-location seam for config/state/cache roots plus named default file paths for current persisted artifacts (`global config`, `auth tokens`, `local DB`, `agent trace DB`) used by config discovery, token storage, database adapters, and doctor diagnostics; its internal `roots` seam now owns the platform-aware root-directory resolution so non-test production modules consume shared path accessors instead of resolving owned roots directly. - `cli/src/services/agent_trace.rs` defines the canonical Rust SCE web base URL and helpers for Agent Trace conversation URLs, persisted Agent Trace trace URLs, Agent Trace session URLs, and setup-created repo-local config schema URLs. - `cli/src/services/token_storage.rs` defines WorkOS token persistence (`save_tokens`, `load_tokens`, `delete_tokens`) via the encrypted `AuthDb` `auth_credentials` table using a `OnceLock` lazy singleton with constant integer row ID `1`. `TokenStorageError` exposes `PathResolution` and `Database` variants. No JSON file I/O remains. -- `cli/src/services/auth_command/mod.rs` defines the auth command orchestration surface (`AuthRequest`, `AuthSubcommand`, `run_auth_subcommand`) for `login`, `logout`, and `whoami`, including shared text/JSON rendering, login's stored-token validation and refresh path for any stored credential, device-flow fallback after absent or unsuccessfully renewed credentials, source-level typed `CliError` propagation through `run_login_with_stored_credentials` for renewal, token-save, and device-login failures, token-storage-backed logout deletion with idempotent absent-credential success, unauthenticated whoami state reporting, Control Plane `/me` profile retrieval for authenticated whoami, flat safe-field rendering with optional-name/null-value handling, exact logged-out text guidance, precedence-aware client-ID guidance sourced from the shared auth-runtime resolver instead of env-only assumptions, and directly awaited WorkOS/Control Plane HTTP on the application runtime; credential load/save/delete remain synchronous APIs invoked through awaited owned `spawn_blocking` closures; `cli/src/services/auth_command/command.rs` owns the `AuthCommand` payload used by the static `RuntimeCommand` enum. +- `cli/src/services/auth_command/mod.rs` defines the auth command orchestration surface (`AuthRequest`, `AuthSubcommand`, `run_auth_subcommand`) for `login`, `logout`, and `whoami`, including shared text/JSON rendering, login's stored-token validation and refresh path for any stored credential, device-flow fallback after absent or unsuccessfully renewed credentials, source-level typed `CliError` propagation through `run_login_with_stored_credentials` for renewal, token-save, and device-login failures, token-storage-backed logout deletion with idempotent absent-credential success, unauthenticated whoami state reporting, Control Plane `/me` profile retrieval for authenticated whoami, flat safe-field rendering with optional-name/null-value handling, exact logged-out text guidance, precedence-aware client-ID guidance sourced from the shared auth-runtime resolver instead of env-only assumptions, and directly awaited WorkOS/Control Plane HTTP on the application runtime; credential load/save/delete are async APIs directly awaited on the application runtime; `cli/src/services/auth_command/command.rs` owns the `AuthCommand` payload used by the static `RuntimeCommand` enum. - `cli/src/app.rs` parses `auth`, `config`, `setup`, `doctor`, `hooks`, `policy`, `sync`, `version`, and `completion` into service-owned runtime command handlers so runtime messages are sourced from domain modules instead of inline strings. ## Local and Agent Trace Turso adapter behavior @@ -177,8 +181,8 @@ An interactive `setup` run instead resolves the selection through an `inquire` m ## Dependency baseline - `cli/Cargo.toml` currently declares runtime dependencies on `anyhow`, `chrono`, `clap`, `clap_complete`, `dirs`, `hmac`, `inquire`, `jsonschema`, `keyring-core`, `murmur3`, `owo-colors`, `rand`, `reqwest`, `serde`, `serde_json`, `sha2`, `tokio`, `tracing`, `turso`, and `uuid`, plus target-specific keyring backend dependencies for Linux/FreeBSD, macOS, and Windows. No CLI dev-dependencies are currently declared. -- `tokio` is pinned with `default-features = false` and enables explicit `rt`, `io-util`, `sync`, `time`, `macros`, and `rt-multi-thread` for application execution and retained synchronous DB bridging, plus timer-backed bounded retry/timeout behavior in resilience-wrapped operations. -- `cli/src/services/auth.rs` provides HTTP-only async functions for the WorkOS Device Authorization Flow (`request_device_authorization`, `poll_for_device_token`, `complete_device_auth_flow_returning_token`) and token-refresh operations (`ensure_valid_token_returning_token`, `renew_stored_token_from_refresh_token`, `is_stored_token_expired`): it requests device codes, polls at fixed API interval (adding 5 seconds on `slow_down`), maps RFC 8628 terminal errors to actionable `Try:` guidance, checks token expiry from persisted `stored_at_unix_seconds + expires_in` with a bounded skew guard, refreshes expired access tokens through `/oauth/token` using `grant_type=refresh_token`, and retries transient refresh failures via the shared resilience wrapper. Token storage (`save_tokens`, `load_tokens`) is called by the caller in `cli/src/services/auth_command/mod.rs` in awaited owned `spawn_blocking` closures, isolating the synchronous DB/keyring work and retaining storage error classification. +- `tokio` is pinned with `default-features = false` and enables explicit `rt`, `io-util`, `sync`, `time`, `macros`, and `rt-multi-thread` for application execution and timer-backed bounded retry/timeout behavior in resilience-wrapped operations. +- `cli/src/services/auth.rs` provides HTTP-only async functions for the WorkOS Device Authorization Flow (`request_device_authorization`, `poll_for_device_token`, `complete_device_auth_flow_returning_token`) and token-refresh operations (`ensure_valid_token_returning_token`, `renew_stored_token_from_refresh_token`, `is_stored_token_expired`): it requests device codes, polls at fixed API interval (adding 5 seconds on `slow_down`), maps RFC 8628 terminal errors to actionable `Try:` guidance, checks token expiry from persisted `stored_at_unix_seconds + expires_in` with a bounded skew guard, refreshes expired access tokens through `/oauth/token` using `grant_type=refresh_token`, and retries transient refresh failures via the shared resilience wrapper. Token storage (`save_tokens`, `load_tokens`, `delete_tokens`) is async and directly awaited by AuthCommand on the application runtime, preserving typed storage error classification. The control-plane client uses a concrete generic async `CredentialStore`; DB work crosses no `spawn_blocking` boundary. ## Scope boundary for this phase diff --git a/context/cli/codex-mutation-scope-integration.md b/context/cli/codex-mutation-scope-integration.md index a72ebff88..30084c317 100644 --- a/context/cli/codex-mutation-scope-integration.md +++ b/context/cli/codex-mutation-scope-integration.md @@ -145,7 +145,7 @@ state-to-ingress-to-state across hook processes (boundary lock then state lock); file existence alone is never ownership. Both locks are `AdapterLockSpec` identities (`state::STATE_LOCK`, `state::BOUNDARY_LOCK`) over the single OS advisory-lock implementation in `cli/src/services/hooks/mutation_scope_lock.rs`, -shared with the OpenCode and Pi adapters. +shared with the OpenCode and Pi adapters. `BOUNDARY_LOCK` is held as a task-local lease (`run_under_boundary`) that `run_locked_blocking` workers clone, so a started state transition outlives caller cancellation under boundary protection. The persistence and recovery-bookkeeping contracts common to the Codex, OpenCode, and Pi state files (missing/malformed/wrong-version reads, durable replace, remove, normalize, complete, relinquish, single flush claimant) are asserted once diff --git a/context/cli/default-path-catalog.md b/context/cli/default-path-catalog.md index 2e4ef6e60..50c44cf6c 100644 --- a/context/cli/default-path-catalog.md +++ b/context/cli/default-path-catalog.md @@ -25,13 +25,12 @@ - `.opencode/`, `.opencode/opencode.json` - `.claude/` - `.pi/` -- `.git/`, `.git/hooks/`, `.git/COMMIT_EDITMSG` - `context/`, `context/plans/`, `context/decisions/`, `context/handovers/`, `context/tmp/`, `context/tmp/.gitignore` via `RepoPaths::context_tmp_gitignore_file()` ### Install paths - OpenCode plugin/catalog targets under `.opencode/` -- required git hook install targets under `.git/hooks/` +- required git hook install targets are resolved from git truth by the setup service, not by this catalog (the former `.git/`, hooks and `COMMIT_EDITMSG` path helpers were removed as unused) Compile-time payload locations are build-script-owned `OUT_DIR` paths rather than default runtime paths and are intentionally absent from this catalog. @@ -41,7 +40,7 @@ Compile-time payload locations are build-script-owned `OUT_DIR` paths rather tha - `cli/src/services/config/mod.rs` now resolves the default repo-local config path through `RepoPaths::sce_config_file()` during config discovery. - `cli/src/services/doctor/inspect.rs` now resolves the repo-local config path through `RepoPaths::sce_config_file()` for local-config health reporting and validation. - `cli/src/services/doctor/inspect.rs` also resolves OpenCode manifest/plugin/preset locations through shared `RepoPaths` and `InstallTargetPaths` accessors instead of owning those paths locally. -- `cli/src/services/setup/mod.rs` now resolves setup target directory names and required hook identifiers through `default_paths.rs` constants/accessors instead of owning those path literals locally. +- `cli/src/services/setup/mod.rs` now resolves setup target directory names through `default_paths.rs` constants/accessors instead of owning those path literals locally. - `cli/src/services/default_paths.rs` includes a regression test that scans non-test Rust source under `cli/src/` and fails when new centralized production path literals appear outside the default-path service. - Active hook runtime no longer resolves or writes collision-safe JSON artifacts under `context/tmp/`; `context/tmp/` remains a repo-relative scratch/session path owned by the default path catalog. - Active hook runtime and `cli/src/services/agent_trace_db/lifecycle.rs` resolve repository-scoped Agent Trace DB files through `agent_trace_storage` and `agent_trace_db_path_for_repository(repository_id)`. Outside a Git repository, lifecycle code returns an actionable "requires a Git repository" diagnostic instead of resolving any sentinel path. diff --git a/context/cli/doctor-ref-reconciliation.md b/context/cli/doctor-ref-reconciliation.md new file mode 100644 index 000000000..1d53a602c --- /dev/null +++ b/context/cli/doctor-ref-reconciliation.md @@ -0,0 +1,46 @@ +# Doctor snapshot-ref reconciliation + +`sce doctor --fix` is the only production caller of snapshot-ref reconciliation. +Plain `sce doctor` only reports whether a pass is recommended. Automatic +reclamation is not delivered; it is a documented follow-up (see +[`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md) and +[`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)). + +## Runtime facade + +`cli/src/services/mutation_trace/runtime/ref_doctor.rs` is the only crate-visible +surface doctor uses; it keeps the maintenance types private to `runtime`: + +- `run_explicit_reconciliation(repository_root)` wraps `reconcile_explicit` + (one `WorktreeLock` acquisition, verified-existing Agent Trace DB, state + recorded under the same lock) and returns a plain `ReconciliationFix`. +- `inspect_reconciliation_recommendation(repository_root)` derives the Git dir + from the filesystem, reads `/sce/ref-maintenance.json`, and applies the + pure `evaluate_recommendation`. It takes no lock and writes no state or ref. It + returns `None` unless reconciliation is recommended or the state is unusable. + +Doctor receives both as injected seams (`execute_doctor_with_lifecycle_providers`). + +## `--fix` reporting + +After the existing lifecycle, config and mutation-scope repairs, `--fix` always +runs one pass, regardless of the recommendation, failure streak, or timestamp +validity, and appends one fix-result row (category `mutation_scope_health`): + +| Pass result | Outcome | Detail | +| --- | --- | --- | +| `Completed` | `fixed` | deleted / retained / locally required counts | +| `CompletedStatePersistFailed` | `fixed` | counts plus a diagnostics-not-recorded warning | +| `Failed` | `failed` | typed kind (for example `agent_trace_db_missing`, `agent_trace_db_incompatible_schema`, `agent_trace_db_missing_metadata`, `agent_trace_db_repository_mismatch`, `malformed_ref`, `missing_required_pins`, `durable_roots_unavailable`, `delete_transaction_failed`) and message, plus any state warning | +| `Skipped(Busy)` | `skipped` | another boundary or pass holds the worktree lock; retry | + +A busy pass leaves state unchanged and is not a failure. While a pass runs, a +mutation boundary may wait up to the 10 s coordinator lock timeout. + +## Plain `sce doctor` + +When reconciliation is recommended (stale since the last success, an outstanding +failed attempt, invalid timestamps, or an unreadable state file) the text report +adds a `Snapshot ref reconciliation` section and the JSON report carries a +`ref_reconciliation` object (`null` otherwise). The recommendation does not depend +on whether a hook reminder was ever seen. diff --git a/context/cli/mutation-scope-hook-ingress.md b/context/cli/mutation-scope-hook-ingress.md index 561fc6718..1f7dc40ee 100644 --- a/context/cli/mutation-scope-hook-ingress.md +++ b/context/cli/mutation-scope-hook-ingress.md @@ -258,4 +258,5 @@ The Codex mapping and partial coverage are in [`codex-mutation-scope-integration - [Mutation-trace scope abandonment](mutation-trace-scope-abandonment.md) - [Mutation-trace protected worktree](mutation-trace-protected-worktree.md) - [Mutation-scope provenance](mutation-scope-provenance.md) +- [Mutation-scope ref-reconciliation advisory](mutation-scope-ref-advisory.md) - [Agent Trace hooks command routing](../sce/agent-trace-hooks-command-routing.md) diff --git a/context/cli/mutation-scope-ref-advisory.md b/context/cli/mutation-scope-ref-advisory.md new file mode 100644 index 000000000..b7bfd2c5b --- /dev/null +++ b/context/cli/mutation-scope-ref-advisory.md @@ -0,0 +1,95 @@ +# Mutation-scope ref-reconciliation advisory (hook trigger) + +The shared mutation-scope ingress (`cli/src/services/hooks/mutation_scope.rs`, +see [`mutation-scope-hook-ingress.md`](mutation-scope-hook-ingress.md)) emits a +lightweight, advisory-only signal that snapshot-ref reconciliation is +recommended. It never reconciles, lists refs, opens the Agent Trace DB, or spawns +Git. Reconciliation itself is operator-driven; this signal is only a reminder. + +## Trigger policy + +`drive_mutation_scope` takes an injected advisory seam (production: +`runtime::advise_after_completed_boundary`) and a generic diagnostic writer +`E: Write` (production: `std::io::stderr()`, supplied by +`run_mutation_scope_from_payload`). The seam runs exactly once, and only when all +of these hold: + +- the boundary is `Close` or `Flush`; +- the `coordinate_boundary(...)` result is `Ok` or + `CoordinateError::MarkerClearAfterCommit` (a durably completed boundary). + +It runs after the coordinator future has returned, so the coordinator's +`WorktreeLock` is already released. It never runs for `Start`, `Advance`, +`Abandon`, or a failed coordinate. The hook result is computed before the seam +runs and is never altered by it, including when the diagnostic write fails. The +advisory never creates, clears, or modifies the external-taint marker. + +## Outcomes, logging and diagnostics + +Every outcome is logged as `sce.hooks.mutation_scope.ref_reconciliation_advisory` +with an `outcome` field (plus `persistence_warning` when present). Actionable +outcomes are additionally written, one `SCE: ...` line each, to the diagnostic +writer. Stdout stays empty and never carries advisory text. + +| Outcome | Log | Diagnostic (stderr) | +|---|---|---| +| `Anchored` | debug | none | +| `NoAction` | debug | none | +| `Busy` | debug | none | +| `Advised` | warn | `SCE: Reconciliation is recommended. Run sce doctor --fix.` | +| `AdvisedDurabilityUncertain` | warn, with `persistence_warning` | the line above, then `SCE: Advisory-state durability could not be confirmed: .` | +| `StateWriteFailed` + `NotApplied` | warn, with `persistence_warning` | `SCE: Reconciliation advisory state could not be written: . Run sce doctor to check reconciliation status.` (replacement did not occur; previous state preserved; no advice) | +| `StateWriteFailed` + `DurabilityUncertain` | warn, with `persistence_warning` | `SCE: Reconciliation advisory-state durability could not be confirmed: . Run sce doctor to check reconciliation status.` (anchor or normalization write; replacement occurred, new state may be visible, crash durability unconfirmed; no advice) | +| `StateUnavailable` | warn | `SCE: Reconciliation advisory state is unavailable. Run sce doctor to check reconciliation status.` | + +`StateWriteFailed` means no confirmed successful advisory-state update, not +necessarily that atomic replacement failed; it carries the `PersistFailure` so the +two phases stay distinguishable. `AdvisedDurabilityUncertain` is reserved for a +due recommendation whose advice-state replacement occurred with uncertain +durability (the recommendation is still emitted). `StateUnavailable` means the +state could not be accessed or evaluated and is not a persistence-write outcome. + +The diagnostic does not depend on the logger (it is emitted with `logger = None`), +and `Logger::warn` and the observability defaults are unchanged: observability +routes only ERROR records to stderr, and only when file logging is off, so WARN +alone would not reach the operator. A write or flush error on the diagnostic +writer is ignored. Only this path writes the advisory diagnostic to stderr. + +## Advisory behavior and cadence + +`ref_advisory::advise_if_due` (`cli/src/services/mutation_trace/runtime/`) takes +no Git, DB, store, or snapshot parameters. It derives the Git dir from the +filesystem (no Git subprocess), makes one zero-wait `try_lock` of +`WorktreeLock`, and reads/writes the small per-worktree state file +`/sce/ref-maintenance.json` (capped, versioned, past-event timestamps +only, atomic staging-and-swap writes). It persists `last_advised` before +reporting `Advised`. A busy lock or unreadable state is reported, never an error. + +- **Time-based reminders:** the first use anchors the state. A stale state + becomes eligible after `RECONCILIATION_ADVISORY_AFTER` (24 h); further + stale-state reminders are suppressed for 24 h after `last_advised`. +- **Failure-triggered reminders:** an outstanding failed explicit reconciliation + is recommended immediately. A newly recorded failure (recorded after + `last_advised`) may be advised even inside the previous 24 h window; once + recorded, that reminder is suppressed. Repeated reminders for the same + unresolved failure follow the normal 24 h cadence. A successful explicit + reconciliation clears the failure-based recommendation. +- Anchoring and clock-skew normalization are the only other state writes, so + state is not written "at most once per window" unconditionally. +- `sce doctor` reports the recommendation from state regardless of whether a hook + reminder was ever delivered. + +## Delivery guarantee and limits + +- Best-effort, not exactly-once, and not guaranteed to be seen by a human. + `last_advised` is persisted before the diagnostic, so a crash or an unseen + stderr can still consume one reminder; the next reminder follows the cadence. +- Claude Code and Codex run the hook as a command and can capture its stderr, but + rendering of exit-0 hook stderr is up to the harness. The OpenCode plugin and + the Pi extension discard the child's stderr, so there the structured log file + and `sce doctor` are the supported surfaces. Changing the plugins is out of + scope here. +- Time-based, not evidence-based: orphan detection needs the Git/DB scan this + path deliberately avoids. +- Orphan pins are reclaimed only by an explicit reconciliation pass. Automatic + reclamation is not provided. diff --git a/context/cli/mutation-scope-runtime.md b/context/cli/mutation-scope-runtime.md index d07fbfdc8..d099e1384 100644 --- a/context/cli/mutation-scope-runtime.md +++ b/context/cli/mutation-scope-runtime.md @@ -33,7 +33,7 @@ The mechanics live in [`mutation-trace-runtime-coordinator.md`](mutation-trace-r `ExternalTaintOperation` crosses the boundary because it is part of `CoordinateError`'s public shape. It lives in `protected_worktree.rs` and is re-exported by `coordinator.rs` without making that module public. -Every runtime `mod` stays private, including `ProtectedWorktree`, its error, `WORKTREE_LOCK_TIMEOUT`, and `reconcile_worktree`. Adapters drive only the two entrypoints and never assemble the safety prefix. +Every runtime `mod` stays private, including `ProtectedWorktree`, its error, `WORKTREE_LOCK_TIMEOUT`, and `reconcile_explicit`. Adapters drive only the two entrypoints and never assemble the safety prefix. The re-exports are the intentional crate-visible seam consumed by the generic ingress; runtime internals stay private. The two re-export statements retain `#[allow(unused_imports)]` because no consumer names the completing types yet. diff --git a/context/cli/mutation-trace-agent-attribution.md b/context/cli/mutation-trace-agent-attribution.md index b714ae4df..ee0ab64fb 100644 --- a/context/cli/mutation-trace-agent-attribution.md +++ b/context/cli/mutation-trace-agent-attribution.md @@ -186,8 +186,6 @@ observable tree difference. committed `src/lib.rs` even with identical added text and line number; an exact nested path still attributes; multiple similar nested mutation paths stay independent of the target; a shared basename alone is not a match. -- `runtime/tests.rs` — a still-relevant event behind 128 newer events is never - loaded or reconstructed. - `hooks/mod.rs` (`mutation_attribution_e2e`) — real Git/DB: mutation-only `ai` without provenance; direct+mutation completion; a newer non-exclusive event keeps a line non-AI; adversarial linked-worktree isolation; the three diff --git a/context/cli/mutation-trace-external-mutation-guard.md b/context/cli/mutation-trace-external-mutation-guard.md index aa7570a7e..a345c9525 100644 --- a/context/cli/mutation-trace-external-mutation-guard.md +++ b/context/cli/mutation-trace-external-mutation-guard.md @@ -1,6 +1,6 @@ # External-mutation guard -`run_external_mutation_guard`, in +`arm_external_mutation_guard` (which yields an `ArmedExternalMutationGuard` whose `exec` runs the shell), in `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, is a harness-neutral runtime primitive — **not** Pi-specific, despite being built alongside the Pi adapter to back Pi's `!`/`!!` `user_bash` — that lets a @@ -27,8 +27,8 @@ composition rather than attributing the ambiguous interval to AI. ```text caller starts the hidden supervisor with {"operation":"arm"} -> acquire ProtectedWorktree (WorktreeLock + ExternalTaintMarker, write-ahead) - -> create a Unix pipe: supervisor owns the read end; the writer is - CLOEXEC-clear and remains supervisor-owned while waiting + -> create a Unix pipe (CLOEXEC set atomically via pipe2 on Linux only; pipe+fcntl elsewhere leaves an unresolved fork/exec window): supervisor + owns both ends while waiting -> durably establish the lifetime-token infrastructure -> write and flush {"status":"armed"} -> WAIT in ArmedWaitingForExec; no shell exists yet @@ -91,7 +91,7 @@ therefore emits no `Armed` event and spawns no shell. The lifetime pipe is separate from the flock. The lock serializes runtime access for the whole protected interval; the pipe tells the still-live supervisor when ordinary shell descendants are gone. The pipe read end is -CLOEXEC, the writer is explicitly CLOEXEC-clear before spawn, the supervisor +CLOEXEC and so is the writer in the parent; only the guarded child clears CLOEXEC on the lock and writer descriptors in a `pre_exec` (`fcntl` only), so unrelated subprocesses never inherit them. The supervisor closes its writer immediately after successful spawn, and only the shell's inherited writer references remain. Normal Unix `fork`/`exec` fd inheritance passes that writer to ordinary descendants, so EOF means no ordinary @@ -232,7 +232,7 @@ and its helpers: **The lifetime pipe proves ordinary descendant completion.** The supervisor creates a pipe before spawning. Its read end stays in the supervisor and its -write end is explicitly made CLOEXEC-clear before `Command::spawn()`. The +write end is created close-on-exec (`pipe2(O_CLOEXEC)` on Linux; elsewhere `pipe` then `fcntl`, leaving a small fork/exec window) and the child clears CLOEXEC on it in `pre_exec`. The supervisor closes its own writer after spawn. Because the shell inherits the writer across `exec`, and ordinary descendants inherit it under normal Unix fd inheritance, a readable EOF is equivalent to the kernel having closed the @@ -268,7 +268,7 @@ eventual explicit unlock. wrapper around the existing (still-private) `coordinate_protected`, plus ordinary OS process/fd mechanics. - No Windows support: the guard is Unix-only; a non-Unix build's - `run_external_mutation_guard` unconditionally returns + `arm_external_mutation_guard` unconditionally returns `GuardError::UnsupportedPlatform`. - No recovery beyond the guard's own crash semantics above — a stale, never-armed marker left by some other failure mode is out of scope here. diff --git a/context/cli/mutation-trace-external-taint.md b/context/cli/mutation-trace-external-taint.md index 4141be60b..5dbe935cb 100644 --- a/context/cli/mutation-trace-external-taint.md +++ b/context/cli/mutation-trace-external-taint.md @@ -73,9 +73,15 @@ resolve git_dir → acquire WorktreeLock ┐ → open_db() ← DB acquired INSIDE the fence → GitSnapshotService::new → coordinate_boundary(&db, .., inherited_external_taint) - → ProtectedWorktree::complete() ← clears; only on a successful outcome + → ProtectedWorktree::complete().await ← clears; only on a successful outcome ``` +`exists()`/`persist()` run in an arming `spawn_blocking` worker that owns the +already-acquired `WorktreeLock`, and `clear()` runs in a completion worker that +owns the whole `ProtectedWorktree`. No marker I/O runs on a Tokio worker, and +no marker operation can outlive the lock that protects it. See +[`mutation-trace-protected-worktree.md`](mutation-trace-protected-worktree.md#blocking-worker-ownership). + **Safety invariant:** no failure after the marker is armed — including a failure to open the Agent Trace DB, or to resolve the Git-derived worktree identity — can disappear without leaving the worktree-local signal for the next invocation. @@ -170,7 +176,7 @@ window: the recovery is durable (cursor rebaselined, scope abandoned, revision advanced), the triggering boundary is unprocessed with no `MutationEvent`, the on-disk marker survives, and a later `coordinate()` inherits it and recovers conservatively again without resurrecting the abandoned scope. -`runtime/tests.rs`'s +The former `runtime/tests.rs` test `a_marker_clear_failure_after_a_durable_boundary_keeps_the_marker_for_a_later_recovery` proves an attributable `Advance` commits durably, the returned `MarkerClearAfterCommit` carries the matching `committed` outcome diff --git a/context/cli/mutation-trace-protected-worktree.md b/context/cli/mutation-trace-protected-worktree.md index f08d35df4..d8a61bf9a 100644 --- a/context/cli/mutation-trace-protected-worktree.md +++ b/context/cli/mutation-trace-protected-worktree.md @@ -19,13 +19,39 @@ only piece they hold in common. ```mermaid flowchart TD A["resolve git_dir
(runtime::git_snapshot::resolve_git_dir)"] --> B["acquire WorktreeLock
(bounded WORKTREE_LOCK_TIMEOUT, 10s)"] - B --> C["ExternalTaintMarker::exists()
→ inherited_external_taint"] + B --> C["ExternalTaintMarker::exists()
→ inherited_external_taint
(arming worker owns the lock)"] C --> D["ExternalTaintMarker::persist()
fence armed, write-ahead"] D --> E["resolve_worktree_id from Git topology
→ WorktreeId"] E --> F["caller's runtime operation
(DB provider, snapshot, protocol, CAS)"] - F --> G["complete() clears the marker
(lock still held)"] + F --> G["complete() clears the marker
(completion worker owns the guard and its lock)"] ``` +### Blocking-worker ownership + +The marker's synchronous filesystem I/O (`symlink_metadata`, `create_dir_all`, +`open`, `sync_data`, `remove_file`, parent-directory `sync_all`) never runs on +a Tokio worker. It runs on Tokio's blocking pool behind two ownership +boundaries, each of which owns the worktree lock for its whole duration: + +1. **Arming.** Lock acquisition stays a separate `acquire_inner_async` worker, + so a caller cancelled while waiting for the lock still writes no marker. + Only after acquisition succeeds does the private `arm_marker_with_lock` move + the acquired `WorktreeLock` into a second `spawn_blocking` worker that runs + `exists()` then `persist()` and returns the lock, marker, and inherited flag. + If the awaiting caller is cancelled mid-arming, the worker still owns the + lock until persistence finishes, so the marker is left armed and the lock + releases afterwards. +2. **Completion.** `complete(self)` is `async`. It moves the entire + `ProtectedWorktree` (through the by-value `complete_blocking(self)` method, + not just the `marker` field) into a `spawn_blocking` worker, so the lock is + released only after `clear()` returns. A cancelled completion therefore + cannot release the lock early and let an older clear delete a newer + operation's marker. If `complete()` is dropped before it spawns its worker, + the guard just drops and the marker stays armed. + +Git, worktree-identity, and Turso work stay on the async side. Neither worker +runs DB futures. + The fence is armed **write-ahead of every fallible step that follows it**, including the DB acquisition and any durable-state lookup. A process that dies anywhere past that point leaves the worktree-local signal behind for the next @@ -43,9 +69,11 @@ ProtectedWorktreeError>` runs the whole prefix. The guard then exposes: - `inherited_external_taint() -> bool` — whether a marker was already present on entry, i.e. whether some earlier invocation never proved a trustworthy durable completion. -- `complete(self) -> anyhow::Result<()>` — clears the marker while the worktree - lock is still held, then releases the lock as the guard is consumed. This is - the **only** thing that clears the marker. +- `async complete(self) -> anyhow::Result<()>` — moves the whole guard into a + blocking worker that clears the marker while the worktree lock is still held, + then releases the lock as the guard is consumed. This is the **only** thing + that clears the marker. A completion-worker panic surfaces as an error and + leaves the marker state unspecified, with no retry. `WORKTREE_LOCK_TIMEOUT` (10s) is owned by this module. `pub(super) acquire_inner(repository_root, on_lock_contention)` carries the lock-contention @@ -66,6 +94,7 @@ map it onto its own error surface without losing which safety step failed: | `GitDirResolution(anyhow::Error)` | before the lock | untouched | | `LockAcquisition(WorktreeLockError)` | lock acquire/timeout | untouched | | `ExternalTaintMarker { operation: Inspect \| Persist, source }` | fence inspect/arm | left as it was | +| `MarkerWorkerFailed(JoinError)` | arming blocking worker panicked or was cancelled by the runtime while holding the lock | unknown; never auto-cleared | | `CheckoutIdentity(anyhow::Error)` | Git-derived worktree identity resolution, after the fence is armed | **armed** | `ExternalTaintOperation` lives here, beside the fence step that produces it, and @@ -75,7 +104,10 @@ keeps naming it; `AbandonScopeError::ExternalTaintMarker` carries the same type. onto `CoordinateError::Other`, `LockAcquisition` onto `CoordinateError::LockAcquisition`, and the fence variant onto `CoordinateError::ExternalTaintMarker` with the same `operation` — the exact -variants that step produced before the extraction. +variants that step produced before the extraction. `MarkerWorkerFailed` maps +onto `CoordinateError::Other` / `AbandonScopeError::Other`, keeping the +`JoinError` cause, and the external mutation guard passes it through +`GuardError::Acquire`. ## Testing boundary @@ -88,6 +120,16 @@ lock; the lock is proven held for the guard's whole lifetime through the `on_lock_contention` seam; and a prefix that times out against a held lock fails with `LockAcquisition` having armed nothing. +Two one-worker (`multi_thread`, `worker_threads = 1`) regressions use static +`FnOnce() + Send + 'static` hooks that run inside the blocking workers to pause +them deterministically. They prove that an unrelated Tokio timer keeps making +progress while the hook holds the worker; that aborting the awaiting caller +leaves the worktree lock held, so another acquisition times out; that cancelled +arming leaves the marker armed, so the next `acquire()` reports inherited +taint; and that a cancelled completion clears its marker before the next +`acquire()` can arm, which then reports no inherited taint and keeps its own +marker. + The coordinator's own pre-existing fence and lock regressions ([`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md#testing-boundary)) pass unchanged through the guard, which is what proves `coordinate()`'s diff --git a/context/cli/mutation-trace-protocol.md b/context/cli/mutation-trace-protocol.md index 9653c37e7..83e949fbf 100644 --- a/context/cli/mutation-trace-protocol.md +++ b/context/cli/mutation-trace-protocol.md @@ -172,7 +172,7 @@ owns the lock, the external-taint fence, the caller-supplied DB provider, one Git snapshot, and a bounded CAS-retry loop; `store.rs` never remaps an existing `ScopeId`'s `actor_kind`/`worktree_id`; `protocol.rs` assumes referenced scopes already exist and stays free of any Git object, DB row, or CAS transaction -concept. `runtime::ref_reconciliation` +concept. `runtime::ref_reconciliation` / `runtime::ref_maintenance` ([`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)) is imperative durability maintenance *outside* the verified protocol — it never advances the cursor, chooses attribution, changes scope state, or creates a diff --git a/context/cli/mutation-trace-quint-connect.md b/context/cli/mutation-trace-quint-connect.md index 1cd5e78f1..00e244921 100644 --- a/context/cli/mutation-trace-quint-connect.md +++ b/context/cli/mutation-trace-quint-connect.md @@ -183,52 +183,47 @@ generate traces toward that boundary. The dedicated no-wrap regressions in `mutation-trace-revision-refinement.md` remain the sole targeted coverage for it; this harness's coverage is incidental, not a substitute. -## CI: two Nix checks, both need Quint +## CI: full flake check plus an opt-in focused runner -`mutation_trace::mbt` is registered as an ordinary `#[cfg(test)] mod mbt;` -(gated behind `#[cfg(test)]` in `mutation_trace/mod.rs`), so it is compiled -and run by *any* `cargo test` over the CLI crate — including the pre-existing -generic `checks.cli-tests` in `flake.nix`, not only a dedicated focused -check: +`mutation_trace::mbt` is an ordinary `#[cfg(test)] mod mbt;` module in +`mutation_trace/mod.rs`. The full Rust check already exercises it, so the +default flake check must not compile and run it again as a second derivation: ```text checks.cli-tests - -> full `cargo test`, including mutation_trace::mbt - -> needs the pinned Quint binary on PATH + -> full cargo test, including mutation_trace::mbt + -> pinned Quint on PATH -checks.mutation-trace-quint-connect - -> focused `cargo test ... mutation_trace::mbt` only - -> needs the pinned Quint binary on PATH +packages.mutation-trace-quint-connect (explicit invocation only) + -> focused cargo test ... mutation_trace::mbt + -> pinned Quint on PATH ``` -Both checks list the Nix `quint` package in `nativeCheckInputs`. Quint's -presence alone is not sufficient, though: `quint run`/`quint test` resolve -the spec by a path relative to the `cli/` crate root -(`../spec/mutation_cursor.qnt`), and `craneLib.fileset.commonCargoSources` -only covers each crate's own Cargo-referenced sources, not files outside any -crate. `workspaceSrc`'s Nix fileset therefore lists the top-level `spec/` -directory explicitly; without it, the spec never reaches either check's -sandbox and every MBT test fails with an opaque `"Quint returned non-zero -code."` (`quint-connect`'s error formatting is `Display`-only, so the -underlying Quint stderr explaining *why* — file not found — never surfaces -in the Rust test panic). - -`checks.mutation-trace-quint-connect` follows the same `craneLib.cargoTest` -pattern as `cli-tests`/`cli-clippy`/`cli-fmt`, reusing the pinned -`rustToolchain`/`cargoArtifacts`, scoped via `cargoTestExtraArgs`, printing -`rustc`/`cargo`/`quint --version` in `preCheck`. Both checks are part of -ordinary `nix flake check` (not Linux-only); `.github/workflows/quint.yml` -additionally invokes the dedicated check directly (`nix build -.#checks.x86_64-linux.mutation-trace-quint-connect`) for fast, targeted -feedback without waiting on the full Nix CI matrix — the entire invocation -comes from Nix, never a second, unpinned Rust toolchain stitched together -with the runner's own Cargo. That workflow's change detector also watches -`cli/src/services/mutation_trace/mbt/**`, `.../protocol.rs`, `.../types.rs`, -`cli/Cargo.toml`, `cli/Cargo.lock` (`flake.nix`/`flake.lock` were already -watched), so a Rust-only refinement/driver change triggers Quint CI without -touching the spec. Its "Run Quint tests" step passes `--match '^test.*'` — -omitting `--match` silently selects zero tests on this spec rather than -running the named scenarios. +Both runners use Crane's separate test-profile dependency artifacts, the same +pinned Rust toolchain, and `nativeCheckInputs = [ pkgs.git pkgs.quint ]`. +The test profile is unoptimized and uses line-table-only debug information; +production binaries continue using their existing release-profile artifacts. + +Run all checks (including MBT **once**) with `nix flake check`. To run only +the focused MBT suite, use: + +```sh +nix build .#mutation-trace-quint-connect --print-build-logs +``` + +The focused package is intentionally absent from `checks`, but remains in +`packages` for explicit, targeted MBT validation. `.github/workflows/quint.yml` +runs Quint spec checks only, while `cli-tests` performs Rust refinement testing +on both Linux and macOS in the ordinary Nix CI matrix. + +Quint's presence alone is not sufficient: `quint run` and `quint test` +resolve the model via `../spec/mutation_cursor.qnt` relative to the +`cli/` crate root. Therefore `workspaceSrc` must explicitly include the +top-level `spec/` tree (it is not part of Crane's common Cargo sources). + +Quint CI continues to pin Quint via the flake and separately runs the named +`test.*` scenarios, the typecheck, and the randomized safety check. The workflow uses `--match '^test.*'` because omitting `--match` +silently selects zero named scenarios on this spec. ## Non-goals diff --git a/context/cli/mutation-trace-ref-maintenance-state.md b/context/cli/mutation-trace-ref-maintenance-state.md new file mode 100644 index 000000000..dd53b4067 --- /dev/null +++ b/context/cli/mutation-trace-ref-maintenance-state.md @@ -0,0 +1,81 @@ +# Snapshot-ref maintenance state and verified-existing DB opener + +Supporting contracts for [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md): the durable per-worktree maintenance state file shared by the hook advisory, `sce doctor`, and `reconcile_explicit`, and the dedicated Agent Trace DB opener reconciliation uses. Related: [`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md), [`doctor-ref-reconciliation.md`](doctor-ref-reconciliation.md), [`agent-trace-storage.md`](agent-trace-storage.md). + +## Verified-existing Agent Trace DB opener + +Reconciliation never uses the hook opener +(`open_agent_trace_db_for_hook_runtime`), which can create directories and files, +repair migration metadata, and insert repository metadata. The dedicated path is +`resolve_existing_agent_trace_storage_for_maintenance` plus +`RepositoryAgentTraceDb::open_verified_existing_at`, built on +`TursoDb::open_existing_without_migrations_at`: + +- The Turso open is non-creating and read-only (`read_only(true)` with + `experimental_multiprocess_wal(true)` preserved), so existence and open are one + OS call. A failed open is classified `Missing` only when the path is then + absent. No pre-existence check and no parent-directory creation. +- It never runs migrations, `repair_missing_repository_schema_migration_metadata` + or `verify_or_initialize_repository_metadata`. +- Before any ref listing it verifies schema readiness (all expected migrations), + metadata presence, stored `repository_id` against the resolved identity, and a + valid `source_instance_id` (`verify_existing_repository_metadata`). +- Typed failures `ExistingRepositoryDbError`: `Missing`, `Unreadable`, + `IncompatibleSchema`, `MissingMetadata`, `RepositoryMismatch`. They stay + downcastable from `ReconcileError::AgentTraceDbUnavailable`. + +Per-DB-state guarantees: a **missing** DB creates no DB file, parent directory, +WAL or shared-memory file and no ref is listed or deleted; an **existing invalid** +DB may incur incidental Turso WAL/shared-memory bookkeeping but no migration, +repair, initialization, or application-table write and no ref deletion; an +**existing valid** DB is verified before reconciliation with no initialization. +Logical immutability is the guarantee, not filesystem byte-identity. + +Limitations: Turso opens `-wal`/`-tshm` sidecars by path, so only a coherent +replacement of main file and sidecars is meaningful; the process-wide Turso +registry may share an already-open instance (the read-only flag is then not +enforced); a DB replaced between verification and the delete transaction is not +detected, the guard being that roots are read from the verified connection under +the held lock. + +Plain `sce doctor` shares the non-creating read-only open through +`RepositoryAgentTraceDb::open_existing_schema_ready_at`, which inspects schema +readiness without requiring metadata. + +## Maintenance state file + +`/sce/ref-maintenance.json`, beside `mutation-cursor.lock` and the +taint marker. Versioned, capped at `MAINTENANCE_STATE_MAX_BYTES` (4096), written +only under `WorktreeLock` by atomic staging-and-swap (temp file in the same +directory, rename, best-effort parent-directory sync; the filesystem is the +statically dispatched `StateFilesystem` seam). No in-memory registry or +process-global state; the wall clock is an injected `Fn() -> i64` (unix ms). + +Fields: past-event timestamps `anchor`, `last_success`, `last_attempt`, +`last_advised`; `last_attempt_outcome` (`completed` | `failed`); `last_report` +counts; `consecutive_failures`; `last_failure`. **No field is a deadline.** +Eligibility is computed at read time as `now − reference ≥ window`, so rereading +cannot extend it. Constants: `RECONCILIATION_ADVISORY_AFTER` 24 h, +`FUTURE_SKEW_TOLERANCE` 5 min. + +Transitions: explicit success sets `last_attempt` = `last_success` = now, stores +the report, and clears the failure streak and fields; explicit failure sets +`last_attempt`, outcome `failed`, increments `consecutive_failures`, and leaves +`last_success` untouched; lock contention changes nothing. A missing, unparsable, +wrong-version, or oversized file is treated as default and rewritten by the next +recorded operation; it never fails an explicit pass. + +Timestamps: a value strictly more than `FUTURE_SKEW_TOLERANCE` ahead of `now` is +invalid; one within the tolerance (inclusive) is valid, preserved byte-for-byte, +and evaluated as age 0. **Successful state-writing operations** (a successful +advisory write, a successful explicit record) normalize invalid timestamps once +under the lock and persist them, so afterwards every stored timestamp satisfies +`t ≤ now + FUTURE_SKEW_TOLERANCE`. **Read-only or unsuccessful operations** +(plain `sce doctor`, lock-busy passes, unreadable state, a `NotApplied` write, an +interrupted write) never mutate the file, evaluate invalid timestamps +conservatively as eligible/recommended, and never report them as normalized. + +`evaluate_recommendation(state, now)` is the pure function shared by the advisory +and read-only doctor. An outstanding failed explicit pass is recommended +independently of staleness and of `last_advised`. + diff --git a/context/cli/mutation-trace-ref-reconciliation.md b/context/cli/mutation-trace-ref-reconciliation.md index dc74874cb..6c53a0809 100644 --- a/context/cli/mutation-trace-ref-reconciliation.md +++ b/context/cli/mutation-trace-ref-reconciliation.md @@ -1,131 +1,108 @@ -# Mutation-cursor snapshot-ref reconciliation (`runtime::ref_reconciliation`) - -`cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` is a -conservative, per-worktree maintenance pass that removes orphaned / unreferenced -SCE-owned snapshot pins **within the ref namespace of a checkout identity a -current worktree still owns** while retaining every tree any current or -historical durable mutation-cursor state in the repository still references. -Built by the `mutation-cursor-ref-reconciliation` plan -(`context/plans/mutation-cursor-ref-reconciliation.md`). It is deliberately -**not** a guarantee that SCE snapshot refs never accumulate: a namespace whose -checkout identity no current worktree derives is beyond its reach (see -[Scope: an owned checkout-identity namespace only](#scope-an-owned-checkout-identity-namespace-only)). - -`GitSnapshotService::pin_tree` is **create-only per invocation** — a crash, -failed transition, or other interrupted `coordinate()` path can leave a -`refs/sce/mutation-cursor//` pin with no corresponding -durable root. Reconciliation is the reclamation step for exactly that state. It -is **not** a bound on storage growth: every retained `mutation_trace_events` -row keeps its `before_tree` / `after_tree` as durable roots, so a successful -`A → B → C → D` history keeps all four pins. Bounding historical snapshot -storage needs a separate future retention/compaction lifecycle this module does -not design. - -The design is deliberately asymmetric and this bias governs every rule below: -**keeping an unnecessary ref costs disk; deleting a required ref destroys -durable evidence.** False retention is acceptable; false deletion is not. - -`mod ref_reconciliation;` is private in `runtime/mod.rs` (like `coordinator`); -`reconcile_worktree` is `pub` only within `runtime` — no `pub(crate)` re-export, -no hook / command / `diff_traces` wiring yet (invocation timing is the -harness-wiring PR's). - -## Entry point and identity - -```rust -pub fn reconcile_worktree( - repository_root: &Path, - open_db: impl FnOnce() -> anyhow::Result, -) -> Result -``` - -A one-line delegation to `pub(super) fn reconcile_worktree_inner(.., -on_lock_contention: impl FnOnce())` — the deterministic test seam (mirroring -`coordinate` / `coordinate_inner`), visible only within `runtime`. - -Like `coordinate()`, it never accepts a `WorktreeId`, `TreeId`, or ref name and -never opens the DB itself: identity is -`repository_root → resolve_git_dir → read_checkout_id`, `open_db` is -caller-supplied. `read_checkout_id → Ok(None)` returns -`Ok(SkippedNoCheckoutIdentity)` — an observable skip, **not** a zero-count -report: nothing inventoried, `open_db` never called, no ref touched, lock -released, no identity created or recovered; it makes **no** claim the repository -holds no SCE refs for a prior checkout identity (those are the unowned -namespaces below). `Err` is `ReconcileError::CheckoutIdentity` (a corrupt id, not an absent one). - -## Scope: an owned checkout-identity namespace only - -Reconciliation reclaims orphan / unreferenced pins **only** under -`refs/sce/mutation-cursor//` where `` is a checkout identity a **current -worktree still derives** (`resolve_git_dir → read_checkout_id`). The unsupported -case is identity-based, not path-based: - -```text -refs/sce/mutation-cursor//* — does a current worktree derive ? - yes → active namespace (reclaimed here under the WorktreeLock) - no → unowned namespace (no per-worktree pass can ever reach it) -``` - -High-frequency harness traffic against a worktree with a **stable** checkout -identity — where interrupted `coordinate()` runs leave orphan pins — is exactly -the active-namespace case this pass covers. - -A namespace becomes **unowned** whenever no current worktree can derive its -checkout id. Two lifecycles cause this: - -**Case A — deleted linked worktree.** `git worktree remove W` deletes W's -worktree-specific git dir, so its `/sce/checkout-id` is gone; the -`refs/sce/mutation-cursor//*` it created survive in the shared repository -ref namespace. - -**Case B — checkout-identity metadata loss / recreation.** A **present** -worktree's `/sce/checkout-id` disappears (id `A`). `reconcile_worktree` -returns `SkippedNoCheckoutIdentity` and does nothing. A later -`get_or_create_checkout_id` (e.g. the next `coordinate()`) mints a **new** id -`B`; the worktree now operates as `B` while `refs/.../A/*` are unowned. This is -**not** normal operation: it is a metadata-loss / recreation lifecycle the -maintenance model must handle conservatively — reconciliation never recreates -`A` or adopts its namespace. - -**Harness gate.** A persistent / current-worktree harness whose **checkout -identity stays stable** relies on this pass for active-namespace orphan cleanup -and is storage-cleanup complete for this module's scope. Any lifecycle that can -**retire, replace, lose, or recreate** checkout identities (ephemeral linked -worktrees among them, but not only those) can leave unowned namespaces and needs -the future repository-scoped operation below. This is a scope limit, not a bug, -orthogonal to the "reconciliation ≠ historical retention policy" boundary above. - -### Future work: repository-scoped unowned-namespace reconciliation - -Not implemented here — this module adds **no** repository-global namespace scan -and **no** repository-global ref deletion. Recorded shape: - -```text -enumerate refs/sce/mutation-cursor//* (git for-each-ref on the namespace) - ↓ -active checkout ids (enumerate current worktrees → read each one's checkout-id) - ↓ -unowned ids (namespace present, no current worktree derives it) - ↓ -for each unowned namespace, each pinned tree T: - T ∈ durable_roots(repository) → retain - T ∉ durable_roots(repository) → safe deletion candidate -``` +# Mutation-cursor snapshot-ref reconciliation (`runtime::ref_reconciliation`, `runtime::ref_maintenance`) + +Snapshot-ref reconciliation is a conservative, per-worktree maintenance pass that +removes orphaned SCE-owned snapshot pins under +`refs/sce/mutation-cursor//` while retaining every tree any current +or historical durable mutation-cursor state in the repository still references. +It is built by the `mutation-cursor-ref-reconciliation` plan and wired by the +`mutation-cursor-ref-reconciliation-wiring` plan +(`context/plans/mutation-cursor-ref-reconciliation-wiring.md`). + +`GitSnapshotService::pin_tree` is **create-only per invocation**: a crash, failed +transition, or other interrupted `coordinate()` path can leave a pin with no +corresponding durable root. Reconciliation is the reclamation step for exactly +that state. It is **not** a bound on storage growth: every retained +`mutation_trace_events` row keeps its `before_tree` / `after_tree` as durable +roots, so a successful `A → B → C → D` history keeps all four pins. Bounding +historical snapshot storage needs a separate retention/compaction lifecycle that +does not exist. + +The design is deliberately asymmetric: **keeping an unnecessary ref costs disk; +deleting a required ref destroys durable evidence.** False retention is +acceptable; false deletion is not. + +## Who runs it + +Reconciliation is **explicit only**. `sce doctor --fix` is the single production +caller (see [`doctor-ref-reconciliation.md`](doctor-ref-reconciliation.md)). +Orphan pins are reclaimed only when an operator runs it. **Automatic reclamation +is not delivered** and nothing may claim otherwise; it is a follow-up (see +[Deferred follow-ups](#deferred-follow-ups)). + +The mutation-hook completion path performs no reconciliation, ref listing, DB +I/O, or Git subprocess. After a durably completed `Close`/`Flush` it only runs a +lightweight advisory check that may record "reconciliation is recommended" in a +small state file (see [`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md)). +Inline reconciliation was rejected because `git for-each-ref`, DB open, +repository-wide durable-root queries, and a started `git update-ref` cannot be +bounded, the Tokio runtime joins in-flight blocking workers, and the pass holds +`WorktreeLock`, which a coordinator waits on for at most 10 s before failing the +boundary. + +## Module layout and entrypoints + +All runtime modules are private to `mutation_trace::runtime`; only the +`ref_doctor` facade and the advisory seam are re-exported `pub(crate)`. + +| Module | Role | +| --- | --- | +| `ref_reconciliation.rs` | The single reconciliation algorithm, `reconcile_with_held_lock` | +| `ref_maintenance.rs` | `reconcile_explicit`: the production maintenance entrypoint | +| `ref_advisory.rs` | `advise_if_due`: hook-path advisory | +| `maintenance_state.rs` | `/sce/ref-maintenance.json` state, normalization, atomic writes, `evaluate_recommendation` | +| `ref_doctor.rs` | Plain-data facade used by `sce doctor` | + +### Single lock acquisition + +`WorktreeLock` is a non-reentrant advisory file lock. The algorithm is split so +each maintenance operation acquires it **exactly once**: + +- `reconcile_with_held_lock` borrows the held `&WorktreeLock`, never acquires, + and derives deletion leases from `lock.lease()`. +- `reconcile_explicit(repository_root) -> ExplicitOutcome` acquires the lock once + with the bounded 10 s `EXPLICIT_LOCK_TIMEOUT`, resolves the repository identity + and canonical Agent Trace DB path itself, opens the DB only through the + verified-existing opener, runs `reconcile_with_held_lock`, records the result + in the state file while still holding the same lock, then drops it. It takes + no opener parameter; the injection seam is private to `ref_maintenance`. +- `advise_if_due` takes one zero-wait `try_lock`, has no Git/DB/store + parameters, and never reaches reconciliation. +- `reconcile_with_held_lock` is the only reconciliation entrypoint; it never + acquires the lock. The former test-only lock-acquiring wrappers were removed. + +Mutating Git work receives `lock.lease()` and moves it into the `spawn_blocking` +worker, so dropping the caller future cannot release the file lock while a ref +deletion is running. + +### Concurrency contract + +An explicit pass holds the lock for its whole duration, because the +pin-then-root ordering that makes deletion safe requires inventory, root reads, +and deletion not to interleave with `coordinate()`'s pin-then-CAS. A mutation +boundary arriving meanwhile waits up to `WORKTREE_LOCK_TIMEOUT` (10 s) and then +fails **before durable completion** with the existing lock-timeout error. This is +an operator-initiated trade-off of `sce doctor --fix`. The hook advisory never +waits on the lock; contention yields `Busy`. + +## Worktree identity + +Identity is Git-topology-derived (`resolve_worktree_id`): `main` for the primary +worktree, `worktrees/` for a linked worktree. Failure is +an `Err`. The former checkout-identity service was removed (see +[`checkout-identity.md`](checkout-identity.md)); there is no skip-on-missing-identity +outcome. `ReconcileError::CheckoutIdentity` remains the error variant for a +worktree identity that cannot be resolved. + +The dedicated DB opener and the `ref-maintenance.json` state file are documented in +[`mutation-trace-ref-maintenance-state.md`](mutation-trace-ref-maintenance-state.md). -It must inherit this module's guarantees: the repository-wide durability -invariant (`delete /T` only if `T ∉ durable_roots(repository)`, read -through `load_all_tree_roots()`) because an unowned namespace may still hold the -only SCE ref protecting historical `mutation_trace_events` trees other tooling -needs; the false-retention-over-false-deletion bias; and a hard prohibition on -the shortcut "checkout id is unowned → delete its whole namespace". It is a -separate PR (a repository-global scan and cross-worktree active-id inventory this -module omits), gated behind the same harness-wiring work as invocation timing. ## Two invariants -Conflating them — deciding deletion from the target worktree's roots alone — is -the cross-worktree safety bug this design avoids: linked worktrees share one -object database, so an `A`-owned ref can be the last SCE ref protecting a tree only `B` durably requires. +Conflating them, i.e. deciding deletion from the target worktree's roots alone, +is the cross-worktree safety bug this design avoids: linked worktrees share one +object database, so an `A`-owned ref can be the last SCE ref protecting a tree +only `B` durably requires. ```mermaid flowchart TD @@ -142,111 +119,131 @@ flowchart TD ds -- "retained" --> keep["keep the pin"] ``` -- **Local consistency** (`load_tree_roots(W)`) is strictly per-worktree. A - missing pin in some *other* worktree never makes `W`'s pass fail — that would - let one worktree's degradation block maintenance everywhere. -- **Deletion safety** (`load_all_tree_roots()`) is repository-wide. `W/T` is - deleted only when `T` is in **no** worktree's durable root set. If `B` durably - needs `T` and `A` also has a `T` pin, `A` retains it as accidental backup - reachability for `B`'s degraded state. +- **Local consistency** (`load_tree_roots(W)`) is per-worktree and checked before + any deletion. If it fails the pass deletes nothing and returns + `MissingRequiredPins`. A missing pin in some *other* worktree never makes `W`'s + pass fail. +- **Deletion safety** (`load_all_tree_roots()`) is repository-wide: `delete(W, T)` + only if `T` is in **no** worktree's durable root set. A pin another worktree + requires is retained. + +Claimed: these two invariants. **Not claimed:** that every repository-wide durable +root is pinned. Reconciliation does not repair, recreate, or verify missing pins +belonging to other worktrees. A DB `TreeId` is a *logical* durability requirement, not itself a Git reachability edge: it obliges reconciliation to keep at least one SCE ref -protecting that tree, and that retained ref supplies physical Git reachability. -Each root-set query is [one SQL statement over one DB -snapshot](mutation-trace-store.md), which keeps a concurrent atomic -`cursor T → X` + `event T → X` commit on another worktree from tearing the -repository-wide read — no repository-global lock is needed. - -## Locking - -Reconciliation holds the **same** `/sce/mutation-cursor.lock` -`WorktreeLock` that `coordinate()` holds across `pin → CAS → return`, acquired -via `worktree_lock::acquire_inner` and bounded by the module-owned -`RECONCILIATION_LOCK_TIMEOUT` (`Duration::from_secs(10)`, matching -`runtime::protected_worktree`'s private `WORKTREE_LOCK_TIMEOUT` by intent, not a -shared constant — there is no semantic reason the two must stay identical, so -each module owns its own). -Mutual exclusion on that one file makes the pin → DB-CAS race structurally -impossible: the reconciler's inventory → diff → delete runs wholly before -`coordinate()` takes the lock (nothing pinned yet) or wholly after it releases -it (tree committed → durable root → retained; never committed → true orphan → -deletable). The lock stays per-worktree; only the durable-root *read* is repository-wide. +protecting that tree. Each root-set query is [one SQL statement over one DB +snapshot](mutation-trace-store.md), so a concurrent atomic `cursor T → X` + +`event T → X` commit on another worktree cannot tear the repository-wide read. + +## Locking and the pin → CAS boundary + +The pass holds the **same** `/sce/mutation-cursor.lock` that +`coordinate()` holds across `pin → CAS → return`. Mutual exclusion on that one +file makes the pin → DB-CAS race structurally impossible: the inventory → diff → +delete runs wholly before `coordinate()` takes the lock (nothing pinned yet) or +wholly after it releases it (tree committed → durable root → retained; never +committed → true orphan → deletable). The lock stays per-worktree; only the +durable-root *read* is repository-wide. ## Algorithm and error contract -Every step runs under the lock, and every fallible step maps to one dedicated -`ReconcileError` variant — there is no `Other` catch-all: +Every step runs under the lock and every fallible step maps to one dedicated +`ReconcileError` variant; there is no catch-all: | Step | Error on failure | | --- | --- | | `resolve_git_dir` | `GitDir` | -| `acquire_inner(RECONCILIATION_LOCK_TIMEOUT)` | `Lock(WorktreeLockError)` | -| `read_checkout_id` → `Err` (corrupt id) | `CheckoutIdentity` | -| `open_db()` provider | `AgentTraceDbUnavailable` | +| lock acquisition (`EXPLICIT_LOCK_TIMEOUT`) | `Lock(WorktreeLockError)`; surfaced as `Skipped(Busy)` by `reconcile_explicit` | +| worktree identity resolution | `CheckoutIdentity` | +| verified-existing DB open | `AgentTraceDbUnavailable` (downcastable `ExistingRepositoryDbError`) | | `GitSnapshotService::new` | `SnapshotService` | | `list_pins` → `PinInventoryError::Git` | `PinInventory` | | `list_pins` → `PinInventoryError::MalformedRef` | `MalformedPin { ref_name, reason }`, delete nothing | -| `load_tree_roots` / `load_all_tree_roots` | `DurableRoots` | +| `load_tree_roots` / `load_all_tree_roots` | `DurableRoots` (durable-root queries belong to migration `004_mutation_trace_protocol`) | | a target-worktree root has no pin | `MissingRequiredPins { missing }`, delete nothing | | `delete_pins` transaction | `DeleteTransaction`, delete nothing (atomic-or-nothing) | `AgentTraceDbUnavailable` here is a **maintenance** error only: reconciliation never arms `ExternalTaintMarker`, calls `protocol::*`, or writes a `mutation_trace_*` row, and never becomes -`CoordinateError::AgentTraceDbUnavailable` — no mutation boundary is being -coordinated (contrast [`mutation-trace-external-taint.md`](mutation-trace-external-taint.md)). +`CoordinateError::AgentTraceDbUnavailable` (contrast +[`mutation-trace-external-taint.md`](mutation-trace-external-taint.md)). +`delete_pins` is the only ref-deletion path; there is no `git gc`, object +removal, or independent deletion. + +## Outcomes and report + +`reconcile_with_held_lock` returns `Result` +where `ReconciliationOutcome` has the single variant `Reconciled(ReconciliationReport)`. + +`ReconciliationReport { local_required, retained, deleted }`: +`local_required = load_tree_roots(W).len()`, `deleted` = stale pins removed, +`retained = actual.len() − deleted`. `retained == local_required` is not an +invariant; only `report.local_required ≤ report.retained` holds. -## Outcome and report +`reconcile_explicit` returns the typed, never-merged `ExplicitOutcome`: -Both entrypoints return `Result` — -`Reconciled(ReconciliationReport)` | `SkippedNoCheckoutIdentity`, the skip an -`Ok` (never `Err`) and distinct from a zero-work `Reconciled(.. { deleted: 0 })`. +- `Completed(report)` +- `CompletedStatePersistFailed { report, warning }`: the cleanup stands and is + reported as done +- `Failed { error, state_warning: Option }`: keeps both the + reconciliation error and any persistence warning +- `Skipped(Busy)`: lock contention; state untouched; not a failure -`ReconciliationReport { local_required: usize, retained: usize, deleted: usize }` -— `local_required = load_tree_roots(W).len()`, `deleted` = stale pins removed, -`retained = actual.len() − deleted`. `retained == local_required` is **not** an -invariant (a pin another worktree needs counts toward `retained` only); for -`Reconciled(report)` the only relation is `report.local_required ≤ report.retained`, -and `SkippedNoCheckoutIdentity` carries no report. +`StatePersistWarning` is `PreviousStateUnreadable(io::Error)` (update not +attempted) or `WriteFailed(PersistFailure)`. `PersistFailure::NotApplied` means +the atomic replacement did not occur and the previous state is preserved; +`PersistFailure::DurabilityUncertain` means the rename occurred (new state may be +visible) but crash durability could not be confirmed. A completed ref deletion is +never rolled back. ## Model boundary Ref reconciliation is imperative durability maintenance **below** the verified -`spec/mutation_cursor.qnt` protocol — it never advances the cursor, chooses -attribution, changes scope state, or creates a `MutationEvent`, so no Quint -change is warranted. It deletes only SCE-owned refs, never Git objects, and runs -no `git gc` / `git prune` / `git reflog expire`; Git reclaims unreachable -objects on its own schedule. - -## Testing boundary - -`ref_reconciliation.rs`'s inline `#[cfg(test)] mod tests` uses a RAII `Fixture` -(`tempfile::TempDir`, real `git init` repo, checkout id via -`get_or_create_checkout_id`, a schema `RepositoryAgentTraceDb` beside the -worktree so it never perturbs a captured tree) with raw-SQL row seeders, -following the filesystem-touching inline-test precedent (`context/patterns.md`). -Coverage: orphan pin deleted (with/without a worktree row); current-cursor pin -retained without a referencing event; historical `before`/`after` pins retained -after the cursor advances; a pin another worktree durably requires retained -(the `retained > local_required` case); `MissingRequiredPins` fail-closed; a -malformed / symbolic ref fail-closed; idempotence; refs deleted without object -reclamation; and the no-checkout-identity skip (`SkippedNoCheckoutIdentity`; -`open_db` never called; the pin ref byte-identical across the skip). - -`runtime/tests.rs` holds the cross-module integration suite (T09), driven -through the public `reconcile_worktree` (and, for setup, `coordinate`) against -real Git and a real `RepositoryAgentTraceDb`: active-worktree orphan deletion; -current-cursor retention with no referencing event; historical `before`/`after` -retention through a real `A → B → C → D` `coordinate()` history; idempotence; -linked-worktree isolation and cross-worktree degraded-tree retention -(byte-identical pins); a missing required pin failing closed despite another -worktree pinning the same tree; a malformed / symbolic ref failing closed; no -protocol/DB/marker write; no object reclamation; the no-checkout-identity skip; -and two deterministic lock-race regressions (no sleeps): -`reconciliation_blocks_on_the_worktree_lock_and_retains_a_pin_that_becomes_durable_under_it` -(the generic `WorktreeLock` happens-before edge) and -`reconciliation_blocks_until_a_real_coordinate_cas_commits_the_pinned_tree` -(the same edge across the real `coordinate()` `pin → store CAS` path, test-only). - -See also: [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md), [`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md) (`list_pins` / `delete_pins`), [`mutation-trace-store.md`](mutation-trace-store.md) (`load_tree_roots` / `load_all_tree_roots`), [`mutation-trace-protocol.md`](mutation-trace-protocol.md). +`spec/mutation_cursor.qnt` protocol. It never advances the cursor, chooses +attribution, changes scope state, or creates a `MutationEvent`, so no Quint or +`protocol.rs` change was made. It deletes only SCE-owned refs, never Git objects. + +## Testing + +Coverage lives in `runtime/ref_maintenance/tests.rs` and its submodules +(`support`, `coordination`, `worktrees`, `cross_process`, `fail_closed`, +`state_recovery`), the advisory and state unit tests, `git_snapshot` cancellation +tests, `hooks::mutation_scope::advisory_trigger_tests`, and +`app::doctor_reconciliation_cli_tests`. Race and ordering scenarios use +deterministic coordination (barriers, parked phase hooks via `ReconcilePhase`, +the worker-entered seam, injected clocks), not sleeps. The plan's coverage map +records each scenario's verification status and the guard-removal results. + +## Deferred follow-ups + +Not implemented and not to be described as behavior: + +- **Automatic out-of-band reclamation.** Binding prerequisites: (1) a + lifecycle-owned execution point with defined ownership, lifetime, scheduling, + and failure recovery (for example an explicitly invoked `sce` maintenance + command run by a user-managed scheduler, or a future shared session-end + mechanism), never a detached task, daemon, fire-and-forget subprocess, or + timeout-cancelled Git/DB work; (2) a durable next-attempt reservation persisted + under `WorktreeLock` **before** opening the DB, listing pins, or loading + durable roots, bounded, never interpreted as success, superseded by the success + cadence or exponential failure backoff, and bypassed by explicit repair, with + the automatic pass skipped if the reservation cannot be persisted; (3) + distinct outcomes for completed, completed-but-state-not-persisted, failed, and + skipped/deferred; (4) bounded or split lock-hold analysis against the 10 s + coordinator timeout; (5) the same invalid-timestamp normalization as above. +- **Retired-worktree cleanup.** The pass only inventories the namespace of a + currently resolvable worktree, so refs owned by removed linked worktrees + survive. A repository-scoped operation would need active-worktree inventory, + repository-wide durable-root retention (`delete /T` only if `T` is + outside `durable_roots(repository)`, never "namespace unowned → delete all"), + and protection against worktree removal/recreation races. +- **Evidence-based advice.** The advisory is time-based; detecting actual orphans + requires the Git/DB scan kept off the hook path. +- **Historical retention.** Trees referenced by retained mutation events are + never deleted; bounding history needs a separate event-retention and + snapshot-compaction policy. + +See also: [`mutation-trace-ref-maintenance-state.md`](mutation-trace-ref-maintenance-state.md), [`doctor-ref-reconciliation.md`](doctor-ref-reconciliation.md), [`mutation-scope-ref-advisory.md`](mutation-scope-ref-advisory.md), [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md), [`mutation-trace-snapshot-service.md`](mutation-trace-snapshot-service.md) (`list_pins` / `delete_pins`), [`mutation-trace-store.md`](mutation-trace-store.md) (`load_tree_roots` / `load_all_tree_roots`), [`mutation-trace-protocol.md`](mutation-trace-protocol.md). diff --git a/context/cli/mutation-trace-runtime-coordinator.md b/context/cli/mutation-trace-runtime-coordinator.md index 07683b292..b0845aee5 100644 --- a/context/cli/mutation-trace-runtime-coordinator.md +++ b/context/cli/mutation-trace-runtime-coordinator.md @@ -13,8 +13,8 @@ submodule is declared privately in `runtime/mod.rs`, which re-exports `coordinate` and `abandon_scope` at `pub(crate)` — reachable crate-wide, contract in [`mutation-scope-runtime.md`](mutation-scope-runtime.md). `coordinate()` and `abandon_scope()` are now driven by the generic `sce hooks mutation-scope` CLI -ingress. `reconcile_worktree` stays `runtime`-internal and unwired, and the -mutation runtime does not itself insert into `diff_traces`. Concrete Claude Code, +ingress. Snapshot-ref reconciliation stays `runtime`-internal and runs only through +`sce doctor --fix`, and the mutation runtime does not itself insert into `diff_traces`. Concrete Claude Code, Codex, and OpenCode adapters are wired to the seam (OpenCode reachable in production via a generated plugin installed by `sce setup`); Pi remains future work. @@ -30,11 +30,11 @@ The per-worktree runtime lock, the isolated Git snapshot service (which also derives `WorktreeId` from Git topology), the coordinator's internal protocol-integration pipeline, and the public, lock-wrapped `coordinate()` entrypoint that owns the external-taint fence and DB provider around that -pipeline all exist, with cross-module integration tests in `runtime/tests.rs` -exercising the public API end to end. Only harness/command wiring remains. +pipeline all exist (the former cross-module integration tests in `runtime/tests.rs` were removed; +see Testing boundary). Only harness/command wiring remains. - `cli/src/services/mutation_trace/runtime/worktree_lock.rs` — - `WorktreeLock::acquire(git_dir: &Path, timeout: Duration) -> + `WorktreeLock::acquire_async(git_dir: &Path, timeout: Duration) -> Result` opens/creates `/sce/mutation-cursor.lock` and polls `std::fs::File::try_lock()` on a 100ms interval against the caller-supplied bounded `timeout`, rather @@ -106,16 +106,16 @@ exercising the public API end to end. Only harness/command wiring remains. `refs/sce/mutation-cursor//...` ref. This derivation reads only Git's own state — no file is created or read under `/sce/`, and no identity is generated or persisted anywhere. -- `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` — the - conservative per-worktree snapshot-ref maintenance pass — `reconcile_worktree` - / `pub(super) reconcile_worktree_inner` return `Result` (`ReconciliationOutcome` = `Reconciled(ReconciliationReport)` - | `SkippedNoCheckoutIdentity`). Under the worktree's `WorktreeLock` it deletes - only pins whose tree is a durable root of **no** worktree, fails closed if any - local root lacks a pin, and writes no `mutation_trace_*` row or taint marker - (only the namespace of a checkout id a current worktree still derives — a - namespace no current worktree owns, via a deleted worktree or checkout-id - metadata loss/recreation, is future repository-scoped work). Full contract in +- `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` and + `ref_maintenance.rs` — the conservative per-worktree snapshot-ref maintenance + pass. `reconcile_with_held_lock` is the single lock-held algorithm and + `reconcile_explicit` (single `WorktreeLock` acquisition, verified-existing + Agent Trace DB, state recorded under the same lock) is its only production + entrypoint, called by `sce doctor --fix`. It deletes only pins whose tree is a + durable root of **no** worktree, fails closed if any local root lacks a pin, + and writes no `mutation_trace_*` row or taint marker. Only the namespace of the + currently resolvable worktree is inventoried; refs owned by removed linked + worktrees survive (deferred follow-up). Full contract in [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md). - `cli/src/services/mutation_trace/runtime/protected_worktree.rs` — the shared safety prefix every runtime entrypoint runs behind (`ProtectedWorktree`: @@ -199,9 +199,12 @@ The runtime lock guards every runtime entrypoint's critical section (external-taint marker arming/clearing, snapshot capture where applicable, worktree/scope materialization, recovery, and the CAS retry loop): `coordinate()` and `abandon_scope()` acquire it before their protected work and -hold it until the call returns. The separate `ref_reconciliation::reconcile_worktree` -pass acquires this same lock before inventorying pins, reading durable roots, -or deleting refs, with its own bounded timeout. `/sce/mutation-cursor.lock` +hold it until the call returns. The separate `ref_maintenance::reconcile_explicit` +pass acquires this same lock exactly once, before inventorying pins, reading +durable roots, or deleting refs, with a bounded 10 s wait; a boundary arriving +meanwhile waits up to `WORKTREE_LOCK_TIMEOUT` and then fails before durable +completion. The hook-path advisory takes the lock with a zero-wait `try_lock` +only after `coordinate()` has returned. `/sce/mutation-cursor.lock` remains worktree-specific because `git_dir` itself is worktree-specific for linked worktrees (`resolve_git_dir` resolves each worktree's own `--absolute-git-dir`), so each worktree has an independent critical section. @@ -217,7 +220,7 @@ On-disk layout so far: (runtime::git_snapshot writes here directly) -└── refs/sce/mutation-cursor// (runtime::git_snapshot, create-only per invocation; orphan/unreferenced pins reclaimed by runtime::ref_reconciliation only for a checkout id a current worktree still derives, every pin for a current or historical durable mutation-cursor root retained; a namespace no current worktree owns — deleted worktree or checkout-id metadata loss/recreation — is unreachable, future repository-scoped work) +└── refs/sce/mutation-cursor// (runtime::git_snapshot, create-only per invocation; orphan/unreferenced pins reclaimed by runtime::ref_maintenance only through sce doctor --fix and only for the namespace of a currently resolvable worktree, every pin for a current or historical durable mutation-cursor root retained; refs owned by removed linked worktrees are unreachable, future repository-scoped work) ``` ## Testing boundary @@ -289,28 +292,15 @@ durable, the boundary unprocessed with no `MutationEvent`, the marker still present, and a later `coordinate()` re-recovering off it; another proves an attributable `Advance` that commits then fails its trailing `marker.clear()` surfaces `MarkerClearAfterCommit` with the matching committed outcome. The -`after_load` seam is exercised by the reconciliation pin→CAS lock-race regression -([`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)), pausing a real `coordinate()` between `pin` and CAS. +The `after_load` seam was used by the removed reconciliation pin→CAS lock-race regression and is unused while that suite is gone. -`runtime/tests.rs` is `runtime`'s own `#[cfg(test)] mod tests`, holding -cross-module integration tests that drive the public `coordinate()` and -`abandon_scope()` APIs against real Git repositories (`git init`, `git worktree -add`) and real temp-file `RepositoryAgentTraceDb`s, following the same -unique-temp-path precedent: two linked worktrees of one repository (different -`git_dir` → different lock paths → different `WorktreeId`s) are proven -independently locked by holding one worktree's `WorktreeLock` across a -synchronous `coordinate()` call for the other and observing that call return -`Ok` before the held guard is dropped — a shared lock could not be acquired -while the guard is alive, and no wall-clock timing is used. Each call is handed -a provider closure that opens the one shared repository-scoped DB path -(`coordinate()` never resolves the DB), and both distinct worktree rows then -coexist in it. A full failure/recovery cycle and the cross-runtime abandonment -regressions also run through the public entrypoints. The same module covers the -public `reconcile_worktree` integration suite and the -`coordinate_inner` / `reconcile_worktree_inner` / `abandon_scope_inner` -lock-race seams, including orphan reclamation, durable-root retention, -missing-pin fail-closed behavior, malformed refs, linked-worktree scoping, and -the real coordinator and abandonment CAS races. +There is no `runtime/tests.rs` cross-module integration suite. Coverage of +`coordinate()` / `abandon_scope()` rests on the Quint MBT suite in +`mutation_trace/mbt`, the `mutation_trace` unit tests and the inline +`worktree_lock.rs` / `git_snapshot.rs` tests. Reconciliation has its own +production-path and concurrency suite under `ref_maintenance/tests/` (see +[`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md)), +including tests that drive the real coordinator against a parked pass. ## Status @@ -319,7 +309,7 @@ pipeline, the public `coordinate()` entrypoint (prefix → DB provider → pipel → `complete()` on success), and the second `abandon_scope()` entrypoint sharing that prefix are all implemented and both `pub(crate)` re-exported from `runtime/mod.rs` ([`mutation-scope-runtime.md`](mutation-scope-runtime.md)), -with `runtime/tests.rs` covering `coordinate()` end to end and both entrypoints +with the former `runtime/tests.rs` end-to-end coverage removed (see Testing boundary) and both entrypoints driven together; an inherited external-taint marker is overlaid onto `database_failure` recovery on the next invocation. The generic `sce hooks mutation-scope` CLI ingress now drives both entrypoints diff --git a/context/cli/mutation-trace-scope-abandonment.md b/context/cli/mutation-trace-scope-abandonment.md index 247b0c089..9993f2cda 100644 --- a/context/cli/mutation-trace-scope-abandonment.md +++ b/context/cli/mutation-trace-scope-abandonment.md @@ -163,7 +163,7 @@ whole transition back (below); and a `clear()` failure returning A `pub(super) abandon_scope_inner(.., after_load)` seam (mirroring `coordinate_inner`) fires once per CAS attempt after the projection loads, so a -test — in this module or in `runtime/tests.rs` — can land a competing write +test in this module can land a competing write inside the CAS window. It is invisible outside `runtime`; production passes a no-op. @@ -188,8 +188,8 @@ exists, and the external-taint marker is still armed — the general rule that ### Cross-runtime regressions -`runtime/tests.rs` ([`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md#testing-boundary)) -drives `coordinate()` and `abandon_scope()` together over real `git init` / +The former `runtime/tests.rs` cross-module suite was removed ([`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md#testing-boundary)); it +drove `coordinate()` and `abandon_scope()` together over real `git init` / `git worktree add` repositories and one real repository-scoped Agent Trace DB, covering what a single-module test with no real Git cannot: diff --git a/context/cli/mutation-trace-snapshot-service.md b/context/cli/mutation-trace-snapshot-service.md index 2ba710f5d..e14684a31 100644 --- a/context/cli/mutation-trace-snapshot-service.md +++ b/context/cli/mutation-trace-snapshot-service.md @@ -119,8 +119,8 @@ second, read-only caller: the bounded mutation-history attribution consumer transition and reads baseline file content through its own `TreeReadSource` trait (see [`mutation-trace-agent-attribution.md`](mutation-trace-agent-attribution.md)). -`list_pins` / `delete_pins` are consumed by the per-worktree -ref-reconciliation maintenance pass. +`list_pins` / `delete_pins` are consumed only by the per-worktree +ref-reconciliation maintenance pass, which runs only through `sce doctor --fix`: pins orphaned by crashed or failed `coordinate()` paths are reclaimed when an operator runs it, and automatic reclamation does not exist. `delete_pins` is the only ref-deletion path and moves the `WorktreeLockLease` into its blocking worker, so dropping the caller future cannot release the lock while `git update-ref` runs. See [`mutation-trace-ref-reconciliation.md`](mutation-trace-ref-reconciliation.md). ## Testing boundary diff --git a/context/cli/mutation-trace-store.md b/context/cli/mutation-trace-store.md index 50ed63725..5f19aea2e 100644 --- a/context/cli/mutation-trace-store.md +++ b/context/cli/mutation-trace-store.md @@ -136,7 +136,7 @@ worktree is still an `Err` there. cold-path, read-only queries — siblings of `load_mutation_event`, never reached from `load_worktree` or a hook-boundary path — that expose the set of Git tree SHAs the mutation-cursor protocol still durably depends on, for the -per-worktree ref-reconciliation pass (see +per-worktree ref-reconciliation pass, which runs only through `sce doctor --fix` (the single-event loaders `load_mutation_event` / `load_mutation_event_active_scopes` and their SQL/row-mapping helpers have no production reader and carry declaration-level `dead_code` allows saying so; attribution history uses `load_mutation_event_page`; see [`mutation-trace-runtime-coordinator.md`](mutation-trace-runtime-coordinator.md)). - `load_tree_roots(worktree) -> BTreeSet` returns one worktree's roots: @@ -174,8 +174,8 @@ These are pure reads: no schema change, no migration, no write path. ## Write path -`MutationTraceStore::commit(transition: &DurableTransition) -> Result` -translates the transition into one worktree CAS `UPDATE`, zero or more scope +`MutationTraceStore::commit(&mut self, transition: &DurableTransition) -> Result` +(the store holds an exclusive `&mut` to the DB) translates the transition into one worktree CAS `UPDATE`, zero or more scope status `UPDATE`s, an optional processed-event `INSERT`, and an optional mutation-event `INSERT` plus its active-scope `INSERT`s — all run through `TursoDb::execute_transactional_cas_batch` inside exactly one diff --git a/context/cli/opencode-mutation-scope-adapter-lifecycle.md b/context/cli/opencode-mutation-scope-adapter-lifecycle.md index 5c30c3ca3..70faaad4e 100644 --- a/context/cli/opencode-mutation-scope-adapter-lifecycle.md +++ b/context/cli/opencode-mutation-scope-adapter-lifecycle.md @@ -9,7 +9,7 @@ identity encoding, wire contract, plugin, and attribution boundary, see The adapter processes one hook event at a time under a per-`git-dir` boundary lock (`opencode-mutation-scope-boundary.lock`), serialising boundary work across -concurrent OpenCode processes. +concurrent OpenCode processes. The lock is a lease (`AdapterLockSpec::run_under_boundary`): a state transition already started in `spawn_blocking` keeps it until the worker finishes, so a cancelled boundary cannot be overtaken. - **Attempt phases.** An attempt is `PendingStart` → `Active` → `PendingAbandon`. `PendingAbandon` means exact terminal evidence has been observed, the OpenCode diff --git a/context/cli/patch-service.md b/context/cli/patch-service.md index 20d8ae307..4ede0648b 100644 --- a/context/cli/patch-service.md +++ b/context/cli/patch-service.md @@ -38,7 +38,6 @@ All types derive `Clone, Debug, Deserialize, Eq, PartialEq, Serialize` and suppo Storage-agnostic helpers for reconstructing `ParsedPatch` from serialized JSON content: - `load_patch_from_json(input: &str) -> Result` — loads a `ParsedPatch` from a JSON string; callers who have already read JSON from a database or file can pass the string directly -- `load_patch_from_json_bytes(input: &[u8]) -> Result` — loads a `ParsedPatch` from JSON bytes; convenient when the caller has raw bytes (for example, from a database BLOB column or file read) rather than a UTF-8 string Both functions wrap `serde_json::from_str`/`serde_json::from_slice` and map serde errors to actionable `PatchLoadError` messages. `PatchLoadError` carries a `message` field describing why the JSON payload could not be reconstructed into a valid `ParsedPatch`. @@ -96,7 +95,7 @@ builder, pending state, or schema migration is introduced. | Operation | Wired into | Notes | |-----------|-----------|-------| | `parse_patch` | Hook runtime, Agent Trace DB recent-row parsing, tests | Consumed by `post-commit` capture flow and `recent_diff_trace_patches` to parse stored raw `diff_traces.patch` text | -| `load_patch_from_json` / `load_patch_from_json_bytes` | Storage-agnostic JSON reconstruction callers | Reconstructs serialized `ParsedPatch` JSON when callers already have JSON payloads; not used for raw `diff_traces.patch` text in `recent_diff_trace_patches` | +| `load_patch_from_json` | Storage-agnostic JSON reconstruction callers | Reconstructs serialized `ParsedPatch` JSON when callers already have JSON payloads; not used for raw `diff_traces.patch` text in `recent_diff_trace_patches` | | `intersect_patches` | Post-commit hook runtime | Combines recent patches then intersects with current commit patch | | `combine_patches` | Post-commit hook runtime | Combines chronological recent patches before intersection | diff --git a/context/cli/repository-identity.md b/context/cli/repository-identity.md index e14b35b7a..890d78a49 100644 --- a/context/cli/repository-identity.md +++ b/context/cli/repository-identity.md @@ -12,8 +12,6 @@ The root module (`mod.rs`) performs no I/O: it never opens databases, reads Git - `canonicalize_remote_url(raw) -> Result` — canonicalization without hashing. - `derive_repository_id(canonical_identity) -> String` — domain-separated SHA-256 hex. - `REPOSITORY_ID_HASH_DOMAIN` — the `b"sce-repository-id-v1\0"` prefix constant. -- `repository_dir_segment(canonical_identity) -> String` — pure human-readable on-disk directory segment `-`, where `slug` is the lowercased canonical identity with non-alphanumeric runs collapsed to a single `-` and leading/trailing `-` trimmed, and `short` is the first 4 hex chars of `SHA256(canonical_identity)` with **no** domain prefix (deliberately distinct from `repository_id`, which keeps its `sce-repository-id-v1\0` prefix). All-non-alphanumeric input slugs to empty and the segment falls back to just `short`. Display/layout helper only — the authoritative identity stays `repository_id`. Not yet wired into path construction (T02 of `human-readable-repo-db-directory`). -- `RepositoryIdentity::dir_segment(&self) -> String` — convenience wrapper over `repository_dir_segment` for `self.canonical_identity`. ## Canonicalization rules (remote URLs) diff --git a/context/cli/service-lifecycle.md b/context/cli/service-lifecycle.md index 4727f4874..586506cbe 100644 --- a/context/cli/service-lifecycle.md +++ b/context/cli/service-lifecycle.md @@ -5,15 +5,15 @@ ## Current contract - `ServiceLifecycle: Send + Sync` exposes three default no-op generic methods against any context implementing the narrow repo-root accessor: - - `diagnose(&self, ctx: &C) -> Vec` - - `fix(&self, ctx: &C, problems: &[HealthProblem]) -> Vec` - - `setup(&self, ctx: &C) -> anyhow::Result` + - `async diagnose(&self, ctx: &C) -> Vec` + - `async fix(&self, ctx: &C, problems: &[HealthProblem]) -> Vec` + - `async setup(&self, ctx: &C) -> anyhow::Result` - `HealthProblem`, `HealthCategory`, `HealthSeverity`, `HealthFixability`, and `HealthProblemKind` are lifecycle-owned types that mirror the current doctor taxonomy without making the trait depend on `doctor` module types. - `FixResultRecord` and `FixOutcome` are lifecycle-owned fix result types. - `SetupOutcome` is a minimal lifecycle-owned carrier for current setup result shapes: - generic setup `messages` - optional lifecycle-owned `RequiredHooksInstallOutcome` -- `LifecycleProvider` is a static enum over the concrete provider implementations, and `lifecycle_providers(include_hooks)` is the shared provider catalog/factory used by command orchestrators. The enum owns inherent generic `id`, `diagnose`, `fix`, and `setup` dispatch methods and does not allocate boxed provider trait objects or erase the lifecycle context to `&dyn HasRepoRoot`; provider dispatch remains compile-time typed over the narrow repo-root accessor. +- `LifecycleProvider` is a static enum over the concrete provider implementations, and `lifecycle_providers(include_hooks)` is the shared provider catalog/factory used by command orchestrators. The enum owns inherent generic `id`, `diagnose`, `fix`, and `setup` dispatch methods; callers await the async lifecycle operations and does not allocate boxed provider trait objects or erase the lifecycle context to `&dyn HasRepoRoot`; provider dispatch remains compile-time typed over the narrow repo-root accessor. - Provider order is deterministic: `ConfigLifecycle` → `LocalDbLifecycle` → `AuthDbLifecycle` → `AgentTraceDbLifecycle` → `HooksLifecycle` when hooks are included. ## Current boundaries @@ -34,7 +34,7 @@ - `cli/src/services/agent_trace_db/lifecycle.rs` defines `AgentTraceDbLifecycle`, the Agent Trace DB-owned provider. - `AgentTraceDbLifecycle::diagnose` resolves repository identity from `ctx.repo_root()` plus `agent_trace.repository_id` / `agent_trace.repository_remote`, and emits repository-scoped Agent Trace DB path and parent-directory readiness lifecycle health problems. Outside repository context there is no global/checkout fallback path (removed by the `retire-legacy-agent-trace-db` plan); `resolve_lifecycle_agent_trace_db_path` returns an actionable "requires a Git repository" diagnostic, surfaced as a manual-only `UnableToResolveStateRoot` problem. - `AgentTraceDbLifecycle::fix` bootstraps the resolved repository Agent Trace DB parent directory for auto-fixable DB parent readiness problems; outside repository context it returns the same actionable no-repository diagnostic rather than probing a sentinel path. -- `AgentTraceDbLifecycle::setup` resolves repository storage from `ctx.repo_root()` when available, resolves `/sce/repos//agent-trace.db`, opens/creates that database through `RepositoryAgentTraceDb` so the repository schema is applied and metadata is validated, and returns setup messaging with the repository ID and initialized DB path. Hook runtime uses the same repository storage resolver lazily when setup has not run or schema metadata is incomplete. +- `AgentTraceDbLifecycle::setup` resolves repository storage from `ctx.repo_root()` when available, resolves `/sce/repos//agent-trace.db`, opens/creates that database through `RepositoryAgentTraceDb` so the repository schema is applied and metadata is validated, and returns setup messaging with the repository ID and initialized DB path. Hook runtime shares the storage identity/path resolution but opens through the separate no-migration path (`resolve_agent_trace_storage_for_hook_runtime`); it never runs migrations and surfaces `Run 'sce setup'.` guidance when the schema is missing or stale. See [agent-trace-storage.md](agent-trace-storage.md). - `doctor` runtime execution now aggregates lifecycle providers for diagnosis and repair: - `cli/src/services/doctor/command.rs` accepts any context implementing `ContextWithRepoRoot`. - `cli/src/services/doctor/mod.rs` resolves the repository root once, creates a repo-root-scoped borrowed context using `with_repo_root()`, and requests the full provider catalog with hooks included. diff --git a/context/cli/sync-command.md b/context/cli/sync-command.md index 4b96cd2be..7a49d7f7f 100644 --- a/context/cli/sync-command.md +++ b/context/cli/sync-command.md @@ -6,7 +6,7 @@ control-plane ingestion API. The former `sce trace` command group and its database discovery, shell, list, status, and nested sync invocations are no longer available; no compatibility alias is retained. -Sync is directly awaited on the application-level multi-thread Tokio runtime. Its storage constructor uses a blocking scope; a concrete guard keeps the synchronous DB alive across HTTP awaits and destroys it in a blocking scope before return, including error, cancellation and unwind cleanup. Progress remains alive through the await and presentation finalization occurs only on success. See [application execution runtime](../architecture.md#application-execution-runtime). +Sync and its repository storage, export-reader queries, and credential operations are directly awaited on the application-level multi-thread Tokio runtime. Repository storage remains alive across HTTP awaits through ordinary ownership; no blocking constructor/destructor scope or runtime-drop guard is needed. Progress remains alive through the await and presentation finalization occurs only on success. See [application execution runtime](../architecture.md#application-execution-runtime). The Clap surface is defined in `cli/src/cli_schema.rs` and dispatched through the static `RuntimeCommand::Sync` variant. The sync-owned command boundary lives @@ -60,7 +60,10 @@ resolved `control_plane_base_url`, then performs one authoritative `/state` request before starting the three concurrent remote stream state machines: `messages`, `parts`, and `agent_traces`. Batches and cursor refreshes remain sequential within each stream, and final reporting retains the fixed stream -order. +order. The three remote streams execute concurrently and are all driven to +terminal completion once started. The first observed stream error is returned +after the remaining started streams finish, preventing sibling cancellation from +interrupting credential persistence or other in-flight stream cleanup. The local database has four capture streams (`messages`, `parts`, `diff_traces`, `agent_traces`); `sce sync` remotely synchronizes three of them. `diff_traces` diff --git a/context/context-map.md b/context/context-map.md index 97c46ac5a..b971f89ea 100644 --- a/context/context-map.md +++ b/context/context-map.md @@ -2,7 +2,7 @@ Primary context files: -Application runtime ownership: [architecture](architecture.md#application-execution-runtime), [accepted decision](decisions/2026-10-07-application-owned-async-command-runtime.md). +Application runtime ownership: [architecture](architecture.md#application-execution-runtime), [accepted decision](decisions/2026-10-07-application-owned-async-command-runtime.md), [directly awaited persistence](decisions/2026-10-07-directly-awaited-turso-persistence.md), [PR2 scope and cancellation protections](decisions/2026-10-08-pr2-scope-and-cancellation-protections.md). - `context/overview.md` - `context/architecture.md` @@ -11,36 +11,40 @@ Application runtime ownership: [architecture](architecture.md#application-execut Feature/domain context: -- `context/cli/cli-command-surface.md` (CLI command surface including top-level help with ASCII art banner and gradient rendering, setup install flow with the repeatable `sce setup --workflow ` optional-workflow selection and its interactive post-target multi-select, WorkOS device authorization flow + token storage behavior including stored-credential renewal through `sce auth login`, attribution-only hook routing with validated post-commit `--remote-url` plumbing plus DB-backed `diff-trace` dual persistence and post-commit Agent Trace payload persistence including range `content_hash`, setup-owned local DB + repository-scoped Agent Trace DB bootstrap plus doctor DB health coverage with credential-safe repository identity diagnostics, centralized Rust SCE web URL helpers in `services::agent_trace`, nested flake release package/app installability, Cargo local install + crates.io readiness policy, hidden `sce policy bash` command adapter for bash-policy hook callers, and top-level `sce sync` command wiring for current-repository Agent Trace synchronization; directly awaited static `RuntimeCommand` enum dispatch under the application runtime lives in `services/command_registry.rs`, command payload structs for help/version/completion/auth/config/setup/doctor/hooks/policy/sync are owned by their respective `services/{name}/command.rs` files, and clap-to-runtime conversion lives in `services/parse/command_runtime.rs`) +- `context/cli/cli-command-surface.md` (CLI command surface including top-level help with ASCII art banner and gradient rendering, setup install flow with the repeatable `sce setup --workflow ` optional-workflow selection and its interactive post-target multi-select, WorkOS device authorization flow + token storage behavior including stored-credential renewal through `sce auth login`, attribution-only hook routing with validated post-commit `--remote-url` plumbing plus DB-backed `diff-trace` dual persistence and post-commit Agent Trace payload persistence including range `content_hash`, setup-owned local DB + repository-scoped Agent Trace DB bootstrap plus doctor DB health coverage with credential-safe repository identity diagnostics, centralized Rust SCE web URL helpers in `services::agent_trace`, nested flake release package/app installability, Cargo local install + crates.io readiness policy, hidden `sce policy bash` command adapter for bash-policy hook callers, and top-level `sce sync` command wiring for current-repository Agent Trace synchronization; directly awaited static `RuntimeCommand` enum dispatch under the application runtime and the async persistence path through Turso live in `services/command_registry.rs` and `services/db/mod.rs`, command payload structs for help/version/completion/auth/config/setup/doctor/hooks/policy/sync are owned by their respective `services/{name}/command.rs` files, and clap-to-runtime conversion lives in `services/parse/command_runtime.rs`) - `context/cli/default-path-catalog.md` (canonical production CLI path-ownership contract centered on `cli/src/services/default_paths.rs`, including persisted auth/config files, named DB paths for auth/local/repository-scoped Agent Trace databases, the default observability log-dir accessor consumed by config resolution with Linux `${XDG_STATE_HOME:-~/.local/state}/sce/logs` fallback semantics, repo-relative, embedded-asset, install, hook, and context-path families plus the regression guard that keeps production path ownership centralized) - `context/cli/repository-identity.md` (repository identity module in `cli/src/services/repository_identity/`: pure scheme-neutral `host[:port]/path` canonicalization for SCP/`ssh://`/HTTPS/`git://` remote URLs with credential stripping, hostname lowercasing, default-port removal, and query/fragment/trailing-`.git` cleanup, trim-only explicit-identity handling, `sha256("sce-repository-id-v1\0" + canonical_identity)` repository IDs, credential-safe fieldless errors, plus the `resolve` runtime submodule applying explicit-config-then-configured-remote precedence with `git config --get remote..url` lookup, `RepositoryIdentitySource` provenance, and `.sce/config.json`-guidance resolution errors that never echo URLs; consumed by the T04 `agent_trace_storage` resolver) -- `context/cli/agent-trace-storage.md` (repository-scoped Agent Trace storage resolver in `cli/src/services/agent_trace_storage/`: `AgentTraceStorageContext` inputs mirroring the `agent_trace.*` config keys, shared degradation of invalid default-discovered config before identity fallback while explicit selections remain fatal, `ResolvedAgentTraceStorage` carrying repository identity + `/sce/repos//agent-trace.db` path + open `RepositoryAgentTraceDb` + typed `RepositoryMetadata`, `resolve_agent_trace_storage{,_at_state_root}` setup/lifecycle entrypoints with idempotent concurrent-safe first open via bounded fast-path-then-migrate retry plus narrow one-file schema migration-metadata repair and repository metadata validation, a separate no-migration `resolve_agent_trace_storage_for_hook_runtime{,_at_state_root}` pair that high-frequency hook callers use exclusively and that never runs migration `002` or any migration, path-unsafe repository ID rejection in `default_paths::agent_trace_db_path_for_repository{,_at}`, strict never-touch boundary for any pre-migration checkout-scoped/global DB files, and active hook/runtime plus Agent Trace lifecycle setup call-site consumption after T08) +- `context/cli/agent-trace-storage.md` (async repository-scoped Agent Trace storage resolver in `cli/src/services/agent_trace_storage/`: `AgentTraceStorageContext` inputs mirroring the `agent_trace.*` config keys, shared degradation of invalid default-discovered config before identity fallback while explicit selections remain fatal, `ResolvedAgentTraceStorage` carrying repository identity + `/sce/repos//agent-trace.db` path + open `RepositoryAgentTraceDb` + typed `RepositoryMetadata`, `resolve_agent_trace_storage{,_at_state_root}` setup/lifecycle entrypoints with idempotent concurrent-safe first open via bounded fast-path-then-migrate retry plus narrow one-file schema migration-metadata repair and repository metadata validation, a separate no-migration `resolve_agent_trace_storage_for_hook_runtime{,_at_state_root}` pair that high-frequency hook callers use exclusively and that never runs migration `002` or any migration, path-unsafe repository ID rejection in `default_paths::agent_trace_db_path_for_repository{,_at}`, strict never-touch boundary for any pre-migration checkout-scoped/global DB files, and active hook/runtime plus Agent Trace lifecycle setup call-site consumption after T08) - `context/cli/checkout-identity.md` (checkout identity infrastructure, `cli/src/services/checkout/` and its `/sce/checkout-id` UUIDv7 storage, was removed by the `remove-checkout-id` plan; documents the removal plus the never-touch on-disk handling of any pre-existing checkout-id/pre-migration checkout-scoped DB files left behind) -- `context/cli/patch-service.md` (standalone patch domain model, parser, JSON load helpers, and set operations in `cli/src/services/patch.rs` for in-memory parsed unified-diff representation, capturing only touched lines plus minimal per-file/per-hunk metadata, supporting both `Index:` SVN-style and `diff --git` git-style formats, with `ParseError` for actionable malformed-input diagnostics, `PatchLoadError`/`load_patch_from_json`/`load_patch_from_json_bytes` for storage-agnostic JSON reconstruction, `intersect_patches` for target-shaped overlap with exact-match-first and historical `kind`+`content` fallback semantics plus matched-constructed-line `session_id` and matched-constructed-hunk `model_id` provenance inheritance, and `combine_patches` for ordered patch combination with later-wins conflict resolution plus winning-hunk `model_id` provenance inheritance; repository structured-row reconstruction supplies persisted hunk-model and canonical touched-line-session provenance before these operations; `parse_patch`, `intersect_patches`, and `combine_patches` are consumed by the active post-commit hook runtime) +- `context/cli/patch-service.md` (standalone patch domain model, parser, JSON load helpers, and set operations in `cli/src/services/patch.rs` for in-memory parsed unified-diff representation, capturing only touched lines plus minimal per-file/per-hunk metadata, supporting both `Index:` SVN-style and `diff --git` git-style formats, with `ParseError` for actionable malformed-input diagnostics, `PatchLoadError`/`load_patch_from_json` for storage-agnostic JSON reconstruction, `intersect_patches` for target-shaped overlap with exact-match-first and historical `kind`+`content` fallback semantics plus matched-constructed-line `session_id` and matched-constructed-hunk `model_id` provenance inheritance, and `combine_patches` for ordered patch combination with later-wins conflict resolution plus winning-hunk `model_id` provenance inheritance; repository structured-row reconstruction supplies persisted hunk-model and canonical touched-line-session provenance before these operations; `parse_patch`, `intersect_patches`, and `combine_patches` are consumed by the active post-commit hook runtime) - `context/cli/structured-patch-service.md` (Claude structured editor-hook derivation in `cli/src/services/structured_patch.rs`, including `Write` structured-update hunks, `Write` `tool_input.content` create fallback, `Edit` structured patches, deterministic skip reasons, `ParsedPatch` output semantics, Rust golden fixture coverage, and repository read-time enrichment that assigns persisted row `model_id` to each hunk and canonical row `session_id` to each touched line) - `context/cli/styling-service.md` (CLI text-mode output styling with `owo-colors`, TTY/`NO_COLOR` policy, shared helper API for human-facing surfaces including sync completion markers, and per-column right-to-left RGB gradient banner rendering) - `context/cli/agent-trace-auto-sync.md` (setup-enabled post-commit Agent Trace synchronization with explicit-false opt-out plus doctor readiness reporting: a newly created repo-local config supplies the explicit `true` opt-in while omitted runtime values resolve to `false`; the existing `sce sync` command is launched once through the current executable after local persistence, with detached null-standard-stream behavior, fail-open startup, canonical managed-block proof of hook readiness, stable text/JSON enabled/disabled/not-ready/not-applicable states, no daemon/queue/high-frequency trigger, and retryability through manual sync and control-plane cursor authority) - `context/cli/sync-command.md` (the top-level `sce sync` command: repository-scoped Agent Trace storage resolution, WorkOS-authenticated three-stream control-plane synchronization (`messages`, `parts`, `agent_traces`; `diff_traces` stays local with a `streams.diffTraces` compatibility no-op entry) through the sync-owned consumer-typed `services::sync::progress` reporter contract with sync-owned events, its generic/no-op contract and `indicatif` presentation adapter for aligned stderr progress with independent stream completion, explicit successful finalization, JSON stdout silence, and rejection of the removed `sce trace` command group) +- `context/cli/async-persistence-guarantees.md` (verified guarantees and named limits of the directly awaited Turso persistence migration — exclusive `&mut` transactions, deferred rollback, credential-rotation and terminal-sync behavior, lease/descriptor protection — plus PR1–PR4 staging and the PR3 cleanup ledger) - `context/cli/agent-trace-sync-command.md` (composed local-to-control-plane `sce sync` architecture: the `hooks/plugins → repository Agent Trace DB → AgentTraceExportReader → sce sync → HTTPS + WorkOS Bearer → control plane` data flow, the `sce auth login` / `cd ` / `sce sync` user flow, the no-local-cursor/no-`agent-trace-sync.db`/no-Turso-Sync/no-`BridgeLock`/no-local-DWH invariants, the four-local-capture-streams versus three-remote-streams boundary with its `diff_traces` release gate, and `401`/`409`/ambiguous-batch-failure recovery semantics) - `context/cli/config-precedence-contract.md` (implemented `sce config` show/validate command contract, deterministic `flags > env > config file > defaults` resolution order, focused `config/resolver.rs` ownership for config discovery/merge/runtime precedence plus default-discovered invalid-file degradation, focused `config/render.rs` ownership for `show`/`validate` text+JSON output construction, canonical `$schema` acceptance for startup-loaded `sce/config.json` files, shared auth-key env/config/optional baked-default support starting with `workos_client_id`, shared runtime resolution for flat logging observability keys including config-file/default `log_to_file`, `log_dir` / `SCE_LOG_DIR` with `/sce/logs` defaulting plus config-file/default-only positive `log_file_retention_limit`, config-file-only `agent_trace.repository_id`/`agent_trace.repository_remote` repository-identity keys with default remote `origin`, `agent_trace.auto_sync` resolution with omitted-value fallback `false`, explicit-false opt-out, and setup-written explicit `true` bootstrap behavior for the post-commit trigger boundary, the catalog-derived `integrations.optional_workflows` optional-workflow selection key, JSON-pointer-prefixed schema-validation errors, canonical Pkl-generated `sce/config.json` schema ownership plus CLI embedding/reuse contract including `policies.attribution_hooks.enabled` default-true/explicit-false opt-out metadata, config-file selection order, `show` provenance output, and trimmed `validate` output contract) -- `context/cli/capability-traits.md` (current broad CLI capability seam in `cli/src/services/capabilities.rs`, including `FsOps`/`StdFsOps`, `GitOps`/`ProcessGitOps`, git root/hooks resolution behavior, compile-time-typed borrowed AppContext wiring with scoped borrowing across directly awaited commands with associated-type narrow capability accessors plus `ContextWithRepoRoot` repo-root-scoped context derivation, generic command execution bounds, and test-only unimplemented stubs; current service internals do not consume fs/git traits until later lifecycle migration tasks) -- `context/cli/service-lifecycle.md` (current compile-safe lifecycle seam in `cli/src/services/lifecycle.rs`, including default no-op `ServiceLifecycle` diagnose/fix/setup methods against narrow `HasRepoRoot`, lifecycle-owned health/fix/setup result types with generic setup messages, doctor/setup adapter boundaries, the static `LifecycleProvider` enum catalog/dispatcher, hook/config/local_db/auth_db/agent_trace_db lifecycle providers including setup-time repository-scoped Agent Trace DB initialization plus checkout identity diagnostics, implemented doctor aggregation over diagnose/fix providers, and implemented setup aggregation over `setup` providers in order config → local_db → auth_db → agent_trace_db → hooks when requested) +- `context/cli/capability-traits.md` (current broad CLI capability seam in `cli/src/services/capabilities.rs`, including `FsOps`/`StdFsOps` (method-less marker), `GitOps`/`ProcessGitOps` (command execution and availability), compile-time-typed borrowed AppContext wiring with scoped borrowing across directly awaited commands with associated-type narrow capability accessors plus `ContextWithRepoRoot` repo-root-scoped context derivation, generic command execution bounds, and test-only unimplemented stubs; current service internals do not consume fs/git traits until later lifecycle migration tasks) +- `context/cli/service-lifecycle.md` (current async lifecycle seam in `cli/src/services/lifecycle.rs`, including default no-op `ServiceLifecycle` diagnose/fix/setup methods against narrow `HasRepoRoot`, lifecycle-owned health/fix/setup result types with generic setup messages, doctor/setup adapter boundaries, the static `LifecycleProvider` enum catalog/dispatcher, hook/config/local_db/auth_db/agent_trace_db lifecycle providers including setup-time repository-scoped Agent Trace DB initialization plus checkout identity diagnostics, implemented doctor aggregation over diagnose/fix providers, and implemented setup aggregation over `setup` providers in order config → local_db → auth_db → agent_trace_db → hooks when requested) - `context/cli/mutation-trace-protocol.md` (pure, dependency-free `cli/src/services/mutation_trace/` refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol: the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — plus cross-action sequence/invariant tests and a module-level Quint refinement matrix are all complete (`types.rs`, `protocol.rs`, `mod.rs`, `tests.rs`, registered with `#[allow(dead_code)]`); opaque-newtype identity refinement decisions vs. the Quint model's bounded enums, and the `coordinator.rs` target end-state seam this layout leaves room for but does not create — `store.rs`, `runtime/git_snapshot.rs`, and `runtime/coordinator.rs` (including its public `coordinate()` entrypoint) now exist, built out by the `mutation-cursor-store-persistence` and `mutation-cursor-runtime-coordinator` plans respectively; `protocol.rs` is invoked only through the runtime, and the runtime is now driven by the generic `sce hooks mutation-scope` CLI ingress and a first Claude Code adapter driver, registered by `sce setup` and reachable by a real Claude Code session, now proven against real Git repositories and a real Agent Trace DB by the `claude-mutation-scope-integration` plan's T08 regressions) - `context/cli/mutation-trace-runtime-materialization.md` (how the finite, pre-populated Quint `SCOPES`/`WORKTREES` universes map onto the Rust protocol's unbounded identity space: the coordinator/store layer materializes scope and worktree state, scope identity is immutable once materialized, and a missing scope is distinct from a `NeverSeen` scope; `protocol.rs` only transitions already-known identities) -- `context/cli/mutation-trace-agent-attribution.md` (causal mutation-lineage attribution: direct target-shaped coverage is resolved first, then the newest-128 events for the invoking worktree (bounded also by a commit attribution cut — `revision <= latest_mutation_event_revision` captured under the worktree lock) are replayed oldest-to-newest as one ordered sequence of tree transitions. `lineage.rs` is a pure module propagating per-line `LineProvenance` (`Unknown` / `MutationAi{scope}` / `MutationNonAi`) forward through structurally-applied hunks: context carries, removed is permanently deleted, added takes only the introducing transition's origin, replacement never transfers by text. A committed line is AI only if an AI event's line survives every later transition — including a history-gap reload and the unobserved latest-tree→commit-tree tail — into the committed tree; anything unproven stays `Unknown`. Historical patches are never matched against the committed patch. `attribution.rs` keeps only `exclude_direct_coverage` + `patch_for_locations`. The `runtime/mutation_attribution.rs` consumer drives it over `MutationEventPageSource` + `TreeReadSource` (`diff_trees` + `file_at_tree`) seams with conservative fail-closed reloads; wired into post-commit via the read-only `runtime::resolve_post_commit_mutation_ai_patch` entrypoint — existing checkout identity only, direct-only fallback on absent identity/history/unavailable cut, no identity creation, no mutation-cursor write, `diff_traces` / `post_commit_patch_intersections` unchanged; see `agent-trace-hooks-command-routing.md` and `agent-trace-minimal-generator.md` — with real Git/DB post-commit regressions in `cli/src/services/hooks/mod.rs` and a real 128/129-horizon regression in `runtime/tests.rs`) +- `context/cli/mutation-trace-agent-attribution.md` (causal mutation-lineage attribution: direct target-shaped coverage is resolved first, then the newest-128 events for the invoking worktree (bounded also by a commit attribution cut — `revision <= latest_mutation_event_revision` captured under the worktree lock) are replayed oldest-to-newest as one ordered sequence of tree transitions. `lineage.rs` is a pure module propagating per-line `LineProvenance` (`Unknown` / `MutationAi{scope}` / `MutationNonAi`) forward through structurally-applied hunks: context carries, removed is permanently deleted, added takes only the introducing transition's origin, replacement never transfers by text. A committed line is AI only if an AI event's line survives every later transition — including a history-gap reload and the unobserved latest-tree→commit-tree tail — into the committed tree; anything unproven stays `Unknown`. Historical patches are never matched against the committed patch. `attribution.rs` keeps only `exclude_direct_coverage` + `patch_for_locations`. The `runtime/mutation_attribution.rs` consumer drives it over `MutationEventPageSource` + `TreeReadSource` (`diff_trees` + `file_at_tree`) seams with conservative fail-closed reloads; wired into post-commit via the read-only `runtime::resolve_post_commit_mutation_ai_patch` entrypoint — existing checkout identity only, direct-only fallback on absent identity/history/unavailable cut, no identity creation, no mutation-cursor write, `diff_traces` / `post_commit_patch_intersections` unchanged; see `agent-trace-hooks-command-routing.md` and `agent-trace-minimal-generator.md` — with real Git/DB post-commit regressions in `cli/src/services/hooks/mod.rs`) - `context/cli/mutation-trace-revision-refinement.md` (the Quint `revision: int` → Rust `WorktreeState::revision: u64` bounded-integer refinement: the private `next_revision` checked-arithmetic helper `commit`/`taint`/`abandon`/`recover` all route through instead of a raw `+ 1`, so a worktree at `revision: u64::MAX` is a guarded no-op/rejection rather than a silent wrap to `0`) - `context/cli/mutation-trace-quint-connect.md` (`#[cfg(test)]`-only Quint Connect model-based-testing harness in `cli/src/services/mutation_trace/mbt/` continuously checking `protocol.rs` against `spec/mutation_cursor.qnt`: the verification-only `mbtAction`/`MbtAction` record-payload transport excluded from comparison, the operation-identity-vs-`MbtStutter` distinction on guarded/no-op branches with its two deterministic regressions, finite ID mapping, the AC5 comparable-state field list, `randomPrepare` staying a single `step` branch, deterministic/generated (500×30, seed-reproducible) test coverage, and the two Nix checks — generic `checks.cli-tests` and dedicated `checks.mutation-trace-quint-connect` — that both require the pinned Quint binary plus the top-level `spec/` directory in `workspaceSrc`'s Nix fileset) - `context/cli/mutation-trace-store.md` (durable persistence for the mutation-cursor protocol in `cli/src/services/mutation_trace/store.rs`, built by the `mutation-cursor-store-persistence` plan: the one-directional `protocol.rs` -> `DurableTransition::between` (pure structural diff) -> `store.rs` (SQL translation) -> `RepositoryAgentTraceDb` boundary; migration `004_mutation_trace_protocol.sql`'s five tables; migration `006_mutation_trace_health_invariant.sql`'s transactional rebuild of the worktree and event tables with a cross-column `tainted`/`failure_kind` health `CHECK`, `failure_kind`-authoritative legacy normalization, and a re-runnable SQL body, plus the shared fail-closed `validate_health_encoding` used by all three health-row decoders; `AttemptState`/`external_taint` non-persistence; the 8-byte big-endian `BLOB` revision encoding plus explicit non-`Debug` enum codecs; the bounded hot-path `load_worktree` read vs. the cold-path `load_mutation_event` and descending, exact-worktree, cursor-paged `load_mutation_event_page` reader capped at 32 rows; the public cold-path single-row `load_scope` scope seam that returns one `mutation_trace_scopes` row as `Option` without widening into a projection and, unlike `load_worktree`, never adjudicates worktree identity (a cross-worktree scope is returned as-is, and the mismatch is the caller's decision); the cold-path read-only `load_tree_roots` (one worktree) / `load_all_tree_roots` (repository-wide) durable-tree-SHA queries for ref reconciliation, each a single-statement `UNION` of `cursor_tree`/`before_tree`/`after_tree` read from one DB snapshot; `commit`'s single-`BEGIN IMMEDIATE` CAS batch via `TursoDb::execute_transactional_cas_batch`, distinguishing `Conflict`/retryable-transient/deterministic-`Err` outcomes; and the store's non-goals — no Git/filesystem I/O, no attribution/boundary-kind decisions, no retry-after-`Conflict` loop, no row deletion) - `context/cli/mutation-scope-provenance.md` (the insert-once `mutation_trace_scope_provenance` table from migration `005_mutation_scope_provenance.sql` and its `MutationTraceStore::register_scope_provenance` / `load_scope_provenance` seams: `scope_id -> session_id + model_id?` metadata *about* an established scope, stored beside but never inside protocol state — outside `ProtocolState` and the CAS batch, and never an input to `IneligibleUnscoped` / `AiExclusive` / `AiContended`; `session_id` is immutable identity and `model_id` immutable first-observed metadata, so the first persisted row wins in both directions and only a differing `session_id` for an existing `scope_id` errors; the supported `MutationTraceStore` write seam requires an existing owning `mutation_trace_scopes` row and never creates one implicitly, enforced in the store rather than by a `FOREIGN KEY`, leaving `004_mutation_trace_protocol.sql` unchanged; creation is additionally admission-bounded in the runtime — a row may be created only while the durable scope is `NeverSeen`, so after admission absent provenance stays absent permanently and a later `Start` replay cannot backfill it, while an existing row is still validated on every provenance-carrying `Start`; producers own canonicalization — both Codex and Claude adapters supply provenance today, with Claude resolving exact `(cc_, agent_id)` model state at admission and degrading unavailable models to `NULL`) -- `context/cli/mutation-trace-runtime-coordinator.md` (imperative-shell runtime layer in `cli/src/services/mutation_trace/runtime/`, built by the `mutation-cursor-runtime-coordinator` plan: the per-worktree OS advisory lock, `runtime::worktree_lock::WorktreeLock::acquire(git_dir, timeout)` at `/sce/mutation-cursor.lock`, bounded `try_lock()` polling with a distinct matchable timeout error and RAII release, and its distinction from the separate checkout-identity-creation lock; the isolated Git snapshot service `runtime::git_snapshot::GitSnapshotService` (documented in `mutation-trace-snapshot-service.md`); and `runtime::coordinator`'s protocol-integration pipeline (`RuntimeBoundary`/`CoordinateOutcome`/`CoordinateError`, the load → recover-if-needed → prepare/commit CAS-retry loop, and its own bounded snapshot-failure taint-retry loop) plus the public `coordinate(repository_root, boundary, open_db)` entrypoint that runs the shared `runtime::protected_worktree` prefix (`WorktreeLock` -> external-taint write-ahead fence -> `WorktreeId`, extracted so a second entrypoint cannot drift from it and documented in `mutation-trace-protected-worktree.md`), invokes the caller-supplied `open_db` provider so DB acquisition falls inside that fence, drives the pipeline under one held lock, and clears the marker through `ProtectedWorktree::complete()` only on success; `runtime/tests.rs` covers the public `coordinate()` API end to end against real linked worktrees and a real Agent Trace DB; an inherited external-taint marker is overlaid onto `protocol::database_failure` recovery on the next invocation, against the single captured snapshot and re-injected across a losing recovery CAS; the private `runtime::ref_reconciliation` per-worktree snapshot-ref maintenance pass (documented in `mutation-trace-ref-reconciliation.md`) shares the same `WorktreeLock`; `coordinate()` and `abandon_scope()` are now `pub(crate)` re-exported from `runtime/mod.rs` (documented in `mutation-scope-runtime.md`) while the runtime submodules themselves stay private; a generic `sce hooks mutation-scope` CLI ingress now drives both entrypoints, joined by current Claude Code and Codex adapter drivers (registered by `sce setup`/`sce setup --codex` and reachable) and an OpenCode adapter driver (lifecycle + recovery wired, and now generated as the final `sce-mutation-scope.ts` plugin and installed by `sce setup`), documented in `mutation-scope-runtime.md`) +- `context/cli/mutation-trace-runtime-coordinator.md` (imperative-shell runtime layer in `cli/src/services/mutation_trace/runtime/`, built by the `mutation-cursor-runtime-coordinator` plan: the per-worktree OS advisory lock, `runtime::worktree_lock::WorktreeLock::acquire_async(git_dir, timeout)` at `/sce/mutation-cursor.lock`, bounded `try_lock()` polling with a distinct matchable timeout error and RAII release, and its distinction from the separate checkout-identity-creation lock; the isolated Git snapshot service `runtime::git_snapshot::GitSnapshotService` (documented in `mutation-trace-snapshot-service.md`); and `runtime::coordinator`'s protocol-integration pipeline (`RuntimeBoundary`/`CoordinateOutcome`/`CoordinateError`, the load → recover-if-needed → prepare/commit CAS-retry loop, and its own bounded snapshot-failure taint-retry loop) plus the public `coordinate(repository_root, boundary, open_db)` entrypoint that runs the shared `runtime::protected_worktree` prefix (`WorktreeLock` -> external-taint write-ahead fence -> `WorktreeId`, extracted so a second entrypoint cannot drift from it and documented in `mutation-trace-protected-worktree.md`), invokes the caller-supplied `open_db` provider so DB acquisition falls inside that fence, drives the pipeline under one held lock, and clears the marker through `ProtectedWorktree::complete()` only on success; an inherited external-taint marker is overlaid onto `protocol::database_failure` recovery on the next invocation, against the single captured snapshot and re-injected across a losing recovery CAS; the private `runtime::ref_reconciliation` per-worktree snapshot-ref maintenance pass (documented in `mutation-trace-ref-reconciliation.md`) shares the same `WorktreeLock`; `coordinate()` and `abandon_scope()` are now `pub(crate)` re-exported from `runtime/mod.rs` (documented in `mutation-scope-runtime.md`) while the runtime submodules themselves stay private; a generic `sce hooks mutation-scope` CLI ingress now drives both entrypoints, joined by current Claude Code and Codex adapter drivers (registered by `sce setup`/`sce setup --codex` and reachable) and an OpenCode adapter driver (lifecycle + recovery wired, and now generated as the final `sce-mutation-scope.ts` plugin and installed by `sce setup`), documented in `mutation-scope-runtime.md`) - `context/cli/mutation-trace-protected-worktree.md` (the shared safety prefix every mutation-cursor runtime entrypoint runs behind, in `cli/src/services/mutation_trace/runtime/protected_worktree.rs`, extracted from `coordinate()` by the `mutation-scope-runtime-integration` plan so a second entrypoint cannot drift from it: `ProtectedWorktree::acquire(repository_root)` running the safety-critical fixed order resolve `git_dir` → `WorktreeLock` (module-owned 10s `WORKTREE_LOCK_TIMEOUT`) → `ExternalTaintMarker::exists()` → `persist()` (fence armed write-ahead of every fallible step that follows, including DB acquisition) → `get_or_create_checkout_id` as `WorktreeId`; the `worktree_id()` / `inherited_external_taint()` / consuming `complete()` surface, where `complete()` clears the marker under the still-held lock and is the only thing that ever clears it while `Drop` releases only the lock; and the one-variant-per-prefix-step `ProtectedWorktreeError` (`GitDirResolution` | `LockAcquisition` | `ExternalTaintMarker { operation, source }` | `CheckoutIdentity`) each entrypoint maps onto its own error surface — `coordinate()` onto exactly the `CoordinateError` variants that step produced before the extraction) - `context/cli/mutation-trace-scope-abandonment.md` (the mutation-cursor runtime's second protected entrypoint in `cli/src/services/mutation_trace/runtime/scope_runtime.rs`, built by the `mutation-scope-runtime-integration` plan and the first production call site for `protocol::abandon`: `abandon_scope(repository_root, scope, open_db) -> Result` retires a scope whose final worktree boundary was never observed, sharing `coordinate()`'s `ProtectedWorktree` prefix but deliberately capturing **no** Git snapshot, pin, diff, reconciliation, scope registration, or worktree initialization — so abandonment is not a `RuntimeBoundary` and needs no Quint change; the classify-before-transition order that recovers what `protocol::abandon`'s uniform guarded no-op cannot report (`load_scope` first for the missing-row and cross-worktree cases the projection seam treats as errors, then `load_worktree` for the `NeverSeen` / terminal / `Active` split); the `Abandoned` / `AlreadyTerminal` / `RecoveryRequired` outcomes and the `InheritedExternalTaint` | `MissingScope` | `NeverSeenScope` | `MissingWorktreeState` recovery reasons; the inherited-marker short-circuit that returns before the DB provider is ever invoked; the fence-completion rule that clears the marker only for a settled abandonment or proven-terminal no-op and leaves it armed for every recovery-required outcome and every error, with `MarkerClearAfterCompletion` carrying the already-settled outcome; the CAS retry bounded by the coordinator's shared `MAX_CAS_RETRY_ATTEMPTS`, settling on a competitor's terminal status rather than overwriting it; and the deliberate false-negative tradeoff whereby a missing or `NeverSeen` target forces conservative strong recovery that may abandon unrelated live scopes, because attribution safety outranks preserving potentially valid evidence) -- `context/cli/mutation-scope-runtime.md` (the crate-visible mutation-trace runtime seam and the lifecycle contract every current or future harness adapter must uphold, recorded by the `mutation-scope-runtime-integration` plan: the ten `pub(crate)` re-exports in `runtime/mod.rs` (`coordinate`, `RuntimeBoundary`, `StartProvenance`, `CoordinateOutcome`, `CoordinateError`, `ExternalTaintOperation`, `abandon_scope`, `AbandonScopeOutcome`, `AbandonRecoveryReason`, `AbandonScopeError`), with `ExternalTaintOperation` riding through `coordinator.rs`'s own `pub use` because `CoordinateError::ExternalTaintMarker` carries it — so the type becomes crate-visible without `protected_worktree` becoming a public module — while every `mod` declaration stays private and `ProtectedWorktree`/`ProtectedWorktreeError`/`WORKTREE_LOCK_TIMEOUT`/`reconcile_worktree` stay internal, kept clippy-clean by `#[allow(unused_imports)]` on the re-exports per the `services/style.rs` precedent rather than a placeholder consumer; and the adapter obligations themselves — a scope is one independently mutation-capable execution so a concurrent main agent and subagent need distinct `ScopeId`s, `Start`/`Advance`/`Close` semantics including that only `Start` may carry the optional `StartProvenance`, registered after scope registration and before the protocol commits and never inside `ProtocolState` or the CAS transition, with that registration conditional on the `ScopeState` `register_scope` returns so a provenance row may only be created while the scope is `NeverSeen` (an admission-time snapshot that a post-admission replay cannot backfill) while an existing row is still validated on every provenance-carrying `Start`, that a failed tool still requires `Advance` (the boundary is an observation, not a successful edit) and that a `ScopeId` is never reused after a terminal status (the `NeverSeen` guard silently refuses to reactivate it), `abandon_scope()` requiring positive staleness evidence and never inferring it from `ActorKind`, the `abandon` → `coordinate(Start(successor))` sequence with what each outcome implies for it including that a failed abandonment must never be treated as a safely started successor, that abandonment is not a `RuntimeBoundary` and needs no Quint change, the D1 tradeoff whereby a missing or `NeverSeen` target deliberately forces conservative strong recovery that may invalidate unrelated live scopes because attribution safety outranks preserving potentially valid evidence, and the attribution boundary that `AiExclusive(scope)` means scope exclusivity only and is not standalone proof that no human edited the worktree; a generic `sce hooks mutation-scope` ingress now drives the seam, joined by current Claude Code and Codex adapter drivers (both registered by `sce setup` and reachable) and an OpenCode adapter driver (full lifecycle + recovery, now generated as the final `sce-mutation-scope.ts` plugin and registered by `sce setup`, so reachable by a real session) — a Pi adapter driver now also exists (`cli/src/services/hooks/pi_mutation_scope/`, hidden `sce hooks pi-mutation-scope`) and is driven by the canonical generated Pi extension in ordinary Pi sessions; its human `user_bash` path uses the separate guarded external-mutation supervisor) +- `context/cli/mutation-scope-runtime.md` (the crate-visible mutation-trace runtime seam and the lifecycle contract every current or future harness adapter must uphold, recorded by the `mutation-scope-runtime-integration` plan: the ten `pub(crate)` re-exports in `runtime/mod.rs` (`coordinate`, `RuntimeBoundary`, `StartProvenance`, `CoordinateOutcome`, `CoordinateError`, `ExternalTaintOperation`, `abandon_scope`, `AbandonScopeOutcome`, `AbandonRecoveryReason`, `AbandonScopeError`), with `ExternalTaintOperation` riding through `coordinator.rs`'s own `pub use` because `CoordinateError::ExternalTaintMarker` carries it — so the type becomes crate-visible without `protected_worktree` becoming a public module — while every `mod` declaration stays private and `ProtectedWorktree`/`ProtectedWorktreeError`/`WORKTREE_LOCK_TIMEOUT`/`reconcile_explicit` stay internal, kept clippy-clean by `#[allow(unused_imports)]` on the re-exports per the `services/style.rs` precedent rather than a placeholder consumer; and the adapter obligations themselves — a scope is one independently mutation-capable execution so a concurrent main agent and subagent need distinct `ScopeId`s, `Start`/`Advance`/`Close` semantics including that only `Start` may carry the optional `StartProvenance`, registered after scope registration and before the protocol commits and never inside `ProtocolState` or the CAS transition, with that registration conditional on the `ScopeState` `register_scope` returns so a provenance row may only be created while the scope is `NeverSeen` (an admission-time snapshot that a post-admission replay cannot backfill) while an existing row is still validated on every provenance-carrying `Start`, that a failed tool still requires `Advance` (the boundary is an observation, not a successful edit) and that a `ScopeId` is never reused after a terminal status (the `NeverSeen` guard silently refuses to reactivate it), `abandon_scope()` requiring positive staleness evidence and never inferring it from `ActorKind`, the `abandon` → `coordinate(Start(successor))` sequence with what each outcome implies for it including that a failed abandonment must never be treated as a safely started successor, that abandonment is not a `RuntimeBoundary` and needs no Quint change, the D1 tradeoff whereby a missing or `NeverSeen` target deliberately forces conservative strong recovery that may invalidate unrelated live scopes because attribution safety outranks preserving potentially valid evidence, and the attribution boundary that `AiExclusive(scope)` means scope exclusivity only and is not standalone proof that no human edited the worktree; a generic `sce hooks mutation-scope` ingress now drives the seam, joined by current Claude Code and Codex adapter drivers (both registered by `sce setup` and reachable) and an OpenCode adapter driver (full lifecycle + recovery, now generated as the final `sce-mutation-scope.ts` plugin and registered by `sce setup`, so reachable by a real session) — a Pi adapter driver now also exists (`cli/src/services/hooks/pi_mutation_scope/`, hidden `sce hooks pi-mutation-scope`) and is driven by the canonical generated Pi extension in ordinary Pi sessions; its human `user_bash` path uses the separate guarded external-mutation supervisor) - `context/cli/mutation-scope-hook-ingress.md` (the one harness-neutral CLI ingress that drives the mutation-scope runtime, in `cli/src/services/hooks/mutation_scope.rs`, built by the `mutation-scope-hook-ingress` plan: the hidden `sce hooks mutation-scope` command routing through the normal `cli_schema::HooksSubcommand::MutationScope` → `convert_hooks_subcommand_request` → `services::hooks::HookSubcommand::MutationScope` → `run_hooks_subcommand_in_repo` stack, reading one normalized JSON lifecycle object from STDIN; the strict `parse_mutation_scope_payload` contract supporting exactly `start`/`advance`/`close`/`flush`/`abandon` with a local `MutationScopePayload` transport enum, `claude_code`/`codex`/`opencode`/`pi` actor mapping, non-blank `scope_id`/`event_id`, exact per-operation key sets, an optional `provenance` object accepted on `start` only (a required non-blank `session_id`, an optional `model_id` where an absent key and an explicit `null` both mean no model, and no other key), and a dedicated hard rejection for any `worktree_id` key; the operation mapping to `RuntimeBoundary::Start`/`Advance`/`Close`/`Flush` through `coordinate()` or a direct `abandon_scope()` call, forwarding `ScopeId`/`EventId`/`ActorKind` and `start`'s optional `StartProvenance` verbatim because `EventId` equality is the runtime replay/idempotency key and provenance values arrive already canonical; identity ownership — the adapter owns `scope_id`/`event_id`/`actor_kind`, SCE owns `worktree_id`/Git tree identities/revisions/attempt IDs, and worktree identity is derived only by the runtime from the invoking checkout; the lazy `FnOnce` DB provider reusing `open_agent_trace_db_for_hook_runtime` so DB acquisition stays inside the runtime's protected-worktree ordering; the non-fail-open error classification by durable completion — malformed payload or any pre-completion `CoordinateError`/`AbandonScopeError` → `CliError`/non-zero, while `CoordinateError::MarkerClearAfterCommit` and `AbandonScopeError::MarkerClearAfterCompletion` are treated as durable success with empty stdout, the marker-cleanup failure logged via `sce.hooks.mutation_scope.marker_clear_after_durable_completion`, and the transition not retried; the empty-stdout success contract; abandonment ownership keeping no-snapshot semantics; and the generic-ingress vs harness-adapter boundary — a Claude Code adapter driver (`cli/src/services/hooks/claude_mutation_scope/`) and a Codex adapter driver (`cli/src/services/hooks/codex_mutation_scope/`, hidden `sce hooks codex-mutation-scope`) now exist and are registered by `sce setup`, both consuming this seam's own `pub(crate)` in-process entrypoint; an OpenCode adapter driver (`cli/src/services/hooks/opencode_mutation_scope/`, hidden `sce hooks opencode-mutation-scope`) also consumes it with a full `Start`/`Close`/`Abandon` lifecycle and checkout-local recovery state, and is now driven in production by the generated `sce-mutation-scope.ts` plugin installed last by `sce setup`; a Pi adapter driver (`cli/src/services/hooks/pi_mutation_scope/`, hidden `sce hooks pi-mutation-scope`) also consumes it now, with `(session-id, tool-call-id)`→`ScopeId` derivation and getppid()/`/proc`-start-time-based stale-process recovery; the canonical generated Pi extension drives it in ordinary sessions — see `context/cli/pi-mutation-scope-integration.md`) - `context/cli/claude-mutation-scope-integration.md` (the first concrete harness lifecycle adapter, `cli/src/services/hooks/claude_mutation_scope/`, hidden command `sce hooks claude-mutation-scope`, built by the `claude-mutation-scope-integration` plan: one independently mutation-capable Claude tool execution = one SCE mutation `ScopeId` (a session/prompt/main agent/subagent is never a scope); the `classify_tool` table (mutation-capable including unknown names, read-only `Read`/`Glob`/`Grep`/`WebFetch`/`WebSearch`/`AskUserQuestion`, `Agent` = delegation) plus the model-only `is_explicit_background_shell` predicate; the length-prefixed hash-free `cc-tool-v1|n=|s=..|a=..|t=..` `ScopeId` keyed on a monotonic checkout-local `attempt_seq` (never reused after terminal) with deterministic `|start` / `|close` `EventId`s; the `/sce/claude-mutation-scope-state.json` bookkeeping store (never attribution evidence, never synced) with its own separate lock never held across a seam call; `PreToolUse` write-ahead `pending_start` → seam `start` → `active` and its fail-closed Claude `permissionDecision: "deny"` on any failure (never `allow`, detail logged via `sce.hooks.claude_mutation_scope.pre_tool_use_fail_closed`); `PostToolUse`/`PostToolUseFailure` → `close`, with `pending_start`+terminal → abandon-not-late-start (D11) and failed-`close` → abandon-not-replay (D12); the abandonment cleanup signals (`PermissionDenied`, `Stop`/`StopFailure`, `UserPromptSubmit`, `SubagentStop`, `SessionEnd`, best-effort `WorktreeRemove` — the last two not observed to fire on Claude Code `2.1.258`); the `recovery_pending` barrier that denies new mutation-capable `PreToolUse` until quiescent then runs one seam `flush`; raw hook `cwd` (or `worktree_path` for `WorktreeRemove`) as authoritative repository root with no adapter-constructed `WorktreeId`; `run_in_background = true` handling — native Claude background Bash is operationally supported but unsupported for mutation attribution (allowed in `PreToolUse` before repository resolution with no mutation scope, adapter attempt, or lifecycle tracking, per the effective decision `context/decisions/2026-10-02-claude-background-bash-untracked.md`), while background `PowerShell` stays denied — and the separate self-detaching-descendant unsupported boundary (D20, with T04's Git-observable evidence — full detail split out to [`claude-mutation-scope-background-execution.md`](cli/claude-mutation-scope-background-execution.md) for the repository's per-file line budget); the ten unmatched `sce setup` registrations; the strict `claude_mutation_scope → hooks::mutation_scope → mutation_trace::runtime` dependency direction through the single `run_mutation_scope_from_payload` seam import (T05); admission-time exact model-state snapshot into `ScopeProvenance`; real Git/Agent Trace persistence coverage shared with the Codex path; and, added by the `doctor-mutation-scope-fix` plan's T04, a third `pending_abandon` attempt phase durably persisted before any abandon seam call — established evidence that abandonment has already been decided, distinct from "may still be running" — with every attempt a broad cleanup signal (`Stop`/`UserPromptSubmit`/`SubagentStop`/`SessionEnd`/`WorktreeRemove`) matches now marked `pending_abandon` in one write before any of their seam calls run, so one attempt's seam failure can never leave a batch sibling without its own retryable evidence; health classification detail split out to [`claude-mutation-scope-health.md`](cli/claude-mutation-scope-health.md)) - `context/cli/claude-mutation-scope-background-execution.md` (native Claude background Bash is operationally supported but unsupported for mutation attribution: `Bash(run_in_background=true)` is allowed in `PreToolUse` with no mutation scope, adapter attempt, lifecycle tracking, or attribution guarantee — effective record `context/decisions/2026-10-02-claude-background-bash-untracked.md` — its later terminal hooks are no-ops, and the accepted A/B contamination consequence is pinned by a regression; background `PowerShell` stays denied with the existing deny text; the T01 `2.1.284` probe result with lifecycle completion still unproven; and the separate self-detaching-descendant unsupported boundary D20 — Git-observable evidence that a foreground `setsid` command's detached descendant can still mutate the repository after `PostToolUse` returns — detail split out of `claude-mutation-scope-integration.md` for the repository's per-file line budget) - `context/cli/claude-mutation-scope-health.md` (the `doctor-mutation-scope-health` plan's Claude task, extended by the `doctor-mutation-scope-fix` plan's T04: `classify_health` in `cli/src/services/hooks/claude_mutation_scope/health.rs` maps `recovery_pending`/`attempts` onto `healthy | recovering | blocked | invalid` — `recovery_pending == false` is `healthy` regardless of live attempts; `recovery_pending == true` with `attempts` empty is `recovering` (the barrier's own next-`PreToolUse` flush clears it); `recovery_pending == true` with `attempts` non-empty is `blocked`, the exact incident shape this classifier exists to surface, since nothing in the ordinary hook lifecycle retries, removes, flushes, or clears from that shape; a read/parse failure is `invalid`. T04 adds a separate, `doctor`-invoked repair path (`assess_repairability`/`repair_blocked` in `health.rs`/`lifecycle.rs`, wired into `sce doctor --fix` by the same plan's T05) for the `blocked` shape: `AutoFixable` only when every currently persisted attempt is already `pending_abandon` (no established-abandon-intent `pending_start`/`active` attempt present); `repair_blocked` re-reads and re-proves that same condition inside one lock-protected read-only state transaction before releasing the lock and retrying each attempt's already-established seam `abandon` call independently, clearing the `recovery_pending` barrier only once no attempts remain — this classifier's own read-only behavior is unaffected by that repair path's existence) -- `context/cli/mutation-trace-ref-reconciliation.md` (the conservative per-worktree snapshot-ref reconciliation pass in `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs`, built by the `mutation-cursor-ref-reconciliation` plan: `reconcile_worktree(repository_root, open_db)` → `pub(super) reconcile_worktree_inner(.., on_lock_contention)`, both returning `ReconciliationOutcome` (`Reconciled(ReconciliationReport { local_required, retained, deleted })` for a pass that ran | `SkippedNoCheckoutIdentity` — an `Ok`, not an `Err` — when no current checkout identity could be derived), a variant-per-fallible-step `ReconcileError` with no `Other`, and the module-owned `RECONCILIATION_LOCK_TIMEOUT`; the two-invariant model — a strictly per-worktree fail-closed local-consistency check via `load_tree_roots(W)` vs. a repository-wide deletion-safety set via `load_all_tree_roots()` so an `A`-owned ref is retained whenever any worktree still durably needs its tree — run entirely under the same `/sce/mutation-cursor.lock` `WorktreeLock` `coordinate()` holds, with the repository-wide read kept coherent by being one SQL statement / one DB snapshot rather than a repository-global lock; deletes only SCE-owned refs via one atomic `git update-ref --no-deref --stdin`, writes no `mutation_trace_*` row, never arms `ExternalTaintMarker`, runs no `git gc`; imperative durability maintenance below the verified Quint protocol; reclaims orphan/unreferenced refs only for the namespace of a checkout id a current worktree still derives — a namespace no current worktree owns (identity-based: a deleted linked worktree, or checkout-id metadata loss followed by `get_or_create_checkout_id` minting a fresh id on a still-present worktree) is beyond every per-worktree pass and left to a recorded future repository-scoped unowned-namespace operation, so the current pass does not bound all orphan-ref growth; `reconcile_worktree` has no `pub(crate)` re-export and no harness/command wiring yet) +- `context/cli/mutation-trace-ref-reconciliation.md` (the conservative per-worktree snapshot-ref reconciliation pass, built by the `mutation-cursor-ref-reconciliation` plan and wired by `mutation-cursor-ref-reconciliation-wiring`: the single lock-held algorithm `reconcile_with_held_lock` in `ref_reconciliation.rs`, the explicit production entrypoint `reconcile_explicit` in `ref_maintenance.rs` (one `WorktreeLock` acquisition, verified-existing Agent Trace DB, typed `ExplicitOutcome`), the documented 10 s concurrency contract, the `ref-maintenance.json` state file (past-only timestamps, skew normalization, atomic writes), the D4 verified-existing DB opener, the two-invariant durability model (per-worktree fail-closed local consistency vs repository-wide deletion safety, never a claim that every repository-wide root is pinned), and the deferred follow-ups (automatic out-of-band reclamation with its binding prerequisites, retired-worktree cleanup, evidence-based advice, historical retention); reconciliation runs only through `sce doctor --fix` and is never automatic) +- `context/cli/mutation-trace-ref-maintenance-state.md` (supporting contracts for snapshot-ref reconciliation: the per-worktree `/sce/ref-maintenance.json` state file with past-only timestamps, `FUTURE_SKEW_TOLERANCE` normalization on successful writes only, atomic staging-and-swap writes, the pure `evaluate_recommendation`, and the non-creating verified-existing Agent Trace DB opener with typed `ExistingRepositoryDbError` and per-DB-state guarantees) +- `context/cli/mutation-scope-ref-advisory.md` (the advisory-only hook trigger in `cli/src/services/hooks/mutation_scope.rs` built by T03 of the `mutation-cursor-ref-reconciliation-wiring` plan: a lightweight `advise_if_due` check after a durably completed `Close`/`Flush`, logged and, when actionable, written as an `SCE:` line to an injectable stderr writer (best-effort delivery), with the per-worktree `ref-maintenance.json` state; no reconciliation, Git, or DB work on the hook path) +- `context/cli/doctor-ref-reconciliation.md` (the `sce doctor --fix` caller of snapshot-ref reconciliation built by T04 of the `mutation-cursor-ref-reconciliation-wiring` plan: the `runtime::ref_doctor` facade, typed fix-result rows for completed/completed-with-persist-warning/failed/busy passes, and the read-only recommendation section/`ref_reconciliation` JSON object in plain `sce doctor`) - `context/cli/mutation-trace-snapshot-service.md` (the isolated Git snapshot and ref-pinning service `runtime::git_snapshot::GitSnapshotService` in `cli/src/services/mutation_trace/runtime/git_snapshot.rs`: `new` resolving an absolute `git_dir`, `capture_tree` snapshotting staged/unstaged/untracked/deleted worktree state into the repository's normal object database via a throwaway temp index, `pin_tree` protecting a durable tree with a create-only idempotent **direct** `refs/sce/mutation-cursor//` ref, `diff_trees` emitting `patch.rs`-parseable raw diff text; plus the callerless reconciliation substrate — worktree-scoped `list_pins` inventory returning `Result, PinInventoryError>` that rejects a symbolic ref inside the namespace (mutation-cursor pins are direct refs) as `MalformedRef`, matchable separately from a `git for-each-ref` execution failure, and conditional-atomic `delete_pins` running one `git update-ref --no-deref --stdin` transaction of SHA-conditioned deletes — no-dereference so an inventory→delete direct-ref→symref race cannot escape the inventoried namespace, plus a fail-closed pre-check — that aborts whole if any ref changed since inventory; `REF_NAMESPACE` + `pin_ref_prefix` as the single source of truth for the pin path) - `context/cli/mutation-trace-external-taint.md` (the worktree-local mutation-cursor durability boundary in `cli/src/services/mutation_trace/runtime/external_taint.rs`, built by the `mutation-cursor-external-taint` plan: the `ExternalTaintMarker` primitive — `new(git_dir)`/`exists()`/`persist()`/`clear()` over an empty file at `/sce/mutation-cursor-tainted` whose existence is its entire state, `checkout::persist_checkout_id_inner`-style durability (`sync_data` plus best-effort `#[cfg(unix)]` parent-dir `sync_all`), idempotent persist/clear, `NotFound`-on-clear as success, no `Drop` deletion — as the concrete runtime refinement of the abstract `ProtocolState.external_taint`; armed by the reshaped `coordinate()` entrypoint write-ahead after the `WorktreeLock` and before Agent Trace DB acquisition (a caller-supplied DB provider closure), cleared only on a successful `CoordinateOutcome`, with dedicated fail-closed pre-commit `CoordinateError::ExternalTaintMarker` (`Inspect`/`Persist` only)/`AgentTraceDbUnavailable` variants plus a post-commit `MarkerClearAfterCommit { source, committed }` that carries the durable outcome so a failed trailing clear never hides a committed `MutationEvent`; an inherited marker seeds an invocation-local `external_taint_pending` flag that overlays `protocol::database_failure` onto each freshly loaded projection so `recover` runs once against the captured snapshot, held across a losing recovery CAS and cleared once it lands) - `context/sce/cli-exit-code-contract.md` (stable class-based `sce` process exit-code contract in `cli/src/app.rs`, so automation can branch on failure category without parsing error text) @@ -83,12 +87,12 @@ Feature/domain context: - `context/sce/cli-security-hardening-contract.md` (T06 CLI redaction contract, setup `--repo` canonicalization/validation, and setup write-permission probe behavior) - `context/sce/agent-trace-post-rewrite-local-remap-ingestion.md` (current post-rewrite no-op baseline plus historical remap-ingestion reference) - `context/sce/agent-trace-rewrite-trace-transformation.md` (current post-rewrite no-op baseline plus historical rewrite-transformation reference) -- `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited retry-backed blocking `execute`/`query`/`query_map` methods using the shared Turso adapter) -- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven constructor/open-connect retry via `run_with_retry_sync`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven operation retry for `execute`/`query`/`query_values`/`query_map` (retry-policy budget, not a wall-clock bound), per-`DbSpec` Turso busy timeout on connect (1000 ms for the Agent Trace DB, unset for other databases), the Agent Trace-only outer write-contention retry for replay-safe write units including the opt-in `execute_idempotent_write` and its structured contention-exhaustion error plus `sce.agent_trace_db.contention_exhausted` tracing event, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, non-mutating-data `passive_checkpoint()` PASSIVE WAL checkpoint method on `TursoDb` (not on `EncryptedTursoDb`, fail-open, no truncation guarantee, called once by `sce hooks post-commit` after successful Agent Trace persistence — see `context/sce/agent-trace-hooks-command-routing.md`), and concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`) +- `context/sce/local-db.md` (implemented `cli/src/services/local_db/mod.rs` local database spec with `LocalDb = TursoDb`, canonical local DB path resolution, zero local migrations, and inherited async `execute`/`query`/`query_map` methods using the shared Turso adapter) +- `context/sce/shared-turso-db.md` (current shared `cli/src/services/db/mod.rs` Turso database infrastructure seam, including `DbSpec`, generic `TursoDb`, encrypted `EncryptedTursoDb`, build-time generated migration constants from `cli/build.rs`/Cargo `OUT_DIR`, config-driven async constructor/open-connect retry via `run_with_retry`, no-migration `TursoDb::open_without_migrations()` / explicit-path `open_without_migrations_at(path)` for hot runtime paths, migration-running `new()` / explicit-path `new_at(path)` / `run_migrations()` with per-database `__sce_migrations` tracking, config-driven async operation retry for `execute`/`query_map` (retry-policy budget, not a wall-clock bound), per-`DbSpec` Turso busy timeout on connect (1000 ms for the Agent Trace DB, unset for other databases), the Agent Trace-only outer write-contention retry for replay-safe write units including the opt-in `execute_idempotent_write` and its structured contention-exhaustion error plus `sce.agent_trace_db.contention_exhausted` tracing event, raw-value row fetching for deterministic operator-facing rendering, row-mapping excluded from retry, generic embedded migration execution, non-mutating `migration_metadata_problems()` and `ensure_schema_ready(setup_guidance)` readiness methods on `TursoDb`, non-mutating-data `passive_checkpoint()` PASSIVE WAL checkpoint method on `TursoDb` (not on `EncryptedTursoDb`, fail-open, no truncation guarantee, called once by `sce hooks post-commit` after successful Agent Trace persistence — see `context/sce/agent-trace-hooks-command-routing.md`), and async concrete wrappers for `LocalDb`, `AuthDb`, plus `RepositoryAgentTraceDb`; no adapter runtime bridge) - `context/sce/agent-trace-db-write-contention-evidence.md` (measured Agent Trace DB write-contention behavior under the busy-timeout/outer-retry/contention-deadline contract: the `lock_contention_tests.rs` suite with its lock-budget boundary, strict N=2–4 in-process and real-hook gates and N=8 stress characterization, policy history from 500 / 1250 to the current 1000 / 2250 defaults, the supported-load failure that motivated the tuning, before-vs-after comparison, and the bounded-policy exhaustion failure mode) -- `context/sce/auth-db.md` (encrypted `AuthDb = EncryptedTursoDb` adapter, canonical `/sce/auth.db` path, build-time generated `AUTH_MIGRATIONS` from `cli/migrations/auth/`, auth credential schema and updated-at trigger baseline, lifecycle setup/doctor integration, encrypted token-storage persistence, and `SCE_AUTH_DB_ENCRYPTION_KEY`/OS credential-store key handling) -- `context/sce/agent-trace-db.md` (implemented Agent Trace database adapter: the sole repository-scoped `RepositoryAgentTraceDb` backed by the fresh multi-statement baseline schema plus additive `source_instance_id` and `claude_model_state` migrations, with `repository_metadata` carrying both `repository_id` and a concurrency-safe atomic-claim `source_instance_id` (physical database identity, independent of `repository_id`), narrow concurrent-first-open repair for missing one-file baseline migration metadata after all required schema tables exist, no trace-table `checkout_id` columns, repository-level typed insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, parts, and the non-exported exact-scope Claude model-state register, repository-level recent diff-trace reads without checkout filtering including persisted hunk-model plus canonical touched-line-session enrichment for structured rows, on-demand command/hook initialization with no daemon/background service, and the never-touch on-disk boundary for any pre-migration checkout-scoped/global DB files; the checkout-scoped `AgentTraceDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook writers/readers and Agent Trace setup/lifecycle resolve repository storage through `agent_trace_storage`) -- `context/sce/agent-trace-export-readers.md` (implemented `AgentTraceExportReader<'a>` in `cli/src/services/agent_trace_export/mod.rs`: the read-only local export boundary over `RepositoryAgentTraceDb` — `read_messages_after`/`read_parts_after`/`read_diff_traces_after`/`read_agent_traces_after`, each cursor/limit/JS-safe-integer validated, materialized into owned camelCase `serde::Serialize` DTOs; composes directly with `ResolvedAgentTraceStorage` without owning `source_instance_id`; `sce sync` consumes three of the four readers and `read_diff_traces_after` is a retained compatibility surface; no local sync cursor, no `agent-trace-sync.db`, no Turso Sync, no ETL, no DWH) +- `context/sce/auth-db.md` (encrypted `AuthDb = EncryptedTursoDb` adapter, canonical `/sce/auth.db` path, build-time generated `AUTH_MIGRATIONS` from `cli/migrations/auth/`, auth credential schema and updated-at trigger baseline, lifecycle setup/doctor integration, async encrypted token-storage persistence through a Result-valued Tokio OnceCell, and `SCE_AUTH_DB_ENCRYPTION_KEY`/OS credential-store key handling) +- `context/sce/agent-trace-db.md` (implemented Agent Trace database adapter: the sole repository-scoped `RepositoryAgentTraceDb` backed by the fresh multi-statement baseline schema plus additive `source_instance_id` and `claude_model_state` migrations, with `repository_metadata` carrying both `repository_id` and a concurrency-safe atomic-claim `source_instance_id` (physical database identity, independent of `repository_id`), narrow concurrent-first-open repair for missing one-file baseline migration metadata after all required schema tables exist, no trace-table `checkout_id` columns, repository-level typed insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, parts, and the non-exported exact-scope Claude model-state register, repository-level recent diff-trace reads without checkout filtering including persisted hunk-model plus canonical touched-line-session enrichment for structured rows, async on-demand command/hook initialization with no daemon/background service, and the never-touch on-disk boundary for any pre-migration checkout-scoped/global DB files; the checkout-scoped `AgentTraceDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook writers/readers and Agent Trace setup/lifecycle resolve repository storage through `agent_trace_storage`) +- `context/sce/agent-trace-export-readers.md` (implemented `AgentTraceExportReader<'a>` in `cli/src/services/agent_trace_export/mod.rs`: the read-only async local export boundary over `RepositoryAgentTraceDb` — `read_messages_after`/`read_parts_after`/`read_diff_traces_after`/`read_agent_traces_after`, each cursor/limit/JS-safe-integer validated, materialized into owned camelCase `serde::Serialize` DTOs; composes directly with `ResolvedAgentTraceStorage` without owning `source_instance_id`; `sce sync` consumes three of the four readers and `read_diff_traces_after` is a retained compatibility surface; no local sync cursor, no `agent-trace-sync.db`, no Turso Sync, no ETL, no DWH) - `context/sce/agent-trace-core-schema-migrations.md` (historical reference for removed local DB schema bootstrap behavior; T03 now implements the actual local DB with migrations) - `context/sce/agent-trace-retry-queue-observability.md` (inactive local-hook retry path plus historical retry/metrics reference) - `context/sce/agent-trace-local-hooks-mvp-contract-gap-matrix.md` (T01 Local Hooks MVP production contract freeze and deterministic gap matrix for `agent-trace-local-hooks-production-mvp`) @@ -123,7 +127,7 @@ Additional mutation-scope integration context: - `context/cli/opencode-mutation-scope-health.md` (the `doctor-mutation-scope-health` plan's T04: `classify_health` in `cli/src/services/hooks/opencode_mutation_scope/health.rs` maps the adapter's `RecoveryState`/`AttemptPhase` combinations onto `healthy | recovering | blocked | invalid`, proven by driving the real dispatch/recovery functions — `Clear` + `PendingAbandon` is `invalid`; any state containing a `PendingStart` attempt is `blocked` (except that `Clear` + `PendingAbandon` retains `invalid` precedence), because no boundary sweeps a stale `PendingStart` on behalf of an unrelated call, unlike Pi's dead-owner sweep or Codex's same-lane retry, so a crashed owning process leaves it a permanent wedge for new admissions even while a `Pending`/`Flushing` recovery for another attempt is also outstanding, since `resolve_recovery` never retires `PendingStart`; `Pending`/`Flushing` with no `PendingStart` outstanding is `recovering`; `Clear` with ordinary `Active`/no attempts is `healthy`; any read/parse failure is `invalid`) - `context/cli/pi-mutation-scope-integration.md` (the fourth concrete harness adapter, `cli/src/services/hooks/pi_mutation_scope/`, hidden command `sce hooks pi-mutation-scope`, driven by the canonical generated extension in ordinary Pi sessions: Pi's `bash`/`edit`/`write` tracked-tool allowlist with `tool_call` as the single universal pre-execution gate for every tool including `bash`; the `pi-tool-v1|n=|s=:|c=:` `ScopeId` whose checkout-local monotonic attempt sequence stops a reused `toolCallId` from ever reactivating a terminal scope; the `PendingStart`→`Executed`→`Closed`/`PendingAbandon` attempt-phase machine with no `Active` phase, because `PendingStart` is Pi's normal resting state for an entire in-flight execution (`tool_execution_start` fires unconditionally before the fail-closed `tool_call` gate and proves nothing; `tool_result` is the sole `Executed` evidence, and `tool_execution_end` with no preceding `tool_result` abandons rather than closes); why admission's fail-closed "uncertain attempt" check covers only a lingering `PendingAbandon` or non-`Clear` recovery state and never a sibling's `PendingStart`, unlike the other three adapters, so concurrent Pi tool calls stay distinct live scopes; a positive-process-death-only stale-`PendingStart`/`Executed` recovery (`getppid()`-captured owner plus Linux `/proc`-start-time PID-reuse proofing, no TTL/sweep), run on every tracked Start admission by independently inspecting persisted attempts rather than only ones matching the incoming key, and adapter reconciliation with the external-mutation guard's forced recovery, both added by a later task in this same plan; and the harness-neutral `external-mutation-guard` mechanism this task also built for Pi's `user_bash`, detailed separately in `context/cli/mutation-trace-external-mutation-guard.md`) - `context/cli/pi-mutation-scope-health.md` (the `doctor-mutation-scope-health` plan's T05: `classify_health` in `cli/src/services/hooks/pi_mutation_scope/health.rs` maps the adapter's `RecoveryState`/`AttemptPhase`/owner-liveness combinations onto `healthy | recovering | blocked | invalid`, proven by driving the real dispatch/recovery functions — unlike Codex and OpenCode, Pi's investigation found no persisted shape reachable through ordinary production dispatch that classifies `blocked`: `admit_tracked_attempt` never gates admission on a sibling's `PendingStart`/`Executed` phase (only on a co-existing `PendingAbandon`, itself proven unreachable under `Clear`), and the D10 dead-owner sweep (`reconcile_stale_owners`) unconditionally retires a dead-owner `PendingStart`/`Executed` attempt on the next tracked `tool_call` from any session, so such an attempt is `recovering`, not `blocked`; `Pending`/`Flushing` are always `recovering` regardless of attempt composition, since a recovery-capable fresh tracked Start — one whose key is not already represented by a nonterminal attempt — can claim and retry the flush, not lane- or key-scoped like Codex, though a duplicate Start for an already-tracked `PendingStart`/`Executed` key is instead reused idempotently before the recovery-state gate and so does not itself advance recovery; `Clear` with a `PendingAbandon` attempt and any read/parse failure are `invalid`) -- `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral, not-Pi-specific D13 external-mutation supervisor process, `run_external_mutation_guard` in `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only: acquires a `ProtectedWorktree`, reports it armed, spawns the human shell command as its own child in its own process group with a `dup()`-before-`spawn()` fd handed to the child so the `WorktreeLock` flock survives the supervisor's own death via the child's independent descriptor, finishes exclusively on the shell's own `wait()` (never on a cancel/control-channel signal), and only then forces `database_failure`+`recover` on the already-held worktree and calls `ProtectedWorktree::complete()` — reusing existing runtime primitives with zero `protocol.rs`/Quint change; called by the canonical Pi extension for human `user_bash`) +- `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral, not-Pi-specific D13 external-mutation supervisor process, `arm_external_mutation_guard`/`ArmedExternalMutationGuard::exec` in `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only: acquires a `ProtectedWorktree`, reports it armed, spawns the human shell command as its own child in its own process group with a `dup()`-before-`spawn()` fd handed to the child so the `WorktreeLock` flock survives the supervisor's own death via the child's independent descriptor, finishes exclusively on the shell's own `wait()` (never on a cancel/control-channel signal), and only then forces `database_failure`+`recover` on the already-held worktree and calls `ProtectedWorktree::complete()` — reusing existing runtime primitives with zero `protocol.rs`/Quint change; called by the canonical Pi extension for human `user_bash`) Working areas: diff --git a/context/decisions/2026-10-07-directly-awaited-turso-persistence.md b/context/decisions/2026-10-07-directly-awaited-turso-persistence.md new file mode 100644 index 000000000..16359790f --- /dev/null +++ b/context/decisions/2026-10-07-directly-awaited-turso-persistence.md @@ -0,0 +1,90 @@ +# Decision: Directly awaited Turso persistence + +Date: 2026-10-07 +Status: Accepted +Plan: `context/plans/cli-async-turso-persistence-pr2.md` +Task: `T02` + +## Context + +PR1 established directly awaited static command dispatch under the application +multi-thread Tokio runtime while retaining synchronous Turso adapters that +owned separate runtimes. PR2 migrated the shared adapters and every connected +production caller so database futures can run on that application runtime. +The production build passes, the test target compiles, and the preserved +mutation-trace Quint MBT suite passes. Ordinary Rust test-only code in the +affected closure was disabled under the user's direction after its async +migration build failed; the task plan records that verification reduction. + +## Decision + +`TursoDb` and `EncryptedTursoDb` expose native async persistence APIs and +own no Tokio runtime. The application runtime directly awaits Turso operations +through statically dispatched commands, services, lifecycle providers, hooks, +sync/export, credentials, and token storage. + +## Rationale + +This completes PR1's application-owned runtime boundary without changing SQL, +migrations, transaction scope, encryption, WAL, retry classification, or +credential error handling. Native futures remove the nested executor bridge and +the caller-side runtime lifetime scopes while preserving concrete database +types and static command/service dispatch. + +## Alternatives considered + +- **Keep synchronous adapters and caller-side blocking scopes** — preserves a + second runtime owner and executor bridge in every connected database path. +- **Run database operations in `spawn_blocking`** — keeps Turso futures behind + a worker boundary and complicates borrowing and cancellation behavior. +- **Use boxed async database or service traits** — erases the repository's + established static dispatch and introduces unnecessary type erasure. + +## Compatibility and risks + +- SQL, migrations, one-connection transactions, encryption, WAL, retries, + errors, and foreground hook persistence ordering remain unchanged. +- Native async transactions can be cancelled at await points. T03 owns focused + characterization of transaction acquisition/body cancellation, connection + reuse, retry timing, and contention instrumentation. +- The ordinary Rust test suites for the affected closure were disabled after + async migration compile failures at the user's direction. The preserved + Quint MBT suite passes; ordinary behavioral coverage remains reduced and is + recorded in the plan. + +## Guardrails + +- Keep database and caller dependencies concrete, generic, associated-type + based, or represented by static enums; do not introduce dynamic database or + service traits, boxed futures, or a replacement executor bridge. +- Preserve retry and transaction policies when moving operations to async + control flow. +- Keep pure parsing, formatting, filesystem, and process operations synchronous + unless they directly await database work. + +## Consequences + +- The application Tokio runtime owns the execution of local Turso futures. +- Auth token persistence and control-plane credential operations are directly + awaited; they do not use database `spawn_blocking` wrappers. +- Setup, Doctor, Hooks, sync, and export propagate async calls to persistence. + +## Follow-up + +- T03 characterizes cancellation and retry boundaries introduced by native + async persistence. +- T04 audits and removes any remaining DB-runtime lifetime blocking scopes. +- PR3 remains cleanup and audit; it does not own further caller propagation. + +## References + +- Plan: [CLI async Turso persistence PR2](../plans/cli-async-turso-persistence-pr2.md) +- Task: `T02` +- Current-state context: [Architecture](../architecture.md), + [Shared Turso adapter](../sce/shared-turso-db.md), + [Service lifecycle](../cli/service-lifecycle.md), + [Agent Trace sync](../cli/agent-trace-sync-command.md) +- Evidence: [Turso adapters](../../cli/src/services/db/mod.rs), + [Application entrypoint](../../cli/src/main.rs), + [Static command dispatch](../../cli/src/services/command_registry.rs) +- Related decision: [Application-owned async command runtime](2026-10-07-application-owned-async-command-runtime.md) diff --git a/context/decisions/2026-10-08-pr2-scope-and-cancellation-protections.md b/context/decisions/2026-10-08-pr2-scope-and-cancellation-protections.md new file mode 100644 index 000000000..cf3e354ef --- /dev/null +++ b/context/decisions/2026-10-08-pr2-scope-and-cancellation-protections.md @@ -0,0 +1,55 @@ +# Decision: PR2 scope includes cancellation-protected blocking boundaries and a focused test set + +Date: 2026-10-08 +Status: Accepted +Plan: `context/plans/cli-async-turso-persistence-pr2.md` +Task: `T05` +Supersedes: [Directly awaited Turso persistence](2026-10-07-directly-awaited-turso-persistence.md) (scope and verification posture only) + +## Context + +The 2026-10-07 decision described PR2 as Turso conversion plus its connected +callers, and recorded that ordinary Rust suites were disabled. The implemented +PR2 also moved Git snapshot subprocesses, protected-worktree marker I/O, +lock acquisition, Git ref mutation and guarded-shell supervision onto +`spawn_blocking` boundaries, and the PR #301 review required fixes for +transaction exclusivity, credential rotation, terminal sync failure, adapter +boundary leases and guarded-shell descriptor inheritance. + +## Decision + +- Native awaited Turso persistence is unchanged: no DB-owned runtime, executor + bridge, or Turso future inside a blocking worker. +- PR2 includes the Git/process/filesystem boundaries above. Each retained + blocking worker owns its protected resource (lock, lease, marker, child) until + it finishes after caller cancellation; `spawn_blocking` that provides this is + retained, not removed mechanically. +- Transactions require exclusive `&mut` access with checked transaction + construction. +- Correctness evidence is the Quint MBT suite, surviving targeted tests and a + small set of focused regression tests (transaction, cancellation, contention, + credential rotation, terminal sync, boundary lease, descriptor inheritance). + The deleted ordinary suites are not restored. +- PR3 is cosmetic/maintainability cleanup only; PR4 is OpenTelemetry. + +## Alternatives considered + +- **Move the blocking boundaries back to async workers** — cancellation could + release locks or leases while blocking work is still running. +- **Restore the deleted suites** — large cost; the focused set targets the + distinct new regressions. + +## Compatibility and risks + +- Runtime shutdown waits for retained workers, each bounded by a lock timeout, + file I/O or a child process. +- A killed process can leave a temporary Git index file behind. +- A crash between remote refresh-token rotation and the local save can still + lose the rotated token. +- Ordinary behavioral coverage remains lower than before the migration. + +## References + +- [Architecture: application execution runtime](../architecture.md#application-execution-runtime) +- [Shared Turso adapter](../sce/shared-turso-db.md) +- [Application-owned async command runtime](2026-10-07-application-owned-async-command-runtime.md) diff --git a/context/glossary.md b/context/glossary.md index cb456b320..9bd073f8c 100644 --- a/context/glossary.md +++ b/context/glossary.md @@ -38,7 +38,9 @@ - `canonical OpenCode plugin registration source`: Shared Pkl-authored plugin-registration definition in `config/pkl/base/opencode.pkl`, re-exported from `config/pkl/renderers/common.pkl` as the canonical plugin list/path JSON consumed by OpenCode renderers before they emit generated `opencode.json` manifests; the current entries, in order, are `sce-bash-policy`, `sce-agent-trace`, and `sce-mutation-scope` (list order is load-bearing — see `sce-mutation-scope plugin`). - `checkout identity`: Stable UUIDv7 identifier assigned to a cloned repository or linked Git worktree, stored in `/sce/checkout-id` (never committed) and resolved via `git rev-parse --git-dir`. The identity is created or reused by `sce setup` through `AgentTraceDbLifecycle::setup()` and also auto-created by hook runtime when `sce setup` has not been run. Checkout identity is now diagnostic metadata for repository-scoped Agent Trace storage; it does not select the active DB and is not stored on Agent Trace rows. Any pre-migration per-checkout DB files at `/sce/agent-trace-{checkout_id}.db` are never touched by SCE and are no longer inspectable via the CLI (the checkout-scoped DB surface was removed by the `retire-legacy-agent-trace-db` plan). See `context/cli/checkout-identity.md`. - `repository identity`: Stable identity of a logical Git repository used to select the active repository-scoped Agent Trace DB path `/sce/repos//agent-trace.db` through the `agent_trace_storage` resolver. Resolved by `cli/src/services/repository_identity/` with precedence: explicit `agent_trace.repository_id` config value, then the URL of the configured Git remote (`agent_trace.repository_remote`, default `origin`), otherwise an actionable `.sce/config.json` error. Remote URLs canonicalize to a scheme-neutral, credential-free `host[:port]/path` form so equivalent SSH/SCP/HTTPS remotes converge, and the repository ID is `sha256("sce-repository-id-v1\0" + canonical_identity)` hex. Distinct from `checkout identity`, which stays per clone/worktree for diagnostics. Setup's named-remote preflight uses a strict lookup that returns `Ok(None)` only for a missing/empty URL, preserves other failures as runtime errors, and discards the URL, retaining only the remote name in technical failure sources. See `context/cli/repository-identity.md`. -- `refresh single-flight guard`: Client-owned async coordination for Agent Trace control-plane authentication. Concurrent callers whose stored access token is expired, or whose request rejected the same token, serialize only refresh-and-save work, re-check credentials after acquiring the guard, and reuse the token saved by the first refresher; valid-token requests do not acquire the guard. +- `refresh single-flight guard`: Client-owned async coordination for Agent Trace control-plane authentication. Concurrent callers whose stored access token is expired, or whose request rejected the same token, serialize only refresh-and-save work, re-check credentials after acquiring the guard, and reuse the token saved by the first refresher; valid-token requests do not acquire the guard. The guard is held by the spawned task that performs renewal and save, so caller cancellation cannot release it mid-persistence; `wait_for_pending_refresh` acquires it as a completion barrier before `sce sync` returns. +- `snapshot-ref reconciliation`: Explicit, per-worktree maintenance pass (`runtime::ref_maintenance::reconcile_explicit`, run only by `sce doctor --fix`) that deletes orphaned SCE snapshot pins under `refs/sce/mutation-cursor//` while retaining every tree in any worktree's durable roots; never automatic. See `context/cli/mutation-trace-ref-reconciliation.md`. +- `ref-reconciliation advisory`: Lightweight hook-path check (`advise_if_due`) after a durably completed `Close`/`Flush` that may record and print a "reconciliation is recommended" reminder; it never reconciles, lists refs, opens the DB, or spawns Git. See `context/cli/mutation-scope-ref-advisory.md`. - `repository-scoped Agent Trace DB`: Active Agent Trace storage shape where one logical Git repository maps to `/sce/repos//agent-trace.db`. The current seam is `RepositoryAgentTraceDb = TursoDb` in `cli/src/services/agent_trace_db/repository.rs`, backed by the fresh multi-statement `001_repository_schema` baseline plus the additive `002_repository_source_instance_id` migration, with `repository_metadata` (`repository_id` plus `source_instance_id`) plus repository-level trace tables, no `checkout_id` columns, and typed repository-level insert helpers for diff traces, post-commit intersections, Agent Trace rows, messages, and parts. Hook runtime, Agent Trace setup/lifecycle, and `sce sync` resolve repository-scoped storage through `agent_trace_storage`. This is the sole Agent Trace DB adapter; the checkout-scoped adapter and former trace inspection surface were removed by the `retire-legacy-agent-trace-db` plan. - `source_instance_id`: Physical-database identity column on `repository_metadata`, independent of the logical `repository_id`. Added by the additive `002_repository_source_instance_id` migration (existing/placeholder rows default to an empty string); generated once per physical `agent-trace.db` by application code (`generate_source_instance_id()`, UUID v4 today) and validated with `is_valid_source_instance_id()` (non-empty once trimmed) — never generated in SQL and never derived from `repository_id`, remote URL, checkout ID, filesystem path, hostname, or user/workspace identity. `RepositoryAgentTraceDb::verify_or_initialize_repository_metadata` claims it with a concurrency-safe `UPDATE ... WHERE source_instance_id = ''`, so concurrent first opens of the same physical database converge on one winner and an already-valid value is never overwritten; the value stays stable across reopen and repeated `sce setup` runs. Two independently created databases for the same logical repository (for example two clones) get different `source_instance_id` values. See `context/sce/agent-trace-db.md`. - `checkout registry` (removed): The central JSON registry at `/sce/checkout-registry.json` was removed in the `remove-checkout-registry` plan. The later trace inspection surface was also removed; current checkout IDs are diagnostic metadata only and are not discovered as command targets. Any pre-migration `/sce/agent-trace-*.db` files remain untouched. See `context/cli/checkout-identity.md`. @@ -74,40 +76,40 @@ - `progress reporter contract`: Library-independent, consumer-typed sync seam in `cli/src/services/sync/progress.rs` where `ProgressReporter` carries an arbitrary consumer event type, supports explicit successful finalization and closure-based collectors, and provides `NoopProgressReporter` for callers without human progress output. The sync-owned `SyncProgressEvent` in `cli/src/services/sync/sync.rs` carries lifecycle, accepted-batch, and stream-completion payloads for the three concurrent remote Agent Trace streams (`messages`, `parts`, `agent_traces`); `services::sync::progress` also owns the fixed `indicatif` terminal adapter and focused contract tests, while `sync/command.rs` selects text versus JSON behavior. No top-level `services::progress` module exists. - `remote sync streams`: The three Agent Trace streams `sce sync` uploads to the control plane — `messages`, `parts`, `agent_traces` — as distinct from the four local capture streams, which also include the local-only `diff_traces`. `streams.diffTraces` in `sce sync --format json` is a compatibility no-op entry echoing the server cursor, not a synchronized stream. See `context/cli/agent-trace-sync-command.md`. - `local Turso adapter`: Module in `cli/src/services/local_db/mod.rs` that defines `LocalDbSpec` and exposes `LocalDb` as a `TursoDb` alias. It resolves the canonical local DB path with `local_db_path()`, currently declares zero migrations, and inherits retry-backed `new()`, `execute()`, `query()`, and `query_map()` behavior from the shared generic adapter. -- `encrypted Turso adapter`: Generic adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` (connection, tokio runtime bridge, spec typing). Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; the adapter also exposes retry-backed synchronous `execute`, `query`, `query_map`, and `run_migrations` helpers with `__sce_migrations` tracking parity. +- `encrypted Turso adapter`: Generic native async adapter seam in `cli/src/services/db/mod.rs` exposed as `EncryptedTursoDb`, structurally parallel to `TursoDb` and owning no Tokio runtime. Its constructor resolves the encryption key via `encryption_key::get_or_create_encryption_key(&db_path, db_name)`, which derives a Turso-compatible 64-character hex key from non-empty `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret text before falling back to OS credential-store keyring get-or-create behavior; credential-store default registration is guarded by stable `OnceLock` plus an atomic in-progress flag so errors or panics leave initialization retryable without mutex poisoning. The adapter enables Turso local encryption with strict `aegis256` cipher selection through `turso::EncryptionOpts`, wraps encrypted local open/connect in the default DB connection-open retry policy, and runs embedded migrations after retry has produced a connection; execute, query, query_map, and migration operations are async and preserve `__sce_migrations` tracking parity. - `auth DB adapter`: Module in `cli/src/services/auth_db/mod.rs` that defines `AuthDbSpec` and exposes `AuthDb` as an `EncryptedTursoDb` alias. It resolves the canonical `/sce/auth.db` path with `auth_db_path()`, keeps encryption mandatory with `SCE_AUTH_DB_ENCRYPTION_KEY` env-secret precedence before OS keyring fallback and no plaintext mode, and embeds ordered auth migrations where baseline SQL creates `auth_credentials` without `user_id`, with `updated_at`, and a trigger that auto-refreshes `updated_at` on row updates. Auth runtime token-storage is now wired through `cli/src/services/token_storage.rs`, which persists tokens via the `auth_credentials` table in the encrypted auth DB instead of a JSON file. - `AuthDbLifecycle`: Lifecycle provider in `cli/src/services/auth_db/lifecycle.rs` that implements `ServiceLifecycle` for encrypted auth DB setup/doctor integration. `diagnose` collects auth DB path health problems, `fix` bootstraps missing auth DB parent directory, and `setup` calls `AuthDb::new()`. Registered as `LifecycleProviderId::AuthDb` in the shared lifecycle catalog. - `agent trace DB adapter`: Modules under `cli/src/services/agent_trace_db/` that define the sole repository-scoped `RepositoryAgentTraceDb = TursoDb` adapter (the checkout-scoped `AgentTraceDb`/`AgentTraceDbSpec` adapter and its 15-file migration chain were removed by the `retire-legacy-agent-trace-db` plan). The repository adapter uses the `agent-trace-repository` migration set (fresh baseline schema plus additive `source_instance_id` and `claude_model_state` migrations) with `repository_metadata`, repository-level `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts` tables, no row-level `checkout_id`, typed parameterized insert helpers, the non-exported `claude_model_state` latest-locally-observed register with exact `(session_id, agent_id)` reads and guarded writes, and chronological recent `diff_traces` query/parse support. `AgentTraceDbLifecycle` initializes/checks repository-scoped storage through `agent_trace_storage`. - `post-commit Agent Trace auto-sync readiness`: The doctor report fact that explains whether the enabled post-commit trigger is ready without invoking it. Doctor compares the installed `post-commit` hook's SCE managed block using the same currency semantics as setup and resolves config-file-only `agent_trace.auto_sync` with source metadata. JSON states are `ready`, `disabled`, `not_ready`, and `not_applicable`; explicit disable is healthy, while existing hook problems continue to own overall readiness and remediation. See [automatic Agent Trace synchronization](cli/agent-trace-auto-sync.md) and [doctor human text](sce/doctor-human-text-contract.md). - `structured patch service`: Pure synchronous Rust service in `cli/src/services/structured_patch.rs` that derives supported structured editor hook payloads into canonical `ParsedPatch` values. The current implemented source is Claude `PostToolUse` payloads for `Write` creates and `Edit` structured patches; wired into `sce hooks diff-trace` for Claude payload classification at intake and into `RepositoryAgentTraceDb::recent_diff_trace_patches` for post-commit structured payload parsing, where persisted row `model_id` is assigned to every hunk and persisted canonical row `session_id` to every touched line before downstream reconstruction. - `MutationLineage` / `causal mutation-lineage attribution`: `cli/src/services/mutation_trace/lineage.rs` is a pure module holding, per repo path, a vector of `(content, LineProvenance)` where `LineProvenance` is `Unknown`, `MutationAi { scope_id }`, or `MutationNonAi`. `MutationLineage::apply(patch, origin)` advances it one reconstructed tree transition at a time, applying hunks structurally: context lines carry provenance forward, removed lines are deleted permanently, added lines take only the introducing `TransitionOrigin` (`MutationAi` for healthy untainted `AiExclusive`, `MutationNonAi` for any other recorded event, `Unobserved` → `Unknown` for gap reloads and the latest-tree→commit-tree tail); a structurally inconsistent transition is a `LineageError` and the caller reloads that file to an all-`Unknown` baseline. Historical mutation patches are never matched independently against the committed patch — that earlier `MutationPatchEvidence` / strict-matcher model let stale evidence resurrect and is removed. `attribution.rs` now only supplies `exclude_direct_coverage` (drop directly covered lines) and `patch_for_locations`. The store supplies events through an exact-worktree-scoped `mutation-event page` ordered by descending 8-byte big-endian revision, continued with an exclusive revision cursor, capped at 32 rows; plus `latest_mutation_event_revision` for the commit attribution cut. The `bounded mutation-history consumer` (`resolve_bounded_mutation_attribution` in `cli/src/services/mutation_trace/runtime/mutation_attribution.rs`) loads the newest ≤128 in-cut events through `MutationEventPageSource` + `TreeReadSource` (`diff_trees` + `file_at_tree`) seams, replays them oldest-to-newest from a conservative baseline through the tail into `commit_tree`, then projects surviving provenance onto the committed patch; it is current-worktree-only and timestamp-independent, owns `MAX_MUTATION_ATTRIBUTION_EVENTS = 128` (event 128 may contribute, 129 is never loaded), tracks database (`loaded_pages`/`loaded_rows`) apart from Git (`inspected_events`/`reconstructed_events`/`gap_resets`) work, and records a `mutation-attribution barrier` on any page-query or reconstruction/tail failure while still returning conservative results. See [mutation-trace store](cli/mutation-trace-store.md) and [mutation-trace Agent Trace attribution](cli/mutation-trace-agent-attribution.md). -- `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace(...)` under `metadata.sce`, carrying `version` (sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`) and `line_changes` (exact `{ai,mixed,unknown}` × `{added,removed}` `u64` touched-line attribution counts derived from canonical `post_commit_patch` hunks, reusing each hunk's existing `Conversation.contributor.type` classification with no independent second classification pass, `#[serde(default)]` for backward-compatible deserialization of pre-existing payloads); the whole object is schema-validated with the rest of the payload and persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. +- `Agent Trace SCE metadata`: Implementation-owned top-level metadata emitted by `build_agent_trace_from_evidence(...)` under `metadata.sce`, carrying `version` (sourced from the compiled `sce` CLI package version via `env!("CARGO_PKG_VERSION")`) and `line_changes` (exact `{ai,mixed,unknown}` × `{added,removed}` `u64` touched-line attribution counts derived from canonical `post_commit_patch` hunks, reusing each hunk's existing `Conversation.contributor.type` classification with no independent second classification pass, `#[serde(default)]` for backward-compatible deserialization of pre-existing payloads); the whole object is schema-validated with the rest of the payload and persisted in AgentTraceDb `agent_traces.trace_json` without changing the top-level Agent Trace payload/schema `version`. - `ScopeProvenance`: Optional insert-once metadata keyed by a mutation `ScopeId`, storing canonical SCE `session_id` and nullable first-observed `model_id`. It is registered only at `Start` admission while the owning scope is `NeverSeen`, remains outside `ProtocolState` and the CAS transition, and is resolved during mutation projection. `ScopeId` proves mutation ownership; `ScopeProvenance` describes the owning scope. Model/session provenance is observational and never changes `AiExclusive`/`AiContended`/`IneligibleUnscoped` protocol decisions. -- `Agent Trace range content_hash`: Per-range `content_hash` emitted by `build_agent_trace(...)` inside every `ranges[]` entry as `murmur3:`, computed from the touched-line kind/content of the `post_commit_patch` or embedded-patch hunk used to emit that range while excluding positions, paths, metadata, and database IDs. -- `external-mutation guard` / `external-mutation supervisor`: `run_external_mutation_guard` (`cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only), the harness-neutral, not-Pi-specific mechanism that lets a human-initiated shell command (Pi's `!`/`!!` `user_bash`) mutate a worktree without ever creating a mutation scope: it acquires the same `ProtectedWorktree` guard `coordinate()` uses, reports it armed, itself spawns the human shell as its own child (never signaled or stopped by control-channel closure — only by an explicit cancel request, which it enacts by signaling the shell's own process group), finishes exclusively on the shell's real process termination, and only then forces the existing `database_failure`+`recover` composition on the already-held worktree via `coordinate_on_held_worktree` before clearing the marker with `ProtectedWorktree::complete()`. See [`mutation-trace-external-mutation-guard.md`](cli/mutation-trace-external-mutation-guard.md). +- `Agent Trace range content_hash`: Per-range `content_hash` emitted by `build_agent_trace_from_evidence(...)` inside every `ranges[]` entry as `murmur3:`, computed from the touched-line kind/content of the `post_commit_patch` or embedded-patch hunk used to emit that range while excluding positions, paths, metadata, and database IDs. +- `external-mutation guard` / `external-mutation supervisor`: `arm_external_mutation_guard` + `ArmedExternalMutationGuard::exec` (`cli/src/services/mutation_trace/runtime/external_mutation_guard.rs`, hidden `sce hooks external-mutation-guard`, Unix-only), the harness-neutral, not-Pi-specific mechanism that lets a human-initiated shell command (Pi's `!`/`!!` `user_bash`) mutate a worktree without ever creating a mutation scope: it acquires the same `ProtectedWorktree` guard `coordinate()` uses, reports it armed, itself spawns the human shell as its own child (never signaled or stopped by control-channel closure — only by an explicit cancel request, which it enacts by signaling the shell's own process group), finishes exclusively on the shell's real process termination, and only then forces the existing `database_failure`+`recover` composition on the already-held worktree via `coordinate_on_held_worktree` before clearing the marker with `ProtectedWorktree::complete()`. See [`mutation-trace-external-mutation-guard.md`](cli/mutation-trace-external-mutation-guard.md). - `Claude diff-trace attribution`: Diff-trace enrichment rule where one Claude `PostToolUse` event resolves its model with `direct > exact transcript > exact session/agent state > bridge-derived chain state > NULL`: direct top-level/nested metadata first, then that event's `transcript_path` matched by `tool_use_id` to an assistant envelope's `tool_use.id`, then one exact `(cc_, agent_id)` lookup in local `claude_model_state`. Only for a raw structured Claude payload in the exact main-session scope (`agent_id` empty), a state miss then falls back to `bridge session correlation` across the transcript's bridge-linked chain; when that resolves, persistence writes a `claude_model_state` row for the current session (`source="bridge_inherited"`) — the one diff-trace resolution path that writes state — and uses that model, so later traces of the same session resolve from their own exact-scope row. Model sources receive one `claude/` normalization step, ephemeral agent context is never exported, and subagents do not inherit main-session state. - `bridge session correlation`: Claude-specific local correlation using the `bridgeSessionId` in leading transcript records to relate a Claude session's transcript to its sibling transcripts sharing that ID. Selection resolves the newest `claude_model_state` observation across all bridge-linked chain members by `observed_at_ms` (deterministic session-ID tie-break), not the chain origin and not the most recently modified sibling transcript. Discovery is bounded (leading records only) and fail-open, uses existing exact-scope `claude_model_state` rows without persisting the bridge ID, and does not claim authoritative session ordering. One selection rule serves both call sites — the `diff-trace` state-miss path and the `sce hooks claude-model-state` `SessionStart` path — with no most-recently-modified-sibling pick left in the code. See [the read-path bridge-seeding decision](decisions/2026-09-10-claude-bridge-seeding-on-diff-trace.md) and [the bridge-session inheritance decision](decisions/2026-09-08-claude-bridge-session-model-inheritance.md). - `DiffTraceInsert`: Insert payload in `cli/src/services/agent_trace_db/mod.rs` carrying `time_ms`, tool-prefixed `session_id`, `patch`, `model_id`, `tool_name`, nullable `tool_version`, and `payload_type` for parameterized writes to the `diff_traces` table; `payload_type` uses `PAYLOAD_TYPE_PATCH` (`"patch"`) for `OpenCode` unified-diff payloads and `PAYLOAD_TYPE_STRUCTURED` (`"structured"`) for `Claude` `PostToolUse` structured payloads. - `diff_traces payload_type discriminator`: `TEXT NOT NULL DEFAULT 'patch'` column in `diff_traces` added by migration `015_add_diff_traces_payload_type`; values are `PAYLOAD_TYPE_PATCH` (`"patch"`) for `OpenCode` unified-diff source payloads and `PAYLOAD_TYPE_STRUCTURED` (`"structured"`) for `Claude` `PostToolUse` structured source payloads; existing rows default to `"patch"` for backward compatibility. - `bash policy satisfied_by`: Optional field on a custom `policies.bash` entry listing wrapper argv prefixes that already satisfy the policy. When the matched command was unwrapped from one of these wrappers (outermost first, tracked by `NormalizedSegment.wrappers` in `cli/src/services/bash_policy.rs`), the policy does not fire, so a policy steering `rg` toward nix stays quiet for `nix shell nixpkgs#ripgrep -c rg ...` while still blocking a bare `rg`. Custom-policy-only; presets cannot declare satisfying wrappers. Exact argv-prefix matching only. See `context/sce/bash-tool-policy-enforcement-contract.md`. - `DbSpec`: Service-specific database metadata trait in `cli/src/services/db/mod.rs` that supplies a diagnostic database name, canonical path resolver, ordered embedded migration list, and config-file lookup key (`db_config_key()`) for `TursoDb`. -- `TursoDb`: Generic unencrypted Turso database adapter in `cli/src/services/db/mod.rs`; owns parent-directory creation and Turso local open/connect flow wrapped in config-driven connection-open retry, then delegates synchronous `execute()`/`query()`/`query_map()` wrappers with config-driven query retry and migration execution through the shared internal `TursoConnectionCore` for a `DbSpec` implementation. -- `TursoConnectionCore`: Internal shared operation core in `cli/src/services/db/mod.rs` used by both `TursoDb` and `EncryptedTursoDb`; owns the Turso connection and tokio current-thread runtime bridging used by the public adapter methods; generic embedded migration execution with per-database `__sce_migrations` metadata is delegated to `run_embedded_migrations` helpers. +- `TursoDb`: Generic unencrypted Turso adapter in `cli/src/services/db/mod.rs`; owns the concrete connection and service marker but no Tokio runtime. Its construction, open, migration, execute/query/materialization, checkpoint, readiness, and transaction APIs are native async operations awaited by connected callers; retry and migration policies remain adapter-owned. +- `TursoConnectionCore`: Removed with the async persistence migration. `TursoDb` and `EncryptedTursoDb` now share async helper functions without a runtime-owning core or synchronous executor bridge. - `no-migration DB open path`: `TursoDb::open_without_migrations()` / `TursoDb::open_without_migrations_at(path)` plus Agent Trace adapter-specific no-migration seams; opens/connects a local Turso database with parent-directory creation and configured connection-open retry but does not create `__sce_migrations` or run embedded schema migrations. Active Agent Trace hook callers first try the repository-scoped no-migration path and then fall back to migration-running initialization when readiness or repository metadata validation fails. - `TursoDb migration readiness check`: Public methods on `TursoDb` in `cli/src/services/db/mod.rs` for non-mutating schema-readiness verification: `migration_metadata_problems(&self) -> Result>` queries `__sce_migrations` metadata and compares applied IDs against `M::migrations()`, returning problems (missing table, incomplete migrations, unexpected migrations) or an empty list when ready; `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>` calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found. `RepositoryAgentTraceDb::ensure_schema_ready_for_hooks()` delegates to `TursoDb::ensure_schema_ready()` with the Agent Trace–specific `AGENT_TRACE_SCHEMA_SETUP_GUIDANCE` constant. - `database_retry config namespace`: Nested config namespace under `policies.database_retry` in `sce/config.json`, authored in `config/pkl/base/sce-config-schema.pkl` and parsed/resolved in `cli/src/services/config/`. Supports per-database overrides (`local_db`, `agent_trace_db`, `auth_db`) each with optional `connection_open` and `query` objects containing `max_attempts`, `timeout_ms`, `initial_backoff_ms`, `max_backoff_ms`; `agent_trace_db` alone also accepts `busy_timeout_ms` and `contention_deadline_ms`. Validated against JSON Schema at config load and surfaced in `sce config show`/`validate`. Parsed into `DatabaseRetryConfig` (`local_db`/`auth_db` as `Option`, `agent_trace_db` as `Option` wrapping a `PerDbRetryConfig` plus the two Agent Trace keys), initialized at app startup via the `DATABASE_RETRY_CONFIG` `OnceLock`, and consumed by config-aware retry and busy-timeout resolution in DB adapters with fallback to hardcoded defaults. - `busy timeout` / `contention deadline`: Agent Trace DB contention settings (`policies.database_retry.agent_trace_db.busy_timeout_ms`, default `1000`, max `10000`; `contention_deadline_ms`, default `2250`, max `30000`). The busy timeout is Turso's connection-wide wait for a contended lock before it reports `Busy` (`0` disables the handler). The contention deadline is the cutoff after which SCE starts no further outer write-contention retry (`0` starts none); it is not a hard timeout on an operation already running. The outer write-contention retry (at most `2` attempts, full-jitter `0..=100ms` backoff, typed `Busy`/`BusySnapshot` only) covers only Agent Trace write units that are safe to replay whole: the transactional insert-pair and CAS-batch primitives and `TursoDb::execute_idempotent_write`. See `context/sce/shared-turso-db.md`. -- `DB connection-open/query retry policies`: Connection-open retry applies to `TursoDb::new()` and `EncryptedTursoDb::new()` through `policies.database_retry..connection_open`, with fallback to `3` attempts, `1s` timeout, and `25..200ms` backoff; query retry applies to `execute()`/`query()`/`query_map()` (and every Agent Trace operation outside the write-contention set) through `policies.database_retry..query`, with fallback to `5` attempts, `200ms` `timeout_ms`, and `25..100ms` backoff. `timeout_ms` only labels an attempt that ran long after it returns; it never interrupts a synchronous operation, so the policy's configured budget is not a wall-clock bound (Agent Trace attempts can also include Turso busy waiting). Both resolve through `DATABASE_RETRY_CONFIG` and `run_with_retry_sync`; embedded migrations stay outside connection-open retry and `query_map()` keeps caller row mapping outside retry. -- `__sce_migrations`: Per-database migration metadata table created by the shared `TursoConnectionCore` migration path behind public adapter `run_migrations()` methods; records applied migration IDs after successful execution so later setup/lifecycle initialization applies only migrations not yet recorded, while existing metadata-less DBs are brought forward by re-applying the current idempotent migration set and recording each ID. +- `DB connection-open/query retry policies`: Connection-open retry applies to async `TursoDb::new()` and `EncryptedTursoDb::new()` through `policies.database_retry..connection_open`, with fallback to `3` attempts, `1s` timeout, and `25..200ms` backoff; async operations use `policies.database_retry..query` with fallback to `5` attempts, `200ms` `timeout_ms`, and `25..100ms` backoff. `timeout_ms` only labels an attempt that ran long after it returns; it does not cancel the operation. Agent Trace contention retries preserve their separate typed Busy policy. Embedded migrations stay outside connection-open retry and `query_map()` keeps caller row mapping outside retry. +- `__sce_migrations`: Per-database migration metadata table created by async shared adapter migration helpers; records applied migration IDs after successful execution so later setup/lifecycle initialization applies only migrations not yet recorded, while existing metadata-less DBs are brought forward by re-applying the current idempotent migration set and recording each ID. - `CLI generated migration manifest`: Build-time Rust source at `OUT_DIR/generated_migrations.rs` written by `cli/build.rs` from immediate `cli/migrations//*.sql` directories after staging SQL under `OUT_DIR/static/migrations`; constants are named from the database directory (for example `AGENT_TRACE_REPOSITORY_MIGRATIONS`, `AUTH_MIGRATIONS`), sorted by the numeric filename prefix before `_`, and embed staged SQL via `include_str!`. - `sync command deferral` (historical): Former plan/state note that a user-invocable sync command was deferred to `0.4.0`; superseded first by nested `sce trace sync` and now by top-level `sce sync` (see `context/cli/sync-command.md`). Local DB bootstrap and setup-time repository-scoped Agent Trace DB initialization still flow through lifecycle providers aggregated by the setup command, hook runtime still keeps a lazy repository Agent Trace DB fallback for repositories where setup has not run or schema metadata is incomplete, and DB health/repair still flows through the doctor surface. -- `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, async `run_with_retry`, sync `run_with_retry_sync`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. The sync helper is currently wired into shared database constructors for local open/connect retry and into `TursoDb`/`EncryptedTursoDb` operation retry for `execute()`/`query()`/`query_map()`. +- `CLI bounded resilience wrapper`: Shared policy in `cli/src/services/resilience.rs` (`RetryPolicy`, async `run_with_retry` and non-cancelling `run_with_retry_elapsed`) that applies deterministic retries/timeouts/capped backoff to transient operations, emits retry observability events, and returns actionable terminal failure guidance. Async database construction and operations use `run_with_retry`; there is no synchronous variant. - `setup service orchestration`: Setup execution logic in `cli/src/services/setup/command.rs` that resolves the repository root, always ensures the durable-context baseline, then for normal modes derives a repo-root-scoped `AppContext` from the runtime command context, dispatches `setup` through the static lifecycle provider catalog (config → local_db → auth_db → agent_trace_db → hooks when requested), handles interactive target selection for config asset installation, and emits deterministic success messaging per target. - `setup target flags` and `setup mode contract`: Mutually-exclusive `sce setup` target selectors (`--opencode`, `--claude`, `--pi`, `--codex`, `--all`) force non-interactive `SetupMode::NonInteractive(SetupTarget)` when exactly one is provided; otherwise `SetupMode::Interactive` is the default. `--all` expands to opencode+claude+pi+codex and replaced the removed `--both` flag. - `setup interactive target prompt`: `inquire::Select` flow in `cli/src/services/setup/mod.rs` (`InquireSetupTargetPrompter`) that presents OpenCode, Claude, Pi, Codex, and All (OpenCode + Claude + Pi + Codex) when `sce setup` runs without target flags. - `setup dispatch outcome`: Execution model in `cli/src/services/setup/mod.rs` (`SetupDispatch`) where setup either proceeds with a selected/non-interactive target or exits as cancelled without file changes. Interactive dispatch also carries independent `agent_trace.auto_sync` and `policies.attribution_hooks.enabled` booleans from Yes-default confirmations; non-interactive dispatch carries neither, allowing persistence to preserve existing keys and omissions. - `setup embedded asset manifest`: Compile-time generated file index emitted by `cli/build.rs` into `OUT_DIR/setup_embedded_assets.rs`, embedding bytes from Pkl-generated `OUT_DIR/pkl-generated/config/.{opencode,claude,pi}/**` plus staged `OUT_DIR/static/hooks/**` as deterministic normalized relative-path entries consumed by `cli/src/services/setup/mod.rs`; `OPENCODE_EMBEDDED_ASSETS`, `CLAUDE_EMBEDDED_ASSETS`, and `PI_EMBEDDED_ASSETS` all back live setup targets. The manifest also carries `CODEX_EMBEDDED_ASSETS`, embedding Codex's two Pkl-generated output roots (`config/.agents/**`, `config/.codex/**`) merged by `cli/build.rs` into a build-time-only `OUT_DIR/pkl-generated/config/codex-target/` staging tree so its relative-path entries keep their `.agents/`/`.codex/` prefixes; `SetupTarget::Codex` now backs it as a fourth live setup target via `sce setup --codex`/`--all`, installing directly at the repository root (via `InstallTargetPaths::codex_target_dir()`) since its asset paths already carry their own output-root prefix, unlike the other three targets' single-subdirectory destinations. -- `setup required-hook embedded assets`: Setup-service accessors in `cli/src/services/setup/mod.rs` (`iter_required_hook_assets`, `get_required_hook_asset`) that expose canonical embedded templates for `pre-commit`, `commit-msg`, and `post-commit` without runtime config reads. +- `setup required-hook embedded assets`: Setup-service accessor in `cli/src/services/setup/mod.rs` (`iter_required_hook_assets`) that expose canonical embedded templates for `pre-commit`, `commit-msg`, and `post-commit` without runtime config reads. - `SCE managed block`: The CLI-presence check plus `sce hooks ` invocation in each canonical hook template (`cli/assets/hooks/{pre-commit,commit-msg,post-commit}`), delimited by `# >>> sce managed block (do not edit) >>>` / `# <<< sce managed block <<<` comment markers so the same block content can be embedded inside a foreign hook without disturbing content around it (see `context/sce/setup-githooks-hook-asset-packaging.md`). The block propagates an available `sce` command's exit status by capturing `$?` and calling `exit` explicitly rather than by `exec`, so it terminates the script deterministically even when appended after other content. A pure merge module computes hook install bytes against this marker pair (see `setup hook-merge seam`); both `sce setup --hooks` install (see `setup required-hook install orchestration`) and `sce doctor` hook inspection decide currency against this marker pair rather than whole-file byte comparison, so a hook a repository has extended around the block still reports current. - `setup hook-merge seam`: Pure module `cli/src/services/setup/hook_merge.rs`, covering `pre-commit`, `commit-msg`, and `post-commit`. `merge_or_create_hook(existing: Option<&[u8]>, canonical: &[u8], hook_name: &str) -> Result` returns `canonical` verbatim (`HookMergeKind::Created`) when no hook exists; otherwise it locates the `SCE managed block` marker pair by exact line match. A hook already carrying a balanced marker pair identical to the canonical block returns its bytes unchanged (`AlreadyCurrent`); one whose block differs gets that block spliced in place between the same marker lines, leaving surrounding content untouched (`ManagedBlockReplaced`); a marker-free hook containing the legacy pre-marker guidance URL (`https://sce.crocoder.dev/docs/getting-started#install-cli`) is treated as SCE-owned wholesale and replaced entirely with `canonical` (also `ManagedBlockReplaced`); any other marker-free hook is foreign and kept as an exact byte prefix with the canonical block appended after it (`AppendedToForeign`). An unbalanced or partial marker pair is a hard, deterministic error naming `hook_name`, with no bytes returned. For the `AppendedToForeign` case, `HookMerge.unreachable_block_advisory` is set when the foreign hook's last non-blank, non-comment line sits at zero indentation and starts with `exec ` or `exit` — a narrow heuristic (no shell parsing) flagging that the appended block would never run. This module is pure and filesystem-free per "Unit testing in Nix sandbox"; required-hook install calls it (see `setup required-hook install orchestration`), and doctor hook inspection (`cli/src/services/hooks/lifecycle.rs`, `cli/src/services/doctor/inspect.rs`) also calls it, reporting a hook `Current` only when merging the canonical template into its on-disk bytes is a no-op — including treating an unbalanced or partial marker pair as `Stale` rather than `Unknown`, so `sce doctor --fix` repairs it. - `setup required-hook install orchestration`: Setup-service flow in `cli/src/services/setup/mod.rs` (`install_required_git_hooks`, backed by the rename-injectable `install_required_git_hooks_with_rename`) that resolves repository root + effective hooks directory via git truth, then for each hook computes the bytes to stage with the `setup hook-merge seam` (`hook_merge::merge_or_create_hook`) instead of writing the canonical asset verbatim, reports deterministic per-hook outcomes (`Installed`, `Updated`, `Skipped`) against that merged content plus the executable bit, enforces executable permissions, sets `RequiredHookInstallResult.unreachable_block_advisory` (rendered as a named advisory line) when an appended block would be unreachable, and uses the `setup atomic-swap` policy (see `setup atomic-swap`) — staged content is renamed directly over an existing hook without unlinking it first — with deterministic recovery guidance on swap failure. @@ -142,9 +144,9 @@ - `sce shell completion contract`: Deterministic CLI completion contract where `sce completion --shell ` emits parser-aligned Bash/Zsh/Fish completion scripts for current top-level commands and supported options/subcommands. - `CLI default path catalog`: Canonical production path-ownership contract in `cli/src/services/default_paths.rs`; it is the shared owner for non-test production CLI path definitions, covering per-user persisted paths plus repo-relative, install, hook, and context-path definitions used by production CLI code; build-time payload paths are `OUT_DIR`-owned and excluded. - `SCE web URL owner`: Rust service module at `cli/src/services/agent_trace.rs`; owns `SCE_WEB_BASE_URL = "https://sce.crocoder.dev"` and helper functions for SCE-owned URL construction, including Agent Trace conversation URLs, persisted Agent Trace trace URLs, Agent Trace session URLs, and setup-created versioned config schema URLs. It does not own the `control_plane_base_url` used by `sce sync`, whose baked default is `https://sce.crocoderlab.dev`. -- `CLI capability traits`: Broad capability seam in `cli/src/services/capabilities.rs` consumed by the borrowed, compile-time-typed `AppContext`. `FsOps`/`StdFsOps` wrap filesystem operations and `GitOps`/`ProcessGitOps` wrap git process execution plus repository-root and hooks-directory resolution; current service internals do not consume them directly yet. -- `FsOps`: Filesystem capability trait in `cli/src/services/capabilities.rs` with `read_file`, `write_file`, `metadata`, and `exists`, implemented in production by `StdFsOps`. -- `GitOps`: Git capability trait in `cli/src/services/capabilities.rs` with `run_command`, `resolve_repository_root`, `resolve_hooks_directory`, and `is_available`, implemented in production by `ProcessGitOps`. +- `CLI capability traits`: Broad capability seam in `cli/src/services/capabilities.rs` consumed by the borrowed, compile-time-typed `AppContext`. `FsOps`/`StdFsOps` is a method-less filesystem capability marker and `GitOps`/`ProcessGitOps` wrap git command execution and availability; current service internals do not consume them directly yet. +- `FsOps`: Filesystem capability trait in `cli/src/services/capabilities.rs` with no methods (the previous unused file operations were removed), implemented in production by `StdFsOps`. +- `GitOps`: Git capability trait in `cli/src/services/capabilities.rs` with `run_command` and `is_available`, implemented in production by `ProcessGitOps`. - `SCE default path policy seam`: Canonical path resolver in `cli/src/services/default_paths.rs` that owns config/state/cache root resolution through an internal `roots` helper seam, named default paths, and an explicit inventory for the current default persisted artifacts (`global config`, `auth tokens`); named DB/log paths include `auth DB`, `local DB`, `Agent Trace DB`, and the default observability log directory. On Linux those defaults resolve to `$XDG_CONFIG_HOME/sce/config.json`, `$XDG_STATE_HOME/sce/auth/tokens.json`, `$XDG_STATE_HOME/sce/auth.db`, `$XDG_STATE_HOME/sce/local.db`, `$XDG_STATE_HOME/sce/repos//agent-trace.db`, and `$XDG_STATE_HOME/sce/logs`, with platform-equivalent `dirs` fallbacks elsewhere. The same module is also the canonical owner for broader production CLI path definitions, including repo-local `context/tmp/` scratch/session access, and is protected by a regression test that fails when new non-test production path literals are introduced outside `default_paths.rs`. - `cli config precedence contract`: Deterministic runtime value resolution in `cli/src/services/config/resolver.rs` with precedence `flags > env > config file > defaults`; `log_dir` resolves through `SCE_LOG_DIR` > config-file `log_dir` > `/sce/logs`. Config discovery order is `--config`, `SCE_CONFIG_FILE`, then default discovered global+local paths, with the canonical schema declaration accepted in parsed config files. - `shared runtime/config primitives seam`: Canonical ownership in `cli/src/services/config/types.rs` for the CLI's shared observability/config enums (`LogLevel`, `LogFormat`, `LogFileMode`), request/response primitives (`ConfigSubcommand`, `ConfigRequest`, `ReportFormat`), source metadata types (`ValueSource`, `ConfigPathSource`, `LoadedConfigPath`, `ResolvedValue`, `ResolvedOptionalValue`), resolved runtime config types (`ResolvedAuthRuntimeConfig`, `ResolvedObservabilityRuntimeConfig`, `ResolvedHookRuntimeConfig`), the `NAME` constant, observability env-key constants, and shared bool parsing helpers; re-exported through `cli/src/services/config/mod.rs` via `pub use types::*` so downstream modules continue importing through `services::config` unchanged. @@ -222,19 +224,16 @@ - `ParseError`: Error type in `cli/src/services/patch.rs` produced when `parse_patch` encounters malformed input (missing file headers, invalid hunk headers, missing `@@` delimiters); carries an actionable `message` field - `FileChangeKind`: Enum in `cli/src/services/patch.rs` classifying a file change as `Added`, `Modified`, `Deleted`, or `Renamed`; serialized as `snake_case` JSON strings - `TouchedLineKind`: Enum in `cli/src/services/patch.rs` classifying a touched line as `Added` or `Removed`; serialized as `snake_case` JSON strings; derives `Hash` to support set-based intersection operations -- `PatchLoadError`: Error type in `cli/src/services/patch.rs` produced when `load_patch_from_json` or `load_patch_from_json_bytes` encounters invalid JSON or a payload that does not match the expected `ParsedPatch` structure; carries an actionable `message` field +- `PatchLoadError`: Error type in `cli/src/services/patch.rs` produced when `load_patch_from_json` encounters invalid JSON or a payload that does not match the expected `ParsedPatch` structure; carries an actionable `message` field - `load_patch_from_json`: Public function in `cli/src/services/patch.rs` that reconstructs a `ParsedPatch` from a JSON string; storage-agnostic entrypoint for callers who have already read serialized JSON content from a database, file, or other source -- `load_patch_from_json_bytes`: Public function in `cli/src/services/patch.rs` that reconstructs a `ParsedPatch` from JSON bytes; bytes-oriented counterpart to `load_patch_from_json` for callers working with raw byte data - `intersect_patches`: Public function in `cli/src/services/patch.rs` that computes target-shaped touched-line intersection between two `ParsedPatch` values; takes `constructed_patch` and `post_commit_patch` as inputs, matches files by post-change path identity (exact `new_path` equality or absolute-vs-relative suffix-equivalent path segments), prefers exact touched-line matching by `kind` + `line_number` + `content`, falls back to historical matching by `kind` + `content` when line numbers drift across intermediate edits, and returns a `ParsedPatch` shaped from `post_commit_patch` file/hunk ranges while inheriting result-hunk `model_id` from matched `constructed_patch` hunk provenance when available; consumed by the active post-commit hook runtime - `combine_patches`: Public function in `cli/src/services/patch.rs` that merges multiple `ParsedPatch` values into one deterministic result with later-input-wins semantics; groups files by `new_path`, deduplicates touched lines by identity (`kind` + `line_number` + `content`) with later patches winning, preserves file metadata and hunk metadata from the last contributing patch, orders files by first encounter and hunks by `old_start`; consumed by the active post-commit hook runtime before intersection - `HunkContributor`: Enum in `cli/src/services/agent_trace.rs` classifying a `post_commit_patch` hunk from the union of direct AI coverage (`intersection_patch = intersect_patches(direct_patch, post_commit_patch)`) and any mutation-derived AI coverage supplied via `AgentTraceEvidence.mutation_ai_patch`: `Ai` (every touched line covered), `Mixed` (a non-empty proper subset covered), `Unknown` (no line covered); with no mutation evidence this reduces exactly to the direct-only slot rule (`Ai` exact line-by-line match, `Mixed` same slot different content, `Unknown` no matching `old_start` slot); serialized as `snake_case` JSON strings - `Conversation`: Struct in `cli/src/services/agent_trace.rs` representing one per-hunk entry in the minimal agent-trace payload, carrying a nested `contributor` object (`type` plus optional `model_id`) plus `ranges`, where the current implementation emits exactly one `{ start_line, end_line }` entry derived from the `post_commit_patch` hunk - `TraceFile`: Struct in `cli/src/services/agent_trace.rs` representing one per-file entry in the minimal agent-trace payload, carrying `path` (from `post_commit_patch`'s `new_path`) plus `conversations` (one per `post_commit_patch` hunk) - `AgentTraceVcs`: Top-level VCS metadata struct in `cli/src/services/agent_trace.rs` carrying `type` and `revision`; builder behavior maps `type` from caller metadata (`AgentTraceMetadataInput.vcs_type`, enum-backed) and maps revision from caller metadata when VCS metadata is present. -- `AgentTrace`: Top-level struct in `cli/src/services/agent_trace.rs` representing the minimal agent-trace payload, carrying top-level `version` (fixed to `0.1.0`, strict numeric `x.y.z`), `id` (UUIDv7 string derived from the same commit-time moment used for `timestamp` in `build_agent_trace(...)`), `timestamp` (caller-provided commit timestamp via `AgentTraceMetadataInput.commit_timestamp`, validated as RFC 3339), optional `vcs` (`Option`, omitted from serialized JSON when `None`), and `files` (`Vec`, one per `post_commit_patch` file); `serde`-serializable with `snake_case` field naming -- `classify_hunk`: Public function in `cli/src/services/agent_trace.rs` implementing the direct-only slot rule (retained primitive; the builder itself classifies through the internal combined direct+mutation line-coverage rule): matches a `post_commit_patch` hunk against `intersection_patch` hunks on `old_start` slot, returning `HunkContributor::Ai` for exact line-by-line match, `Mixed` for same-slot-but-different-content, or `Unknown` when no matching slot exists +- `AgentTrace`: Top-level struct in `cli/src/services/agent_trace.rs` representing the minimal agent-trace payload, carrying top-level `version` (fixed to `0.1.0`, strict numeric `x.y.z`), `id` (UUIDv7 string derived from the same commit-time moment used for `timestamp` in `build_agent_trace_from_evidence(...)`), `timestamp` (caller-provided commit timestamp via `AgentTraceMetadataInput.commit_timestamp`, validated as RFC 3339), optional `vcs` (`Option`, omitted from serialized JSON when `None`), and `files` (`Vec`, one per `post_commit_patch` file); `serde`-serializable with `snake_case` field naming - `AgentTraceMetadataInput`: Metadata input struct in `cli/src/services/agent_trace.rs` that carries `commit_timestamp` (RFC 3339 commit-time value used as `AgentTrace.timestamp`), `commit_revision` (mapped to `AgentTrace.vcs.revision` when VCS metadata is emitted), and optional `vcs_type` (`Option`, mapped to `AgentTrace.vcs.type` and controlling whether top-level `vcs` is emitted); `AgentTraceVcsType` is the schema-aligned `Git`/`Jj`/`Hg`/`Svn` enum serialized as `git`/`jj`/`hg`/`svn`. -- `build_agent_trace`: Public function in `cli/src/services/agent_trace.rs` — the direct-only compatibility path — that delegates to `build_agent_trace_from_evidence` with an empty `mutation_ai_patch`: computes `intersection_patch = intersect_patches(constructed_patch, post_commit_patch)`, iterates over `post_commit_patch` files and hunks, classifies each hunk, validates `AgentTraceMetadataInput.commit_timestamp` as RFC 3339, derives UUIDv7 `AgentTrace.id` from that same commit-time moment, and returns `Result` with top-level metadata fields plus one `Conversation` per `post_commit_patch` hunk; consumed by the active post-commit hook flow, with no standalone `sce agent-trace` command surface. `build_agent_trace_from_evidence(AgentTraceEvidence { direct_patch, mutation_ai_patch }, post_commit_patch, metadata)` is the separated-evidence entrypoint: identical top-level metadata behavior, combined direct+mutation hunk classification, deduplicated direct+mutation session links, and a contributor model only when the present evidence sources agree; top-level `tool` remains derived from the direct intersection only. - `agent-trace plugin diff extraction seam`: Helper `extractDiffTracePayload` in `config/lib/agent-trace-plugin/opencode-sce-agent-trace-plugin.ts` that accepts a typed `message` event and returns `{ sessionID, diff, time, model_id }` only for user-role messages with non-empty `summary?.diffs`; it joins present object-entry `patch` fields with `\n`, skips entries without `patch`, returns `undefined` when no usable patches remain, uses `Date.now()` for `time`, and builds `model_id` as `providerID/modelID` from `event.properties.info.model`. - `get_or_create_encryption_key`: Public keyring-backed helper in `cli/src/services/db/encryption_key.rs` that retrieves or generates a 64-character hex encryption key from the OS credential store (macOS Keychain, Linux Secret Service via zbus, Windows Credential Store); uses `keyring_core::Entry` with service name `"sce"` and the database name as username. Actively consumed by `EncryptedTursoDb::new()` via the shared adapter constructor. - `conversation-trace mixed batch`: Rust `sce hooks conversation-trace` STDIN contract accepting `{ payloads: [{ type: "message" | "message.part", ... }] }` with top-level `type` ignored and malformed-item skipping. See `context/sce/agent-trace-hooks-command-routing.md` and `context/sce/agent-trace-db.md`. diff --git a/context/overview.md b/context/overview.md index d0917981f..d1e89615f 100644 --- a/context/overview.md +++ b/context/overview.md @@ -45,7 +45,7 @@ for `/validate`. See This repository maintains shared assistant configuration for OpenCode, Claude, Pi, and Codex from a single canonical Pkl authoring source. One typed workflow catalog owns the six workflows' shared identity and target routing metadata, while canonical workflow/phase modules own behavior and migrated package-local documents, and target renderers own formatting. Generated target layouts are ephemeral: repository builds consume a pre-Cargo generated payload through `SCE_CLI_GENERATED_INPUT_DIR`, crates.io and Flatpak stage packaging-only fallbacks, and `config/.opencode`, `config/.claude`, `config/.pi`, `config/.agents`, and the generated working-tree SCE config schema are not committed; versioned SCE config schema snapshots live under `schema/v/`. The catalog also marks a workflow `optional` — currently only `brownfield` — which changes nothing about generation and is projected into a generated `config/optional-workflows.json` manifest for install-time consumers. `nix run .#pkl-check-generated` preserves its exact 142-path artifact, metadata/package, phase-reference, internal-reference, optional-workflow-manifest, workflow-orchestration, OpenCode-permission, required-path, and forbidden-path checks while delegating deterministic payload production and inventories to the shared generated-input producer; `nix flake check` runs the same contract. The target matrix contains one manual OpenCode profile plus Claude, Pi, and Codex; the former automated OpenCode profile has been removed. A fourth Pkl renderer, `config/pkl/renderers/codex-content.pkl`, also consumes the same canonical workflow composition to emit skills-only Codex output under `config/.agents/skills/**` (no per-target frontmatter, matching Pi, and no command/prompt layer). Each of the six catalog workflow skills (not `sce-decision`, which has no user-facing entrypoint on any target) additionally carries a Codex-only `agents/openai.yaml`, rendered by `config/pkl/renderers/codex-metadata.pkl` from the shared catalog's `title`/`description` plus an authored `default_prompt`, with `policy.allow_implicit_invocation: false` so these stateful lifecycle workflows activate only from explicit `$sce-` invocation or Codex's `/skills` discovery, never from conversational relevance alone. The Codex renderer also emits a Codex hook registration file (`config/.codex/hooks.json`) and its install-guidance helper (`config/.codex/hooks/run-sce-or-show-install-guidance.sh`). The existing conversation/diff registrations (`UserPromptSubmit`, `Stop`, `PreToolUse` for `Bash`, `PostToolUse` for `apply_patch`) route through `sce hooks codex` and retain their fail-open behavior where designed. The separate mutation-scope registrations route through `sce hooks codex-mutation-scope`: `PreToolUse` and `PostToolUse` use matcher `^(Bash|apply_patch)$`, while `Stop`, `Interrupt`, `SubagentStop`, and `SessionEnd` omit matcher and are unmatched. The tracked mutation `PreToolUse` bootstrap fails closed if Git-root resolution, helper or `sce` discovery, adapter startup, runtime `Start`, or Bash-policy evaluation cannot establish attribution; mutation-scope `PostToolUse` and cleanup hooks do not use this bootstrap behavior. Delegation, MCP, and unknown tools do not match these generated mutation Pre/Post registrations. `sce setup --codex` (and `--all`) now installs it as a fourth `SetupTarget`; setup merges the user-owned `.codex/hooks.json` through the shared structural Codex hook-config service, preserving unrelated valid handlers and rejecting malformed or Codex-invalid documents before the atomic swap. `sce doctor` diagnoses that file per required registration (`PresentAndCurrent`/`Missing`/`Stale`, or `Malformed` for the whole document) rather than by whole-document equality, reporting the mutation-scope registrations under distinct `#(mutation-scope)` rows separate from the `sce hooks codex` rows, and separately reports whether Codex has actually marked each structurally current registration trusted and whether its effective hook-discovery policy allows project hooks, by reading (never writing) Codex's own `$CODEX_HOME/config.toml` and policy state; `sce doctor --fix` repairs structurally unhealthy registrations through the same merge service but never touches trust or policy state (see `context/sce/doctor-human-text-contract.md`). `sce hooks codex` now exists as a typed dispatcher (`cli/src/services/hooks/codex/`): it parses the raw hook JSON into a `CodexHookEvent` and classifies `(hook_event_name, tool_name)` into `UserPromptSubmit`, `Stop`, `PreToolUse(Bash)`, `PostToolUse(apply_patch)`, or a fail-open `NoOp` fallthrough covering every other combination — including `apply_patch` under `PreToolUse`. `UserPromptSubmit` and `Stop` each capture one `messages`/`parts` row (`role="user"`/`role="assistant"`) into the repository Agent Trace DB under the idempotent `cx_` session prefix, `PreToolUse(Bash)` delegates to the existing Bash policy engine, and `PostToolUse(apply_patch)` parses Codex's own `apply_patch` text, resolves paths from the event `cwd` against the real Git root into safe repository-relative paths, normalizes Add/Update evidence into an SCE unified diff under deterministic event-scoped synthetic line identities derived from `tool_use_id`, and persists it as one `diff_traces` row when non-empty (see `context/sce/codex-integration-runtime.md`) — while invalid cwd/path resolution and malformed STDIN payloads fail open with empty stdout, reported model IDs are preserved without fabricated provider prefixes, and missing/blank apply_patch sessions produce no evidence. Delete-File operations and Bash filesystem effects do not become `diff_traces` in this existing conversation/diff pipeline, and its apply_patch evidence has separate operation limitations. Separately, the Codex mutation-scope adapter tracks `Bash` and `apply_patch` executions as `TrackedMutation` scopes; `AiExclusive` means tracked-scope exclusivity, not that Bash authored every mutation in the interval. -It also includes a Rust CLI (`sce`) for Shared Context Engineering workflows: auth, config inspection, setup, doctor, Agent Trace hooks and synchronization, bash-policy evaluation, and repository-scoped Agent Trace storage infrastructure. See `context/architecture.md` for module-level boundaries and `context/context-map.md` for the full domain file index. A new pure, dependency-free `cli/src/services/mutation_trace/` module now exists as a Rust refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol; it currently carries domain types and the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — is registered with `#[allow(dead_code)]`. A separate `cli/src/services/mutation_trace/store.rs` persistence layer (built out by the `mutation-cursor-store-persistence` plan) now provides a real, CAS-guarded database call site against the repository-scoped Agent Trace DB, and a `cli/src/services/mutation_trace/runtime/` coordinator layer (built out by the `mutation-cursor-runtime-coordinator` plan) adds the per-worktree advisory lock, the isolated Git snapshot/ref-pinning service, and a public `coordinate()` entrypoint that drives the protocol against a real worktree under that lock, plus a per-worktree `ref_reconciliation` maintenance pass (built out by the `mutation-cursor-ref-reconciliation` plan) that, only for the namespace of a checkout id a current worktree still derives, reclaims orphaned SCE-owned snapshot refs under that same lock while retaining every tree any durable mutation-cursor state still references (a namespace no current worktree owns — via a deleted linked worktree or checkout-id metadata loss/recreation — is out of reach and left to future repository-scoped work). That runtime now has a second protected entrypoint alongside `coordinate()`: `cli/src/services/mutation_trace/runtime/scope_runtime.rs`'s `abandon_scope()` (built out by the `mutation-scope-runtime-integration` plan) retires a scope whose final worktree boundary was never observed — a dead agent process leaves no `Close` behind — sharing `coordinate()`'s extracted `runtime/protected_worktree.rs` safety prefix but deliberately capturing no Git snapshot, so abandonment is not a `RuntimeBoundary` and needs no Quint change. Both entrypoints are now reachable crate-wide through nine `pub(crate)` re-exports in `runtime/mod.rs` while every module behind them stays private, and the lifecycle contract every current or future harness adapter must uphold is recorded in `context/cli/mutation-scope-runtime.md`. The protocol runtime entrypoints (`coordinate()` / `abandon_scope()`) are now driven by one harness-neutral CLI ingress, the hidden `sce hooks mutation-scope` command (`context/cli/mutation-scope-hook-ingress.md`), which reads a single normalized JSON lifecycle object from STDIN, strictly parses it into exactly `start`/`advance`/`close`/`flush`/`abandon`, maps it to a `RuntimeBoundary` (`start`/`advance`/`close`/`flush`) or an `abandon_scope()` call forwarding `scope_id`/`event_id`/`actor_kind` verbatim (any `worktree_id` key is refused — worktree identity is derived by the runtime from the invoking checkout), and invokes the runtime with a lazy DB provider closure so DB acquisition stays inside the protected-worktree ordering. Unlike the fail-open `diff-trace`/`conversation-trace` intakes, this ingress never discards a valid boundary: it classifies results by durable completion, returning a non-zero `CliError` for a malformed payload or a pre-completion runtime error while treating the two carried-outcome variants (`MarkerClearAfterCommit` / `MarkerClearAfterCompletion` — the durable transition succeeded and only the trailing external-taint marker cleanup failed) as empty-stdout success without retrying the transition. Every successful execution emits empty stdout and writes `mutation_trace_*` rows only. It is the generic transport seam. Concrete harness lifecycle adapters for Claude Code and Codex are wired to it in-process via a `pub(crate)` seam rather than by re-invoking the ingress command; they map each harness's lifecycle events onto one mutation `ScopeId` per mutation-capable tool execution with write-ahead fail-closed `Start`, terminal `Close`, cleanup, recovery barriers, and harness-specific unsupported-boundary handling. A Pi adapter driver (`cli/src/services/hooks/pi_mutation_scope/`) also now consumes this same `pub(crate)` in-process seam through its hidden `sce hooks pi-mutation-scope` command and canonical generated Pi extension, with a `PendingStart`→`Executed`→`Closed`/`PendingAbandon` attempt-phase lifecycle keyed on Pi's `tool_result` event rather than `tool_execution_start` (see `context/cli/pi-mutation-scope-integration.md`); ordinary Pi sessions now reach it, and its human `user_bash` path uses the separate guarded external-mutation supervisor. Codex execution identity and `ScopeId`/`EventId` derivation are owned by `context/cli/codex-mutation-scope-integration.md`. A separate **read-only** consumer of the same module's persisted mutation-event history is wired, though: the post-commit Agent Trace flow calls `mutation_trace::runtime::resolve_post_commit_mutation_ai_patch(...)` to attribute committed touched lines that direct `diff_traces` evidence does not cover, by replaying the newest 128 events for the invoking worktree (bounded also by a commit attribution cut captured under the worktree lock) oldest-to-newest as one causal per-line provenance lineage rather than matching historical patches independently, resolving the worktree's *existing* checkout identity only, creating no identity and writing no mutation-cursor state, and never inserting into `diff_traces` or `post_commit_patch_intersections` (see `context/cli/mutation-trace-agent-attribution.md` and `context/sce/agent-trace-hooks-command-routing.md`). See also `context/cli/mutation-trace-protocol.md`, `context/cli/mutation-trace-runtime-coordinator.md`, `context/cli/mutation-trace-ref-reconciliation.md`, `context/cli/mutation-trace-protected-worktree.md`, `context/cli/mutation-trace-scope-abandonment.md`, `context/cli/mutation-scope-runtime.md`, `context/cli/mutation-scope-hook-ingress.md`, `context/cli/claude-mutation-scope-integration.md`, and `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral process supervisor that lets Pi's `user_bash` mutate a worktree, guarded by the same `WorktreeLock`, without ever creating a scope). +It also includes a Rust CLI (`sce`) for Shared Context Engineering workflows: auth, config inspection, setup, doctor, Agent Trace hooks and synchronization, bash-policy evaluation, and repository-scoped Agent Trace storage infrastructure. See `context/architecture.md` for module-level boundaries and `context/context-map.md` for the full domain file index. A new pure, dependency-free `cli/src/services/mutation_trace/` module now exists as a Rust refinement of the verified `spec/mutation_cursor.qnt` mutation-cursor protocol; it currently carries domain types and the full pure action set — `prepare`/`commit` transition logic, attribution/mutation-event materialization, snapshot-failure/database-failure taint actions, scope abandonment, and recovery with an explicit observed-tree input — is registered without a module-wide dead-code allowance (only declaration-level, reasoned exceptions remain, and snapshot-ref reconciliation is wired explicitly through `sce doctor --fix`; automatic reclamation is a deferred follow-up). A separate `cli/src/services/mutation_trace/store.rs` persistence layer (built out by the `mutation-cursor-store-persistence` plan) now provides a real, CAS-guarded database call site against the repository-scoped Agent Trace DB, and a `cli/src/services/mutation_trace/runtime/` coordinator layer (built out by the `mutation-cursor-runtime-coordinator` plan) adds the per-worktree advisory lock, the isolated Git snapshot/ref-pinning service, and a public `coordinate()` entrypoint that drives the protocol against a real worktree under that lock, plus a per-worktree `ref_reconciliation` maintenance pass (built out by the `mutation-cursor-ref-reconciliation` plan) that, only for the namespace of a checkout id a current worktree still derives, reclaims orphaned SCE-owned snapshot refs under that same lock while retaining every tree any durable mutation-cursor state still references (a namespace no current worktree owns — via a deleted linked worktree or checkout-id metadata loss/recreation — is out of reach and left to future repository-scoped work). That runtime now has a second protected entrypoint alongside `coordinate()`: `cli/src/services/mutation_trace/runtime/scope_runtime.rs`'s `abandon_scope()` (built out by the `mutation-scope-runtime-integration` plan) retires a scope whose final worktree boundary was never observed — a dead agent process leaves no `Close` behind — sharing `coordinate()`'s extracted `runtime/protected_worktree.rs` safety prefix but deliberately capturing no Git snapshot, so abandonment is not a `RuntimeBoundary` and needs no Quint change. Both entrypoints are now reachable crate-wide through nine `pub(crate)` re-exports in `runtime/mod.rs` while every module behind them stays private, and the lifecycle contract every current or future harness adapter must uphold is recorded in `context/cli/mutation-scope-runtime.md`. The protocol runtime entrypoints (`coordinate()` / `abandon_scope()`) are now driven by one harness-neutral CLI ingress, the hidden `sce hooks mutation-scope` command (`context/cli/mutation-scope-hook-ingress.md`), which reads a single normalized JSON lifecycle object from STDIN, strictly parses it into exactly `start`/`advance`/`close`/`flush`/`abandon`, maps it to a `RuntimeBoundary` (`start`/`advance`/`close`/`flush`) or an `abandon_scope()` call forwarding `scope_id`/`event_id`/`actor_kind` verbatim (any `worktree_id` key is refused — worktree identity is derived by the runtime from the invoking checkout), and invokes the runtime with a lazy DB provider closure so DB acquisition stays inside the protected-worktree ordering. Unlike the fail-open `diff-trace`/`conversation-trace` intakes, this ingress never discards a valid boundary: it classifies results by durable completion, returning a non-zero `CliError` for a malformed payload or a pre-completion runtime error while treating the two carried-outcome variants (`MarkerClearAfterCommit` / `MarkerClearAfterCompletion` — the durable transition succeeded and only the trailing external-taint marker cleanup failed) as empty-stdout success without retrying the transition. Every successful execution emits empty stdout and writes `mutation_trace_*` rows only. It is the generic transport seam. Concrete harness lifecycle adapters for Claude Code and Codex are wired to it in-process via a `pub(crate)` seam rather than by re-invoking the ingress command; they map each harness's lifecycle events onto one mutation `ScopeId` per mutation-capable tool execution with write-ahead fail-closed `Start`, terminal `Close`, cleanup, recovery barriers, and harness-specific unsupported-boundary handling. A Pi adapter driver (`cli/src/services/hooks/pi_mutation_scope/`) also now consumes this same `pub(crate)` in-process seam through its hidden `sce hooks pi-mutation-scope` command and canonical generated Pi extension, with a `PendingStart`→`Executed`→`Closed`/`PendingAbandon` attempt-phase lifecycle keyed on Pi's `tool_result` event rather than `tool_execution_start` (see `context/cli/pi-mutation-scope-integration.md`); ordinary Pi sessions now reach it, and its human `user_bash` path uses the separate guarded external-mutation supervisor. Codex execution identity and `ScopeId`/`EventId` derivation are owned by `context/cli/codex-mutation-scope-integration.md`. A separate **read-only** consumer of the same module's persisted mutation-event history is wired, though: the post-commit Agent Trace flow calls `mutation_trace::runtime::resolve_post_commit_mutation_ai_patch(...)` to attribute committed touched lines that direct `diff_traces` evidence does not cover, by replaying the newest 128 events for the invoking worktree (bounded also by a commit attribution cut captured under the worktree lock) oldest-to-newest as one causal per-line provenance lineage rather than matching historical patches independently, resolving the worktree's *existing* checkout identity only, creating no identity and writing no mutation-cursor state, and never inserting into `diff_traces` or `post_commit_patch_intersections` (see `context/cli/mutation-trace-agent-attribution.md` and `context/sce/agent-trace-hooks-command-routing.md`). See also `context/cli/mutation-trace-protocol.md`, `context/cli/mutation-trace-runtime-coordinator.md`, `context/cli/mutation-trace-ref-reconciliation.md`, `context/cli/mutation-trace-protected-worktree.md`, `context/cli/mutation-trace-scope-abandonment.md`, `context/cli/mutation-scope-runtime.md`, `context/cli/mutation-scope-hook-ingress.md`, `context/cli/claude-mutation-scope-integration.md`, and `context/cli/mutation-trace-external-mutation-guard.md` (the harness-neutral process supervisor that lets Pi's `user_bash` mutate a worktree, guarded by the same `WorktreeLock`, without ever creating a scope). The generated `/next-task` workflow persists task-level context-synchronization lifecycle state in each plan (`pending`, `synced`, or `blocked`) so unresolved task synchronization debt survives a session boundary and gates new implementation. Successful `/next-task` execution hands task synchronization an explicit, pre-edit-Git-baseline-relative changed-file list plus implementation, verification, done-check, plan-update, and context-impact evidence, recorded directly on the completed task (`Completed`, `Files changed`, `Result`, `Verify`, `Context impact`, `Context synchronization`); the five-file root context pass remains mandatory. A later-session sync-debt retry reads that same completed task record directly from the plan by plan path and task ID, with no separate persisted synchronization handoff. `/validate` is validation-only: it runs final checks, writes the Validation Report, and reports `validated`, `failed`, or `blocked` without plan-level context synchronization. @@ -66,10 +66,10 @@ The current doctor presentation contract supersedes the earlier output-shape sca The command loop now enforces a stable exit-code contract in `cli/src/app.rs`: `2` parse failures, `3` invocation validation failures, `4` runtime failures, and `5` dependency startup failures. Auth command orchestration routes authenticated Control Plane authentication failures, token-storage failures, and approved user-facing fallback failures through typed `CliError::User` catalog entries (`NotAuthenticated`, `AuthStorageUnavailable`, `NotGitRepository`, and `UnexpectedFailure`) while retaining technical sources for observability; missing-credential logout/whoami state queries succeed with their documented text/JSON reports, and internal auth failures remain runtime errors. The same runtime also emits stable CLI stderr diagnostics: internal failures use `SCE-ERR-PARSE`, `SCE-ERR-VALIDATION`, `SCE-ERR-RUNTIME`, or `SCE-ERR-DEPENDENCY` in deterministic `Error []: ...` diagnostics with class-default `Try:` remediation appended when missing, while expected failures emit only their redacted, unstyled catalog messages. The command boundary's former flat, string-only `ClassifiedError` has been replaced by typed `CliError` in `cli/src/services/error.rs`: `CliError::User` carries a closed catalog (`NotAuthenticated`, `NotGitRepository`, `NotGitRemote`, authentication-storage `AuthStorageUnavailable`, and general `UnexpectedFailure`) for expected, deliberately-explained failures rendered without the technical source, wrapper, styling, or automatic `Try:` guidance; the general entry renders one fixed static log-files guidance sentence without dynamic path text. `CliError::Internal` carries a live `anyhow::Error` source rendered as the real error chain with the existing styled wrapper and class-default remediation; `app_support` is the sole owner of the distinct terminal paths, and `sce sync` classifies authentication, credential-storage, and all other sync failures as cataloged user errors while preserving their technical sources for observability. Setup preflight errors preserve technical sources, including the configured remote name, while raw remote URLs remain out of user-facing diagnostics. See `context/sce/cli-error-code-taxonomy.md` for the full contract. The app runtime now also includes a structured observability baseline in `cli/src/services/observability.rs`: deterministic env-controlled log threshold/format (`SCE_LOG_LEVEL` defaults to `error`; `SCE_LOG_FORMAT` defaults to `text`), default-backed log-directory routing (`SCE_LOG_DIR` / config-file `log_dir` / `/sce/logs`) with per-operation machine-local dated file selection, optional session filename partitioning, and creation-triggered retention of direct regular `*.log` files to 10 entries, stable lifecycle event IDs, normal logger records routed only to files when file logging is enabled and only error-level logger records routed to `stderr` when file logging is disabled, direct file-write diagnostics remaining on `stderr`, and `observability::traits` boundaries for logger and telemetry behavior. -One application-level multi-thread Tokio runtime directly awaits static command dispatch and auth/sync HTTP before output rendering. Ordinary synchronous commands remain direct; DB-backed paths use temporary blocking lifetime scopes while Turso remains synchronous with DB-owned runtimes. See [application execution runtime](architecture.md#application-execution-runtime). +One application-level multi-thread Tokio runtime directly awaits static command dispatch through async credentials, lifecycle providers, hooks, sync/export, and Turso persistence. The shared Turso adapters own no runtime or executor bridge, and foreground hooks return after required persistence completes. See [application execution runtime](architecture.md#application-execution-runtime). The app command dispatcher now enforces a centralized stdout/stderr stream contract in `cli/src/app.rs`: command success payloads are emitted on stdout only, while redacted user-facing diagnostics and text-mode sync progress are emitted on stderr; JSON sync remains silent. `cli/src/app.rs` also now runs through explicit startup phases — dependency check, observability config resolution, runtime initialization, command parse/execute, and output rendering — with the app runtime carrying logger/telemetry plus static command-catalog state across those phases while preserving the existing exit-code and degraded-startup contracts. Within that lifecycle, `parse_command_phase` delegates clap-to-runtime conversion to `cli/src/services/parse/command_runtime.rs`, which returns a static `RuntimeCommand` enum, `services::app_support::execute_command_phase` logs around enum-owned `execute_with_stderr(...)` dispatch, and generic `RunOutcome` rendering logs classified errors through the logger trait boundary without coupling render support to the production logger type. Command payload structs for `help`, `version`, `completion`, `auth`, `config`, `setup`, `doctor`, `hooks`, and `sync` live in service-owned `command.rs` files; `cli/src/services/command_registry.rs` owns the deterministic static command-name catalog and enum variants instead of boxed command trait objects. The CLI now also enforces a shared output-format parser contract in `cli/src/services/output_format.rs`, with canonical `--format ` parsing and command-specific actionable invalid-value guidance reused by `config` and `version` services. A compile-safe service lifecycle seam also exists in `cli/src/services/lifecycle.rs`: `ServiceLifecycle` exposes default no-op `diagnose`, `fix`, and `setup` methods against the narrow `HasRepoRoot` accessor, uses lifecycle-owned health/fix/setup result types, and owns the shared static `LifecycleProvider` enum catalog/factory with deterministic config → local*db → auth_db → agent_trace_db → hooks ordering and no boxed provider aggregation. Hooks has a `services/hooks/lifecycle.rs` provider for hook rollout diagnosis/fix/setup, config has a `services/config/lifecycle.rs` provider for global/repo-local config validation plus repo-local config bootstrap, local_db has a `services/local_db/lifecycle.rs` provider for canonical local DB path health, parent-directory readiness/bootstrap, and `LocalDb::new()` setup, auth_db has a `services/auth_db/lifecycle.rs` provider for canonical auth DB path health, parent-directory readiness/bootstrap, and `AuthDb::new()` setup, and agent_trace_db has a `services/agent_trace_db/lifecycle.rs` provider for repository-scoped Agent Trace DB setup and repository DB path health/parent readiness from resolved repository identity, returning an actionable "requires a Git repository" diagnostic outside repository context (no global/checkout fallback path). Doctor runtime aggregates the full shared provider catalog for `diagnose` and `fix` and adapts lifecycle records into doctor-owned output records; setup command aggregates the shared provider catalog for `setup` with hooks included only when requested and adapts lifecycle setup outcomes before rendering setup-owned messages. -Agent Trace lifecycle setup now resolves repository storage and initializes `/sce/repos//agent-trace.db` via `RepositoryAgentTraceDb`; hook runtime lazy initialization uses the same repository storage resolver when setup has not prepared the DB or schema metadata is incomplete. +Agent Trace lifecycle setup now resolves repository storage and initializes `/sce/repos//agent-trace.db` via `RepositoryAgentTraceDb`; hook runtime shares the identity/path resolution but opens through a separate no-migration path and never migrates (missing or stale schema yields `Run 'sce setup'.` guidance). The CLI now also includes a shared text styling service in `cli/src/services/style.rs` that provides deterministic color enablement via `owo-colors`, automatic TTY detection, and `NO_COLOR` compliance for human-facing text output; stdout help/text surfaces, stderr diagnostics, and interactive prompt-adjacent text now reuse that shared styling policy while JSON, completion, and other non-interactive/machine-readable flows remain unstyled. The service exports color-detection, conditional styling, help/diagnostic/label/prompt styling, and `banner_with_gradient()` helpers for use across command surfaces while preserving pipe-safe output for non-interactive environments. The `setup` command includes an `inquire`-backed target-selection flow: default interactive selection for OpenCode/Claude/Pi/Codex/All with required-hook installation in the same run, explicit non-interactive target flags (`--opencode`, `--claude`, `--pi`, `--codex`, `--all`), standalone `--bootstrap-context` for additive durable-context baseline creation without integration installs, deterministic mutually-exclusive validation, and non-destructive cancellation exits; the former `--both` flag was removed in favor of `--all` (opencode+claude+pi+codex). Every normal successful setup path also ensures the same context baseline after the Git gate. Workflows the catalog marks optional are installed only when a repository opts in: the repeatable `sce setup --workflow ` flag names the selection for a run, an omitted flag reuses the selection persisted in `integrations.optional_workflows`, and the resolved selection filters the installed target assets and is written back to repo-local config. Interactive runs ask for the selection instead: a multi-select prompt follows target selection with every row unchecked on a first run and pre-checked from the persisted selection afterwards, cancelling it exits non-destructively like the target prompt, and the prompt is skipped when the catalog marks no workflow optional. `brownfield` is currently the only optional workflow, so a default run installs the five core workflows and no brownfield assets. `sce doctor` scopes its integration checks to that same recorded selection, so it never reports an unselected optional workflow's files as missing. For repository generation consumers, `config/pkl/generator-inputs.txt` declares the canonical Pkl/plugin input set and `scripts/produce-cli-generated-input.sh` owns its discovery, two-pass `config/pkl/generate.pkl` evaluation, determinism comparison, payload/input inventories, in-flight input-mutation rejection, atomic handoff publication, and staging cleanup. `scripts/run-cli-cargo.sh` creates a fresh temporary destination, delegates generation to that producer, invokes the requested Cargo workflow with `SCE_CLI_GENERATED_INPUT_DIR`, and removes the handoff after Cargo success, failure, or handled signals. `config/pkl/check-generated.sh` delegates the same production mechanics while retaining contract and path assertions. The root flake also runs `codex-hook-command`, which generates the Codex assets and verifies root, nested-cwd, spaced-path, stdin-forwarding, and fail-open invocation behavior against a stub `sce`. `scripts/prepare-cli-generated-assets.sh` moves the producer-validated Pkl payload and checksums into the unchanged package fallback, adds hooks, migrations, and the Agent Trace schema, and appends only those static checksums to the combined inventory. The root flake's pre-Cargo `cliGeneratedInput` derivation invokes the same producer from a declarative source containing the producer plus its declared inputs. `cli/build.rs` rejects missing, incomplete, modified, or stale repository handoffs, copies the validated payload into Cargo `OUT_DIR/pkl-generated`, stages static inputs under `OUT_DIR/static`, and writes setup-asset, optional-workflow-catalog, and migration Rust manifests into `OUT_DIR`; it never invokes Pkl. Published crates carry the ignored packaging-only fallback, and unpacked downstream builds validate and copy it into their own `OUT_DIR` without requiring Pkl or parent repository paths. @@ -77,7 +77,7 @@ The setup service also provides repository-root install orchestration: it resolv The CLI now also applies baseline security hardening for reliability-driven automation: diagnostics/logging paths use deterministic secret redaction,`sce setup --hooks --repo ` canonicalizes and validates repository paths before execution, and setup write flows run explicit directory write-permission probes before staging/swap operations. The config service now provides deterministic runtime config resolution with explicit precedence (`flags > env > config file > defaults`), strict config-file validation (`$schema`, `log_level`, `log_format`, `log_to_file`, `log_dir`, `workos_client_id`, and nested `policies.bash`, `policies.attribution_hooks.enabled`, plus `policies.database_retry` with per-DB `connection_open`/`query` retry policy specs and Agent Trace-only `agent_trace_db.busy_timeout_ms`/`contention_deadline_ms`), deterministic default discovery/merge of global+local config files (`${config*root}/sce/config.json`then`.sce/config.json`with local override, where`config_root` comes from the shared default-path seam with XDG/`dirs::config_dir()` config-root resolution), defaults for the resolved observability value set (`log_level=error`, `log_format=text`, `log_dir=/sce/logs`), shared auth-key resolution with optional baked defaults starting at `workos_client_id`, first-class bash-policy preset/custom parsing with deterministic conflict and duplicate-prefix validation, custom-policy `satisfied_by`wrapper exemption (a policy does not fire when the matched command was unwrapped from a declared wrapper such as`nix shell nixpkgs#ripgrep`), and a canonical Pkl-authored `sce/config.json`JSON Schema generated beneath Cargo`OUT_DIR`and embedded by`cli/src/services/config/mod.rs`for both`sce config validate`and doctor-time config checks. Runtime startup config loading keeps parity with that schema by accepting its`$schema`declaration in repo-local and global config files, so startup commands such as`sce version`no longer fail before dispatch on that field; the canonical declaration is versioned as`"https://sce.crocoder.dev/v/config.json"` using the CLI release version; this schema URL is separate from the `https://sce.crocoderlab.dev` baked default used by `sce sync` for control-plane ingestion. App-runtime observability now consumes flat logging keys through the shared resolver, so env values still override config-file values while config files provide deterministic fallback for `log_dir`; positive-integer `log_file_retention_limit` uses config-file/default precedence, defaults to `10`, and controls creation-triggered cleanup for primary and v2 log files; `sce config show` reports resolved observability/auth/policy values with provenance, while `sce config validate` is now a trimmed validation surface that reports only pass/fail plus validation errors or warnings in text and JSON modes. The canonical preset catalog and matching contract live in `config/pkl/base/bash-policy-presets.pkl` and `context/sce/bash-tool-policy-enforcement-contract.md`. Invalid default-discovered config files now also degrade gracefully at startup: `sce` keeps running with degraded observability defaults, logs `sce.config.invalid_config` warnings, and reserves hard failures for explicit `--config` / `SCE_CONFIG_FILE` targets or other truly invalid runtime observability inputs. -`cli/src/services/config/mod.rs` is now a module facade that declares focused config submodules (`types`, `schema`, `policy`, `resolver`, private `render`, `command`, and `lifecycle`), re-exporting `pub use types::*`and`pub(crate) use schema::validate_config_file`. Shared config primitive ownership is delegated to `cli/src/services/config/types.rs`; schema loading and file parsing to `cli/src/services/config/schema.rs`; bash-policy semantic validation and policy-specific formatting to `cli/src/services/config/policy.rs`; runtime discovery/precedence to `cli/src/services/config/resolver.rs`; and `sce config show`/`sce config validate`text+JSON output construction to`cli/src/services/config/render.rs`. Downstream modules continue importing through `services::config`unchanged. The CLI now has a generic borrowed`AppContext`dependency view in`cli/src/app.rs`; `AppRuntime`owns concrete production logger/telemetry/fs/git dependencies, and command execution receives context views that borrow those dependencies plus an optional`repo_root: Option`. `AppContext::with_repo_root(...)`/`ContextWithRepoRoot`derives repo-root-scoped views while preserving the borrowed runtime dependencies, and command execution is generic over associated-type narrow accessor traits where practical. The broad capability seam lives in`cli/src/services/capabilities.rs`, where `FsOps`/`StdFsOps`wrap filesystem operations and`GitOps`/`ProcessGitOps`wrap git process execution plus repository-root/hooks-directory resolution. +`cli/src/services/config/mod.rs` is now a module facade that declares focused config submodules (`types`, `schema`, `policy`, `resolver`, private `render`, `command`, and `lifecycle`), re-exporting `pub use types::*`and`pub(crate) use schema::validate_config_file`. Shared config primitive ownership is delegated to `cli/src/services/config/types.rs`; schema loading and file parsing to `cli/src/services/config/schema.rs`; bash-policy semantic validation and policy-specific formatting to `cli/src/services/config/policy.rs`; runtime discovery/precedence to `cli/src/services/config/resolver.rs`; and `sce config show`/`sce config validate`text+JSON output construction to`cli/src/services/config/render.rs`. Downstream modules continue importing through `services::config`unchanged. The CLI now has a generic borrowed`AppContext`dependency view in`cli/src/app.rs`; `AppRuntime`owns concrete production logger/telemetry/fs/git dependencies, and command execution receives context views that borrow those dependencies plus an optional`repo_root: Option`. `AppContext::with_repo_root(...)`/`ContextWithRepoRoot`derives repo-root-scoped views while preserving the borrowed runtime dependencies, and command execution is generic over associated-type narrow accessor traits where practical. The broad capability seam lives in`cli/src/services/capabilities.rs`, where `FsOps`/`StdFsOps` form a method-less filesystem marker and `GitOps`/`ProcessGitOps` wrap git command execution and availability. The shared default path service in`cli/src/services/default_paths.rs`is now the canonical owner for production CLI path definitions. It resolves per-user config/state/cache roots through a dedicated internal`roots`seam, exposes the current persisted-artifact inventory (global config and auth tokens), and also defines named DB paths (auth DB, local DB, Agent Trace DB) plus the repo-relative, install, hook, and context-path accessors consumed across current CLI production code. Non-test production modules should consume this shared catalog instead of hardcoding owned path literals. No default cache-backed persisted artifact currently exists, so cache-root resolution remains available without speculative cache-path features and no legacy default-path fallback is supported. The Rust CLI also centralizes SCE-owned web URI construction in`cli/src/services/agent_trace.rs`, with `SCE_WEB_BASE_URL`as the single Rust owner for`https://sce.crocoder.dev` and helpers consumed by Agent Trace conversation URLs, Agent Trace persisted trace URLs, Agent Trace session URLs, and setup-created repo-local config schema URLs. The config resolver separately owns `control_plane_base_url` and its `https://sce.crocoderlab.dev` baked sync default; the two URL owners must not be conflated. The current user-facing synchronization entrypoint is `sce sync`; references to the former nested spelling in historical records do not describe an available command. @@ -117,7 +117,7 @@ The current supported automated release target matrix is `x86_64-unknown-linux-m The CLI now also includes an approved operator-environment doctor contract documented in `context/sce/agent-trace-hook-doctor.md`; the runtime includes `sce doctor --fix` parsing/help, stable problem/fix-result reporting, canonical hook-repair reuse, bounded doctor-owned local-DB directory bootstrap for the missing SCE-owned DB parent path, and target-scoped integration inventory: Claude reports `Plugins`, `Commands`, and `Skills`; OpenCode reports `Plugins`, `Agents`, `Commands`, and `Skills`; Pi reports `Extensions`, `Prompts`, and `Skills`; and Codex reports `Skills` and `Hooks` (see `context/sce/doctor-human-text-contract.md`). Its non-launching post-commit Agent Trace auto-sync fact reports enabled/current, explicit disabled, not-ready, and not-applicable states using canonical managed-block currency and resolved configuration; existing hook remediation and readiness semantics remain unchanged. The local DB service now provides `LocalDb` as a thin `TursoDb` alias in `cli/src/services/local_db/mod.rs`; `LocalDbSpec` resolves the canonical local DB path from the shared default-path catalog and currently declares zero migrations. Shared Turso infrastructure lives in `cli/src/services/db/mod.rs`, where `DbSpec` and generic `TursoDb` support local or remote sync-mode opens, parent-directory creation, connection setup, synchronous query helpers, embedded migration execution, and shared DB lifecycle helpers. Auth DB persistence uses encrypted `AuthDb = EncryptedTursoDb` and token storage persists credentials through the `auth_credentials` table. Agent Trace persistence uses the sole `RepositoryAgentTraceDb = TursoDb` adapter at `/sce/repos//agent-trace.db`, with a one-file repository schema for `repository_metadata`, `diff_traces`, `post_commit_patch_intersections`, `agent_traces`, `messages`, `parts`, indexes/triggers, and no `checkout_id` columns on trace rows. The checkout-scoped `AgentTraceDb = TursoDb` adapter, its `agent_trace_db_path()`/`agent_trace_db_path_for_checkout()` helpers, and the 15-file `cli/migrations/agent-trace/` chain were removed by the `retire-legacy-agent-trace-db` plan; active hook runtime writes nullable event-local diff-trace attribution without a `session_models` API/table dependency. The hooks command surface now also supports concrete runtime subcommand routing (`pre-commit`, `commit-msg`, `post-commit`, `post-rewrite`, `diff-trace`, `conversation-trace`, and `claude-model-state`) with deterministic argument/STDIN validation; `session-model` is no longer supported. Claude settings install the model-state command for both `SessionStart` and `PostModelSwitch`; compatibility smoke against Claude Code 2.1.251 and immediately older 2.1.250 showed that the older client safely ignores the unknown event, so installation is unconditional without a raised minimum or capability gate. Current runtime behavior keeps commit-msg attribution enabled by default unless explicitly opted out: the attribution gate enables canonical trailer insertion in `commit-msg` only when the staged-diff AI-overlap preflight confirms AI/editor evidence (no trailer is appended when the preflight finds no overlap or encounters any error); `pre-commit`/`post-rewrite` remain deterministic no-ops, `post-commit` requires validated `--remote-url`, threads that URL into the Agent Trace flow, prints it to stderr, remains the active bounded recent-diff-trace intersection path, and after successful Agent Trace persistence optionally launches the detached sync-owned `sync --format json` child when config-file-only `agent_trace.auto_sync` is true; `diff-trace` is the active intake path for parsed STDIN `{ sessionID, diff, time, model_id?, tool_name, tool_version }` payload persistence with optional `model_id`, required non-empty `tool_name`, required nullable/non-empty `tool_version`, `direct > exact transcript > exact session/agent state > bridge-derived chain state > NULL` Claude `model_id` plus direct `tool_version` values (exact local state only; no generic session-model fallback or cache), required `u64` millisecond `time`, same-tool-idempotent stored `session_id` prefixing (`oc_` for OpenCode, `cc_` for Claude, `pi_` for Pi), non-lossy AgentTraceDb `time_ms` conversion, and no parsed-payload artifact fallback. This behavior is documented in `context/sce/agent-trace-hooks-command-routing.md`. The removed `sce hooks claude-capture` raw capture route is documented in `context/sce/claude-raw-hook-capture.md` as a removed feature. - The setup service now also exposes deterministic required-hook embedded asset accessors (`iter_required_hook_assets`, `get_required_hook_asset`) backed by canonical templates in `cli/assets/hooks/` for `pre-commit`, `commit-msg`, and `post-commit`; this behavior is documented in `context/sce/setup-githooks-hook-asset-packaging.md`. + The setup service now also exposes deterministic required-hook embedded asset accessor (`iter_required_hook_assets`) backed by canonical templates in `cli/assets/hooks/` for `pre-commit`, `commit-msg`, and `post-commit`; this behavior is documented in `context/sce/setup-githooks-hook-asset-packaging.md`. The setup service now also includes required-hook install orchestration (`install_required_git_hooks`) that resolves repository root and effective hooks path from git truth, computes the bytes to stage by merging the canonical hook template with any existing hook (preserving a foreign hook's content as an exact prefix with the SCE managed block appended, or bringing an SCE-owned block current in place) rather than writing canonical bytes verbatim, enforces deterministic per-hook outcomes (`Installed`/`Updated`/`Skipped`) against that merged content, surfaces a deterministic advisory when an appended block would be unreachable, and uses a unified atomic-swap policy that renames staged content directly over existing hooks without unlinking them first, with deterministic recovery guidance on swap failures; this behavior is documented in `context/sce/setup-githooks-install-flow.md`. The setup command parser/dispatch now also supports composable setup+hooks runs (`sce setup --opencode|--claude|--pi|--codex|--all --hooks`) plus hooks-only mode (`sce setup --hooks` with optional `--repo `), enforces deterministic compatibility validation (`--repo` requires `--hooks`; target flags remain mutually exclusive), and emits deterministic setup/hook outcome messaging (`installed`/`updated`/`skipped`); this behavior is documented in `context/sce/setup-githooks-cli-ux.md`. diff --git a/context/patterns.md b/context/patterns.md index 8ab42f2eb..ccdcec3ca 100644 --- a/context/patterns.md +++ b/context/patterns.md @@ -2,7 +2,7 @@ ## Awaited command execution -- Keep borrowed context and writers alive through directly awaited static dispatch. Scope blocking only around documented DB construction/destruction lifetimes; isolate credential operations with awaited owned `spawn_blocking` closures. See [application execution runtime](architecture.md#application-execution-runtime). +- Keep borrowed context and writers alive through directly awaited static dispatch. Turso persistence and credential storage are native async operations on the application runtime; do not add executor bridges or DB `spawn_blocking` wrappers. Keep pure parsing, rendering and unrelated service logic synchronous. Use `spawn_blocking` only to wait on a genuinely blocking external API (OS credential store, advisory file-lock polling, Git ref-mutation and marker/state file I/O, guarded-shell supervision), and never move awaited work or a Turso future into the worker. Where cancellation must not release a protected resource (worktree lock, boundary lease, marker), the started worker owns that resource until it finishes; otherwise return the owned result or guard to the async caller. Transactions take `&mut` receivers so no other operation can enter them. Retained workers are enumerated in [application execution runtime](architecture.md#application-execution-runtime). See [application execution runtime](architecture.md#application-execution-runtime). ## Config generation tooling @@ -118,7 +118,7 @@ - Keep production CLI path ownership centralized in `cli/src/services/default_paths.rs`; new non-test path literals or path-shape definitions should be added there as named accessors/constants instead of becoming new path owners in other modules. - Keep SCE-owned web URI construction centralized in `services::agent_trace`; production Rust code should use its helpers instead of repeating `https://sce.crocoder.dev`, host-only variants, or derived path prefixes. -- Prefer localized `#[allow(dead_code)]` on intentionally shared path/setup helper items over file-level dead-code suppression so lint scope stays narrow while keeping catalog seams available to tests and future consumers. +- Dead-code exceptions are declaration-level `#[allow(dead_code, reason = "...")]` only, naming the compatibility, deferral or cross-target reason; never file-wide or module-wide. Unused helpers with no such reason are deleted, not suppressed. - For early CLI foundation tasks, keep the real top-level command catalog/help metadata centralized in one canonical seam (`cli/src/cli_schema.rs` in the current architecture) and let custom top-level help renderers consume that seam instead of maintaining a second parallel command list. - Keep top-level help intentionally curated: command visibility on `sce`, `sce help`, and `sce --help` may differ from parser availability when a command should remain directly invocable but temporarily hidden from operator-facing help. - Keep wrapper-only help rows or banner rendering logic outside the clap catalog, but do not duplicate the real command visibility/purpose metadata in those renderers. @@ -152,10 +152,10 @@ - For setup command messaging, emit deterministic completion output that includes selected target(s) and per-target install counts. - Keep module seams for future domains present and compile-safe even when behavior is deferred. - Keep dependency additions explicit and minimal in `cli/Cargo.toml`, and anchor dependency intent in domain-owned service types/tests rather than a separate compile-time dependency snapshot module. -- Route local Turso access through service adapters so command handlers do not expose low-level `turso` API details. New Turso-backed services should build on `cli/src/services/db/mod.rs` (`DbSpec` + `TursoDb`, or `EncryptedTursoDb` when at-rest encryption is required) for runtime, connection, per-database `__sce_migrations` tracking, and migration infrastructure, then expose domain-specific methods from their own service modules. +- Route local Turso access through service adapters so command handlers do not expose low-level `turso` API details. New Turso-backed services should build on `cli/src/services/db/mod.rs` (`DbSpec` + `TursoDb`, or `EncryptedTursoDb` when at-rest encryption is required) for runtime, connection, per-database `__sce_migrations` tracking, and migration infrastructure, then expose domain-specific async methods from their own service modules. - For current local DB flows, route initialization through the dedicated adapter (`cli/src/services/local_db/mod.rs`) and invoke it from approved orchestration surfaces such as setup or doctor rather than exposing a partial user command before its contract is approved. - For Turso-backed services with setup/doctor ownership, add service-owned lifecycle providers that reuse shared DB path-health and parent-bootstrap helpers, then register them through `lifecycle_providers()` instead of adding command-local database checks. -- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. Use async `run_with_retry` for async operations and sync `run_with_retry_sync` for pure blocking contexts where a Tokio sleep/timeout future cannot be awaited. For Turso database constructors/openers, wrap only the local open/connect operation in retry; keep migration execution outside retry because schema changes must not be replayed. Use `TursoDb::new()` for setup/lifecycle-owned schema initialization and `TursoDb::open_without_migrations()` only for hot runtime paths that verify required schema separately before query/write work. For `TursoDb` and `EncryptedTursoDb` operation retry, convert params to owned cloneable Turso params before retrying `execute()`/`query()`, retry `query_map()` query plus row-fetch failures, and keep caller row-mapping outside retry. Retry policies for both connection-open and query operations can now be configured per database via `policies.database_retry` in `sce/config.json`, parsed and resolved in `cli/src/services/config/mod.rs`, with fallback to hardcoded defaults when the config key is absent. Opt an Agent Trace write into the write-contention retry only when its whole retry unit is replay-safe: a complete `BEGIN IMMEDIATE` → `COMMIT` transaction primitive, or a single statement made idempotent by its SQL (`ON CONFLICT DO NOTHING`, a guarded `UPDATE`) routed through `TursoDb::execute_idempotent_write`; append-only or last-writer statements stay on generic `execute()`. +- For transient local IO/database hotspots, apply bounded resilience wrappers with explicit retry count, timeout, and capped backoff (`cli/src/services/resilience.rs`) and surface terminal failures with deterministic `Try:` remediation guidance. Use async `run_with_retry` (cancels the attempt when the timeout elapses) for Turso and other async operations, and `run_with_retry_elapsed` (measures elapsed time without cancelling the attempt) for operations that must lend an exclusive borrow; there is no synchronous retry variant. For Turso database constructors/openers, wrap only the local open/connect operation in retry; keep migration execution outside retry because schema changes must not be replayed. Use `TursoDb::new()` for setup/lifecycle-owned schema initialization and `TursoDb::open_without_migrations()` only for hot runtime paths that verify required schema separately before query/write work. For `TursoDb` and `EncryptedTursoDb` operation retry, convert params to owned cloneable Turso params before retrying `execute()`/`query()`, retry `query_map()` query plus row-fetch failures, and keep caller row-mapping outside retry. Retry policies for both connection-open and query operations can now be configured per database via `policies.database_retry` in `sce/config.json`, parsed and resolved in `cli/src/services/config/mod.rs`, with fallback to hardcoded defaults when the config key is absent. Opt an Agent Trace write into the write-contention retry only when its whole retry unit is replay-safe: a complete `BEGIN IMMEDIATE` → `COMMIT` transaction primitive, or a single statement made idempotent by its SQL (`ON CONFLICT DO NOTHING`, a guarded `UPDATE`) routed through `TursoDb::execute_idempotent_write`; append-only or last-writer statements stay on generic `execute()`. - For SCE operator-health commands, prefer deterministic local diagnostics over implicit pass/fail behavior: report the inspected environment scope, stable problem categories, severity/fixability classes, actionable remediation text, and any path/location facts needed to repair the issue; when repair mode exists, keep outcome vocabulary deterministic and idempotent (`cli/src/services/doctor/mod.rs`, with focused diagnosis/render/fix helpers under `cli/src/services/doctor/`). - For service-owned operator health, keep command modules as thin aggregators over `ServiceLifecycle` providers once a lifecycle slice is wired: providers own diagnosis/fix problem production through narrow capability accessors, while command-specific report builders preserve existing output facts and rendering contracts. - Keep static lifecycle provider-list construction centralized in the lifecycle service layer so doctor/setup choose provider inclusion without maintaining parallel concrete provider lists. @@ -194,7 +194,7 @@ - Pure unit tests should test in-memory logic, parsing, validation, and data transformations without external dependencies; prefer mocking/faking external dependencies over creating real filesystem or database state when a fake is available and sufficient. - In-memory database tests (e.g., `LocalDatabaseTarget::InMemory`) are acceptable for unit tests since they don't touch the filesystem. -- A filesystem-touching `#[cfg(test)] mod tests` inline in the module under test is an established, Nix-sandbox-safe pattern in this codebase when every path is a unique, process-local path under `TMPDIR` (itself a writable per-build directory inside the Nix sandbox). Prefer an RAII `tempfile::TempDir` fixture, which cleans itself up even when a test panics — as `cli/src/services/mutation_trace/store.rs`'s `RepositoryAgentTraceDb`-backed tests, `runtime/tests.rs`, `runtime/protected_worktree.rs`, `runtime/scope_runtime.rs`, `runtime/mutation_attribution.rs`'s real-store-and-`GitSnapshotService` seam test, and `cli/src/services/hooks/mod.rs`'s `mutation_attribution_e2e` real Git + repository-DB post-commit regressions do. Manually composed unique `std::env::temp_dir()` paths are the older form of the same pattern and remain in `cli/src/services/checkout/mod.rs`'s checkout-identity-lock tests and in `runtime/coordinator.rs`, `runtime/worktree_lock.rs`, and `runtime/external_taint.rs`; they leak their directory on a panicking test, so new tests should use `TempDir`. This is not integration-test-only territory in this repository. +- A filesystem-touching `#[cfg(test)] mod tests` inline in the module under test is an established, Nix-sandbox-safe pattern in this codebase when every path is a unique, process-local path under `TMPDIR` (itself a writable per-build directory inside the Nix sandbox). Prefer an RAII `tempfile::TempDir` fixture, which cleans itself up even when a test panics — as `cli/src/services/mutation_trace/store.rs`'s `RepositoryAgentTraceDb`-backed tests, `runtime/protected_worktree.rs`, `runtime/scope_runtime.rs`, `runtime/mutation_attribution.rs`'s real-store-and-`GitSnapshotService` seam test, and `cli/src/services/hooks/mod.rs`'s `mutation_attribution_e2e` real Git + repository-DB post-commit regressions do. Manually composed unique `std::env::temp_dir()` paths are the older form of the same pattern and remain in `cli/src/services/checkout/mod.rs`'s checkout-identity-lock tests and in `runtime/coordinator.rs`, `runtime/worktree_lock.rs`, and `runtime/external_taint.rs`; they leak their directory on a panicking test, so new tests should use `TempDir`. This is not integration-test-only territory in this repository. - Use integration tests instead of this inline pattern where the behavior genuinely cannot be made deterministic and isolated inside the Nix sandbox this way. - Do not depend on a shared or ambient path (`$HOME`, a fixed `/tmp` file name, the repository's own working tree) from a unit test; each test must construct its own unique, self-cleaning path. - When a unit test needs behavior that cannot be made Nix-sandbox-safe this way (for example, network access, or state shared across the whole test binary), delete it from the unit-test suite and reintroduce that coverage later as an integration test instead of keeping ignored tests in-tree. diff --git a/context/plans/cli-async-persistence-cleanup-pr3.md b/context/plans/cli-async-persistence-cleanup-pr3.md new file mode 100644 index 000000000..9bcb7c97d --- /dev/null +++ b/context/plans/cli-async-persistence-cleanup-pr3.md @@ -0,0 +1,303 @@ +# Plan: cli-async-persistence-cleanup-pr3 + +## Change summary + +PR3 is the dead-code elimination and post-async cleanup that follows PR2 (PR #301, branch `remove-tokio-db-runtimes`, still open at head `f8e9c748`, itself based on `single-tokio-runtime`). PR2 removed the database-owned Tokio runtimes and migrated persistence to native async; its Deferred cleanup ledger in `context/plans/cli-async-turso-persistence-pr2.md` hands PR3 the redundant `async` wrappers, `run_with_retry_sync`, the redundant `drop(storage)`, stale `#[allow(dead_code)]` annotations, orphaned test support, the `join_three_to_completion` re-evaluation, the large-future/`recursion_limit = "256"` measurement and the temporary-index limitation. + +An external investigation reported 114 compiler-reported unused items once dead-code suppressions are removed. That report is a seed investigation, not an authoritative inventory and not a deletion quota: its symbol-by-symbol listing is not committed to the repository, and its figures (114 items, 35 files with suppressions, 37 occurrences, 117 later `rg` matches) may use different counting conventions. **T01's reproducible, source-grounded compiler inventory, taken on the exact PR2 baseline, is the authoritative baseline for PR3.** T01 keeps the seed's classification leads (Category A retained `diff_traces` compatibility surface; Category B unwired mutation-cursor ref reconciliation and its dependencies; Category C superseded Agent Trace, repository identity, setup, sync and retry helpers; Category D unused database, path, doctor, hook, capability, configuration and mutation-trace helpers; and fields written but never read) and classifies every reproduced item. T02 deletes the verified Category C/D code and its suppressions, T03 applies the retain-and-defer disposition to the reconciliation cluster, T04 simplifies the remaining async layers, T05 audits blocking sites, future size and temporary-index cleanup, and T06 re-runs the unsuppressed audit and prepares the validation and PR4 handoff. Full validation is a separate `/validate` gate after T06. + +**Stacking.** PR3 is a stacked PR. The working branch `dead-code-blues` currently sits exactly at the PR2 head (`f8e9c748`), so every PR3 diff and review is taken against `remove-tokio-db-runtimes` (`git diff remove-tokio-db-runtimes...HEAD`), never against `main`, so the PR2 migration is not re-reviewed. If PR2 merges first, rebase onto its integration branch and keep the same commit range. + +**Reconciliation default: retain and defer.** Evidence in the current code and context: `pin_tree` is called only from `coordinator.rs`, nothing in production deletes pins, and `ref_reconciliation` has no caller (`mod ref_reconciliation;` is private; `context/cli/mutation-trace-ref-reconciliation.md` says invocation timing belongs to a later "harness-wiring PR"). The implementation and its integration path are already planned in `context/plans/mutation-cursor-ref-reconciliation.md`, and a separate `ref-rec` branch carries related reconciliation work. So pins left by crashed or failed `coordinate()` paths accumulate until a manual `git update-ref -d`; this documented consequence is accepted. PR3 therefore retains the reconciliation implementation and the helper APIs it needs, narrows suppressions to justified declarations, records that it remains unwired, and does not wire it. No new approval and no new ADR are required for this default; durable context and a reference to the existing reconciliation plan suffice. A new user or architecture decision is needed only if T01 finds material new evidence that invalidates retain-and-defer. + +## Acceptance criteria + +How this plan is proven complete. The final `/validate` gate (run after T01–T06 are complete and synchronized) executes these checks and the Full validation list below; T01–T06 prepare evidence and run only focused checks for their own changes, and none claims full repository validation. + +- [x] AC1: T01's reproduced, symbol-level compiler inventory is the authoritative dead-code baseline: it identifies the exact PR2 baseline commit, was produced from unsuppressed compiler diagnostics in an isolated worktree covering the binary and test targets and accounting for `cfg` and supported platforms, records the actual number of unique unused declarations, counts suppression attributes separately from `rg` matches, and explains differences from the seed report's 114 items where evidence exists. Every inventory row has file, category (A/B/C/D or field), usage evidence and disposition. If the original full report is located it is retained as supporting audit evidence; otherwise the plan states that the reproduced inventory supersedes it. No rows were invented to reach 114. + - Validate: inspect the inventory recorded in T01's completion record (or the file it names under `context/`); confirm the baseline commit, per-target diagnostic counts, unique-declaration count, suppression-attribute count versus `rg` match count versus files-with-suppressions count, the explanation of differences from the seed figures, and that each row has symbol, file, production/test/cross-platform/generated usage, decision/context requirement, category, disposition and risk. +- [x] AC2: Verified Category C/D dead code, written-but-never-read fields and obsolete tests are deleted together with their dead callers, constructors, imports and suppressions, and only rows classified safe-to-remove were deleted; no deletion was made on Linux-only evidence for platform-gated code. + - Validate: `git diff --stat remove-tokio-db-runtimes...HEAD -- cli/src`; each deleted symbol has an inventory row with a `delete` disposition; platform-gated symbols (macOS/Windows variants in `default_paths.rs`, `GuardError::UnsupportedPlatform`) were checked against `cfg` and supported targets rather than the Linux diagnostic. +- [x] AC3: The `diff_traces` compatibility contract from `context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md` is intact. These surfaces are intentionally unused during the staged compatibility window, not accidentally dead, and need no production callers: `ingest_diff_traces` and `read_diff_traces_after` still exist; `AgentTraceDiffTraceExportRow` and its SQL (`SELECT_DIFF_TRACES_AFTER_SQL`) and mapping (`diff_trace_export_row_from_turso`) dependencies remain; `IngestionStream::DiffTraces` keeps its wire representation; `/state.cursors.diffTraces` remains required; `streams.diffTraces` remains in `sce sync --format json` as a zero-upload entry (`uploaded: 0`, `batches: 0`, `initialCursor == finalCursor == state.cursors.diffTraces`); the remote sync path uploads only `messages`, `parts` and `agent_traces` and does not read local `diff_traces` rows; local `diff_traces` capture and consumers are unchanged. Any dead-code exceptions for these declarations are declaration-level and cite the decision. + - Validate: `nix shell nixpkgs#ripgrep -c rg -n "ingest_diff_traces|read_diff_traces_after|AgentTraceDiffTraceExportRow|SELECT_DIFF_TRACES_AFTER_SQL|diff_trace_export_row_from_turso|DiffTraces|diffTraces" cli/src` finds each definition and the wire-name/cursor handling, and `git diff remove-tokio-db-runtimes...HEAD -- cli/src` shows no signature, serialization or `/state` DTO change for them; the existing sync tests that assert the three-stream upload, the required `cursors.diffTraces` decode and the zero-upload `streams.diffTraces` JSON entry pass (names confirmed via `-- --list`; zero matches do not count); local hook-ingress, patch-reconstruction, post-commit intersection and attribution consumers of `diff_traces` show no diff. A symbol-existence grep alone is not sufficient. +- [x] AC4: Ref reconciliation is retained and deferred by default: the implementation and required helper APIs are preserved, suppressions are narrowed to justified declarations, it remains unwired, and PR3 introduces no new production behavior for it (no caller, hook, command or doctor wiring). Durable context records that it is unwired, that orphaned snapshot pins can accumulate, and references `context/plans/mutation-cursor-ref-reconciliation.md` as the intended integration path. A different disposition exists only if T01 recorded material new evidence and a user or architecture decision followed. + - Validate: inspect the T03 record; `rg -n "reconcile_worktree" cli/src` shows no non-test caller outside `ref_reconciliation.rs`; `context/cli/mutation-trace-ref-reconciliation.md` and `context/cli/mutation-trace-snapshot-service.md` state the retained-and-unwired status and the orphaned-pin consequence; no duplicate ADR was added to restate an already accepted decision. +- [x] AC5: No blanket or file-wide `#[allow(dead_code)]` remains that hides unrelated declarations; every remaining exception is declaration-level with a stated compatibility, deferral or cross-target reason; no `#[allow(unused)]`, unrelated warning suppression or weakened lint setting was added. + - Validate: `rg -n "allow\(dead_code\)|allow\(unused" cli/src` (via nix) lists only the justified sites recorded in T06; no `#![allow(dead_code)]` at file level; `git diff remove-tokio-db-runtimes...HEAD -- cli/Cargo.toml cli/src/main.rs flake.nix` shows no lint weakening. +- [x] AC6: Two verification modes are kept distinct. In diagnostic mode (isolated worktree, suppressions temporarily removed, binary and test targets compiled), a nonzero exit caused solely by expected dead-code diagnostics is an expected audit outcome, and any other compiler failure is investigated rather than counted as dead code. In normal verification mode (retained narrowly scoped suppressions restored), the binary and test targets compile, clippy passes with warnings denied, and the normal validation checks pass. The final diagnostic-mode audit reports only known, documented retained declarations. + - Validate: the T06 diagnostic-mode output is recorded with its build-target and platform context and every diagnostic maps to a documented exception; no claim of cross-platform safety is derived from the Linux build alone; normal-mode results come from `/validate`. +- [x] AC7: Retry, cancellation, transaction-ownership, adapter/worktree-lease and terminal-sync/credential-persistence behavior is unchanged; `run_with_retry` (cancelling timeout) and `run_with_retry_elapsed` (elapsed measurement, no cancellation) remain distinct; `run_with_retry_sync` is gone only if it had no active consumer. + - Validate: `/validate` runs the complete Rust suite, including `db::transaction_cancellation_tests`, `agent_trace_db::transaction_tests`, the sync `terminal_stream` tests and the Quint MBT suite in `cli/src/services/mutation_trace/mbt`; `git diff remove-tokio-db-runtimes...HEAD -- cli/src/services/resilience.rs` shows no policy change. +- [x] AC8: `join_three_to_completion`/`JoinSlot` is either simplified without changing cooperative stopping, in-flight batch completion, credential-persistence protection, first-observed-error selection, non-`Send` progress reporting or concurrent execution, or is retained with a recorded reason; no `try_join!` cancellation of siblings, no extra spawned tasks. + - Validate: inspect the `cli/src/services/sync/sync.rs` diff and the T04 record; `terminal_stream` tests pass in `/validate`. +- [x] AC9: No dynamic dispatch, `BoxFuture`/`dyn Future`, or nested Tokio runtime is introduced. T05 measured large-future size and `recursion_limit = "256"` pressure (a reduction is not assumed possible; "no reduction available" is an acceptable recorded outcome), and the temporary-index (`TempIndexGuard`) finding is documented; the recursion-limit override is removed only if the pinned compiler and the full checks pass without it. + - Validate: `git diff remove-tokio-db-runtimes...HEAD -- cli/src` searched for `dyn `, `BoxFuture`, `Runtime::new`, `block_on` finds nothing new; the T05 record states the measured future sizes, the measurement method and the decisions on `#![recursion_limit]` and on temporary-index cleanup (default: no automatic sweeper). +- [x] AC10: No broad test restoration occurred (no deleted ordinary test suite was restored for coverage parity) and the Rust diff is net smaller. + - Validate: `git diff --shortstat remove-tokio-db-runtimes...HEAD -- cli/src` shows deletions exceeding insertions; no previously deleted test module or fixture is re-added (`git diff --diff-filter=A --name-only remove-tokio-db-runtimes...HEAD -- cli`). +- [x] AC11: Durable context and docs match the code, including any docs that claim guarded-shell descriptor creation is atomic on every supported platform; the macOS `pipe()`/`fcntl()` race is described as an unresolved limitation, not fixed. + - Validate: inspect `context/cli/mutation-trace-external-mutation-guard.md` and `context/cli/async-persistence-guarantees.md` against `external_mutation_guard.rs` (`pipe2` on Linux, `pipe`+`fcntl` fallback elsewhere); `context/cli/default-path-catalog.md`, `capability-traits.md`, `repository-identity.md`, `agent-trace-sync-command.md` and the context map no longer describe deleted symbols. +- [x] AC12: Validation ownership is respected: T06 records which checks must run during final validation but does not claim they have passed; the plan is marked fully validated only after `/validate` records actual results, including failures. + - Validate: `/validate`'s report lists each Full validation command with its real outcome. + +### Full validation + +Owned by the separate final `/validate` gate (repository workflow `sce-validate`), run only after T01–T06 are complete and every task is `synced`. Record actual results, including failures. Do not mark PR3 fully validated because T06 is complete. + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` (complete Rust suite) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` +- `nix flake check` +- `nix build .#ci-checks` +- `nix run .#pkl-check-generated` (only if a Pkl or `config/lib` input was touched; generated-input and platform checks relevant to the diff) +- `git diff --check` + +### Verification modes + +- **Diagnostic mode (T01, T06, optionally T05):** in an isolated `git worktree`, temporarily remove dead-code suppressions, compile the non-test binary and the test targets (`cargo test --no-run`), capture warnings/errors, identify unused declarations, and record diagnostics with their build-target and platform context. A nonzero exit caused solely by expected dead-code diagnostics (the project compiles with `-D warnings`) is an expected audit outcome, not a verification failure. Other compiler failures are investigated, not counted as dead code. Discard the worktree safely (`git worktree remove`), never with destructive restore commands against the main worktree. +- **Normal verification mode (T02–T05 focused checks and `/validate`):** with the intentionally retained, narrowly scoped suppressions restored, the binary and test targets compile, clippy passes with warnings denied and the normal validation checks pass. Unrelated warnings are never suppressed and compiler lint settings are never weakened. + +### Context sync + +- `context/cli/async-persistence-guarantees.md` (PR3 staging, known limits: future size, temporary index, macOS descriptor race wording) +- `context/architecture.md`, `context/patterns.md` (only where deleted seams are described) +- `context/cli/mutation-trace-ref-reconciliation.md`, `mutation-trace-snapshot-service.md`, `mutation-trace-store.md`, `mutation-trace-runtime-coordinator.md` (retained-and-unwired status, orphaned-pin consequence, cluster ownership and link to `context/plans/mutation-cursor-ref-reconciliation.md`) +- `context/cli/default-path-catalog.md`, `capability-traits.md`, `repository-identity.md`, `agent-trace-sync-command.md`, `sync-command.md`, `cli-command-surface.md`, `service-lifecycle.md` (only for deleted symbols they name) +- `context/context-map.md`, `context/overview.md`, `context/glossary.md` when entries change +- No new ADR for the default retain-and-defer reconciliation disposition (already covered by the accepted plan and context); a new ADR only if T01 finds material new evidence that forces a different architectural decision + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src` dead-code and suppression cleanup; deleting verified-dead production code and the tests/fixtures that only exercised it; `resilience.rs`, `db/mod.rs`, `sync/sync.rs`, service forwarding and hook/lifecycle helper simplification; measurement/audit of futures, `spawn_blocking` sites and temporary Git indexes; durable context updates for the above. +- **Out of scope:** the macOS guarded-shell `pipe()`/`fcntl()` race fix; implementing or wiring ref reconciliation; new mutation-cursor semantics; database schema migrations; retry policy changes; OpenTelemetry (PR4); broad test-suite restoration; new generic execution or persistence abstractions; removal of the `diff_traces` compatibility surface (separate planned phase); full repository validation (owned by the final `/validate` gate). +- **Constraints:** the baseline is PR2 head `f8e9c748` on `remove-tokio-db-runtimes`; all diffs are measured against that branch. All tools run through Nix per `AGENTS.md` (`nix develop -c ./scripts/run-cli-cargo.sh ...`, `nix shell nixpkgs#ripgrep -c rg`); no host `rg`, `cargo`, `python3`. Reproduction and the unsuppressed build run in a disposable `git worktree`/copy, never with destructive restore commands (`git checkout .`, `git reset --hard`) against the main worktree. Repository conventions apply: no code comments except documented-exception reasons required by this plan's suppression policy (user preference is comment-free code, so an exception's reason lives in the attribute's `reason = "..."` string or the owning context doc rather than prose comments — T01 confirms the repo's lint level accepts `reason`), exact-pinned dependency versions unchanged, minimal focused tests only. +- **Constraints (preservation):** keep static dispatch and associated-type capability boundaries; do not add dynamic dispatch; preserve fail-closed recovery in mutation-scope adapters; do not delete platform-gated code on Linux-only evidence; written-but-never-read state fields are removed only after checking serialization, derives, Debug, Drop, tests and protocol transitions. +- **Non-goal:** preserving unused APIs for hypothetical future flexibility, or converting dead production code to `#[cfg(test)]` merely to keep obsolete tests alive. +- **Non-goal:** meeting a deletion count or reproducing the seed report's 114; the reproduced inventory is the baseline and no rows are invented to reach any number. +- **Non-goal:** restoring deleted ordinary test suites for coverage parity. +- **Non-goal:** deleting the reconciliation cluster to shrink the dead-code count, or wiring it into production in PR3. +- **Constraint (compatibility):** the `diff_traces` compatibility contract in `context/decisions/2026-10-01-retire-diff-traces-remote-sync-compatibility-gate.md` is preserved; intentionally unused compatibility declarations are retained with narrow exceptions rather than deleted. + +## Assumptions + +- The seed report's figures (114 items, 35 files, 37 occurrences) and the later `rg` count of 117 `allow(dead_code)` matches (35 files on the PR2 head) may use different counting conventions (attributes versus annotated declarations versus compiler-reported items). The original symbol-by-symbol listing is not committed and cannot be assumed reproducible; T01 establishes the authoritative counts and does not invent rows. +- Since PR2 is open and `dead-code-blues` equals the PR2 head, no rebase is needed now; the PR base for review is `remove-tokio-db-runtimes`. +- `reason = "..."` on `#[allow(dead_code, reason = ...)]` is acceptable under the pinned toolchain; if not, the exception is documented in the owning context file and named in T06's audit table instead of in a code comment (rests on the user's recorded no-comments-in-code preference). +- Per the user's prefs, final verification is the `/validate` gate using the commands in Full validation; T02–T06 use only narrow focused checks (`cargo test` filtered by names confirmed via `-- --list`; zero-match filters never count as passing). +- The reconciliation default (retain and defer) follows the existing context statement that invocation timing is a later harness-wiring decision, the existing plan `context/plans/mutation-cursor-ref-reconciliation.md` and the "false retention costs disk, false deletion destroys evidence" bias recorded in `mutation-trace-ref-reconciliation.md`. + +## Task stack + +- [x] T01: `Reproduce the authoritative dead-code baseline and classify it` (status:done) + - Task ID: T01 + - Scope: In — (1) identify the exact PR2 baseline commit (`git rev-parse remove-tokio-db-runtimes`; expected `f8e9c748`) and confirm `git diff remove-tokio-db-runtimes...HEAD -- cli/src` is empty; (2) in a disposable `git worktree`, in diagnostic mode, temporarily remove every dead-code suppression and reproduce the unsuppressed compiler diagnostics, repeating until the diagnostic set is stable; (3) inspect the non-test binary target and the test targets (`cargo test --no-run`) separately and record per-target diagnostics; (4) account for conditional compilation and supported platforms (`cfg(target_os)`, Windows/macOS variants, features) rather than treating the Linux build as complete; (5) produce a symbol-level inventory with file, category, usage evidence (production binary, tests, other supported target/platform, generated code or build tooling, context/ADR requiring retention) and disposition (delete, retain with reason, decision-needed) plus risk; (6) record the actual number of unique unused declarations; (7) count suppression attributes, `rg` matches and files-with-suppressions as separate figures; (8) explain differences from the seed report's 114/35/37/117 figures where evidence is available; if the original full report can be located retain it as supporting audit evidence, otherwise state explicitly that the reproduced inventory supersedes it, and do not invent rows to reach 114. The seed leads guide classification: Category A intentionally retained `diff_traces` compatibility surface (protected list per the accepted decision); Category B unwired mutation-cursor ref reconciliation and its dependencies; Category C superseded Agent Trace, repository identity, setup, sync and retry helpers; Category D unused database, path, doctor, hook, capability, configuration and mutation-trace helpers; and fields written but never read (with serialization/derive/Debug/Drop/protocol checks and a separate list of suspected logic defects). Also record the full reconciliation dependency graph (used only by `ref_reconciliation` and its tests versus also by production, and whether any material new evidence invalidates retain-and-defer), baseline measurements (test list names via `-- --list`, LOC of `cli/src`, per-category removable LOC estimate, future/`recursion_limit` baseline method, current `spawn_blocking` sites), and `run_with_retry_sync` consumers. The inventory is recorded as an appendix of this plan's T01 completion record (not under `context/tmp`). Out — any production or test modification in the real worktree, any deletion, wiring, or lint change, and counting expected dead-code diagnostics as failures. + - Dependencies: none + - Done when: the baseline commit, per-target diagnostics, unique-declaration count, separate suppression/match/file counts and the explanation of differences from the seed figures are recorded; every reproduced item has a row with category and disposition; counts of delete-now / Category A / retained-for-tests-or-cross-platform / decision-needed are recorded; the reconciliation graph, the `run_with_retry_sync` consumer check and the D-category platform variants are resolved; the disposable worktree is removed without touching unrelated changes; the main worktree is unchanged. + - Verify: in the disposable worktree, `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml` and `... test --manifest-path cli/Cargo.toml --no-run` with suppressions removed (nonzero exit from dead-code diagnostics alone is expected; any other failure is investigated); `git status --short` in the main worktree is unchanged; `nix shell nixpkgs#ripgrep -c rg -c "allow\(dead_code\)" cli/src` figures recorded. + - Completed: 2026-10-08 + - Files changed: `context/plans/cli-async-persistence-cleanup-pr3.md` (this record and the T01 inventory appendix); no `cli/` changes + - Result: Authoritative baseline reproduced on PR2 head `f8e9c748` (`cli/src` diff vs `remove-tokio-db-runtimes` empty; branch HEAD `2f073f98` adds only context plan commits). 147 unique unused declarations on the binary target (114 compiler warnings, several grouping multiple symbols), 127 symbol entries on the test target (100 warnings). Classified (corrected after PR review): 5 Category A, 30 Category B, 4 cross-platform retains, 7 fields retained pending defect review, 101 delete-now. See "T01 inventory appendix". + - Verify: unsuppressed `build` and `test --no-run` ran in a disposable worktree with `RUSTFLAGS=--cap-lints warn` (so the `-D warnings` lint level reports every diagnostic instead of aborting); both exited 0 with the diagnostic counts above; main worktree `git status --short` empty before and after; disposable worktree removed; `rg` suppression figures recorded in the appendix; `#[allow(dead_code, reason = "...")]` compiles under `-D warnings` on rustc 1.95.0 (open question resolved). + - Post-review correction (PR #303 review): two inventory rows were corrected without a new compiler run or any claimed test execution. (1) `run_external_mutation_guard` stays a delete of the production wrapper only; its two guarded-shell regression tests are preserved by adaptation, not deleted. (2) `WorktreeLock::acquire` moved from Category B to D (delete in T02): callers verified from source are `acquire_async` and `acquire_inner_async` (git_snapshot, mutation_attribution, protected_worktree, ref_reconciliation) while `acquire` has none; `acquire_inner` stays as the blocking implementation of `acquire_inner_async`. Totals: Category B 31 to 30, delete-now 100 to 101; 101+5+30+4+7 = 147 reconciles to the unique-declaration count. Compiler diagnostic counts (114 / 147 / 100 / 127) are unchanged. + - Context impact: none to root context; the appendix is plan-local evidence consumed by T02-T06. + - Context synchronization: synced +- [x] T02: `Delete verified Category C/D dead code and its suppressions` (status:done) + - Task ID: T02 + - Scope: In — only items T01 classified safe to remove (`delete` disposition with usage evidence), in small module-grouped commits: Agent Trace DB convenience inserts; unused `default_paths.rs` helpers/constants (only platform variants T01 proved unused on every supported target); doctor, mutation-scope adapter, capabilities (`FsOps`/`GitOps` methods and the file-wide `#![allow(dead_code)]`), `db/mod.rs` `QueryRows`/`query`/`query_values`, `HasFs::fs`, `AppContext` accessors, config/command-registry/lifecycle/storage-resolver helpers; superseded Category C items T01 verified against their originating plans/ADRs (agent-trace generator and validators, `repository_identity` dir-segment helpers, setup hook-asset and prompt helpers, `run_current_sync*`/`SyncCommand::execute`, `load_patch_from_json_bytes`, the unused `run_external_mutation_guard` production wrapper and its re-exports (its two guarded-shell regression tests are preserved, see below), the unused synchronous `WorktreeLock::acquire` wrapper, `GuardError::UnsupportedPlatform` only if no supported target needs it); genuinely dead fields (decided per T01's field table); dead tests and test support that only exercised deleted code; the `#[allow(dead_code)]` attributes those removals leave unneeded, then a second diagnostic-mode pass to catch newly exposed unreachable items. Out — Category A items (including all `diff_traces` compatibility declarations), the Category B reconciliation cluster, `run_with_retry_sync` and async wrappers (T04), any state field whose non-use may hide a missing invariant (recorded as a suspected defect instead), any row not classified safe-to-remove, restoring tests. + - Dependencies: T01 + - Done when: all T02-owned deletion candidates are removed, including their unused imports, constructors, callers and suppressions. Items explicitly assigned to later tasks, including `run_with_retry_sync` in T04, remain tracked and do not block T02 completion; broad/file-wide suppressions in the touched files are gone; the production and test targets build in normal mode with `-D warnings`; Category A symbols are untouched; each commit is one module/ownership group; suspected logic defects are listed in the completion record, not deleted; `guarded_shell_supervision_leaves_the_runtime_worker_responsive` and `guarded_shell_streams_output_and_finishes_with_exit_code` still exist in `external_mutation_guard.rs`, drive the real guarded-shell supervisor through `arm_external_mutation_guard(...).await` and `ArmedExternalMutationGuard::exec(...).await` (any `#[cfg(test)]` helper is test-only, not a production abstraction), preserve `GuardEvent::Armed` timing, output event order/contents, exit-code and external-taint marker cleanup assertions and the single-worker Tokio responsiveness assertion, use no mocks bypassing the supervisor, and pass; `acquire_inner_async` and `acquire_inner` are retained unchanged (no change to polling, timeouts or cancellation protection). + - Verify: focused normal-mode checks only: `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml`; `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings`; `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml ` for touched modules after confirming names with `-- --list`; `git diff --check`; explicitly run both guarded-shell tests by name (confirmed via `-- --list`; a zero-match filter never counts) after the wrapper is removed. + - Completed: 2026-10-08 + - Files changed: `cli/src/**` (about 60 source files across app, agent_trace, agent_trace_db, agent_trace_storage, capabilities, command_registry, config, db, default_paths, doctor, error, hooks, lifecycle providers, mutation_trace, observability, output_format, patch, repository_identity, setup, sync); deleted orphan fixtures `cli/src/services/agent_trace/fixtures/{average_age,file_rename,hello_world,mixed_change,poem_edit,poem_write,text_file_lifecycle}_reconstruction/`; `context/plans/cli-async-persistence-cleanup-pr3.md`. Working tree diff vs HEAD: 116 files, 111 insertions, 3343 deletions. + - Result: Deleted all T02-owned T01 `delete` rows with their callers, constructors, imports, tests and suppressions: Agent Trace DB inserts, `agent_trace.rs` validators/`classify_hunk`/`hunks_match_exactly`/`build_agent_trace` (+ seven golden tests and fixtures), `default_paths` helpers/constants, doctor chain (`inspect_repository_hooks`, `collect_global_state_health`, `collect_hook_health`, `inspect_hook_content_state`, `display_node`, `domain`/`group`/`branch`, `validate_config_file` dependency field and its generic), capabilities `FsOps`/`GitOps` methods (`FsOps` is now a marker trait), `db/mod.rs` `QueryRows`/`query`/`query_values`/encrypted `query`, `AppContext`/`HasFs` accessors, `parse_env` x2 and `from_env_lookup`, `CliError::user` (tests construct `CliError::User { .. source: None }`), `ServiceLifecycle::id` and its five impls, mutation-scope `is_subagent`/`is_clear`/`mark_recovery_pending`/`arm_recovery`/`arm_and_begin_recovery_flush`/MCP helpers, codex `tool_response` field and `resolve_codex_patch_path`, `OutputFormat::parse`, `load_patch_from_json_bytes`, repository-identity dir-segment helpers, setup hook-asset/prompt helpers, `run_current_sync`/`run_current_sync_with_progress`/`SyncCommand::execute`, `run_external_mutation_guard` (production wrapper, non-unix stub and re-exports), `WorktreeLock::acquire`, `MutationLineage::reset_file`/`tracked_paths`, `is_terminal` x2 and `scope_actor`. The two guarded-shell tests were preserved via a test-only `arm_and_exec_guard` helper driving `arm_external_mutation_guard(...).await` and `ArmedExternalMutationGuard::exec(...).await`. Blanket/file-wide dead-code allows were removed everywhere (including `capabilities.rs`, four `*_mutation_scope/mod.rs`, `generated_migrations`, `agent_trace_storage`, `agent_trace_sync`, `bash_policy`, `repository_identity`); the unsuppressed binary and test builds now warn only on retained items. Retained with declaration-level `reason` allows: Category A `diff_traces` declarations (cite the ADR), `PlatformFamily` and `GuardError::UnsupportedPlatform` (cross-platform), `run_with_retry_sync` (T04 removes). Retained under the pre-existing module-level allow on `mutation_trace` in `services/mod.rs`, now carrying a reason (T03 narrows it): the Category B reconciliation cluster, `WorktreeLockInner::path`, and the suspected-defect fields. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` passed (exit 0, normal mode, binary and test targets); unsuppressed `RUSTFLAGS="--cap-lints warn"` build and `test --no-run` (second diagnostic pass, Linux only) report only the retained items above plus a pre-existing test-only `mbt/model.rs` `IgnoredAny` field warning that also exists at baseline; `test ... guarded_shell` ran both guarded-shell tests (2 passed, names confirmed by output); a filtered run of `agent_trace::`, `mutation_trace::lineage`, `mutation_trace::tests`, `error::`, `observability::`, `repository_identity` passed (126 passed, 0 failed); `git diff --check` clean. The full suite was not run (reserved for `/validate`); test count is now 344 (baseline 354). + - Deviations and notes: tests deleted with their code — seven `build_agent_trace` golden tests (evidence-based golden tests remain), `reset_file`, deleted-file `tracked_paths` assertion test, `dir_segment` test; `scope_status_live_and_terminal_table` was reduced to `scope_status_live_table`. `AppContext` fields `fs`/`git` and the `HasFs` associated type remain because `app_support` bounds still use them. `acquire_inner_async` and `acquire_inner` are unchanged. The `hook_dir` module and related `default_paths` constants were deleted after confirming no other users. Suspected logic defects (kept, not deleted): `CoordinateOutcome` fields `worktree_id`, `observed_tree`, `revision`, `evaluation`, `mutation_event`, and the `committed`/`completed` error-payload fields. + - Context impact: yes — context docs naming deleted symbols (default-path catalog, capability traits, repository identity, agent-trace sync command, patch/agent-trace docs) need verification and update. + - Context synchronization: synced +- [x] T03: `Retain and defer ref reconciliation; narrow its dependency-cluster suppressions` (status:done) + - Task ID: T03 + - Scope: In — default disposition is **retain and defer**; no approval gate. (1) Inspect the actual dependency graph from T01. (2) Preserve the reconciliation implementation and the helper APIs it requires. (3) Replace any file-wide suppression with declaration-level exceptions limited to cluster members used solely by reconciliation (`ref_reconciliation`, `list_pins`/`delete_pins*`/`run_ref_mutation`/`assert_pins_are_unchanged_direct_refs`/`PinnedRef`/`PinInventoryError`, `load_mutation_event*`/`load_tree_roots`/`load_all_tree_roots`/`reconstruct_boundary`/`decode_boundary_kind`, and the `WorktreeLockInner::path` field decision; `acquire_inner_async` and `acquire_inner` are live and untouched, and `WorktreeLock::acquire` is no longer in this cluster), each with a `reason` naming the deferral and `context/plans/mutation-cursor-ref-reconciliation.md` as the integration path; delete only members T01 proves are not part of the reconciliation graph; resolve the unused `WorktreeLock` `path` field per T01's field analysis. (4) Record in the ref-reconciliation, snapshot-service, store and runtime-coordinator context that reconciliation remains unwired and that orphaned snapshot pins can accumulate (pins created by `pin_tree` in `coordinator.rs`, no production deleter), and reference the existing reconciliation plan and the `ref-rec` branch as the intended integration path. (5) Keep production wiring outside PR3. Only if T01 recorded material new evidence that invalidates retain-and-defer: stop, record the evidence as a blocker and require a user or architecture decision (and an ADR only then). Do not create an ADR merely to restate the already accepted decision. Correct inaccurate context found on the way (including descriptor-atomicity claims). Out — deleting the reconciliation cluster to reduce the dead-code count, wiring reconciliation into any command/hook/doctor, adding behavior, new retention/compaction design, the macOS race fix. + - Dependencies: T01, T02 + - Done when: retain-and-defer is recorded in durable context with the unwired status, orphaned-pin consequence and plan reference; no reconciliation code is deleted or wired; exceptions are declaration-level with reasons; the `WorktreeLock` `path` field is resolved per T01. + - Verify: `rg -n "reconcile_worktree" cli/src` shows no new caller; targeted tests in `ref_reconciliation`/`git_snapshot`/`worktree_lock` run (names via `-- --list`; zero matches do not count); `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings`; context inspection against the code. + - Completed: 2026-10-08 + - Files changed: `cli/src/services/mod.rs`, `cli/src/services/mutation_trace/{store.rs,mbt/model.rs}`, `cli/src/services/mutation_trace/runtime/{coordinator,git_snapshot,ref_reconciliation,scope_runtime,worktree_lock}.rs`, `context/cli/mutation-trace-{ref-reconciliation,runtime-coordinator,snapshot-service,store}.md`, `context/plans/cli-async-persistence-cleanup-pr3.md` + - Result: Removed the module-wide `#[allow(dead_code)]` on `pub mod mutation_trace`. Unsuppressed build then reported 26 warnings, all now covered by declaration-level `#[allow(dead_code, reason = ...)]`: the reconciliation cluster (`ref_reconciliation.rs` x6, `git_snapshot.rs` x10, `store.rs` x13) cites `context/plans/mutation-cursor-ref-reconciliation.md`; the suspected-defect fields (`CoordinateOutcome` x5, `committed`, `completed`) cite this plan; `mbt/model.rs` `WireModelState` (test-only, previously hidden by the module allow) is allowed because `deny_unknown_fields` requires the verification-only fields. `WorktreeLockInner::path` was never read, is outside the reconciliation graph, and was deleted (`acquire_inner_async`/`acquire_inner` untouched). No reconciliation code was deleted or wired. Context: added a "retained and unwired" status with the orphaned-pin consequence to the ref-reconciliation, snapshot-service, store and runtime-coordinator docs; corrected them where they described removed tests (`ref_reconciliation` inline tests, `runtime/tests.rs`). + - Verify: `clippy --all-targets --all-features -- -D warnings` passed (exit 0); `rg reconcile_worktree cli/src` outside `ref_reconciliation.rs` found nothing; `test mutation_trace::runtime` ran 11 passed (includes the `git_snapshot` pin tests; `ref_reconciliation` has no tests, a zero-match filter so not counted as coverage); `cargo fmt` applied; `git diff --check` clean. Full suite not run (reserved for `/validate`). + - Deviations and notes: other docs still mention the removed `runtime/tests.rs` (`mutation-trace-agent-attribution.md`, `mutation-trace-scope-abandonment.md`, `mutation-trace-external-taint.md`, `context/patterns.md`, `context/context-map.md`); left for T06 context cleanup. + - Context impact: yes — reconciliation-status docs updated above; remaining stale `runtime/tests.rs` references noted for T06. + - Context synchronization: synced +- [x] T04: `Simplify post-async wrappers and re-evaluate sync joining` (status:done) + - Task ID: T04 + - Scope: In — remove `run_with_retry_sync` (if T01 confirmed no consumer) and any duplicate helper logic it exposes, extracting shared validation/error formatting only if simpler while preserving the distinct semantics of `run_with_retry` (cancelling timeout: the attempt future is cancelled when the timeout elapses) and `run_with_retry_elapsed` (elapsed measurement, non-cancelling); remove redundant `async { ... }.await` wrappers (for example the storage scope in `sync/sync.rs`), redundant `drop(storage)`, trivial forwarding layers and unnecessary temporaries in `db/mod.rs`, sync, hook and lifecycle helpers; evaluate `join_three_to_completion`/`JoinSlot` and simplify only if cooperative stopping, in-flight completion, credential-persistence protection, first-observed-error selection, non-`Send` progress and concurrent execution are all preserved, otherwise retain and record why. Out — retry policy changes, collapsing the cancelling and non-cancelling retry variants, `try_join!` if it cancels siblings, spawned tasks, new async abstractions, dynamic dispatch, new tests unless a replacement introduces an untested invariant. + - Dependencies: T02 + - Done when: the listed leftovers are removed or recorded as retained with a reason; retry timeout/backoff behavior is unchanged in observable semantics and the cancelling versus non-cancelling distinction is intact; the join decision and rationale are recorded; no nested runtime, `BoxFuture` or `dyn` is introduced. + - Verify: focused normal-mode checks: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml terminal_stream` (confirm names with `-- --list`; zero matches do not count), `... agent_trace_db::transaction_tests`, `... db::transaction_cancellation_tests`; clippy with `-D warnings`; `git diff --check`. + - Completed: 2026-10-08 + - Files changed: `cli/src/services/resilience.rs`, `cli/src/services/sync/sync.rs`, `cli/src/services/db/mod.rs`, `context/plans/cli-async-persistence-cleanup-pr3.md` + - Result: Deleted `run_with_retry_sync` (zero consumers) with its `std::thread` import; `run_with_retry` (cancelling timeout) and `run_with_retry_elapsed` (non-cancelling elapsed measurement) are untouched and still distinct. In `sync/sync.rs` removed the redundant `async { ... }.await` storage scope and `drop(storage)` (storage still outlives the client and drops at function end) and the three redundant `async move { ... }.await` wrappers in the ingest closures. In `db/mod.rs` flattened the redundant `async { ... }.await` wrappers in `ensure_migrations_table`, `is_migration_applied` and `apply_migration`. Net 42 insertions, 135 deletions. + - Join decision: `join_three_to_completion`/`JoinSlot` retained unchanged. `try_join!` would cancel siblings on the first error, which breaks the in-flight batch completion and credential-persistence protection; `join!` would not give first-observed-error selection after all siblings finish; the hand-rolled poller keeps cooperative stopping, non-`Send` progress reporting and concurrent execution without spawned tasks, and is covered by the three `services::sync::sync::tests` cases. No simpler combinator preserves all of those behaviors. + - Verify: `clippy --all-targets --all-features -- -D warnings` passed (exit 0); `test terminal_stream` 2 passed; `test services::sync` 3 passed (terminal-stream stop, discard-after-termination, in-flight credential save); `test agent_trace_db::transaction_tests` 2 passed; `test db::transaction_cancellation_tests` 1 passed; `cargo fmt` applied; `git diff --check` clean; added lines contain no `dyn`, `BoxFuture`, `block_on` or `Runtime::new`. Full suite not run (reserved for `/validate`). + - Deviations and notes: other `async { ... }` blocks in `db/mod.rs` (inside retry operations) were left because they scope borrows or cancellation, not mere forwarding. + - Context impact: minor — `context/cli/async-persistence-guarantees.md` may mention `run_with_retry_sync` or the retained join helper; verify. + - Context synchronization: synced +- [x] T05: `Audit blocking sites, future size and temporary Git indexes` (status:done) + - Task ID: T05 + - Scope: In — review each remaining `spawn_blocking` site against the PR2 retained-worker table in `context/architecture.md`; measure future size and `recursion_limit` pressure for `main.rs`/`app.rs` and the largest async dispatch functions (for example compiler future-size diagnostics or `size_of_val` in a disposable worktree) and record the method and numbers; determine why `#![recursion_limit = "256"]` is needed; measure large-future complexity rather than assuming a reduction is possible, and take only safe reductions of nested async composition or captured state if the measurements show one (recording "no safe reduction" is an acceptable outcome); remove the override only if the pinned compiler and the checks pass without it; assess `TempIndexGuard` (`git_snapshot.rs`) and decide: no automatic sweeper unless a safe ownership/lock protocol already exists (filename or age alone is never evidence of abandonment), otherwise document and defer to a separate PR; fix documentation that overstates guarded-shell descriptor atomicity and describe the macOS `pipe()`/`fcntl()` race as an unresolved limitation, not fixed. Out — new concurrency/ownership protocols, a temp-index sweeper without safe ownership, the macOS race fix, `BoxFuture`/`dyn Future`, test restoration. + - Dependencies: T04 + - Done when: measurements and the keep/remove decision for the recursion limit, the temporary-index decision, and any retained blocking complexity are recorded in durable context; any code change is a safe reduction with passing focused checks; the macOS race is described as an open limitation. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml` (and the same build without the override in a disposable worktree if removal is attempted); clippy with `-D warnings`; context inspection. + - Completed: 2026-10-08 + - Files changed: `context/architecture.md`, `context/cli/async-persistence-guarantees.md`, `context/cli/mutation-trace-external-mutation-guard.md`, `context/plans/cli-async-persistence-cleanup-pr3.md`; no `cli/` changes + - Result: Audit only, no code change. The 14 `spawn_blocking` sites match the architecture retained-worker list. Future sizes measured with `RUSTC_BOOTSTRAP=1 RUSTFLAGS=-Zprint-type-sizes` on a Linux debug build in a disposable worktree: `app::run` 14000 bytes, `main` `block_on` future 14016, the dispatch chain 13952 down to `join_three_to_completion` 11832, with the dependency `DatabaseSyncEngine::open_db` future alone at 12704, so the size is dominated by a third-party future and wrappers add about 50-200 bytes each; no safe reduction without boxing. Removing `#![recursion_limit = "256"]` fails with "queries overflow the depth limit" (increased by 130 computing the layout of `app::run`), so the override is kept. `TempIndexGuard`: no automatic sweeper (no ownership or lock record; name or age is not evidence of abandonment); documented and deferred. Docs that claimed the guarded-shell pipe is created close-on-exec atomically now say atomicity holds only on Linux (`pipe2`) and the `pipe`+`fcntl` window elsewhere is an unresolved limitation. + - Verify: normal-mode `build` passed; `clippy --all-targets --all-features -- -D warnings` passed; the no-override build in the disposable worktree failed as described above and the worktree was removed; `git diff --check` clean; context inspected against `external_mutation_guard.rs` (`pipe2` on Linux, `pipe`+`fcntl` otherwise). Full suite not run (reserved for `/validate`). + - Context impact: yes — `architecture.md`, `async-persistence-guarantees.md` and the external-mutation-guard doc updated above. + - Context synchronization: synced +- [x] T06: `Final cleanup audit and validation preparation` (status:done) + - Task ID: T06 + - Scope: In — in diagnostic mode on a disposable worktree, reproduce the final unsuppressed dead-code diagnostic audit and record that each remaining diagnostic maps to a documented declaration-level exception (Category A `diff_traces`, Category B reconciliation, any cross-target cases); classify and list all remaining suppression sites with reasons; verify the `diff_traces` compatibility declarations and contract (AC3) are intact; record removed symbol and LOC counts (before/after, against T01's inventory) and the intentionally retained code; review cross-platform limitations (Linux-only diagnostics never support a cross-platform safety claim); update durable context (`context/cli/async-persistence-guarantees.md` PR3 staging/known limits, context map and overview); prepare the PR4 (OpenTelemetry) handoff and the reconciliation follow-up reference to `context/plans/mutation-cursor-ref-reconciliation.md`; record which tests and checks must run during final validation by pointing to this plan's Full validation section rather than duplicating it. T06 may run focused checks needed to verify its own changes (for example normal-mode build and the targeted regression filters) but must not run or claim the full repository validation. Out — new cleanup beyond T01's inventory, suppression changes beyond what the audit shows, any new behavior, running the Full validation suite, marking the plan validated. + - Dependencies: T03, T05 + - Done when: the audit table, LOC/declaration deltas, retained exceptions, cross-platform limitations, deferred work, PR4 handoff and the list of checks reserved for `/validate` are recorded; no new unexplained suppression exists; the completion record states that full validation has not yet run. + - Verify: focused only: diagnostic-mode build and `--no-run` test build in a disposable worktree; normal-mode `nix develop -c ./scripts/run-cli-cargo.sh build --manifest-path cli/Cargo.toml`; `nix shell nixpkgs#ripgrep -c rg -n "allow\(dead_code\)|allow\(unused" cli/src`; `git diff --check`. + - Completed: 2026-10-08 + - Files changed: `context/plans/cli-async-persistence-cleanup-pr3.md`, `context/cli/async-persistence-guarantees.md`, `context/cli/mutation-trace-{agent-attribution,external-taint,scope-abandonment}.md`, `context/context-map.md`, `context/overview.md`, `context/patterns.md`; no `cli/` changes + - Result: Final diagnostic audit (Linux x86_64, rustc 1.95.0, disposable worktree at `e45e7747`, all 44 `allow(dead_code ...)` attributes in 10 files stripped, `RUSTFLAGS=--cap-lints warn`): the binary target reports 32 warnings and the test target 27 (including a test-only `mbt/model.rs` `WireModelState` warning). Every diagnostic maps to a documented declaration-level exception. Audit table: (1) Category A `diff_traces` compatibility: `AgentTraceDiffTraceExportRow`, `SELECT_DIFF_TRACES_AFTER_SQL`, `read_diff_traces_after`, `diff_trace_export_row_from_turso` (`agent_trace_export/mod.rs`), `ingest_diff_traces` (`control_plane.rs`), all citing the ADR and all present with unchanged signatures. (2) Category B reconciliation, citing `context/plans/mutation-cursor-ref-reconciliation.md`: `ref_reconciliation.rs` x6, `git_snapshot.rs` x10 (`FOR_EACH_REF_PIN_FORMAT`, pin helpers, `PinnedRef`, `PinInventoryError`, `parse_pin_line`, `pin_ref_prefix`) and `store.rs` x13; `RECONCILIATION_LOCK_TIMEOUT` is the only one also flagged in the test target. (3) Cross-platform: `PlatformFamily::{Macos, Windows, Other}`, `GuardError::UnsupportedPlatform`. (4) Suspected-defect fields: `CoordinateOutcome::{worktree_id, observed_tree, revision, evaluation, mutation_event}`, `committed`, `completed`. No unexplained suppression exists; the remaining non-dead-code allows are clippy or `unused_imports` pre-existing exceptions. Deltas against PR2 (`remove-tokio-db-runtimes`): `cli/src` 66 files, 301 insertions, 1569 deletions; LOC 56620 to 55352 (-1268); unique unused declarations 147 to 32 binary-target diagnostics; `dead_code` allow attributes 119 matching lines in 35 files to 44 attributes in 10 files; no file-wide `#![allow(dead_code)]` remains; tests 354 to 344. No test module or fixture was added back (`git diff --diff-filter=A --name-only` over `cli` lists only already-reviewed paths: none are test modules or fixtures). No `dyn`, `BoxFuture`, `block_on` or `Runtime::new` was added in the `cli/src` diff; `Cargo.toml` and `flake.nix` carry no lint changes, and the `main.rs` change only drops a dead-code allow. Context: removed stale `runtime/tests.rs` references from the scope-abandonment, agent-attribution, external-taint, context-map and patterns docs, removed the stale module-wide dead-code claim from `overview.md`, and recorded the PR3 outcome and known limits in `async-persistence-guarantees.md`. + - Cross-platform limits: only the Linux build was compiled. macOS/Windows/non-unix code (`PlatformFamily` variants, `GuardError::UnsupportedPlatform`, the macOS `pipe()`/`fcntl()` descriptor race) was reviewed by reading and is not proven by this audit. + - Deferred work and PR4 handoff: (a) wiring ref reconciliation and a pin-retention or compaction design (plan `context/plans/mutation-cursor-ref-reconciliation.md`, `ref-rec` branch); orphaned snapshot pins accumulate until then. (b) The `TempIndexGuard` sweeper needs an ownership or lock protocol. (c) The macOS descriptor race fix. (d) Removal of the `diff_traces` compatibility surface (separate phase). (e) Resolving the suspected-defect fields. PR4 (OpenTelemetry) starts from a single application-owned Tokio runtime with directly awaited persistence; the dispatch future is about 14 KB with `recursion_limit = "256"` required, so instrumentation that nests more async layers should re-measure. + - Verify: diagnostic-mode `build` and `test --no-run` in the disposable worktree both exited 0 with the diagnostics above, and the worktree was removed; normal-mode `build` passed; `rg` listing of `allow(dead_code` and `allow(unused` recorded above; `git diff --check` clean. **Full validation has not run.** `/validate` must run the commands in this plan's Full validation section (complete Rust suite, clippy with `-D warnings`, `nix flake check`, `nix build .#ci-checks`, `git diff --check`; `pkl-check-generated` only if Pkl or `config/lib` inputs changed). + - Context impact: yes — updated as listed under Files changed. + - Context synchronization: synced + +- [x] Final gate (not an implementation task): `/validate` + - Dependencies: T01–T06 complete and `synced` + - Scope: run the repository's existing `sce-validate` workflow against this plan's acceptance criteria and the Full validation commands; record actual results, including failures, in the validation report. + - Done when: the validation report records every Full validation command outcome and each AC1–AC12 result. +## Open questions + +- None blocking. Reconciliation defaults to retain-and-defer without a gate; T01 raises a decision only if it finds material new evidence that invalidates it. +- Whether the original 114-item report can be located is unknown; T01 treats the reproduced inventory as authoritative either way. +- The no-comments-in-code preference conflicts with "every retained exception should explain its reason"; this plan carries reasons in `#[allow(..., reason = "...")]` or the owning context doc. T01 confirms the pinned toolchain accepts `reason`. + +## T01 inventory appendix + +**Baseline.** Branch `dead-code-blues` HEAD `2f073f98`; PR2 base `remove-tokio-db-runtimes` = `f8e9c748`; `git diff remove-tokio-db-runtimes...HEAD -- cli/src` is empty. Toolchain rustc 1.95.0. Platform: Linux x86_64 only; `cfg` for macOS/Windows/non-unix code was reviewed by reading, not compiled. + +**Method.** Disposable `git worktree` (since removed): deleted every `#[allow(dead_code)]`, `#![allow(dead_code)]` and `#[cfg_attr(not(test), allow(dead_code))]`, and dropped `dead_code, ` from two combined allows in `style.rs` (35 files, 119 deleted and 2 modified lines). Built with `RUSTFLAGS=--cap-lints warn` so every lint is reported in one pass. The nonzero-exit-from-lints outcome did not occur because of the cap; no non-dead-code compiler error appeared. + +**Counts.** + +| Figure | Value | +| --- | --- | +| Binary-target warnings | 114 (equals the seed's 114) | +| Binary-target unique unused declarations | 147 (warnings group multiple symbols) | +| Test-target warnings / symbol entries | 100 / 127 | +| Declarations live only through tests | 19 (9 outside reconciliation, 10 pin helpers) | +| `rg "allow\(dead_code"` matching lines | 119 (117 are `allow(dead_code)`, 2 combined in `style.rs`) | +| Files with suppressions | 35 | +| File-wide `#![allow(dead_code)]` | 5 (`capabilities.rs`, four `*_mutation_scope/mod.rs`) | + +The seed's 114 matches the binary-target warning count, not the declaration count; its 37 occurrences and 117 `rg` matches are not reproduced by any single count here (117 is the bare-attribute line count). The original symbol listing was not found, so this inventory supersedes it. No rows were invented. + +**Disposition totals.** delete-now 101 (corrected from 100); Category A retain 5; Category B retain-and-defer 30 (corrected from 31); cross-platform retain 4; fields retained pending defect review 7; decision-needed 0. Test baseline: 354 tests via `-- --list`. LOC of `cli/src`: 56620. + +**Inventory (grouped by file; all symbols listed).** + +| File | Symbols | Cat | Usage evidence | Disposition | Risk | +| --- | --- | --- | --- | --- | --- | +| app.rs | `fs`, `fs`/`git`/`repo_root` (AppContext) | D | no callers in prod or tests | delete | low | +| agent_trace.rs | `FileRead`, `InvalidJson`, `validate_agent_trace_json`, `validate_agent_trace_file`, `classify_hunk`, `hunks_match_exactly` | C | unused in prod and tests | delete | low | +| agent_trace.rs | `build_agent_trace` | C | tests only (`agent_trace/tests.rs`) | delete with its tests | low | +| agent_trace_db/mod.rs, repository.rs | `insert_message_with`, `insert_part_with`, `insert_message`, `insert_part` | D | only each other | delete | low | +| agent_trace_export/mod.rs, agent_trace_sync/control_plane.rs | `AgentTraceDiffTraceExportRow`, `SELECT_DIFF_TRACES_AFTER_SQL`, `read_diff_traces_after`, `diff_trace_export_row_from_turso`, `ingest_diff_traces` | A | protected by the 2026-10-01 diff_traces ADR | retain, declaration-level exception citing ADR | none | +| agent_trace_storage/mod.rs | `resolve_agent_trace_storage_at_state_root` | D | no callers | delete | low | +| capabilities.rs | `read_file`, `write_file`, `metadata`, `exists`, `resolve_repository_root`, `resolve_hooks_directory` | D | trait methods with no callers; file-wide allow | delete (check trait impls/fakes) | medium | +| command_registry.rs, sync/command.rs | `execute` x2 | C | no callers (T02 confirms against originating plan) | delete | low | +| config/types.rs | `parse_env` x2 | D | referenced only by unused `from_env_lookup` and tests of themselves (T02 re-verifies) | delete | low | +| db/mod.rs | `QueryRows`, `query`, `query_values`, `query` | D | no callers | delete | low | +| default_paths.rs | `PersistedArtifactRootKind`, `PersistedArtifactId`, `PersistedArtifactLocation`, `persisted_artifact_locations`, `cache_root`, `git_dir`, `git_hooks_dir`, `git_hook_file`, `git_commit_editmsg`, `opencode_plugin_target`, `skill_tile_relative_path`, `pre_commit_hook_path`, `commit_msg_hook_path`, `post_commit_hook_path`, constants `GIT`, `GIT_COMMIT_EDITMSG`, `HOOKS`, `PRE_COMMIT`, `COMMIT_MSG`, `POST_COMMIT`, `OPENCODE_DIR`, `PLUGIN_FILE`, `PLUGIN_MANIFEST_ENTRY`, `AGENTS_DIR`, `SKILL_DEFINITION` | D | defined and referenced only inside the unused helpers; constants `PRE_COMMIT`/`COMMIT_MSG`/`POST_COMMIT` also appear in `setup/mod.rs` (T02 re-verifies after setup deletion) | delete | low | +| default_paths.rs | `PlatformFamily::Macos`, `Windows`, `Other` | D | constructed only under `cfg(target_os)` in `current_platform_family`; matched in `encryption_key.rs` | retain (cross-platform, not provable on Linux) | high if deleted | +| doctor/mod.rs, inspect.rs, types.rs | field `validate_config_file`, `inspect_repository_hooks`, `collect_global_state_health`, `collect_hook_health`, `inspect_hook_content_state`, `display_node`, `domain`, `group`, `branch` | D | dead chain; the field is read only by the dead functions | delete the chain together | medium | +| error.rs | `user` | D | tests only | delete with tests | low | +| hooks/claude_mutation_scope/events.rs, codex_mutation_scope/events.rs | `is_subagent` x2, `MCP_TOOL_NAME_PREFIX`, `is_mcp_tool_name` | D | no callers | delete | low | +| hooks/*_mutation_scope/state.rs | `mark_recovery_pending`, `arm_recovery`, `arm_and_begin_recovery_flush`, `is_clear` x3 | D | no callers (T02 reviews recovery semantics before deleting) | delete (review recovery semantics) | medium | +| hooks/codex/mod.rs | field `tool_response` | field | serde payload field, never read | delete after checking serde derive and `deny_unknown_fields` | low | +| hooks/codex/apply_patch/path.rs | `resolve_codex_patch_path` | C | no callers | delete | low | +| lifecycle.rs | `id` | D | no callers | delete | low | +| mutation_trace/lineage.rs | `reset_file`, `tracked_paths` | D | tests only | delete with tests | low | +| mutation_trace/runtime/coordinator.rs | fields `worktree_id`, `observed_tree`, `revision`, `evaluation`, `mutation_event` of `CoordinateOutcome`; `committed` | field | never read; error payload carries outcome | retain, suspected defect list | medium | +| mutation_trace/runtime/scope_runtime.rs | field `completed` | field | never read; same error-payload pattern | retain, suspected defect list | medium | +| mutation_trace/runtime/external_mutation_guard.rs | `UnsupportedPlatform` | C | constructed under `cfg(not(unix))` | retain (cross-platform) | high if deleted | +| mutation_trace/runtime/external_mutation_guard.rs | `run_external_mutation_guard` | C | production wrapper with test-only callers; re-exported in `runtime/mod.rs` (post-review correction) | delete the production convenience wrapper (unix `unix_impl` and `cfg(not(unix))` stub) and its re-exports; preserve `guarded_shell_supervision_leaves_the_runtime_worker_responsive` and `guarded_shell_streams_output_and_finishes_with_exit_code` by adapting them to `arm_external_mutation_guard(...).await` and `ArmedExternalMutationGuard::exec(...).await` (a `#[cfg(test)]` helper is allowed) | low | +| mutation_trace/runtime/git_snapshot.rs | `FOR_EACH_REF_PIN_FORMAT`, `list_pins`, `delete_pins`, `delete_pins_inner`, `run_ref_mutation`, `assert_pins_are_unchanged_direct_refs`, `PinnedRef`, `PinInventoryError`, `parse_pin_line`, `pin_ref_prefix` | B | used by `ref_reconciliation` and tests only | retain, narrowed exception (T03) | none | +| mutation_trace/runtime/ref_reconciliation.rs | `RECONCILIATION_LOCK_TIMEOUT`, `ReconciliationReport`, `ReconciliationOutcome`, `ReconcileError`, `reconcile_worktree`, `reconcile_worktree_inner` | B | no caller outside the module (`mod ref_reconciliation;` private) | retain, narrowed exception (T03) | none | +| mutation_trace/runtime/worktree_lock.rs | `WorktreeLock::acquire` (sync wrapper; post-review correction) | D | no production, test or supported-platform callers; reconciliation calls `acquire_inner_async`, not `acquire` | delete in T02 (unused synchronous convenience wrapper) | low | +| mutation_trace/runtime/worktree_lock.rs | field `WorktreeLockInner::path` | B | never read; classified separately from the wrapper | retain; resolved in T03 | low | +| mutation_trace/store.rs | `decode_boundary_kind`, `SELECT_MUTATION_EVENT_SQL`, `SELECT_MUTATION_EVENT_ACTIVE_SCOPES_SQL`, `SELECT_TREE_ROOTS_BY_WORKTREE_SQL`, `SELECT_ALL_TREE_ROOTS_SQL`, `load_mutation_event`, `load_tree_roots`, `load_all_tree_roots`, `load_mutation_event_active_scopes`, `tree_root_row_from_turso`, `MutationEventRow`, `mutation_event_row_from_turso`, `reconstruct_boundary` | B | only store-internal chain plus reconciliation | retain, narrowed exception (T03) | none | +| mutation_trace/types.rs | `is_terminal` x2, `scope_actor` | D | no production callers; `is_terminal` also exercised in `mutation_trace/tests.rs` (T02 re-verifies) | delete | low | +| observability.rs | `from_env_lookup` | D | no callers | delete | low | +| output_format.rs | `parse` | D | no callers | delete | low | +| patch.rs | `load_patch_from_json_bytes` | C | no callers | delete | low | +| repository_identity/mod.rs | `dir_segment`, `DIR_SEGMENT_SHORT_HASH_LEN`, `repository_dir_segment`, `slugify`, `derive_short_hash` | C | superseded directory-segment scheme; non-method items used by tests only | delete with tests | low | +| resilience.rs | `run_with_retry_sync` | C | zero consumers anywhere in `cli/src` | delete in T04 | low | +| setup/mod.rs, prompt.rs | `RequiredHookAsset`, `get_required_hook_asset`, `setup_prompt_target_label_with_color_policy`, `setup_prompt_title_with_color_policy` x2 | C | superseded setup helpers | delete | low | +| sync/sync.rs | `run_current_sync`, `run_current_sync_with_progress` | C | no callers | delete | low | + +**Reconciliation graph and decision.** `reconcile_worktree` has no caller outside `ref_reconciliation.rs`; its dependency cluster (30 rows after the post-review correction) is used only by itself, `pin_tree` remains called from `coordinator.rs`, and no production code deletes pins. No new evidence invalidates retain-and-defer; no decision is needed. + +**Defects and checks recorded for later tasks.** +- Suspected logic defects (not for deletion): `CoordinateOutcome` fields and the `committed`/`completed` error payloads are carried but never read, so error variants may drop information callers could report. +- `run_with_retry_sync` has zero consumers; T04 owns removal. +- 19 declarations are live only via tests; T02 deletes their tests with them, and T03 keeps the pin helpers. +- `spawn_blocking` baseline (14 sites): `git_snapshot.rs` x2, `protected_worktree.rs` x2, `worktree_lock.rs`, `external_mutation_guard.rs`, `mutation_scope_lock.rs` x3, `db/mod.rs` (encryption key), and the four `*_mutation_scope/state.rs` readers. +- Future-size baseline method (for T05): `#![recursion_limit = "256"]` is in `cli/src/main.rs`; measure with compiler future-size diagnostics in a disposable worktree. +- Platform limits: only the Linux build was compiled. `PlatformFamily` variants and `UnsupportedPlatform` are cfg-gated and must not be deleted on Linux evidence. +- Stale unrelated worktree `.../f4f0ae83-.../scratchpad/wt-t01` at `f8e9c748` exists from another session; left untouched. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-08 + +### Commands run + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` -> exit 0 (356 passed, 0 failed; includes `db::transaction_cancellation_tests`, `agent_trace_db::transaction_tests`, `services::sync::sync` terminal-stream tests, the new `sync::contract_tests`, and the `mutation_trace::mbt` Quint MBT suite) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` -> exit 0 +- `nix flake check` -> exit 0 (all checks passed; aarch64-darwin, aarch64-linux, x86_64-darwin omitted as incompatible systems) +- `nix build .#ci-checks` -> exit 0 +- `nix run .#pkl-check-generated` -> not run (no Pkl or `config/` input in `git diff remove-tokio-db-runtimes`) +- `git diff --check` -> exit 0 + +### Success-criteria verification + +- [x] AC1: T01 appendix records baseline `f8e9c748`, per-target counts (114/147 binary, 100/127 test), suppression/match/file counts, differences from the seed, and per-row dispositions -> inspected appendix +- [x] AC2: `git diff --shortstat remove-tokio-db-runtimes -- cli/src` -> 67 files, +665/-1575; platform-gated `PlatformFamily` and `GuardError::UnsupportedPlatform` retained +- [x] AC3: all listed declarations present with unchanged wire/DTO shapes; the working-tree diff touches only `control_plane.rs`, `render_sync.rs` and `sync.rs`, adding contract tests (required and out-of-range `cursors.diffTraces` decode, zero-upload `streams.diffTraces` JSON entry, `sync_uploads_only_three_streams_and_reports_zero_upload_diff_traces`) that pass in the full suite; the only non-test signature change is a generic credential-store parameter on private `run_sync_async`/`sync_one_stream` +- [x] AC4: no `reconcile_worktree` caller outside `ref_reconciliation.rs`; ref-reconciliation and snapshot-service docs state retained-and-unwired, the orphaned-pin consequence and the `mutation-cursor-ref-reconciliation` plan; no new ADR +- [x] AC5: no `#![allow(dead_code)]` remains; every `dead_code` allow in the diff carries a `reason`; no lint changes in `cli/Cargo.toml`, `flake.nix` +- [x] AC6: T06 records diagnostic-mode output (Linux x86_64, rustc 1.95.0; 32 binary / 27 test diagnostics mapped to documented exceptions); normal-mode build, clippy and tests pass above +- [x] AC7: full suite, transaction and terminal-stream tests pass; `resilience.rs` diff only removes `run_with_retry_sync` +- [x] AC8: `join_three_to_completion`/`JoinSlot` retained with a recorded reason (T04); sync tests pass +- [x] AC9: no added `dyn`, `BoxFuture`, `Runtime::new` or `block_on` in the `cli/src` diff (grep of added lines empty); T05 records future sizes (~14 KB), method, `recursion_limit = "256"` keep decision and temp-index deferral +- [x] AC10: deletions (1575) exceed insertions (665); `git diff --diff-filter=A --name-only remove-tokio-db-runtimes -- cli` is empty +- [x] AC11: `rg build_agent_trace context` no longer hits `context/sce/agent-trace-minimal-generator.md` (now describes `build_agent_trace_from_evidence`); macOS `pipe()`/`fcntl()` limitation documented as unresolved +- [x] AC12: T06 recorded that full validation had not run; this report lists each command with its real result + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Only the Linux build was compiled; macOS/Windows/non-unix code and the macOS descriptor race are unverified +- Orphaned snapshot pins accumulate until ref reconciliation is wired +- Fixes for the earlier AC3/AC11 failure are uncommitted in the working tree +- Historical plans under `context/plans/` still mention `build_agent_trace` diff --git a/context/plans/cli-async-turso-persistence-pr2.md b/context/plans/cli-async-turso-persistence-pr2.md new file mode 100644 index 000000000..32321a637 --- /dev/null +++ b/context/plans/cli-async-turso-persistence-pr2.md @@ -0,0 +1,757 @@ +# Plan: cli-async-turso-persistence-pr2 + +## Change summary + +PR1 owns the application multi-thread Tokio runtime, but shared Turso persistence still constructs a current-thread runtime and synchronously drives native database futures. PR2 replaces that bridge with directly awaited persistence operations, preserving SQL, schema, migrations, transactions, retry/error policies, encryption, WAL, and repository/auth separation. It also removes obsolete runtime-lifetime blocking scopes and the sync storage cleanup guard where ordinary ownership suffices. + +The user explicitly revised the split: "PR2 must contain the mechanical async propagation through all direct DB consumers required to remove the runtime bridge. PR3 becomes cleanup/audit rather than propagation." Because RepositoryAgentTraceDb, LocalDb, and AuthDb are aliases of shared generic adapters, the compile-safe migration includes connected storage, credential-store, hook, mutation-trace, export, token-storage, lifecycle, setup/doctor, and command callers as necessary. Only signatures, awaits, required static adapter wiring, lifetime correctness, and test adaptations change; unrelated synchronous service logic stays synchronous. + +## Current state after the PR #301 review (2026-10-08) + +Re-inspected against branch head `044991d8e7d1117eaa861d4a935844ce05054783` (`remove-tokio-db-runtimes`, clean tree). This section was added by a planning-only update; no production code or tests changed and no new test results are claimed. T01 and T02 history below is unchanged. + +**Testing decision.** The large ordinary Rust suites were deliberately deleted during T02 and that decision stands. It is not treated as a mistake and does not require restoring test-count parity, deleted files, or a coverage-equivalence matrix. T02 established the **mechanical async migration**; T03, T06 and T07 establish the **minimum necessary correctness evidence** for the new execution model with a small set of focused, deterministic regression tests (see Minimal safety test set). Acceptance criteria that named deleted suites or filters that now match zero tests are rewritten as invariant-based checks. + +**Completed architecture changes (T01, T02).** The Turso runtime bridge is gone; `TursoDb`/`EncryptedTursoDb`, token storage, generic `CredentialStore`, lifecycle, hooks, mutation trace, sync/export and command dispatch use native awaited persistence with static dispatch. The implementation also moved Git snapshot, protected-worktree marker I/O, worktree/adapter lock acquisition, cancellation-shielded ref mutations and guarded-shell supervision onto async/`spawn_blocking` boundaries. + +**Confirmed unresolved issues (verified in source at the reviewed head).** +- Transaction exclusivity: `execute_transactional_cas_batch` and `execute_transactional_insert_pair_if_absent` take `&self` and use `Transaction::new_unchecked(&self.core.conn, Immediate)` (`db/mod.rs` ~983–1008, 1117–1128), contradicting T01's exclusive-`&mut` instruction. Owner T03. +- Sync fail-fast: `join_three_to_completion` (`sync/sync.rs` ~323) records the first error but keeps polling every sibling to completion, so a terminal 403 on one stream no longer stops sibling backlogs. Owner T06. +- Guarded-shell descriptors: `LifetimeToken::new` uses plain `pipe` and clears close-on-exec on the writer for its whole lifetime; `spawn_guarded_shell` uses `dup` and clears close-on-exec in the parent before `spawn`, with no `pre_exec` (`external_mutation_guard.rs` ~262–335). Owner T07. +- Credential refresh: `refresh_and_save` awaits the WorkOS renewal and then the save inside the outer `run_with_retry` cancelling timeout (`control_plane.rs` ~342, ~513), with no shield between receiving the replacement token and persisting it. The structural exposure is confirmed; actual token loss depends on WorkOS rotation behavior and is **not** established as production data loss. Owner T06. + +**Plausible concurrency risks needing deterministic characterization (not proven defects).** +- Adapter boundary locks: `with_boundary_lock` holds the guard across `operation().await` while state transitions run in separate `spawn_blocking` calls, so a cancelled boundary may release the lock while an older blocking transition is still running. Owner T07. +- Transaction cancellation: Turso 0.8.1 deferred rollback means cancellation after the first write needs one direct test. Owner T03. +- Worktree/marker/Git-ref cancellation ownership and temporary-index cleanup: audited by T04 against existing tests (for example `protected_worktree.rs` cancellation tests); no new per-site tests. + +**Deferred maintainability work.** Tracked in the Deferred cleanup ledger below as PR3 handoff; none of it blocks PR2 unless a concrete correctness defect is established. + +**Scope reconciliation.** The "Out of scope" line excluding Git/process/filesystem async conversion below is historical: the implemented PR2 includes that work. AC10 and T05 require the final scope to be stated honestly. + +## Architecture decisions + +PR2 preserves and extends SCE's compile-time/static dispatch model. The entire PR2 propagation closure uses concrete types, generic type parameters, associated types, or static enums for both synchronous and asynchronous dependencies. Every function/module mechanically touched to propagate async DB access converts its existing dynamic dependency-injection and callback seams too. No trait-object DI, boxed futures, `async-trait`, dynamic database/service abstractions, or type-erased command/service futures are permitted inside that closure. This is architectural consistency, not a performance optimization or a repository-wide dyn purge. + +```text +application Tokio runtime + ↓ +static async command dispatch + ↓ +static async service/lifecycle/hook dispatch + ↓ +static async persistence + ↓ +Turso futures +``` + +These decisions apply to the entire migrated call graph, including tests' injection seams. T01 inventories their concrete paths; it does not defer architecture selection until implementation. + +### Synchronous dependencies within the propagation closure + +Pure synchronous dependencies outside the PR2 propagation closure remain out of scope. Any synchronous dependency-injection seam inside a function/module that PR2 must touch is converted to static dispatch as part of the migration. Scope depends on the call graph, not whether a callback returns a future. Pure synchronous operations stay synchronous while their dependency dispatch changes. + +Preserve `cli/src/app.rs`'s architecture: `AppRuntime` owns concrete capabilities, `AppContext` borrows concrete capabilities, and `HasLogger`, `HasTelemetry`, `HasFs`, and `HasGit` retain sized associated types. Prefer these existing capability bounds and `GitOps`/`FsOps` in `cli/src/services/capabilities.rs` wherever they already represent the operation; do not duplicate them as callbacks or regress to `Arc`, `Arc`, or `Arc`. + +Replace all four harness adapters' and ingress conformance's `GitDirResolver = &dyn Fn` aliases with direct generic parameters, borrowed or owned as lifetimes require, conceptually `R: Fn(&str) -> Result`. Keep the existing absolute-Git-dir resolver behavior and diagnostics: GitOps currently has no matching dedicated operation, so this focused test seam remains a monomorphized closure rather than a new resolver framework. Convert `BashPolicyEvaluator` to `P: Fn(&Path, &str) -> Result` with borrowed or owned P; policy fakes remain ordinary closures and policy semantics are unchanged. + +Ingress, conformance, model resolution, and doctor repair use direct generic functions and generic future parameters where borrowing permits; use a narrow native async static trait only for a meaningful capability with multiple operations or borrowing requirements that cannot be expressed by a single lifetime-independent future type. Do not invent general callback framework traits. `MutationScopeRepairSeam` becomes fully static async repair with generic repair/future and sized logger types, sharing the ingress operation where natural; no `&dyn Fn`, `&dyn Logger`, `BoxFuture`, or `async-trait` remains. Conformance methods and synchronous rejection/test callbacks also become generic; synchronous test fakes are not exempt. + +All migrated logger arguments become `Option<&L>` with sized `L: Logger`, or obtain the logger through `C: HasLogger` and its associated type. Do not add `?Sized` bounds that admit trait objects. Use a concrete no-op logger type or separate logged/unlogged helpers if None inference is awkward, and introduce shared ownership only when required. Preserve logger output and ordering. + +Control generic growth: prefer a concrete type for one implementation, then existing capability/associated types, then narrow generic test callbacks, then a small static enum for an existing closed domain. A coherent set of operations may use one narrow static capability, but do not create giant generic plumbing structs. Doctor Git calls/checks use `HasGit`/`GitOps`, filesystem operations use `HasFs`/`FsOps`, and path/config/policy-probe test operations remain focused generic closures with concrete production functions. Preserve doctor Git error-to-None, trimming/empty-output handling and once-per-invocation policy probing exactly. + +### Commands and persistence + +`RuntimeCommand` remains an enum matching concrete command types, with awaits at DB-backed branches. No general dynamic async Command abstraction, `Box`, `Arc`, or boxed command/service futures. Keep `TursoDb`, `EncryptedTursoDb`, `RepositoryAgentTraceDb = TursoDb`, `LocalDb = TursoDb`, and `AuthDb = EncryptedTursoDb`. No `dyn Database`, `dyn Repository`, `dyn Persistence`, `Box`, `Arc`, or dynamic async DB traits. Tests use these concrete/generic types or narrow statically typed seams. + +### Credential store and auth token storage + +Replace the existing dynamic credential seam with native async methods and generic injection: + +```rust +pub trait CredentialStore: Send + Sync { + async fn load(&self) -> Result, ControlPlaneError>; + async fn save(&self, token: &TokenResponse) + -> Result; +} + +pub struct AuthenticatedControlPlaneClient { + http: reqwest::Client, + // Existing concrete configuration fields remain. + credential_store: S, + refresh_lock: Arc>, +} +``` + +Client operations use `S: CredentialStore`; constructors and connected sync/test callers propagate concrete generic types. A borrowed store or statically typed shared ownership is acceptable where ownership requires it. Remove `Arc` and `Box` constructors, rather than retaining them with `async-trait`, `BoxFuture`, or `Pin>`. `SystemCredentialStore` directly awaits token storage; delete production credential load/save `spawn_blocking` calls. Preserve valid-token reuse, expired-token refresh, refresh single-flight/exactly-one refresh, unexpected-401 refresh/retry, second-401 terminal behavior, storage error classification, and exact save/no-save assertions. + +Replace `token_storage`'s `std::sync::OnceLock>` with static `tokio::sync::OnceCell>`. Use the API supported by locked Tokio (currently 1.52.3 with `sync` enabled), conceptually: + +```rust +static AUTH_DB: tokio::sync::OnceCell> = + tokio::sync::OnceCell::const_new(); + +async fn get_auth_db() -> Result<&'static AuthDb, TokenStorageError> { + let result = AUTH_DB.get_or_init(|| async { + AuthDb::new().await.map_err(|error| error.to_string()) + }).await; + // Borrow the cached success or map the cached error through existing diagnostics. + // Keep the initializer output Result-valued; do not use get_or_try_init. +} +``` + +This is an API sketch, not a complete function body. Initialization is single-flight: the first completed success is cached and the first completed failure is cached; later calls must not implicitly retry a failed initialization. Cancellation before an initializer completes leaves no completed result and follows Tokio OnceCell cancellation semantics. Verify these semantics and the exact API against locked dependency source, and test cached success/failure and concurrent initialization without resetting the production static. + +Make `save_tokens`, `load_tokens`, and `delete_tokens` async. AuthCommand login/whoami directly await `load_tokens()`, refresh/login directly await `save_tokens(&token)`, and logout directly awaits `delete_tokens()`. Remove PR1's `run_credential_operation` DB wrapper and preserve all typed `CliError` mappings and secret redaction. Convert control-plane and connected sync fake stores to concrete native async implementations injected generically. Retain load/save/refresh counts, single-flight, 401, and failure assertions. Delete tests whose sole purpose was proving execution outside Tokio; replace them with proof that async stores execute correctly on the application runtime. + +### Lifecycle, hooks, mutation trace, and export + +Make `ServiceLifecycle::diagnose`, `fix`, and `setup` native `async fn` with their existing result types. Keep identity/metadata methods synchronous. `LifecycleProvider` remains the static enum dispatcher: its matching concrete providers are awaited through `diagnose().await`, `fix().await`, and `setup().await`. Doctor awaits diagnosis/fixes and Setup awaits setup: `RuntimeCommand → Doctor/Setup → LifecycleProvider → concrete provider → async persistence`. Config/Hooks providers execute their existing pure synchronous work inside the native async methods; no `spawn_blocking` for pure lifecycle work, `Box`, `async-trait`, or boxed lifecycle futures. + +DB hooks follow `RuntimeCommand::Hooks → await HooksCommand → static enum match → specific handler → await RepositoryAgentTraceDb → await Turso`. Pure hook paths may remain synchronous internally. No boxed futures, trait-object handlers, `async-trait`, DB `spawn_blocking`, or executor bridge. Foreground hooks return only after required persistence completes; the existing detached post-commit auto-sync child remains the only authorized detached hook-adjacent execution. + +Propagate awaits from `execute_transactional_cas_batch` and other DB primitives through `MutationTraceStore → mutation-trace runtime/coordinator → provider/closure seams → hook adapter → RuntimeCommand` (read from persistence toward callers). Prefer native async functions; where injection genuinely needs callbacks, use `F: FnOnce(...) -> Fut, Fut: Future` with concrete generic futures and appropriate borrowed lifetimes. No boxed future callbacks, `block_on`, or `spawn_blocking` to stop propagation. + +Make `AgentTraceExportReader` and connected repository reads async as required, and await them directly in the existing async sync engine. Preserve one initial state request, three concurrent remote streams, sequential batches within each stream, cursor reconciliation, retries, `diff_traces` compatibility reporting, and progress ordering. No intermediate executor or behavioral redesign. + +### Blocking and storage lifetime + +No async DB work inside `spawn_blocking`. No production path from the application Tokio runtime to a Turso future crosses a synchronous executor boundary. Prohibit `block_on`, executor bridges, runtime-containing worker threads, and DB futures scheduled inside blocking workers, including token-storage load/save/delete wrappers. Every remaining production `spawn_blocking` or `block_in_place` must terminate in a genuinely synchronous/blocking external API and be individually justified. A synchronous OS keyring API used for encryption-key acquisition may be isolated in `spawn_blocking`; the worker must contain only the synchronous keyring work, never AuthDb construction or Turso futures. + +Remove PR1 DB-runtime-only blocking regions for Setup, Doctor, DB-backed Hooks, sync storage construction, and `SyncStorageGuard` Drop. Ask: **Does SyncStorageGuard still express any invariant ordinary Rust ownership cannot express?** If no, remove it and use scoped ownership. Storage must outlive all futures borrowing it and drop safely on cancellation/errors, without detached cleanup tasks or leaks. A remaining guard requires a documented independent invariant and cannot retain blocking Drop for a removed DB runtime. + +### PR roadmap and handoff + +- PR1: Application-owned Tokio runtime — complete. +- PR2: Async Turso persistence plus mechanically required async propagation: DB core, token/credential storage, hooks, mutation trace, lifecycle, setup/doctor, sync/export DB reads, removal of the DB runtime bridge and obsolete DB-lifetime blocking islands. + - Review amendment (2026-10-08): the implemented PR2 also contains Git/process/filesystem async work and its cancellation protections (see Current state), and PR2 now additionally owns transaction isolation and cancellation evidence (T03), credential-rotation and terminal-sync correctness (T06), and adapter-boundary and guard-descriptor safety (T07), proven by a minimal focused test set rather than restored legacy suites. These are merge prerequisites, not PR3 work. +- PR3: Post-migration cleanup and runtime audit: simplify mechanically awkward signatures, remove obsolete compatibility helpers, classify remaining blocking work, and ensure no accidental dynamic dispatch/executor bridge remains. No behavioral redesign or deferred required propagation. Also owns the Deferred cleanup ledger items marked PR3 (cosmetic and maintainability only; never correctness or coverage). +- PR4: OpenTelemetry. + +T05 updates existing architecture staging documentation to this roadmap. + +## Acceptance criteria + +How this plan is proven complete. Each criterion is observable and names the +check that proves it. `/validate` runs these checks; no task in the stack +performs final validation. + +- [x] AC1: Production persistence owns zero Tokio runtimes; build_current_thread_runtime and block_on_isolated are deleted and no replacement synchronous executor drives DB futures. + - Validate: `nix shell nixpkgs#ripgrep -c rg -n 'build_current_thread_runtime|block_on_isolated|Builder::new_current_thread|runtime\.block_on|Handle::block_on|tokio::runtime::Runtime|block_in_place|try_current|\.block_on\(' cli/src`; classify every match, separating test fixtures from production and confirming only application runtime ownership. Inspect for futures/pollster executors, runtime-containing threads, global runtime singletons, and async DB work inside spawn_blocking. +- [x] AC2: Both shared Turso adapters, their migration/transaction helpers, and all connected production DB consumers required to compile use awaited native operations on the existing caller runtime. + - Validate: Inspect the async API/call-chain inventory against source in `cli/src/services/db/`, `agent_trace_db/`, `auth_db/`, `local_db/`, `agent_trace_storage/`, `token_storage.rs`, auth/control-plane callers, exports, mutation-trace, lifecycle, setup/doctor, hooks, sync, and command dispatch. Tests A–C (multi-thread Tokio, real databases) exercise open, execute/query and transactions without nested runtimes; no deleted suite is required. +- [x] AC3: Transaction integrity — SQL and one-connection scope remain unchanged (BEGIN IMMEDIATE, commit/rollback, idempotent insert pairs, CAS row-count guards, deterministic errors, whole-transaction contention retry) and transaction exclusivity is structurally enforced: exclusive mutable ownership spans an entire transaction (`&mut` receivers and checked `Transaction::new(&mut connection, Immediate)` where the pinned dependency supports it, otherwise a recorded equivalent static proof), so no other operation on the same connection can enter an active transaction. Owner T03. + - Validate: Inspect transactional signatures and caller chains for exclusive borrows; `nix shell nixpkgs#ripgrep -c rg -n 'new_unchecked' cli/src` finds no unjustified transaction construction; review the SQL/statement-order diff; Test A (runtime test or documented compile-time argument); Quint MBT stays green. The deleted `db/mod.rs`, repository and store suites are not required. +- [x] AC4: Migration discovery, IDs, numeric ordering, SQL, metadata tracking, per-file transaction behavior, baseline upgrade/rebuild fixtures, and failure behavior match the current implementation. + - Validate: Migration behavior is verified by source/diff inspection and the retained tests (including the Quint MBT suite); no deleted migration suite is required. `git diff -- cli/build.rs cli/migrations` shows no migration generation/SQL changes; inspect actual migration source paths from the T01 inventory too. Confirm migration execution remains outside open/connect retries, and execute_batch plus separate metadata insertion retain their current boundaries. +- [x] AC5: Generic retries retain existing attempts/backoff, post-attempt timeout diagnostics, logging and exhaustion behavior; Agent Trace contention retains Busy/BusySnapshot classification, exactly two bounded attempts, jitter 0..=100ms, configured busy timeout/deadline, and pre/post-sleep admission. Owner T03. + - Validate: Source/diff confirmation that the numeric policies (attempts, backoff, jitter, busy timeout, deadline admission) are unchanged and that DB paths use the non-cancelling `run_with_retry_elapsed` rather than the cancelling `resilience::run_with_retry`; plus Test C, one focused retry/timeout/contention regression (reuse a surviving test if it already covers this). The deleted retry suite is not recreated. +- [x] AC6: Multiprocess WAL, passive checkpoint policy, lock-overlap behavior and duplicate suppression remain intact. Owner T03. + - Validate: Test C as the representative real contention verification on independent connections (or processes) at one chosen supported tested load, with no lost or duplicate durable writes and no exhausted retries; passive checkpoint mode and WAL flags confirmed by source inspection. The previous writer/round stress matrices (distinct 2/1000, 3/1000, 4/500; duplicate 2/500, 3/500, 4/500; real hook processes 2/500, 3/500, 4/200) are optional characterization, required only if a specific failure makes them necessary. A filter matching zero tests never counts. +- [x] AC7: Repository identity/metadata convergence, no-migration hook readiness, no-touch legacy paths, auth encryption/configuration/secret-safe diagnostics, credential behavior, and LocalDb behavior are unchanged. Credential behavior is covered with AC14. Owner T06 for credentials; other items by source/diff inspection. + - Validate: Inspect AuthCommand's direct awaited load/save/delete paths, unchanged typed `CliError` mappings and secret redaction, and removal of `run_credential_operation`; Test D plus any surviving existing coverage for valid reuse, 401 retry and terminal second-401 (the 28 deleted control-plane tests are not restored). +- [x] AC8: Transaction cancellation — newly cancellable transactions/writes cannot commit partial state or panic on resource Drop; storage outlives borrowing sync futures; error/cancellation cleanup is safe without a DB-owned runtime. Owner T03. + - Validate: Test B (cancel after the first write and before commit: no partial committed state, safe connection reuse, a later transaction succeeds, an independent connection sees the correct durable result) plus the compile-time exclusivity argument from AC3. State exact guarantees and upstream Turso 0.8.1 limitations (deferred rollback; a completed commit may already be durable) rather than assuming immediate Drop rollback. No per-helper or per-await-point tests. +- [x] AC9: Blocking audit — zero DB-runtime-related block_in_place, zero blocking SyncStorageGuard Drop, and zero token-storage DB spawn_blocking. Every remaining production spawn_blocking/block_in_place has an individually documented genuinely synchronous/blocking external API reason independent of DB runtime ownership. **Review amendment (owner T04, after T06/T07):** the audit also covers encryption-key lookup, adapter state transactions, adapter boundary-lock and worktree-lock acquisition, external-taint marker persistence/clearing, cancellation-shielded Git ref mutations, guarded-shell supervision and temporary-index destructor cleanup, recording per site the six ownership/cancellation/shutdown answers listed in T04; `spawn_blocking` that provides necessary cancellation safety is retained, not mechanically removed. + - Validate: Audit `cli/src/services/command_registry.rs` Setup/Doctor/Hooks sites and `cli/src/services/sync/sync.rs` storage construction/SyncStorageGuard Drop against `nix shell nixpkgs#ripgrep -c rg -n 'block_in_place|spawn_blocking|SyncStorageGuard' cli/src` and Source audit. Source inspection plus the focused tests owned by T03/T06/T07 and existing tests (for example `protected_worktree.rs` cancellation tests) cover the affected paths; no separate lifetime test matrix is required. Ordinary synchronous filesystem/process work is assessed independently. +- [x] AC10: The expanded PR2 scope consists only of required mechanical DB propagation and lifetime/cancellation correctness; it introduces no unrelated behavior, schema/protocol change, dynamic DB framework, artificial locking layer, or OpenTelemetry work. **Review amendment — scope honesty (owner T05):** the original wording "Check no new async filesystem/process conversion" is not satisfied by the implementation, which includes Git/process/filesystem async work. The final PR scope is explicitly reconciled with that implemented work (what was migrated, why it was required, which cancellation protections it carries) in the plan, the PR description and context, rather than pretending it did not occur; still no unrelated behavior, schema/protocol change, dynamic DB framework or OpenTelemetry work. + - Validate: Inspect changed files and `nix shell nixpkgs#ripgrep -c rg -n 'BoxFuture|async_trait|dyn .*Db|Arc uses S: CredentialStore with native async load/save and concrete async fake injection; SystemCredentialStore directly awaits production token storage. No credential DB operation runs in spawn_blocking; refresh/single-flight/401 behavior remains unchanged. A received replacement refresh token cannot be lost to caller cancellation or the outer `/state` timeout before durable persistence is attempted; the single-flight guard spans refresh and persistence; the remote-operation ambiguity and crash limitations are documented. Owner T06. + - Validate: Inspect generic client/store implementations and connected sync callers; Test D (cancelled refresh cannot lose persisted credentials, with single-flight where feasible) plus existing coverage where available. The deleted control-plane suite is not restored. +- [x] AC15: Lifecycle — native async diagnose/fix/setup propagate through Doctor/Setup and LifecycleProvider's static enum matches to concrete providers and async DB persistence. Pure Config/Hooks work stays synchronous internally; no boxed providers/futures or pure-work spawn_blocking. + - Validate: Inspect every provider and both command chains; run any surviving lifecycle/setup/doctor tests; otherwise compilation and inspection suffice. +- [x] AC16: Auth token storage — async Result-valued OnceCell initialization caches both completed success and completed failure, never implicitly retrying a returned initialization error. load_tokens/save_tokens/delete_tokens are directly async; run_credential_operation is removed. + - Validate: Check locked Tokio API/source (recorded in T01) and the Result-valued `get_or_init` usage so a cached failure never re-invokes the initializer; no new test unless that cannot be established by inspection. Inspect cancellation-before-completion separately and preserve diagnostics/secret safety. +- [x] AC17: End-to-end executor invariant — No production path from the application Tokio runtime to a Turso future crosses a synchronous executor boundary. + - Validate: Trace every DB caller class (credentials/auth, lifecycle/setup/doctor, hooks/mutation trace, sync/export, repository/local persistence) from static RuntimeCommand dispatch to Turso. Source audit plus inspection must exclude block_on substitutes, runtime-containing workers, boxed callback executors, and DB futures inside spawn_blocking/block_in_place. Foreground hooks await required persistence; only the existing detached post-commit auto-sync child remains authorized. + +The criteria below were added by the 2026-10-08 review update. AC1–AC17 are retained and amended only where they named deleted suites or zero-match filters; safety requirements are not weakened. + +- [x] AC18: **Minimal safety regression coverage (owners T03, T06, T07).** The required evidence is the smallest set of tests that proves distinct, material correctness invariants — Tests A–G in the Minimal safety test set (guidance: roughly 6–10 new tests, fewer where existing tests, types or Quint suffice). Tests execute and carry meaningful assertions. No test-count parity, percentage-based coverage threshold, or automatic restoration of deleted tests. A command that matches zero tests never counts as passing. + - Validate: For each of Tests A–G, ` -- --list` shows the intended test (or the test is explicitly replaced by a documented compile-time argument) and the run passes; each test's completion record states the regression it catches that existing tests, types and Quint do not. +- [x] AC19: **Boundary cancellation (owner T07).** Cancellation cannot permit an older state operation to invalidate or overtake a newer protected boundary. + - Validate: Test F plus inspection of the chosen lock-ownership design against source. +- [x] AC20: **FD inheritance (owner T07).** Unrelated spawned subprocesses cannot inherit the guarded shell's protected descriptors; the intended guarded child does. + - Validate: Test G plus inspection of descriptor creation/duplication and the post-fork child setup. +- [x] AC21: **Terminal sync (owner T06).** Terminal stream errors prevent unnecessary additional batches without cancelling required credential persistence. + - Validate: Test E. +- [x] AC22: **Deferred findings are tracked (owner T05).** Every lower-priority finding appears in the Deferred cleanup ledger as a PR3 handoff or documented limitation, and no cosmetic cleanup is mixed into T03–T07. + - Validate: Inspect the ledger against the final diff and T05's context records. + +### Acceptance criteria ownership map + +| AC | Owning task(s) | Primary verification | +| --- | --- | --- | +| AC1, AC17 | T04 | Source audit; caller trace | +| AC2, AC13, AC15, AC16 | T02 (done); re-checked by T04 | Source audit; inspection | +| AC3, AC8 | T03 | Exclusive-borrow inspection; Tests A, B | +| AC4 | T02 (done); T03 inspection | `git diff -- cli/build.rs cli/migrations`; Quint/flake check | +| AC5, AC6 | T03 | Test C; numeric-policy source diff | +| AC7, AC14 | T06 | Test D; inspection | +| AC9 | T04 | Source audit + retained-site table | +| AC10, AC11, AC22 | T05 | Scope reconciliation; context/ADR inspection; ledger | +| AC12 | `/validate` | Full validation | +| AC18 | T03, T06, T07 | Tests A–G | +| AC19, AC20 | T07 | Tests F, G | +| AC21 | T06 | Test E | + +### Source audit + +Run these through Nix, classify every match by production/test, new/pre-existing, migrated/unrelated, and record exact sites/reasons. Searches are evidence, not a substitute for tracing calls and inspecting aliases or differently spelled executor/type-erasure APIs. + +```bash +nix shell nixpkgs#ripgrep -c rg -n \ + 'build_current_thread_runtime|block_on_isolated|Builder::new_current_thread|runtime\.block_on|Handle::block_on|\.block_on\(|block_in_place|spawn_blocking|tokio::runtime::Runtime' \ + cli/src +nix shell nixpkgs#ripgrep -c rg -n \ + 'dyn CredentialStore|dyn ServiceLifecycle|dyn .*Db|BoxFuture|Pin, boxed async APIs, dynamic traits or artificial Send/Sync bounds. +- **Constraints:** use Nix and repository-supported Cargo wrapper. Generated target trees remain absent. No canonical generation inputs are expected to change; if a mechanically required change touches them, add `nix run .#pkl-check-generated` to relevant checks and full validation without generating committed trees. +- **Non-goal:** making unrelated synchronous service logic async or designing future telemetry integration. The required native async lifecycle seam may wrap existing pure synchronous provider work without changing that work. + +## Assumptions + +- Core async API changes and their connected caller adaptations form one coherent compile-safe migration commit because aliases/direct calls provide no independent synchronous adapter seam; task-local editing may be sequential, but no broken or bridge-bearing intermediate commit is planned. +- PR1 application runtime is complete. PR2 completes async persistence and its mechanically required caller closure, static credential/lifecycle seams, and obsolete DB-runtime blocking removal. PR3 is post-migration cleanup/runtime audit; PR4 is OpenTelemetry. This plan implements neither follow-up and leaves no required propagation or executor bridge for PR3. +- Existing dependencies, schema and test assertions are authoritative. Any discovered unrelated bug is reported separately. + +## T01 source inventory (2026-10-07) + +Audited baseline: `9ceff8414613a4a619cf505ae9ac0e3eb46db69f`, clean staged/unstaged/untracked state. This is the pre-migration inventory; target signatures below are instructions for T02, not claims that async persistence already exists. All paths and line numbers refer to that baseline. + +### Persistence APIs and unchanged policies + +- `cli/src/main.rs:12` owns the application multi-thread runtime through `#[tokio::main]`; `cli/src/app.rs` directly awaits static command dispatch. `cli/src/services/command_registry.rs:26` owns `RuntimeCommand`. Persistence currently owns additional runtimes in `cli/src/services/db/mod.rs:457` (`TursoConnectionCore` contains one `turso::Connection`, one `tokio::runtime::Runtime`, and `PhantomData M>`). `build_current_thread_runtime` at 178 constructs them; `block_on_isolated` at 198 either blocks directly or starts a scoped OS thread to drive the DB runtime when `Handle::try_current()` succeeds. Delete the field, constructor and bridge; retain concrete connection ownership and static `DbSpec` metadata. +- `cli/src/services/db/mod.rs:893–1454`: await `TursoDb::{new,new_at,open_without_migrations,open_without_migrations_at,execute,execute_idempotent_write,query,query_values,query_map,execute_transactional_insert_pair_if_absent,execute_transactional_cas_batch,run_migrations,passive_checkpoint,migration_metadata_problems,ensure_schema_ready}`. `query` returns native `Rows`, whose subsequent `next()` is async; `query_values` fetches owned column/value collections; `query_map` fetches all native rows inside the retry, then runs synchronous mapping outside it. Keep parameter conversion before retries and clone the same owned params per attempt. `TransactionStatement::{new,expect_rows_affected}`, row mappers, SQL builders, config/path resolution and filesystem health helpers remain synchronous. +- `cli/src/services/db/mod.rs:1475–1651`: await `EncryptedTursoDb::{new,execute,query,query_map,run_migrations}` and shared core migrations. Preserve key acquisition before encrypted open, `experimental_encryption(true)`, `aegis256`, path-qualified secret-safe failures, and the absence of a busy handler on encrypted connections. Unlike the unencrypted builder, the encrypted builder currently does **not** set `experimental_multiprocess_wal(true)`; do not silently change its flags. +- Aliases stay static: `cli/src/services/agent_trace_db/repository.rs:114` has `RepositoryAgentTraceDb = TursoDb`; `cli/src/services/local_db/mod.rs:36` has `LocalDb = TursoDb`; `cli/src/services/auth_db/mod.rs:37` has `AuthDb = EncryptedTursoDb`. LocalDb has no service tables/migrations, but shared initialization still creates `__sce_migrations`. No dynamic DB trait exists. +- `cli/src/services/agent_trace_db/repository.rs:120–329`: await hook opens/readiness, narrow metadata repair and its private object queries, metadata select/initialize/claim, diff/intersection/Agent Trace/model-state/message/part/conversation insert/read methods. `cli/src/services/agent_trace_db/mod.rs:292–539` contains the generic typed SQL wrappers `insert_diff_trace_with`, `insert_post_commit_patch_intersection_with`, `insert_agent_trace_with`, `upsert_claude_model_state_with`, `claude_model_state_by_session_and_agent_with`, `insert_message[s]_with`, `insert_part[s]_with`, `insert_conversation_text_event_with`, `recent_diff_trace_patches_with`; all DB work becomes awaited, while parsing/mapping stays synchronous. Preserve multi-row statement construction and idempotent conversation pairing. +- `cli/src/services/db/mod.rs:290,412,1146,1281`: the two transaction primitives use `Transaction::new_unchecked(&conn, Immediate)` on the **same logical connection**. Insert pair: existence check → first insert → optional injected failure → second insert → commit (including no-op duplicate). CAS: guard rows `0` means false/no body, `1` permits ordered body, other counts are deterministic failure; optional body row expectations are enforced. Body failures explicitly await rollback, ignoring rollback errors as today; commit failures remain classified. Retry unit is the whole acquisition/body/commit, never a single constituent statement. Preserve non-Agent-Trace CAS deterministic outcomes carried outside generic retry. +- `cli/src/services/resilience.rs:103` owns generic `run_with_retry_sync`: success wins even after the configured elapsed threshold; only a returned error is labelled `attempt N exceeded ... and failed`. Its async DB counterpart must await a generic `FnMut(u32) -> Fut`, measure elapsed after completion and await backoff without `tokio::time::timeout`. Existing `run_with_retry` at 36 cancels attempts and remains for HTTP; it is not the DB substitute. Open fallback is 3 attempts / 1000ms diagnostic threshold / 25..200ms capped exponential backoff; query fallback is 5 / 200ms / 25..100ms. Actual open sleeps are 25,50ms and query sleeps 25,50,100,100ms. Config overrides are resolved for **all three** aliases by `resolve_connection_open_retry_policy` / `resolve_query_retry_policy` at 477/494 via `config::{get_database_retry_config,DATABASE_RETRY_CONFIG}`; do not change that synchronous config cache. +- `cli/src/services/db/mod.rs:525–746`: Agent Trace busy timeout defaults to 1000ms and contention admission deadline to 2250ms (configurable); other specs resolve zero. Only explicit idempotent writes and both transaction primitives opt into typed Busy/BusySnapshot retries, at most 2 attempts, full jitter `0..=100ms`. Before sleeping require positive remaining deadline and `remaining >= backoff + busy_timeout`; afterward require `remaining >= busy_timeout`. No cancellation deadline, generic retry wrapping, or deterministic-error retry is added. Exhaustion preserves exact error fields and one `sce.agent_trace_db.contention_exhausted` warning. Generic reads/execute/checkpoint remain on generic retry. +- Sleep conversions needed for DB propagation: `resilience.rs:159` generic retry backoff, `db/mod.rs:634` contention backoff, `agent_trace_storage/mod.rs:181` initialization backoff (20 attempts / 100ms between failed initializations). Preserve duration, placement and replay unit; adapt the injected sleep/clock/backoff/attempt callbacks statically. Do not retain `rand::ThreadRng` across an await when a caller requires a Send future: draw before awaiting and reacquire for later draws; the production full-jitter distribution and seeded-test reproducibility must remain. OS-lock polling sleeps in `hooks/mutation_scope_lock.rs`, `hooks/claude_mutation_scope/state.rs`, and `mutation_trace/runtime/worktree_lock.rs` are independent synchronous work, not DB retry conversions. +- `TursoDb::open_without_migrations_at` at 935 sets multiprocess WAL, builds/connects inside open retry, then installs the connection busy timeout. `new`/`new_at` run migrations **after** open retry. `passive_checkpoint` at 1355 queries/fully consumes `PRAGMA wal_checkpoint(PASSIVE)` under generic retry; retain passive mode and required foreground completion. + +### Migration sources and boundaries + +- Canonical SQL is in `cli/migrations/`, not `config/lib` or generated target trees. `cli/build.rs:545,623–694` discovers immediate database directories and immediate `.sql` files, validates numeric prefix before `_`, uses filename stem IDs, sorts by numeric prefix then full ID, and emits `OUT_DIR/generated_migrations.rs` including staged `OUT_DIR/static/migrations/...`. `cli/src/main.rs:5` includes that manifest. Producer staging is owned by `scripts/produce-cli-generated-input.sh` and build.rs; leave generation unchanged. +- Repository list: `cli/migrations/agent-trace-repository/001_repository_schema.sql`, `002_repository_source_instance_id.sql`, `003_claude_model_state.sql`, `004_mutation_trace_protocol.sql`, `005_mutation_scope_provenance.sql`, `006_mutation_trace_health_invariant.sql`. Auth list: `cli/migrations/auth/001_create_auth_tokens.sql`, `002_create_auth_credentials_updated_at_trigger.sql`. LocalDb returns `&[]` rather than an embedded list. +- `cli/src/services/db/mod.rs:211–286` ensures `__sce_migrations`, checks each ID, awaits `execute_batch(sql)`, then separately inserts metadata. There is no shared per-file transaction wrapper: preserve SQL-owned boundaries (notably the transaction/table rebuild in 006), multi-statement behavior, separate metadata insertion and failure diagnostics. Migrations are outside generic open/query retries; do not “fix” this by transactional_batch or wrapping metadata in a new transaction. +- `cli/src/services/agent_trace_storage/mod.rs:150–212`: lifecycle/setup open first tries no-migration open, narrow metadata repair/readiness and repository metadata; falls back to migrated construction with bounded initialization recovery. Hook open at 199 never falls back to migrations. `repository.rs:134` repair verifies all required tables before recording missing baseline metadata, then checks readiness. Initial metadata insertion and guarded source-instance claim converge across independent first openers. Preserve no-touch legacy DB paths and repository/auth separation. + +### Required callers and dispatch conversions + +| Boundary and exact source (relative to `cli/src/services/`) | Current dispatch | Target dispatch | Fixed T02 target mechanism | +| --- | --- | --- | --- | +| `db/mod.rs` core and aliases above | Concrete/generic, synchronous public APIs driving native futures | Static | Concrete/generic native async APIs; no executor boundary | +| `agent_trace_storage/mod.rs:51–212`, including `open_storage_with` at 134 | Concrete storage plus synchronous generic opener callback | Static | Async resolve/open helpers; concrete awaited opener selected statically; where callback retained use generic future bounds and fixed borrowed lifetimes or owned path/ID inputs | +| `token_storage.rs:16–18,73,109,148` | Static `OnceLock>`, sync load/save/delete | Static | Static Result-valued Tokio OnceCell; async initializer and direct async token operations | +| `agent_trace_sync/control_plane.rs:227–255,293–325,523–550` | Dynamic sync `CredentialStore`, `Box` constructor, `Arc` field, DB blocking workers | Static | Native async `CredentialStore`, `AuthenticatedControlPlaneClient` with `S: CredentialStore`; concrete default constructor and generic injected stores; direct awaited load/save | +| `auth_command/mod.rs:64–208,438–449`, `auth_command/command.rs` | Static async command calling blocking `run_credential_operation` | Static | Direct load/login/whoami, save/refresh/device-login, delete/logout awaits; delete worker helper and worker-only tests; retain `auth_storage_error` typed mappings and redaction | +| `lifecycle.rs:165–256`, `config/lifecycle.rs`, `{local_db,auth_db,agent_trace_db,hooks}/lifecycle.rs` | Static generic trait and static enum | Static | Native async diagnose/fix/setup with unchanged result types; enum awaits concrete providers; id/order remain synchronous, pure provider work stays synchronous internally | +| `doctor/{command.rs,mod.rs,inspect.rs,fixes.rs}`, `setup/command.rs` | Concrete sync services; provider calls and Doctor mutation repair | Static | Async orchestration from static commands through lifecycle and DB-capable repair; replace iterator closures that require await with ordered loops | +| `hooks/{command.rs,mod.rs,runtime.rs,commit_hooks.rs,diff_trace.rs,conversation_trace.rs,claude_model_state.rs}`, `hooks/codex/{mod.rs,apply_patch/mod.rs,stop.rs,user_prompt_submit.rs}` | Static subcommand/event matches; concrete DBs plus synchronous generic persistence/query callbacks | Static | Native async DB-capable handlers and static awaits through command; generic future callbacks for retained query/persist seams; pure payload parsing/clock/Git/snapshot/render logic stays synchronous | +| `hooks/{claude,codex,opencode,pi}_mutation_scope/{mod.rs,lifecycle.rs,health.rs}` (where present), `hooks/mutation_scope_ingress_conformance.rs:12,29–34` | Dynamic borrowed `IngressSeam = &dyn Fn(...)->Result`; synchronous adapter state machines | Static | Generic ingress/future parameters and sized generic logger; direct generic functions preferred, narrow native async static capability only for meaningful operations/borrowed lifetimes; await all DB ingress/recovery/repair and convert synchronous conformance fakes too | +| `doctor/inspect.rs:233–293` `MutationScopeRepairSeam` | Dynamic sync callback with `Option<&dyn Logger>` to mutation-scope ingress | Static | Generic async repair/future and sized `L: Logger`, sharing static ingress operation; borrowed or owned repair as lifetimes require; preserve target/order/final-health rendering | +| `hooks/claude_mutation_scope/lifecycle.rs:34–35` `ClaudeModelStateResolver` | Dynamic sync callback doing model-state DB read | Static | Generic async resolver/future with concrete production/fake implementations; native async static method when borrowed lifetimes require it; no boxed resolver | +| `hooks/{claude,codex,opencode,pi}_mutation_scope/lifecycle.rs:18,23,65,62` respectively; `hooks/mutation_scope_ingress_conformance.rs:10` `GitDirResolver` and adapter conformance implementations in each harness `tests.rs` | Dynamic synchronous `&dyn Fn(&str) -> Result` | Static | Direct generic `R: Fn(&str) -> Result`, borrowed `&R` or owned R; ordinary monomorphized closure fakes; retain existing resolver/diagnostics rather than duplicating GitOps with a new trait | +| `hooks/codex_mutation_scope/lifecycle.rs:27–28` `BashPolicyEvaluator`, including preflight at 349 | Dynamic synchronous `&dyn Fn(&Path, &str) -> Result` | Static | Direct borrowed/owned generic P with the same Fn signature; concrete production function and generic policy fakes, unchanged decisions | +| `doctor/mod.rs:55–56` `run_git_command`, `check_git_available` | Dynamic synchronous Git callbacks | Static | Existing `HasGit` associated type / `GitOps::run_command` and `is_available`; retain error-to-None, trimming and empty-output mapping; filesystem work naturally uses `HasFs`/`FsOps`, not duplicate callbacks | +| `doctor/mod.rs:57–59,66` `resolve_state_root`, `resolve_global_config_path`, `validate_config_file`, `probe_codex_hook_policy`; dependent inspection/filesystem helpers | Dynamic synchronous path/config/probe callbacks; concrete filesystem calls | Static | Focused generic test closures with concrete production functions for operations not represented by existing capabilities; existing `HasFs`/`FsOps` for matching filesystem operations; preserve probe-once/order semantics without a giant callback plumbing struct | +| `hooks/{mod.rs,commit_hooks.rs,diff_trace.rs,conversation_trace.rs,claude_model_state.rs,mutation_scope.rs}`, `hooks/codex/{mod.rs,apply_patch/mod.rs}`, all four mutation-scope `lifecycle.rs` and connected health/test fakes, `doctor/inspect.rs:234,260,4784` | Dynamic `Option<&dyn Logger>` (including qualified Logger spellings and closure arguments) | Static | Existing `HasLogger` associated type at context entrypoints; sized generic `L: Logger` and `Option<&L>` at leaf/ingress/repair helpers; concrete logger/unlogged helper for None inference, no trait-object-permitting `?Sized` or unnecessary Arc | +| `mutation_trace/runtime/coordinator.rs:722–748` `HookedFailingCapture` | Dynamic test-only `Box` snapshot fault hook in a migrated module | Static | Generic `H: FnMut()` field/constructor/implementation for pure fault injection; DB-bearing fake operations move to static async test seams; test-only placement does not exempt migrated DI | +| `hooks/mutation_scope.rs:285–315,528–590` | Generic sync open/coordinate/abandon callbacks and guard protocol | Static | Await static generic providers (`FnOnce() -> Fut`), coordinator/abandon and guard DB finalization; retained callbacks with borrowed structured payloads use native async generic seam methods, avoiding impossible lifetime-independent future bounds | +| `mutation_trace/store.rs:530–950` | Concrete borrowed store over RepositoryAgentTraceDb | Static | Store constructor/encoding/transition computation stay sync; initialize/register/load/page/provenance/tree-root/commit queries become async; transactional store holds an exclusive mutable DB borrow and commit takes a mutable store borrow, awaiting the existing CAS primitive | +| `mutation_trace/runtime/{coordinator.rs,scope_runtime.rs,ref_reconciliation.rs}` | Generic synchronous DB providers and injected after-load/recovery hooks; static `SnapshotCapture` | Static | Async entrypoints and DB error/recovery/commit helpers; `P: FnOnce() -> Fut`, concrete futures; DB-using injected hooks get async generic bounds, pure snapshot methods remain sync | +| `mutation_trace/runtime/mutation_attribution.rs:28–58,118–434` | Generic `MutationEventPageSource` and synchronous Git-only `TreeReadSource`, with trait-object-admitting `?Sized` bounds | Static | Native async page/provenance trait methods and awaited window/project/post-commit attribution; sized generic P/R, remove DI `?Sized` bounds, keep R tree operations synchronous | +| `mutation_trace/runtime/external_mutation_guard.rs:530–807` | Concrete `ArmedExternalMutationGuard

` with generic deferred DB opener | Static | Keep static P plus future bounds; `exec` and run helpers await final `coordinate_on_held_worktree` after existing synchronous process supervision. No DB future in control-reader thread or blocking worker; preserve lock/taint ownership and descendant lifetime protocol | +| `agent_trace_export/mod.rs:146–267` | Concrete borrowed reader with four sync DB reads | Static | Async `read_{messages,parts,diff_traces,agent_traces}_after`; constructor, validation and row mappers remain sync | +| `agent_trace_sync/mod.rs:162–179` `SyncFuture`, `sync/sync.rs:346–538` stream callbacks | Dynamic future dispatch: generic closures returning **type-erased** `Pin>`; sync reads wrapped in ready futures | Static | Remove SyncFuture type erasure from the migrated engine. Use a native async statically generic stream-operations seam with associated row type and concrete per-stream adapters; read/ingest/refresh methods can borrow rows through native async signatures. Preserve the existing cursor algorithm and test fake operations. No boxed callbacks or executor substitute | +| `sync/sync.rs` client parameters/test factories, `sync/command.rs`, `command_registry.rs:78–109` | Concrete async sync; static commands with DB compatibility scopes | Static | Propagate generic credential-store type through injected sync clients; await async storage/export/commands directly. Pin concurrent concrete futures without type erasure (stack pinning suffices); no Send/'static/spawn requirement for borrowed progress/writers | + +No dynamic DI seam in the PR2 propagation closure remains a deferred design choice: ingress/repair/model-resolution become static generic async seams, Git-dir/policy/conformance callbacks become generic, doctor reuses existing capabilities and focused generic callbacks, logger arguments use sized generics/HasLogger, SyncFuture becomes static generic stream operations, credentials become generic S, and lifecycle/commands remain enums. Pure synchronous dependencies outside the PR2 propagation closure remain out of scope. Any synchronous dependency-injection seam inside a function/module that PR2 must touch is converted to static dispatch as part of the migration. The already generic HRTB insert callbacks in `hooks/commit_hooks.rs:247,458` need borrowed native async seam methods rather than boxed futures; local snapshot/trace payloads must outlive the awaited insert. + +### Dynamic seam audit classification + +The two dynamic-seam Source audit commands plus the supplemental dyn search cover every current match as follows. Line references describe the T01 baseline; re-run searches after migration and record exact resulting sites. No matching DI/future seam is preserved inside a migrated production module. + +| Current match sites (relative to `cli/src/services/`) | Classification | Reason / T02 disposition | +| --- | --- | --- | +| All matches in `hooks/{mod.rs,commit_hooks.rs,diff_trace.rs,conversation_trace.rs,claude_model_state.rs,mutation_scope.rs}`, `hooks/codex/{mod.rs,apply_patch/mod.rs}`, all four harness `lifecycle.rs`, `doctor/{mod.rs,inspect.rs}` | removed by PR2 | DB-backed caller modules must be mechanically changed; convert every resolver, ingress, policy, repair, doctor dependency and logger DI signature/alias, including qualified names and inline fakes, by the table above | +| All matches in `hooks/mutation_scope_ingress_conformance.rs`, all four harness `tests.rs`, Claude/OpenCode/Pi `health.rs`, Pi `lifecycle_tests.rs` | removed by PR2 | Shared conformance helpers and adapter/health test callbacks exercise migrated ingress; convert aliases, fake closure logger parameters and trait-object-admitting callback bounds even when a fake is synchronous | +| `agent_trace_sync/control_plane.rs:255,313` | removed by PR2 | Replace Arc/Box credential trait objects with generic S and native async concrete stores | +| `agent_trace_sync/mod.rs:162` | removed by PR2 | Delete SyncFuture type erasure in favor of static stream operations | +| `mutation_trace/runtime/coordinator.rs:724` | removed by PR2 | Generic fault hook; this test DI sits in a mechanically migrated module | +| `parse/command_runtime.rs:10` `Option<&dyn LoggerTrait>` | unrelated untouched legacy seam | Argument parsing only constructs a RuntimeCommand before execution (`cli/src/app.rs:408`); it never runs persistence or calls async services, and its signature/module needs no DB propagation change. Leave this exact seam for later cleanup; if PR2 mechanically changes this module, convert it too | + +No `test-only unrelated seam` is matched by the two requested DI/future audits at this baseline. The supplemental whole-word dyn audit also finds `setup/mod.rs:61` and `setup/install.rs:397` std::error::Error::source signatures (classify as `unrelated untouched legacy seam` at the signature level: mandated error-reporting protocol, not injected dependencies or future dispatch), and `db/mod.rs:2613` tracing::field::Visit::record_debug's `&dyn std::fmt::Debug` (classify as `test-only unrelated seam`: mandated diagnostic visitor signature, not test DI). These protocol occurrences remain even if their surrounding modules migrate; they do not exempt any DI seam. Non-code substring hits in `mutation_trace/types.rs:168` and `agent_trace_sync/test_http_server.rs:103` are prose, not dispatch. No dyn ServiceLifecycle/DB/Resolver/Evaluator/Seam trait object, BoxFuture or async_trait implementation was found. Final validation must classify any newly discovered match individually using the same rule. + +Supplemental `?Sized` classification: `mutation_trace/runtime/mutation_attribution.rs:128–129,177,215,257,328,343,370` and `hooks/codex_mutation_scope/tests.rs:749–750` are trait-object-admitting DI bounds to remove by PR2 in favor of sized generics. `bash_policy.rs:617` is an unrelated untouched serialization helper's `T: ?Sized + Serialize`, not DI or future dispatch; leave it unchanged. + +The conversion changes dispatch only: preserve Git resolution behavior, policy decisions, logger output, mutation-scope transitions, doctor repair ordering, hook output, retry semantics, database behavior and test expectations. Do not expand into unrelated subsystems or defer any migrated seam to PR3. + +### Runtime/blocking/lifetime classification + +- Production runtime owner to retain: `cli/src/main.rs:12`. DB runtime creation/isolation and every production bridge call are confined to `cli/src/services/db/mod.rs` (migration calls at 235/250/268; open at 951/1491; operations at 1001/1029/1064/1095/1173/1246/1294/1363/1549/1580/1611). Remove them all; test-only direct core runtime use at 1737 is adapted, not retained as compatibility plumbing. No alternative futures/pollster executor, global DB runtime singleton, dynamic database/repository/persistence object, or dynamic command object was found in cli/src. +- Production `block_in_place`: `command_registry.rs:84` non-context-only Setup, 87 Doctor, 103 DB-capable Hooks; `sync/sync.rs:220` storage construction and 164 guard Drop. All five are PR1 DB runtime lifetime workarounds; remove during native propagation where necessary to keep T02 compile-safe. Test-only instances: `hooks/tests.rs:4111,4137`, `sync/sync.rs:625,652,676,687,705`. Setup's test name `replaces_block_in_place_preserving_surrounding_foreign_content` refers to a managed text block, not Tokio. +- Production `spawn_blocking`: `auth_command/mod.rs:443` `run_credential_operation`, `agent_trace_sync/control_plane.rs:525,540` load/save. All three include DB work and must disappear. There are no other existing production `spawn_blocking` sites to retain. `db/encryption_key.rs:136` is currently synchronous; an async encrypted constructor must isolate only its synchronous OS keyring registration/read/write in an awaited worker, independently justified by keyring-core/platform APIs. Env-key validation can stay synchronous; no AuthDb creation, Turso build or SQL may enter that worker. +- `sync/sync.rs:152–166` `SyncStorageGuard` adds only blocking destruction of an Option-owned storage, with no independent invariant. Ordinary scoped `ResolvedAgentTraceStorage` ownership expresses its lifetime once the runtime field is gone; remove the guard. Borrowed readers and all three stream futures complete or drop before storage. Existing error/cancellation cleanup regressions become ownership regressions, without blocking fixture Drop or detached cleanup. +- Test runtime helpers: `agent_trace_sync/mod.rs:240`, `agent_trace_sync/control_plane.rs:876` currently create current-thread runtimes for sync tests; credential “outside runtime” fixtures at control_plane 1053/1061 and auth_command 569 specifically prove PR1 isolation. Convert affected tests to application-runtime async tests, deleting worker-only proof and join-failure behavior rather than asserting it after worker removal. RuntimeFlavor observations in app/hooks/auth tests are assertions, not extra owners. +- Existing production OS threads outside the bridge: `hooks/mutation_scope.rs:570` reads control-channel stdin and sends cancel signals only; `codex_hook_policy.rs:142` reads child app-server stdout only. Neither owns/drives a runtime or DB future. Existing detached post-commit auto-sync subprocess remains in `cli/src/services/sync/auto_sync.rs`; required DB checkpoint/persistence still completes in the foreground. Synchronous Git/filesystem/OS-lock/process supervision stays synchronous; removing a DB workaround does not establish a new blocking-scope justification for those services. + +### Locked dependency ownership and cancellation + +Inspected actual local registry source selected by `cli/Cargo.lock`: `turso`, `turso_core`, `turso_sdk_kit` 0.8.1 and Tokio 1.52.3. Dependency paths below are relative to their crate source roots (locally `/home/davidabram/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/`); no floating web documentation was used. + +- `turso-0.8.1/src/connection.rs:52–98` asserts Connection Send + Sync. It contains an Arc-owned SDK connection, an atomic dangling-transaction flag and an operation gate. Clone shares the underlying connection/gate but copies the dangling flag into a different atomic; **cloning is not a separate DB connection and cannot be used for independent writers or deferred rollback ownership**. Keep one logical connection per adapter, use borrowed `&Connection` and directly awaited sequential transaction body operations, and use separate opens for contention tests. No extra Arc/Mutex/pool is needed; exclusive mutable transaction borrowing is the static serialization mechanism. +- `connection.rs:100–150,513–566` handles dangling rollback before query/execute/batch. The shared/exclusive operation gate guards statement/batch activity, not a full transaction's application-level lifetime. `transaction.rs:123–132` `new_unchecked` constructs the guard only after awaited `BEGIN IMMEDIATE` completes; unlike `Transaction::new(&mut Connection)`, it does not statically prevent nesting. For the mechanical migration, take exclusive mutable adapter borrows for transaction primitives and propagate mutable store/DB ownership through transactional callers, keeping shared borrows for read-only export and encrypted token operations; prove serialization in focused tests; Send/Sync does not authorize overlapping transaction units. +- `transaction.rs:166–185,232–244`: commit/rollback clear `in_progress` only after their awaited SQL succeeds. Default Drop stores `Rollback` into `conn.dangling_tx`; it runs no SQL and owns no runtime. Next query/execute/checked transaction performs deferred rollback and resets the flag after success. Cancelling a body while a guard exists therefore schedules rollback, **not immediate release**. Cancelling acquisition before a guard exists has no transaction Drop hook; statement cleanup and actual connection state require direct characterization. A completed commit may already be durable when its caller is cancelled; no stronger exactly-once cancellation claim is inferred. +- `turso-0.8.1/src/lib.rs:487–507` executes by polling statements and resets on ordinary completion; `turso_sdk_kit-0.8.1/src/rsapi.rs:1541–1544` statement Drop removes the registered statement. Neither establishes that dropping every pending acquisition/commit future rolls back immediately. T03 must coordinate acquisition/body/commit boundaries where controllable, check zero partial rows plus subsequent same-connection reuse and independent observer state, and state any upstream limitations. Existing explicit-error rollback assertions remain authoritative; no new cancellation timeout is introduced in production. +- `tokio-1.52.3/src/sync/once_cell.rs:168,350–389,400,464–469`: `OnceCell::const_new()` and `get_or_init Fut, Fut: Future>` are supported by locked Tokio with `sync` enabled. A semaphore provides single-flight initialization; completed T is stored and the semaphore closed. With `T = Result`, both Ok and Err are completed cached values. `get_or_try_init` instead leaves Err uncached and is prohibited here. Cancel/panic before completion drops the permit without storing a value, allowing a waiter/later call to initialize again; recursive initialization deadlocks. Static cell requires its T to be Send + Sync, consistent with Connection and static adapter ownership. Test fresh local Result-valued cells for success, failure, concurrent callers and pre-completion cancellation; never reset the production static or require a live user keyring. + +### Test and verification map + +- `cli/src/services/db/mod.rs` inline tests: busy-timeout config/defaults and held-lock overlap; CAS no-op/ordered-success/deterministic rollback/guard and body row counts; Busy/BusySnapshot classification; checkpoint repeated/readability; numeric retry budget; seeded jitter, pre/post-sleep admission/equality/oversleep; exact exhaustion error/event fields and attempt caps. `cli/src/services/resilience.rs` tests include slow-success/slow-error sync semantics; retain those and add awaited non-cancelling DB retry characterization rather than mirroring HTTP timeout tests. +- Thread-local test instrumentation in `db/mod.rs:752–862`: read-statement/write-statement counts, contention counts and attempt/backoff timeline. Callers are `agent_trace_db/{repository.rs,lock_contention_tests.rs}` and `mutation_trace/store.rs`. Synchronous `FnOnce` scopes and tracing `with_default` at 2655 cannot be held blindly across migrated task polling: use task-scoped concrete instrumentation and polling-scoped subscriber context, with instrumentation enclosing the awaited operation. Preserve exact counters, timings and independent connections; keep injected clock/backoff determinism. +- `agent_trace_db/repository.rs` inline tests own baseline/schema/metadata/no-write readiness/no-migration paths, source-instance concurrent convergence, conversation pair duplication/injected rollback, migration 003 upgrades, and 006 fresh-vs-upgraded SQL, normalization, rerun and constraint fixtures. `auth_db/mod.rs` tests use a generic TestAuthDbSpec/path OnceLock for **unencrypted** baseline schema assertions; encrypted/key-policy behavior is covered by `db/encryption_key.rs` tests and must also get representative encrypted async execution coverage. `local_db/lifecycle.rs` tests cover path/bootstrap behavior; the LocalDb module itself has no dedicated inline DB suite. Do not claim a Cargo filter running zero tests proves async LocalDb behavior. +- `agent_trace_storage/mod.rs` tests cover repository/clone/worktree identity, explicit overrides, stable source-instance metadata, legacy-path isolation, no-migration missing/baseline DB rejection, and setup-open/hook-open agreement. `mutation_trace/store.rs`, `mutation_trace/runtime/{tests.rs,coordinator.rs,scope_runtime.rs,ref_reconciliation.rs,external_mutation_guard.rs,mutation_attribution/tests.rs}` cover durable CAS/idempotency and runtime recovery/ownership; convert their threaded DB callbacks/fakes to static async calls without losing independent-connection races. Pure protocol/MBT/lineage tests do not need blanket async conversion. +- `agent_trace_sync/control_plane.rs`: FakeCredentialStore, ConcurrentCredentialStore and RuntimeCheckingCredentialStore at 955–1088, `client_with_store` factory and sync's `AlwaysValidCredentialStore`/factory at `sync/sync.rs:732–771` must be concrete async injections. Retain valid-token no-resave, expired refresh/save, exactly-one concurrent refresh/save, unexpected-401 retry and second-401 terminal, missing credentials and storage/status classification assertions. The concurrent fake's 10ms synchronous sleep at 1026 is test coordination to adapt to async, not production retry policy. `auth_command/mod.rs` tests retain renewal/fallback/redaction and typed storage errors; worker join failure and outside-runtime fixtures become obsolete. +- `agent_trace_export/mod.rs` inline tests own cursor/limit/safe-integer/raw-row/read-only/source-instance contracts. `agent_trace_sync/mod.rs` tests own accepted cursor validation, conflict/ambiguous reconciliation, terminal/no-refresh and bounded convergence. `sync/sync.rs` owns three-stream overlap, per-stream sequential batches, one initial state request, incremental second run, diff_traces compatibility, progress order/end on error, and storage error/cancellation fixtures at 636/659. Preserve borrowed/non-Send progress compatibility; don't spawn streams merely to make them concurrent. +- Command/provider verification paths: `cli/src/app.rs` inline async dispatch/output/telemetry tests; `services/{auth_command,setup,doctor,hooks,sync}/command.rs`; `setup/tests.rs`; `doctor/{mod.rs,inspect.rs}` inline tests; `hooks/tests.rs` (`async_dispatch_doctor_and_hook_persist_before_completion` and `isolated_async_dispatch_boundary`); all four harness `tests.rs`, OpenCode/Claude health inline tests, `hooks/pi_mutation_scope/{runtime_seam_tests.rs,guard_reconciliation_tests.rs}`, and ingress conformance. Adapt DB fixtures and ingress fakes in these suites, keeping fail-open/fail-closed and completion ordering assertions. +- `agent_trace_db/lock_contention_tests.rs` contains **three actual ignored tests** (842/858/1154): `concurrent_duplicate_delivery_persists_each_event_once_under_write_contention`, `concurrent_distinct_events_persist_every_event_under_write_contention`, `concurrent_real_codex_hook_processes_persist_every_distinct_event` (Unix only). Strict distinct/duplicate checks require zero losses/errors/exhaustions plus expected pair counts; process checks require exact message/part counts and zero nonzero exits. Missing SCE_BIN returns early and is not evidence. Preserve held-lock overlap, first-attempt 100ms hold, initialized hook-open zero writes and budget tests. *(Superseded 2026-10-08: this file was deleted in T02 and is not restored; AC6 now relies on Test C, with the former stress matrices optional characterization.)* +- Toolchain/check ownership: `cli/Cargo.toml` enables Tokio rt-multi-thread/sync/time/macros, edition 2021 and deny warnings/Clippy; `flake.nix:93` pins Rust 1.95.0, Crane owns cli-tests/cli-clippy/cli-fmt and ci-checks/release packaging. `scripts/run-cli-cargo.sh` generates a fresh temporary payload through the producer, sets `SCE_CLI_GENERATED_INPUT_DIR`, and cleans it after Cargo. Focused migrated suites use `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml `; formatting autofix uses repo Nix. No .cargo/config.toml or CARGO_TARGET_DIR/BUILD_TARGET override was found in this audited session; wrapper build's default binary is `cli/target/debug/sce`, to be resolved/verified from actual Cargo metadata/build output before AC6 process runs. Full-plan checks remain `/validate`'s `nix flake check`, `nix build .#ci-checks`, `git diff --check`. + +### Context owners, drift and audit evidence + +- Existing owners: `context/{overview,architecture,patterns,glossary,context-map}.md`; `context/sce/{shared-turso-db,agent-trace-db,auth-db,local-db,agent-trace-db-write-contention-evidence,agent-trace-export-readers}.md`; `context/cli/{cli-command-surface,agent-trace-storage,service-lifecycle,agent-trace-sync-command,mutation-trace-store,mutation-trace-runtime-coordinator,mutation-trace-agent-attribution,mutation-trace-ref-reconciliation,mutation-trace-scope-abandonment,mutation-trace-external-mutation-guard,mutation-scope-runtime}.md`; harness integration/health owners linked from context-map; immutable `context/decisions/2026-10-07-application-owned-async-command-runtime.md`. +- Recorded drift for T05/source truth: shared DB contract bullet incorrectly includes migrations in generic operation retry, while its later paragraph and source correctly exclude them; LocalDb owner calls policies hardcoded although shared config overrides exist; service-lifecycle description claims hooks lazily initialize through the setup resolver while the storage owner and runtime opener correctly describe a separate no-migration path; reconcile repository migration descriptions against the six-file list including 006. PR1 ADR documents its historical sync-persistence boundary; preserve it as immutable history and use current-state owners/convention-compliant successor for PR2. T01 establishes no implemented runtime/interface change and does not prematurely rewrite current architecture to the target. +- Verification performed in original T01: the original three Source audit searches succeeded (60 runtime/blocking lines, 4 explicit trait-object/boxed-future lines, 133 named seam lines). Supplemental Nix ripgrep covered dyn Fn/SyncFuture, DB aliases and callers, SQL/transaction/retry/WAL/migration symbols, sleeps/threads, counters, ignored tests and alternative executor spellings. Source reads included Cargo/flake/wrapper/producer and locked Turso/Tokio files. +- Plan-correction verification: the added exact synchronous-DI audit succeeded with 175 matching lines (174 targeted for removal by PR2; parser logger is the one unrelated untouched DI match). The retained credential/boxed-future audit succeeded with 4 matching lines, all targeted for removal, including the coordinator test FnMut. Supplemental whole-word dyn/`?Sized`/SyncFuture search and source inspection classified qualified logger names, conformance implementations, diagnostic protocol signatures and trait-object-admitting bounds above. Inspected AppRuntime/AppContext and capability APIs to select existing HasLogger/HasGit/HasFs dispatch and focused closures without changing semantics. Caller-table shape, T01 done/T02 todo status, absence of the old synchronous exception, plan-only diff and `git diff --check` passed. Counts describe current source, not zero-match migration acceptance. No application tests or full-plan checks are needed or run for this documentation-only correction. + +## Review findings map, minimal safety test set and deferred ledger (added 2026-10-08) + +### Review finding → owner + +| Finding | Status | Owner | Evidence | Criterion | +| --- | --- | --- | --- | --- | +| `&self` + `new_unchecked` transactions | confirmed | T03 | `&mut` compile-time argument + Test A | AC3 | +| Transaction cancellation after first write | plausible | T03 | Test B | AC8 | +| Retry/contention semantics | needs one representative check | T03 | Test C (or surviving test) + source inspection of numeric policies | AC5, AC6 | +| Refresh-token persistence can be cancelled | confirmed structural exposure | T06 | Test D | AC7, AC14 | +| Sync no longer fails fast | confirmed | T06 | Test E (extends existing `sibling_failure_does_not_cancel_in_flight_credential_save`) | AC21 | +| Adapter boundary lock vs. blocking state writes | plausible | T07 | Test F | AC19 | +| Guarded-shell FD inheritance | confirmed | T07 | Test G | AC20 | +| Retained blocking boundaries unaudited | audit gap | T04 | source inspection + T03/T06/T07 tests + existing protected-worktree/guard tests | AC9 | +| Scope drift, stale patterns/architecture/ADR | confirmed | T05 | context inspection | AC10, AC11 | +| Lower-priority maintainability findings | deferred | PR3 | ledger below | AC22 | + +### Minimal safety test set + +Guidance, not a numeric acceptance criterion: roughly 6–10 new tests in total, fewer where existing tests, the type system or Quint already prove the invariant. Test count parity with the deleted suites, restoring deleted files, and a coverage-equivalence matrix are explicitly **not** required. Add a test only if it answers: *what specific regression would this catch that the existing tests, the type system, and the Quint specification would not?* A test filter that matches zero tests never counts as passing; confirm with `-- --list` that the intended test exists before relying on a run. + +| Test | Owner | Invariant | Distinct regression caught | Static / existing alternative | +| --- | --- | --- | --- | --- | +| A | T03 | Another operation on the same adapter cannot run inside an active transaction | someone reintroducing `&self`/`new_unchecked` or a clone-based escape hatch | If `&mut self` makes the overlap unrepresentable in safe Rust, document that compile-time proof (and, if cheap, a `compile_fail` doctest) instead of a runtime test | +| B | T03 | Cancel after the first write, before commit: no partial committed state, connection reusable, a later transaction succeeds, an independent connection sees only the durable result | Turso deferred-rollback behavior leaking an uncommitted write or wedging the connection | Quint MBT covers protocol transitions, not DB cancellation | +| C | T03 | One bounded contention scenario on independent connections: Busy/BusySnapshot classified, bounded attempts/deadline, whole-transaction retry, no duplicate or lost durable writes | retry unit shrinking to a statement, unbounded retry, duplicate/lost rows | Reuse a surviving test if one exists; numeric policies confirmed by source diff inspection | +| D | T06 | Refresh succeeds, save held pending, caller cancelled/timed out, save completes, next request reads replacement credentials (with a concurrent caller proving single-flight) | losing a rotated refresh token to the outer `/state` timeout | none | +| E | T06 | One stream returns 403 while another has several batches: no further batch starts after the terminal error is observed; explicitly protected operation completes | reverting to drain-all-siblings behavior | Extend the existing `sibling_failure_does_not_cancel_in_flight_credential_save` in `sync/sync.rs` | +| F | T07 | Pause a state update inside a blocking worker, cancel the outer boundary caller, attempt a second boundary: it cannot observe an invalid intermediate state or overtake the first transition | a started blocking transition outliving its boundary lock | Test the shared lock primitive in `hooks/mutation_scope_lock.rs`; add a per-adapter test only for a genuinely different transition protocol | +| G | T07 | An unrelated subprocess spawned concurrently with guarded-shell setup does not retain the lock/lifetime descriptors; the guarded child does; the unrelated child does not delay guard completion | parent-side descriptor inheritance window | `FD_CLOEXEC` atomic creation is verified by source inspection; the existing guard tests (`guarded_shell_*`) are reused for lock/lifetime protocol | + +Reused rather than re-created: the Quint model-based suite (`mutation_trace::mbt`, flake check `mutation-trace-quint-connect`) for mutation-cursor/coordinator protocol invariants; surviving pure tests in `mutation_trace/tests.rs`, `lineage`, `agent_trace` golden tests, `protected_worktree.rs` cancellation tests, `external_mutation_guard.rs` and `git_snapshot.rs` tests, and `sync.rs` `sibling_failure_does_not_cancel_in_flight_credential_save`. + +Optional characterization (never a merge gate): the previous writer/round stress matrices, larger contention loads, and per-await-point cancellation variants. + +### Deferred cleanup ledger (PR3 handoff; not PR2 merge blockers unless a concrete correctness defect is established) + +None of these may be mixed into T03–T07. Dead test modules are not restored merely because they exist on disk; unused test infrastructure is deleted in PR3 if no focused test needs it. + +| Finding | Disposition | +| --- | --- | +| Redundant `async { ... }.await` wrappers (for example `sync/sync.rs` storage scope) | PR3 | +| Unused `run_with_retry_sync` and duplicate retry-helper logic | PR3, without blurring elapsed-time vs cancelling semantics | +| Redundant `drop(storage)` (`sync/sync.rs`) | PR3 (T04 may remove it incidentally if it removes `SyncStorageGuard` remnants) | +| Stale `#[allow(dead_code)]` annotations | PR3 | +| Orphaned test-support files and fixtures; unused test seams (for example `fail_before_second`) | PR3: delete if no focused test uses them | +| Custom `join_three_to_completion` complexity | PR3 re-evaluation after T06 settles terminal behavior | +| Large futures and `#![recursion_limit = "256"]` pressure | PR3 measurement; recorded as a known limitation by T05 | +| Possible orphaned temporary Git `index-.lock` files after killed snapshot commands | accepted limitation documented by T04/T05; PR3 may add a sweep | + +## Task stack + +- [x] T01: `Map persistence ownership and the required async caller closure` (status:done) + - Task ID: T01 + - Scope: In — source-grounded runtime/API/transaction/retry/migration/test inventory recorded with exact paths in this plan, connected callers through application dispatch, all PR1 blocking sites, context owners and pinned Turso ownership/cancellation semantics. Explicitly map CredentialStore, AuthenticatedControlPlaneClient, SystemCredentialStore, token_storage AUTH_DB OnceLock, run_credential_operation, ServiceLifecycle, LifecycleProvider, Doctor, Setup, HooksCommand, mutation_trace callbacks/providers, AgentTraceExportReader, SyncStorageGuard, all block_in_place, and all spawn_blocking. Out — application behavior changes or building a replacement bridge. + - Dependencies: none + - Done when: Inventory covers both shared adapters and aliases, direct typed SQL wrappers, synchronous callback/trait/OnceLock boundaries, contention instrumentation and actual ignored suites; classifies every production runtime/blocking site and establishes the mechanically required propagation closure plus unchanged policy baseline. For every dynamic/trait seam record current dispatch (static/dynamic), target dispatch (static), and the concrete/generic/enum conversion. Explicitly confirm no additional dyn seam in the DB propagation closure leaves a design choice unresolved: any discovered seam follows Architecture decisions, never type erasure. Audit Turso Connection Send/Sync and intended same-connection use to select mechanical borrowing/lifetime adaptations within the fixed static design. Verify locked Tokio OnceCell API and Result-valued success/failure caching semantics. + - Verify: Run Source audit and Nix ripgrep searches for runtime/SQL/retry/WAL/migration symbols across persistence and callers; inspect actual tests, Cargo/flake/wrapper configuration and pinned Tokio/Turso transaction/connection source. No tests or final checks are necessary for the inventory-only change. + - Passed: Original three Source audit searches and supplemental Nix ripgrep searches; source inspection of both adapters, aliases, typed wrappers, complete caller/seam closure, policy/migration boundaries, instrumentation and ignored suites. Findings and exact sites are recorded in the T01 source inventory above. + - Passed: Plan-correction synchronous-DI and retained credential/future audits, supplemental dyn/`?Sized`/SyncFuture audit, existing capability/source reads and complete match classification; amended caller table and AC13/T02 static scope. T01 remains done and T02 remains todo; only this plan changes. + - Passed: Locked Turso 0.8.1 Connection/Transaction/statement source and Tokio 1.52.3 OnceCell source inspection, plus Cargo/flake/wrapper/producer review. Deferred rollback, acquisition-before-guard risks, static mutable transaction borrowing and Result-valued OnceCell caching/cancellation semantics are recorded above. + - Passed: Nix-backed Python inventory path check (22 explicit paths, no missing paths), six repository/two auth migration counts and grouped consumer-owner existence checks; `git diff --check`. + - Not required: Application tests and full-plan checks for this inventory-only task; none run. + - Completed: 2026-10-07 + - Files changed: `context/plans/cli-async-turso-persistence-pr2.md` (baseline-relative; clean baseline at `9ceff8414613a4a619cf505ae9ac0e3eb46db69f`). + - Result: Recorded and amended the source inventory above, including all DB-backed callers and both synchronous/asynchronous static conversions: credentials, ingress/conformance, GitDirResolver, BashPolicyEvaluator, model/repair, doctor Git/path/config/probe dependencies, logger arguments, coordinator fault hooks and SyncFuture. Existing AppContext associated-type capabilities remain static. Every dynamic seam audit match has an explicit migration or unrelated-path classification; no synchronous DI exception remains in the migrated closure and no architectural question is deferred. Ordinary storage ownership, retry/WAL/SQL/migration policies, cancellation limitations, relevant tests and context owners/drift remain recorded. No application or test source changed. + - Done-check evidence: Persistence/alias/wrapper/policy/migration sections establish the baseline; caller-conversion table establishes the complete static propagation closure; runtime classification covers every production bridge/blocking site; dependency section selects borrowing/lifetime adaptations and verifies OnceCell semantics; verification map inventories contention instrumentation and all three actual ignored suites. Tests characterize cancellation in T03, not T01. + - Context impact: none — source-grounded inventory and future migration instructions live in this plan; current implementation, interfaces, runtime ownership and terminology are unchanged. Existing owner drift is recorded for T05 without prematurely describing the target as implemented. + - Context synchronization: synced + - Context synchronization evidence: `no_context_change`; verified `context/overview.md`, `context/architecture.md`, `context/glossary.md`, `context/patterns.md`, and `context/context-map.md` against this documentation-only change, plus existing DB/storage/lifecycle owners and the PR1 runtime ADR. Current runtime architecture remains unchanged; no new feature, terminology or implemented qualifying decision requires owner edits or an ADR. Pre-existing migration/retry/hook-readiness drift remains explicitly recorded above for T05. Inventory paths/migration counts and final whitespace check passed; only this plan changed relative to the captured baseline. + +- [x] T02: `Replace the Turso runtime bridge with awaited persistence and callers` (status:done) + - Task ID: T02 + - Scope: In — remove core runtime field/constructor and bridge; convert open/execute/query/materialization/checkpoint/readiness/migration/transaction/encrypted operations and retry helpers; mechanically adapt the complete caller closure and existing tests in one compile-safe migration commit. This includes the resolved credential, token-storage, lifecycle, hook, mutation-trace, and sync/export architectures below and all synchronous DI in mechanically touched functions/modules. No intermediate commit may need a temporary block_on/executor bridge. Out — unrelated untouched legacy seams, unrelated service logic and behavior changes. This plan correction must be committed before T02 implementation begins; T02 remains unstarted during the correction. + - Dependencies: T01 + - Done when: TursoDb/EncryptedTursoDb are async with no DB runtime field, build_current_thread_runtime, or block_on_isolated; all native futures are awaited on the application caller runtime with no synchronous executor. The entire closure compiles using static dispatch, including: + - Remove dynamic dispatch from every migrated seam, including synchronous testability/policy/logging callbacks encountered in the required async caller closure: GitDirResolver, BashPolicyEvaluator, mutation-scope ingress/conformance, doctor repair, model resolver, doctor filesystem/Git/config/probe callbacks, logger argument seams, coordinator fault injection, SyncFuture and CredentialStore. No dynamic seam may remain merely because it does not itself return a future. Preserve AppRuntime/AppContext concrete ownership/borrowing and HasLogger/HasTelemetry/HasFs/HasGit associated types; use existing capabilities where natural, sized logger/generic callback parameters elsewhere, and concrete generic futures/native async static capabilities for borrowed async seams. Zero dynamic DI/future dispatch in changed/migrated production files, with every unrelated occurrence classified by exact site/reason. + - Async token_storage with Result-valued OnceCell caching completed success/failure; direct AuthCommand login/whoami load, refresh/login save, and logout delete awaits; removed run_credential_operation with typed CliError/secret-safe behavior preserved. + - Native async generic CredentialStore and AuthenticatedControlPlaneClient, concrete SystemCredentialStore and async fake-store injection through connected sync/test callers; removed credential trait-object fields/constructors and DB spawn_blocking. Preserve token reuse, refresh/single-flight/exactly-one refresh, 401 retry/terminal behavior, storage failures and exact counts/save assertions; replace outside-runtime tests with application-runtime async-store proof. + - Native async ServiceLifecycle diagnose/fix/setup; LifecycleProvider static enum matches await concrete providers; RuntimeCommand::Doctor/Setup await Doctor/Setup services and lifecycle operations. Pure Config/Hooks work runs synchronously inside those async methods without a blocking worker. + - RuntimeCommand::Hooks awaits HooksCommand static matches and DB-backed handlers; required foreground persistence completes before returning, preserving the sole existing detached post-commit auto-sync child. + - Awaited MutationTraceStore transaction primitives through coordinator/runtime, generic providers/callbacks, hook adapter and command; native async functions or genuinely needed F: FnOnce(...) -> Fut / Fut: Future bounds, never boxed callbacks. + - Async AgentTraceExportReader/connected repository reads awaited in the existing sync engine; unchanged initial state request, three concurrent streams, per-stream sequential batches, cursor reconciliation, retries, diff_traces compatibility and progress ordering. + - Generic retries preserve post-attempt diagnostic timeout semantics rather than adopting cancelling timeout; contention retries preserve whole-operation units and deterministic errors. Each converted sleep is recorded with unchanged policy. Transactions retain the same logical connection/SQL; migrations, encryption, retry and WAL policy are unchanged. Async-aware test instrumentation preserves overlap, rollback and attempt assertions. No dynamic async abstraction or artificial locking layer is introduced. + - Verify: Run focused wrapper suites for `services::db`, `services::agent_trace_db`, `services::agent_trace_storage`, `services::auth_db`, `services::local_db`, `services::mutation_trace`, and affected auth/control-plane/export/hook/lifecycle/command tests from T01 before repository-wide validation. Use `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml `. Run Source audit, trace all caller classes against AC13–AC17, and review SQL/policy diff. Auto-format only through Nix when needed. + - Completed: 2026-10-07 + - Files changed (baseline `d8d1aefb`, T01 complete): + - `cli/src/main.rs` + - `cli/src/services/agent_trace.rs` + - `cli/src/services/agent_trace_db/lifecycle.rs` + - `cli/src/services/agent_trace_db/lock_contention_tests.rs` + - `cli/src/services/agent_trace_db/mod.rs` + - `cli/src/services/agent_trace_db/repository.rs` + - `cli/src/services/agent_trace_export/mod.rs` + - `cli/src/services/agent_trace_storage/mod.rs` + - `cli/src/services/agent_trace_sync/control_plane.rs` + - `cli/src/services/agent_trace_sync/mod.rs` + - `cli/src/services/app_support.rs` + - `cli/src/services/auth_command/mod.rs` + - `cli/src/services/auth_db/lifecycle.rs` + - `cli/src/services/auth_db/mod.rs` + - `cli/src/services/codex_hook_config.rs` + - `cli/src/services/command_registry.rs` + - `cli/src/services/config/lifecycle.rs` + - `cli/src/services/config/render.rs` + - `cli/src/services/db/encryption_key.rs` + - `cli/src/services/db/mod.rs` + - `cli/src/services/doctor/command.rs` + - `cli/src/services/doctor/inspect.rs` + - `cli/src/services/doctor/mod.rs` + - `cli/src/services/doctor/render.rs` + - `cli/src/services/hooks/claude_bridge_session.rs` + - `cli/src/services/hooks/claude_model_state.rs` + - `cli/src/services/hooks/claude_mutation_scope/health.rs` + - `cli/src/services/hooks/claude_mutation_scope/lifecycle.rs` + - `cli/src/services/hooks/claude_mutation_scope/mod.rs` + - `cli/src/services/hooks/claude_mutation_scope/state.rs` + - `cli/src/services/hooks/claude_mutation_scope/tests.rs` + - `cli/src/services/hooks/claude_transcript.rs` + - `cli/src/services/hooks/codex/apply_patch/mod.rs` + - `cli/src/services/hooks/codex/apply_patch/normalize.rs` + - `cli/src/services/hooks/codex/apply_patch/parser.rs` + - `cli/src/services/hooks/codex/apply_patch/path.rs` + - `cli/src/services/hooks/codex/bash_policy.rs` + - `cli/src/services/hooks/codex/mod.rs` + - `cli/src/services/hooks/codex/stop.rs` + - `cli/src/services/hooks/codex/user_prompt_submit.rs` + - `cli/src/services/hooks/codex_mutation_scope/health.rs` + - `cli/src/services/hooks/codex_mutation_scope/lifecycle.rs` + - `cli/src/services/hooks/codex_mutation_scope/mod.rs` + - `cli/src/services/hooks/codex_mutation_scope/state.rs` + - `cli/src/services/hooks/codex_mutation_scope/tests.rs` + - `cli/src/services/hooks/command.rs` + - `cli/src/services/hooks/commit_hooks.rs` + - `cli/src/services/hooks/conversation_trace.rs` + - `cli/src/services/hooks/diff_trace.rs` + - `cli/src/services/hooks/lifecycle.rs` + - `cli/src/services/hooks/mod.rs` + - `cli/src/services/hooks/mutation_scope.rs` + - `cli/src/services/hooks/mutation_scope_ingress_conformance.rs` + - `cli/src/services/hooks/mutation_scope_lock.rs` + - `cli/src/services/hooks/mutation_scope_owner.rs` + - `cli/src/services/hooks/mutation_scope_state_conformance.rs` + - `cli/src/services/hooks/opencode_mutation_scope/health.rs` + - `cli/src/services/hooks/opencode_mutation_scope/lifecycle.rs` + - `cli/src/services/hooks/opencode_mutation_scope/mod.rs` + - `cli/src/services/hooks/opencode_mutation_scope/state.rs` + - `cli/src/services/hooks/opencode_mutation_scope/tests.rs` + - `cli/src/services/hooks/pi_mutation_scope/guard_reconciliation_tests.rs` + - `cli/src/services/hooks/pi_mutation_scope/health.rs` + - `cli/src/services/hooks/pi_mutation_scope/lifecycle.rs` + - `cli/src/services/hooks/pi_mutation_scope/lifecycle_tests.rs` + - `cli/src/services/hooks/pi_mutation_scope/mod.rs` + - `cli/src/services/hooks/pi_mutation_scope/runtime_seam_tests.rs` + - `cli/src/services/hooks/pi_mutation_scope/state.rs` + - `cli/src/services/hooks/pi_mutation_scope/tests.rs` + - `cli/src/services/hooks/runtime.rs` + - `cli/src/services/hooks/tests.rs` + - `cli/src/services/lifecycle.rs` + - `cli/src/services/local_db/lifecycle.rs` + - `cli/src/services/mutation_trace/runtime/coordinator.rs` + - `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs` + - `cli/src/services/mutation_trace/runtime/external_taint.rs` + - `cli/src/services/mutation_trace/runtime/git_snapshot.rs` + - `cli/src/services/mutation_trace/runtime/mod.rs` + - `cli/src/services/mutation_trace/runtime/mutation_attribution.rs` + - `cli/src/services/mutation_trace/runtime/mutation_attribution/tests.rs` + - `cli/src/services/mutation_trace/runtime/protected_worktree.rs` + - `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` + - `cli/src/services/mutation_trace/runtime/scope_runtime.rs` + - `cli/src/services/mutation_trace/runtime/tests.rs` + - `cli/src/services/mutation_trace/runtime/worktree_lock.rs` + - `cli/src/services/mutation_trace/store.rs` + - `cli/src/services/resilience.rs` + - `cli/src/services/setup/command.rs` + - `cli/src/services/setup/mod.rs` + - `cli/src/services/setup/tests.rs` + - `cli/src/services/structured_patch.rs` + - `cli/src/services/sync/auto_sync.rs` + - `cli/src/services/sync/command.rs` + - `cli/src/services/sync/progress.rs` + - `cli/src/services/sync/render_sync.rs` + - `cli/src/services/sync/sync.rs` + - `cli/src/services/token_storage.rs` + - Result: Removed the Turso runtime bridge and propagated native async persistence through the connected production callers, credentials/token storage, lifecycle, hooks, mutation trace, sync/export, setup/doctor, and command dispatch. Kept static dispatch and existing SQL, migration, encryption, retry, and WAL policy. Restored synchronous formatting/counting helpers that had been needlessly made async. `TransactionStatement::new` stays synchronous. + - Test history: the affected ordinary Rust unit-test and conformance code was first disabled with `cfg(any())`. At the user's direction, commit `fc37be7b` then deleted it rather than restoring it, along with production seams used only by those tests. The 14 orphaned external test files (for example `mutation_trace/runtime/tests.rs` and the hook adapter `tests.rs` files) were deleted afterwards. No `cfg(any())` suppression remains. The Quint mutation-trace MBT suite (`mutation_trace::mbt`) stays enabled. Fixture directories are kept because `flake.nix`, docs, and the Pi TypeScript tests reference them. + - Blocking boundaries: no DB or Turso future runs inside `spawn_blocking`. The synchronous OS credential-store lookup in `EncryptedTursoDb::new` is intentionally isolated in `spawn_blocking`. Synchronous advisory file-lock acquisition is isolated the same way: adapter boundary/state locks go through `AdapterLockSpec::acquire_async`, and the mutation-trace worktree lock goes through `worktree_lock::acquire_inner_async`. Each worker returns its owned guard to the async caller. Timeouts, poll intervals, lock paths, contention-callback semantics, external-taint handling, and the external mutation guard's inherited-lock ownership are unchanged. Claude's duplicate private state lock now uses the shared `AdapterLockSpec` with the same path, timeout, and poll interval. + - Verify: Focused wrapper suites listed above; Source audit; review SQL/policy diff; Nix formatting when needed. + - Passed: `nix develop -c sh -c 'cd cli && cargo fmt --all -- --check'`. + - Passed: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml --no-run`. + - Passed: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml`: 318 passed, 0 failed (this includes the 16 `mutation_trace::mbt` Quint Connect tests). + - Passed: `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings`, after fixing five lints that predated the lock change. + - Passed: `quint typecheck spec/mutation_cursor.qnt`, `quint test spec/mutation_cursor.qnt --match '^test.*'` (41 passing), and the randomized `verifyStep` safety run (5000 samples, 20 steps). + - Passed: `nix flake check`, including `cli-tests`, `cli-clippy`, `cli-fmt`, and `mutation-trace-quint-connect`. + - Passed: Source audit found no production runtime bridge, `block_on`, or `block_in_place`; three production `spawn_blocking` sites, all at synchronous external boundaries; no `cfg(any())`; and no new dynamic dispatch. + - Done-check evidence: The production closure compiles without the owned runtime bridge and passes the remaining Rust suite and Quint MBT. Behavioral assertions from the deleted ordinary Rust suites are not covered; this verification reduction was made at the user's direction and is recorded explicitly. + - Context impact: root — persistence execution and async interfaces now follow the application-owned Tokio runtime through the shared command/service/storage closure; this changes cross-domain architecture and belongs in root and domain context. + - Context synchronization: synced + - Post-review clarification (2026-10-08, plan update; the T02 evidence above is unchanged and no new test run is claimed): the deletion of the ordinary Rust suites was an intentional decision and is not being reversed. T02 established the mechanical async migration; T03, T06 and T07 establish the minimum necessary correctness evidence (Tests A–G) for the new execution model, and T01's exclusive-`&mut` transaction instruction is not yet reflected at head `044991d8` (T03). + +- [x] T03: `Enforce transaction isolation and characterize async cancellation` (status:done) + - Task ID: T03 + - Scope: In — make transaction exclusivity structural and add Tests A–C from the Minimal safety test set. Inspect `TursoDb::execute_transactional_insert_pair_if_absent`, `TursoDb::execute_transactional_cas_batch` and the `MutationTraceStore` transactional callers (coordinator, scope runtime, ref reconciliation, hook adapters). Enforce exclusive mutable ownership across an entire transaction: prefer `&mut self` receivers and checked `Transaction::new(&mut connection, Immediate)` where the pinned Turso 0.8.1 supports it; otherwise record the exact upstream reason and the equivalent static proof. No other operation on the same connection may enter an active transaction. Preserve SQL, retries, WAL, error classification and rollback semantics. T03 may make the smallest production changes necessary to establish transaction exclusivity and cancellation correctness. Unrelated architecture redesigns remain out of scope. Out — a connection pool, `Arc>` or connection cloning to solve what static borrowing prevents, a custom transaction framework, restoring deleted suites or test-count parity, per-await-point test variants, the exhaustive strict-load matrix as a merge gate. + - Dependencies: T02 + - Done when: Transaction exclusivity is structurally enforced (signatures take exclusive borrows through the whole transaction; no `new_unchecked` transaction construction remains unless justified in writing). Test A: either a deterministic test (pause transaction A after `BEGIN`, attempt operation B on the same adapter, prove B cannot execute within A) or, where exclusive borrowing makes the overlap unrepresentable in safe Rust, a documented compile-time proof (optionally a `compile_fail` doctest). Test B: cancel transaction A after its first write and before commit, at the most dangerous controllable boundary only; assert no partial committed state, safe subsequent reuse of the connection, a later legitimate transaction succeeds, and an independent connection sees the correct durable result; record exact guarantees and upstream limitations (deferred rollback is not immediate release; a completed commit may already be durable). Test C: one representative bounded contention scenario on independent connections asserting Busy/BusySnapshot classification, bounded attempts/deadline, whole-transaction retry and no duplicate or lost durable writes — reuse a surviving test if one already covers it. No deleted test-suite restoration is required. + - Verify: Inspect signatures and call chains for exclusive borrows; `nix shell nixpkgs#ripgrep -c rg -n 'new_unchecked' cli/src`; review the SQL/statement-order diff and confirm unchanged numeric retry/deadline constants by source diff. Run Tests A–C through the Cargo wrapper (`nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml `), each preceded by the same filter with `-- --list` confirming the test exists (zero matches never pass). Run `nix flake check` (includes the Quint MBT suite). + - Completed: 2026-10-08 + - Files changed (baseline `31489f17`, clean tree): + - `cli/src/app.rs` + - `cli/src/services/agent_trace_db/mod.rs` + - `cli/src/services/agent_trace_db/repository.rs` + - `cli/src/services/agent_trace_db/transaction_tests.rs` (new) + - `cli/src/services/db/mod.rs` + - `cli/src/services/db/transaction_cancellation_tests.rs` (new) + - `cli/src/services/hooks/codex/stop.rs` + - `cli/src/services/hooks/codex/user_prompt_submit.rs` + - `cli/src/services/hooks/commit_hooks.rs` + - `cli/src/services/mutation_trace/runtime/coordinator.rs` + - `cli/src/services/mutation_trace/runtime/mutation_attribution.rs` + - `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs` + - `cli/src/services/mutation_trace/runtime/scope_runtime.rs` + - `cli/src/services/mutation_trace/store.rs` + - `cli/src/services/resilience.rs` + - Result: `execute_transactional_insert_pair_if_absent` and `execute_transactional_cas_batch` now take `&mut self` and begin the transaction with the checked `Transaction::new(&mut Connection, Immediate)` (supported by Turso 0.8.1); no `new_unchecked` remains in `cli/src`. The exclusive borrow propagates through `MutationTraceStore` (`db: &'a mut`, `commit(&mut self)`), the coordinator, scope runtime, ref reconciliation, post-commit attribution, the conversation-text-event insert and the Codex Stop/UserPromptSubmit and post-commit hooks. Because an `FnMut` retry closure cannot return a future borrowing its own captured `&mut`, the retry helpers take a small native-async `RetryOperation` (resilience) / `WriteAttempt` (db) trait with blanket impls for the existing closures; the two transactions are `InsertPairAttempt` / `CasBatchAttempt` structs. Retry numbers, jitter, busy timeout, deadline admission, SQL and statement order are unchanged. `app.rs` test call boxed to satisfy `clippy::large_futures` (the future grew past 16384 bytes). + - Test A (compile-time proof, no runtime test): with `&mut self` and `Transaction::new(&mut conn, ..)` the transaction holds the only borrow of the adapter's connection for its whole lifetime, so another adapter operation inside it does not compile; documented on both primitives. `compile_fail` doctest not added because the crate is a binary (doctests do not run). + - Test B `db::transaction_cancellation_tests::abandoned_transaction_commits_nothing_and_rolls_back_on_next_use_of_its_connection`: a `Transaction` on the adapter's connection with one write is dropped uncommitted (what future cancellation does). Verified: an independent connection sees no row; that connection's writer stays blocked (contention-exhausted error) until the owning connection is next used; the owner's next transaction succeeds and performs the deferred rollback; the observer can then write; abandoned row never appears, later rows appear exactly once. Finding: local Turso futures never return `Pending` (a poll-sweep cancellation test completed on the first poll), so a mid-transaction drop cannot occur in the current primitives; the test therefore drops the guard directly. Limitations (upstream Turso 0.8.1): rollback is deferred to the next use of the same connection and the writer lock is held until then; a completed commit may already be durable when its caller is cancelled. + - Test C `agent_trace_db::transaction_tests::{write_contention_retries_whole_transaction_once_and_persists_one_pair, write_contention_exhausts_bounded_attempts_without_persisting_rows}`: independent connections; lock held 1.3s -> attempt 1 Busy, attempt 2 succeeds, exactly one pair; lock held throughout -> `attempts=2` contention-exhausted error, zero rows, later write succeeds. + - Verify: Passed: `rg new_unchecked cli/src` (no matches). Passed: each filter confirmed with `-- --list` (3 transaction tests listed) then run: 4 passed. Passed: SQL/numeric-policy diff review (no constant or SQL changes). Passed: `nix develop -c sh -c 'cd cli && cargo fmt'`; `git diff --check`. Passed: `nix flake check` (cli-tests, cli-clippy, cli-fmt, mutation-trace-quint-connect). + - Done-check evidence: exclusivity is structural (`&mut` signatures plus checked `Transaction::new`); Tests A (compile-time), B and C as above. + - Context impact: domain — transaction primitives require exclusive borrows and the store/hook callers changed accordingly; belongs in shared-turso-db and mutation-trace-store owners (T05 records the guarantees and limitations). + - Context synchronization: synced + - Context synchronization evidence: `synced`; edited `context/sce/shared-turso-db.md` (exclusive `&mut` transaction primitives, checked `Transaction::new`, retry-operation traits, Turso 0.8.1 cancellation limits, tests) and `context/cli/mutation-trace-store.md` (`commit(&mut self)`). Root files (`overview`, `architecture`, `glossary`, `patterns`, `context-map`) read and unaffected; no qualifying ADR (local ownership change within the accepted directly-awaited persistence decision); both edited files within 250 lines. + +- [x] T06: `Protect credential rotation and stop sync on terminal failure` (status:done) + - Task ID: T06 + - Scope: In — (A) a cancellation-protected native-async refresh-and-persist operation and Test D; (B) cooperative early termination of sibling streams after a terminal failure and Test E. Out — moving any Turso operation into `spawn_blocking`, changing the WorkOS protocol, redesigning the cursor algorithm, restoring the deleted control-plane/sync suites, a generic cancellation framework. + - **A. Refresh-token persistence.** Today the `/state` path (`ingestion_state → run_with_retry → tokio::time::timeout → execute_authenticated → force_refresh_access_token → refresh_and_save → credential save`) lets the outer timeout cancel persistence after WorkOS has rotated the refresh token. Choose a native-async design with explicit ownership and cancellation guarantees (candidates evaluated against source: an owned, shielded refresh-and-save unit awaited before command termination; or restructuring so the timeout cannot wrap post-response persistence — moving the timeout outside one function is not assumed sufficient). Preserve single-flight refresh, exactly one intended refresh per rejected credential generation, durable storage before publishing/reusing the new token, typed errors, secret redaction, bounded HTTP/network operations, and no Turso work in `spawn_blocking`. Explicitly characterize the unavoidable remote-operation ambiguity window (remote refresh succeeds but the process crashes or the response never arrives) and process-crash limitations. + - **B. Terminal sync errors.** `join_three_to_completion` currently waits for every sibling to exhaust its backlog after one stream fails terminally. Require cooperative early termination: stop scheduling new sibling batches after a terminal failure; preserve in-flight credential persistence and do not abandon an irreversible operation (so simply dropping siblings via `try_join!` is not acceptable); preserve cursor/reconciliation semantics and per-stream sequential batching; return the appropriate terminal error (first observed, deterministic) without waiting for unnecessary uploads. + - Dependencies: T02, T03 + - Done when: Credential rotation and persistence have explicit, documented cancellation guarantees and Test D passes: a deterministic fake WorkOS response and controlled async credential store; refresh returns replacement credentials; the save starts and is held pending; the outer request times out or is cancelled; the save completes; a subsequent request reads the replacement credentials; where feasible a concurrent caller proves single-flight (one rotation) in the same test. Terminal sync failure does not unnecessarily drain remaining backlogs and Test E passes: one stream returns 403, another has several batches, no further batch starts after the terminal error is observed, and any explicitly protected operation completes (extend the existing `sibling_failure_does_not_cancel_in_flight_credential_save` rather than duplicating the sync integration suite). Split Test D only if clarity requires. + - Verify: Run Tests D and E through the Cargo wrapper, each preceded by a `-- --list` check that the test exists; `nix shell nixpkgs#ripgrep -c rg -n 'spawn_blocking' cli/src/services/agent_trace_sync cli/src/services/token_storage.rs` shows no credential DB use; inspect logs/errors for secret redaction; `nix flake check`. + - Completed: 2026-10-08 + - Files changed (baseline `9e3067eb`, clean tree): + - `cli/src/services/agent_trace_sync/control_plane.rs` + - `cli/src/services/sync/sync.rs` + - Result: (A) `CredentialStore` methods now return `impl Future + Send` (trait is `Send + Sync + 'static`) and the client holds `Arc`. `refresh_and_save` runs the WorkOS renewal plus credential save as one spawned owned task that also owns the single-flight guard (`lock_owned`); a cancelled or timed-out caller detaches the task instead of cancelling it, and concurrent callers reload the persisted token after the guard releases. No `spawn_blocking`. Limits: renewal stays bounded by the auth module's retry/timeout policy; a runtime shutdown or process crash after WorkOS rotated the token but before the save completes still loses the replacement (remote rotation cannot be atomic with the local save). (B) `sync_one_stream` takes a shared `Cell` terminated flag, set when any stream fails; siblings read it before each batch read and stop with their completed batches, an in-flight ingest/credential save still completes, `join_three_to_completion` still returns the first observed error, and an aborted sibling no longer reports `StreamCompleted` progress. Added `#[allow(clippy::too_many_lines)]` to `sync_one_stream`. Comments I added or touched in these files were removed. + - Test D `agent_trace_sync::control_plane::tests::cancelled_caller_does_not_lose_rotated_refresh_token`: fake WorkOS server, controlled async store with held save; first caller aborted while the save is pending, second caller waits on the guard, save released: exactly one renewal and one save, replacement refresh token persisted, a third call reuses it. Catches losing a rotated token to caller cancellation or a second renewal. + - Test E `sync::sync::tests::terminal_stream_failure_stops_sibling_after_its_in_flight_batch` (alongside the retained `sibling_failure_does_not_cancel_in_flight_credential_save`): 403 on one stream while a sibling batch is in flight; the in-flight batch completes, no further read or ingest starts, first error returned. + - Verify: Passed: both new tests listed with `-- --list` then run (3 passed with the retained join test). Passed: `rg spawn_blocking` over `agent_trace_sync` and `token_storage.rs` (no matches). Passed: no new secret-bearing messages (the task error text contains no token). Passed: `cargo fmt` via Nix, `git diff --check`, `nix flake check` (cli-tests, cli-clippy, cli-fmt, mutation-trace-quint-connect). Test clients use `tls_certs_only(Vec::new())` because the Nix sandbox has no system CA store. + - Done-check evidence: Tests D and E as above; no Turso work in a blocking worker. + - Context impact: domain — credential refresh is now an owned, cancellation-protected unit with a generic `Arc` store, and sync stops sibling batches after a terminal failure; belongs in the sync/auth owners (T05 records the guarantees and limitations). + - Context synchronization: synced + - Context synchronization evidence: `synced`; edited `context/cli/agent-trace-sync-command.md` (owned refresh-and-save unit, `Arc` store with `Send` futures, remaining limits, terminal-failure sibling stop) and `context/glossary.md` (`refresh single-flight guard` held by the spawned task). Root files (`overview`, `architecture`, `patterns`, `context-map`) read and unaffected; no qualifying ADR (local ownership change inside the accepted directly-awaited persistence decision); edited files within 250 lines, `git diff --check` clean. + - Post-review correction (2026-10-08, PR #301 review; the T06 record above is preserved): (1) A detached spawned task only survives while the runtime stays alive; `sce` uses a short-lived `#[tokio::main]` runtime, so after the outer `/state` timeout cancelled the caller, a normal command return could shut the runtime down and cancel the still-running refresh/save. `AuthenticatedControlPlaneClient::wait_for_pending_refresh` now acquires `refresh_lock`; the guard is taken (`lock_owned`) before `tokio::spawn` and moved into the task, which holds it through renewal and `credential_store.save`, so acquiring the mutex proves no started task is still running. `run_current_sync_without_progress` (`sync/sync.rs`) keeps the client alive past `run_sync_against_without_progress`, awaits the drain with no early `?` between them on both success and error, and returns the original result unchanged; the drain has no additional timeout. `run_whoami` needs no drain: it directly awaits `client.me()`, which awaits the spawned task's `JoinHandle`, so an ordinary return implies refresh completion. A refresh/save failure after the caller was cancelled is not observed and not reported as success (stored credentials stay unchanged); no registry or logging was added. (2) `sync_one_stream` checked `terminated` only before the async database read, so a read already in flight when a sibling hit a terminal error could return rows and start a new upload; the flag is now rechecked after the read, returning no rows (already-started uploads still finish; the original terminal error is preserved). Tests: `agent_trace_sync::control_plane::tests::pending_refresh_drain_waits_for_cancelled_callers_save` (save held, caller aborted, drain stays pending, released save completes, replacement refresh token stored, one WorkOS hit and one save) and `sync::sync::tests::terminal_stream_failure_discards_sibling_rows_read_after_termination` (read held until termination, then released with a row: zero sibling ingests, terminal 403 reported; confirmed failing without the post-read check). Existing `cancelled_caller_does_not_lose_rotated_refresh_token` and `terminal_stream_failure_stops_sibling_after_its_in_flight_batch` still pass. Remaining limits: SIGKILL/crash/forced termination can still lose an in-flight save, and a true storage failure leaves the old credentials; the drain is a completion barrier, not proof of persistence success. + +- [x] T07: `Close adapter boundary and guarded-shell descriptor races` (status:done) + - Task ID: T07 + - Scope: In — (A) adapter boundary-lock cancellation safety and Test F; (B) guarded-shell descriptor inheritance and Test G. Inspect `hooks/mutation_scope_lock.rs`, the Codex, OpenCode and Pi lifecycles, and the relevant Claude state boundaries; and `mutation_trace/runtime/external_mutation_guard.rs`. Out — a generic cancellation abstraction unless unavoidable, removing guarded-shell FD inheritance (it is part of the guard protocol), per-adapter test duplication, restoring deleted adapter suites. + - **A. Boundary locks.** The boundary lock is dropped when its async owner is cancelled, but an already-started `run_locked_blocking` state transition may continue. Choose the smallest correct design (for example a lock-ownership lease retained by the started blocking operation, or cancellation-safe ownership transfer) guaranteeing: a state transition cannot outlive the boundary protection it requires; new boundaries cannot observe delayed writes from an older cancelled boundary in an invalid order; lock acquisition and worker-shutdown behavior are understood and documented; existing recovery transitions remain fail-closed. + - **B. Guard descriptors.** Parent lock/lifetime descriptors must stay `FD_CLOEXEC` (created atomically close-on-exec: `pipe2(O_CLOEXEC)` / `F_DUPFD_CLOEXEC` on Linux, with a documented treatment where macOS lacks an atomic variant); only the intended guarded child receives them through post-fork-safe Unix child setup (reviewed `pre_exec` or equivalent); duplication is race-safe; lock and lifetime semantics are unchanged on Linux and macOS. + - Dependencies: T02, T03 + - Done when: Adapter boundary cancellation and guarded-shell descriptor inheritance are safe under the specifically characterized races. Test F: pause a state update inside a blocking worker, cancel the outer boundary caller, attempt a second boundary, and verify it cannot observe or act on an invalid intermediate state or overtake the first transition; use the shared lock primitive if that proves the invariant for all adapters and add a per-adapter test only for a genuinely different transition protocol. Test G: launch an unrelated subprocess concurrently with guarded-shell setup; the intended guarded child retains its lock/lifetime descriptors, the unrelated child does not, and the unrelated child's lifetime does not delay guarded-shell completion (no large subprocess matrix; reuse the existing `guarded_shell_*` tests for the lock/lifetime protocol). + - Verify: Run Tests F and G through the Cargo wrapper, each preceded by a `-- --list` check; inspect descriptor creation/duplication sites for atomic close-on-exec and the `pre_exec` body for post-fork-safe operations; `nix flake check`. + - Completed: 2026-10-08 + - Files changed (baseline `10d4a0c6`, clean tree): + - `cli/src/services/hooks/codex_mutation_scope/lifecycle.rs` + - `cli/src/services/hooks/mutation_scope_lock.rs` + - `cli/src/services/hooks/opencode_mutation_scope/lifecycle.rs` + - `cli/src/services/hooks/pi_mutation_scope/lifecycle.rs` + - `cli/src/services/mutation_trace/runtime/external_mutation_guard.rs` + - Result: (A) New `AdapterLockSpec::run_under_boundary` holds the acquired boundary `OsAdvisoryLock` in an `Arc` exposed through a task-local; `run_locked_blocking` clones that `Arc` into its `spawn_blocking` closure, so a started state transition keeps the boundary lock after the outer caller is cancelled and the lock releases only when the worker finishes. The Codex, OpenCode and Pi `with_boundary_lock` wrappers delegate to it; state-lock protocol and recovery transitions are unchanged (Claude has no boundary lock). Limit: the lease follows `spawn_blocking` calls made inside the boundary scope; a blocked worker delays the next boundary until it finishes or that boundary times out. (B) `LifetimeToken::new` creates its pipe with `pipe2(O_CLOEXEC)` on Linux (non-Linux: `pipe` then `fcntl`, a residual fork/exec window). The parent-side `dup` and `set_cloexec(.., false)` are removed; `spawn_guarded_shell` registers a `pre_exec` that only calls `fcntl` to clear `FD_CLOEXEC` on the lock and lifetime descriptors in the child, so unrelated subprocesses never inherit them. Lock and lifetime protocol is unchanged. + - Test F `hooks::mutation_scope_lock::tests::cancelled_boundary_keeps_its_lock_until_the_started_state_transition_finishes`: a state transition is paused in a blocking worker, the boundary caller is aborted, a second boundary acquisition times out, and succeeds after the worker is released. Catches a started transition outliving its boundary lock. + - Test G `external_mutation_guard::unix_impl::tests::guard_descriptors_are_inherited_only_by_the_guarded_child` (Linux): the lifetime writer is close-on-exec; an unrelated `sh` does not hold the lock or lifetime descriptor targets, the guarded child does, and the parent lock stays close-on-exec. Catches parent-side descriptor inheritance. Existing `guarded_shell_*` tests are reused for the lock/lifetime protocol. + - Verify: Passed: both new tests listed with `-- --list` then run (1 passed each); `guarded_shell_` tests (2 passed). Passed: inspection of descriptor creation sites and the `pre_exec` body (`fcntl` only). Passed: `cargo fmt` via Nix, `git diff --check`, `nix flake check` (cli-tests, cli-clippy, cli-fmt, mutation-trace-quint-connect). + - Done-check evidence: Tests F and G as above. + - Context impact: domain — adapter boundary lease and guard descriptor inheritance semantics; belongs in the mutation-scope adapter and guard owners (T05 records the guarantees and limitations). + - Context synchronization: synced + - Context synchronization evidence: `synced`; edited `context/cli/mutation-trace-external-mutation-guard.md` (atomically close-on-exec lifetime pipe, child-only descriptor inheritance via `pre_exec`), `context/cli/opencode-mutation-scope-adapter-lifecycle.md` and `context/cli/codex-mutation-scope-integration.md` (boundary lock as a lease held by started state transitions). Root files (`overview`, `architecture`, `glossary`, `patterns`, `context-map`) read and unaffected; no qualifying ADR (local ownership change inside the accepted directly-awaited persistence decision); the guard doc was already over 250 lines before this edit; `git diff --check` clean. + +- [x] T04: `Remove obsolete DB runtime lifetime blocking scopes` (status:done) + - Task ID: T04 + - Scope: In — remove obsolete command_registry Setup/Doctor/DB-backed Hooks compatibility scopes, sync storage construction, SyncStorageGuard Drop and runtime-drop test fixtures; confirm T02 removed all credential DB blocking wrappers; audit every retained production `spawn_blocking`/`block_in_place` after the T03/T06/T07 fixes using source inspection and those tasks' focused tests plus existing tests (for example the `protected_worktree.rs` cancellation tests). Out — a separate test for every `spawn_blocking` site; removing a blocking worker that intentionally provides cancellation protection; indiscriminate deletion of independently necessary synchronous external API protection or detached cleanup tasks/leaks. If a blocking scope prevents T02's native async/static closure from compiling, remove it in T02 and record it here; T04 never legitimizes an intermediate executor bridge. + - Dependencies: T02, T03, T06, T07 + - Done when: Zero DB-runtime-related block_in_place, zero blocking SyncStorageGuard Drop, zero token-storage DB spawn_blocking; every scope solely needed for DB runtime construction/destruction is removed; the question whether SyncStorageGuard expresses an invariant ordinary ownership cannot express is answered (remove it if not); scoped storage outlives borrowing futures and drops safely on errors/cancellation without detached cleanup or leaks. An audit table lists each retained site — advisory lock acquisition; adapter state transactions; worktree lock leases; external-taint marker fsync/persistence and clearing; Git ref mutations; guarded-shell supervision; temporary-index cleanup; credential encryption-key lookup; any site added by T06/T07 — with: why blocking execution is required; who owns the operation after caller cancellation; which lock or resource stays held; and its termination/shutdown behavior (including whether runtime shutdown can wait). The original runtime-bridge-removal criteria (AC1, AC9, AC17) stay intact. + - Verify: Run Source audit and inspect all production blocking sites/SyncStorageGuard occurrences; run the focused tests from T03/T06/T07 and any surviving relevant existing tests; confirm no replacement DB bridge or runtime-containing fixture inside async tests. + - Completed: 2026-10-08 + - Files changed (baseline `0cc3e8e3`, clean tree): `context/plans/cli-async-turso-persistence-pr2.md` only; the audit found nothing to remove, so no production or test source changed. + - Result: Audit-only. `SyncStorageGuard`, `block_in_place`, `block_on`, `Builder::new_current_thread`, `tokio::runtime::Runtime` and the `command_registry`/sync-storage compatibility scopes no longer exist in `cli/src` (removed by T02); `app.rs` mentions `RuntimeFlavor::MultiThread` only in test assertions. `token_storage.rs` and `agent_trace_sync` contain no `spawn_blocking`. `SyncStorageGuard` answer: it expressed no invariant beyond ordinary ownership once the DB runtime was gone, and it is already deleted; storage is scoped ownership. Production `spawn_blocking` sites (15 occurrences in 10 files, all at synchronous external APIs; no Turso future runs in any worker): + - `db/mod.rs` `EncryptedTursoDb::new` encryption-key lookup — OS keyring/file API is synchronous. After caller cancellation the worker runs to completion and its result is dropped; holds no lock; short, bounded by the keyring call. Runtime shutdown waits for it. + - `hooks/mutation_scope_lock.rs` `run_locked_blocking` (state transitions for Claude/Codex/OpenCode/Pi) — blocking advisory-lock acquisition plus synchronous state-file I/O. Started worker owns the transition after cancellation and holds the state lock plus a clone of the boundary lease (T07) until it finishes, so it cannot be overtaken (Test F). Bounded by the lock timeout; runtime shutdown waits. + - `hooks/mutation_scope_lock.rs` `acquire_with_timeout_async` (boundary/state lock acquisition) — blocking `flock` polling. If the caller is cancelled the worker still finishes and drops the acquired guard, releasing it; bounded by the timeout; runtime shutdown waits up to that timeout. + - `hooks/{codex,opencode,claude,pi}_mutation_scope/state.rs` `read_state` — synchronous file read; read-only, no lock held, result discarded on cancellation; bounded; shutdown waits briefly. + - `mutation_trace/runtime/worktree_lock.rs` `acquire_inner_async` — blocking worktree advisory lock with timeout; the acquired guard is dropped if the caller is gone; bounded by timeout; shutdown waits. + - `mutation_trace/runtime/protected_worktree.rs` marker arm (`arm_marker_with_lock`) and `complete` — synchronous marker inspect/persist (fsync) and clear. The worker owns the `WorktreeLock` and marker after cancellation, so the lock stays held until the marker is persisted or cleared (covered by `cancelled_marker_arming_keeps_the_lock_until_the_marker_is_persisted` and `cancelled_completion_clears_the_marker_before_a_later_operation_can_arm`); bounded file I/O; shutdown waits. + - `mutation_trace/runtime/git_snapshot.rs` `run_ref_mutation_inner` — blocking Git ref-mutation subprocess. The worker owns the `WorktreeLockLease` after cancellation, so the lock is held until Git exits (covered by `cancelled_pin_keeps_worktree_lock_until_git_ref_mutation_completes`); bounded by the Git process; shutdown waits. + - `mutation_trace/runtime/git_snapshot.rs` `TempIndexGuard::drop` (`Handle::try_current` + `spawn_blocking`, falling back to inline removal) — synchronous `remove_file` in a destructor; the detached task is fire-and-forget with no lock held and its result ignored, so a runtime shutdown may wait for it but a process kill can leave the temporary index file behind (recorded as a limitation for T05; the async `cleanup` path is used on the normal route). + - `mutation_trace/runtime/external_mutation_guard.rs` guarded-shell supervisor — synchronous `Child::wait`/descriptor supervision. The worker owns the `ProtectedWorktree` ownership and guarded child; it ends when the child exits or cancellation is observed through `cancel_rx`; the protected lock stays held until it returns; runtime shutdown waits for the child (covered by `guarded_shell_supervision_leaves_the_runtime_worker_responsive` and Test G). + - Verify: Passed: Source audit searches (`block_in_place|spawn_blocking|SyncStorageGuard|block_on|current_thread|runtime::Runtime|Builder::new`) with every match classified above. Passed: each filter confirmed with `-- --list` then run — `cancelled_` (6 passed, includes Test D, the pending-refresh drain test, Test F and the three protected-worktree/git-ref cancellation tests), `guard_descriptors_are_inherited` (1), `guarded_shell_` (2), `abandoned_transaction` (1, Test B), `terminal_stream` (2, Test E). Passed: no replacement DB bridge or runtime-containing fixture found. Passed: `git diff --check`; `nix flake check` ("all checks passed", the Crane check derivations were already built for this unchanged tree so it ran 0 new checks). + - Done-check evidence: zero DB-runtime `block_in_place`, zero `SyncStorageGuard`, zero token-storage `spawn_blocking`; every retained site has the four answers above (why blocking, post-cancellation owner, held resource, termination/shutdown). + - Context impact: none — audit of existing behavior; the retained-site table is T05's input for the architecture blocking-worker enumeration. + - Context synchronization: synced + - Context synchronization evidence: `no_context_change`; read `context/overview.md`, `context/architecture.md`, `context/glossary.md`, `context/patterns.md` and `context/context-map.md` against the audit. `patterns.md` and `architecture.md` (no DB `spawn_blocking`, no executor bridge, blocking limited to synchronous external boundaries) are not contradicted; no references to `SyncStorageGuard`, `block_in_place` or `block_on` remain. The `architecture.md` blocking-worker list omits the Git ref-mutation, adapter state read/transition, temp-index destructor and guarded-shell supervisor sites; that drift was already assigned to T05 and is intentionally left for it. No decision qualified for an ADR (audit, no new constraint); only this plan changed. + +- [x] T05: `Record application-owned persistence, the actual PR2 scope and the revised PR staging` (status:done) + - Task ID: T05 + - Scope: In — existing architecture/persistence/consumer context owners, relevant drift repairs, ADR update or convention-compliant immutable successor (ADRs are immutable; a changed decision gets a new dated record), context-map references and precise PR3/PR4 handoff. Correct the mismatch between the original PR2 scope and the implementation, which expanded beyond Turso conversion to include async Git subprocesses, filesystem operations off Tokio workers, worktree lock leases, cancellation-shielded Git ref mutation, external-taint marker ownership, adapter state transaction workers and guarded-shell supervision. Fix outdated statements in `context/patterns.md`, `context/architecture.md` and the relevant shared persistence and mutation-trace owners; document the verified cancellation-protection model, exclusive transaction ownership, credential-refresh guarantees and limitations, adapter lifecycle protection, guard FD inheritance, the sync early-termination policy and temporary-index cleanup limitations, claiming nothing stronger than code and tests establish. Record the Deferred cleanup ledger as the PR3 handoff. Out — duplicate architecture documents, claiming all service logic is async, new behavioral tests, final validation execution, telemetry implementation. + - Dependencies: T02, T03, T04, T06, T07 + - Done when: Durable owners describe application Tokio runtime → static async command dispatch → static async service/lifecycle/hook dispatch → static async persistence → Turso and all mechanically propagated consumers, including generic credentials and cached-result async auth initialization; name deleted bridge/helpers and removed/retained blocking sites with reasons (from T04's audit table); preserve database policy facts; stage PR1 (complete), PR2 (async core, required consumers, Git/process/filesystem async work as actually implemented, cancellation protections, obsolete blocking removal), PR3 (post-migration cleanup/runtime audit per the ledger), PR4 OpenTelemetry. Correct only relevant recorded drift, including no-migration hooks, repository lifecycle, migration 006 and configured LocalDb retries. + - Verify: Cross-check context/decision descriptions against source and task evidence (every stated guarantee points to a passing test, a type-level argument or a named limitation), check owner links and document diff whitespace. Repository-wide final checks remain exclusively in Full validation. + - Completed: 2026-10-08 + - Files changed (baseline `5c38185f`, clean tree): `context/architecture.md`, `context/patterns.md`, `context/overview.md`, `context/context-map.md`, `context/cli/service-lifecycle.md`, `context/sce/shared-turso-db.md`, `context/sce/agent-trace-db.md`, new `context/decisions/2026-10-08-pr2-scope-and-cancellation-protections.md`, and this plan. + - Result: Architecture now lists every retained blocking worker from T04's audit (lock/state transitions, `read_state`, marker I/O, Git ref mutation, temp-index destructor, guarded-shell supervisor) with ownership and shutdown behavior, names the deleted bridge symbols, states verified guarantees/limits (exclusive `&mut` transactions, deferred rollback, credential rotation limits, terminal-sync stop, lease/FD protection, deleted suites not restored) and the PR1–PR4 staging with the PR3 ledger. `patterns.md` synchronous-process/filesystem guidance corrected. Drift repaired: migration retry wording, hook no-migration path (service-lifecycle, overview), migrations 004–006 in `agent-trace-db.md` (LocalDb retry wording was already correct). Successor ADR added; the 2026-10-07 ADR stays immutable. Scope reconciliation for AC10: implemented PR2 includes Git/process/filesystem async work and cancellation protections, recorded in the architecture and ADR. + - Verify: Passed: cited symbols and tests confirmed in source (`transaction_cancellation_tests`, `transaction_tests`, `terminal_stream_*`, `run_locked_blocking`, `acquire_with_timeout_async`, `run_ref_mutation_inner`, `pre_exec`, `recursion_limit`, `run_with_retry_sync`); owner links resolve; `git diff --check` clean. Repository-wide checks left to `/validate`. + - Context impact: important — the edited files are the durable context for this change. + - Context synchronization: synced + - Context synchronization evidence: `synced`; the task's own edits are the context update, plus new focused owner `context/cli/async-persistence-guarantees.md` (guarantees, limits, PR1–PR4 staging; linked from `architecture.md` and `context-map.md`) so `architecture.md` did not grow further. Root files read: `glossary.md` and `overview.md` unaffected beyond the hook-path correction, no stale bridge symbols in the glossary. ADR `context/decisions/2026-10-08-pr2-scope-and-cancellation-protections.md` written directly as part of T05 scope (not through `sce-decision`). `agent-trace-db.md` at 250 lines; `architecture.md` was already over 250 before this task. + +## Open questions + +Non-blocking questions added by the 2026-10-08 review update (none changes scope or ordering): + +- Whether pinned Turso 0.8.1 permits checked `Transaction::new(&mut Connection, Immediate)` with the adapter's ownership is unverified; T03 decides from source and records the equivalent static proof if not. +- T06's cancellation-safe refresh design (shielded owned unit vs. restructuring the timeout) and T07's lock-ownership design are selected by those tasks against source evidence; both must satisfy the stated requirements and neither may add a generic cancellation framework. +- Physical stack order is T03, T06, T07, T04, T05 so every task's dependencies precede it while IDs stay stable; selection by plan order with satisfied dependencies yields T03 → T06 → T07 → T04 → T05. + +Original resolution, retained: None. Static dispatch for all synchronous/asynchronous seams in the migrated closure, existing AppContext capabilities, generic credentials/resolvers/policy/conformance/repair/logging, static stream operations, native async/static enum lifecycle and command/hook dispatch, Result-valued async-once success/failure caching, direct auth/mutation/export awaits, blocking policy, lifetime cleanup, and PR staging are resolved above. T01 inventories source paths and T03 proves cancellation/retry behavior within this fixed architecture; neither leaves an architectural choice deferred. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-08 + +### Commands run + +- `nix flake check` -> exit 0 (all checks passed; checks were already built/cached at HEAD `075be01f`, 0 derivations rebuilt) +- `nix build .#ci-checks` -> exit 0 (sce, musl deps, portability audit and ci-checks built) +- `git diff --check` -> exit 0 (no whitespace errors) +- `./scripts/run-cli-cargo.sh test` for each Test A–G filter, confirmed by name in output -> exit 0 (1 passed each; `write_contention_` 2 passed; plus retained `sibling_failure_does_not_cancel_in_flight_credential_save`) +- Source audits via `nix shell nixpkgs#ripgrep -c rg` (runtime/blocking, `new_unchecked`, dyn/BoxFuture/async_trait, OnceCell, CredentialStore, opentelemetry) -> completed and classified +- `git diff --stat main -- cli/build.rs cli/migrations` -> empty (no migration changes) + +### Success-criteria verification + +- [x] AC1: no runtime-bridge symbols -> only `Handle::try_current` in `git_snapshot.rs:718` (Drop-time cleanup via `spawn_blocking`, not DB), `RuntimeFlavor` checks in `app.rs`, and a test name in `hook_merge.rs` +- [x] AC2: `db/mod.rs` persistence methods are native async; flake check passes +- [x] AC3: `new_unchecked` has zero matches; transactional primitives take `&mut` with `Transaction::new(&mut Connection, ..)`; Test A is a documented compile-time argument +- [x] AC4: `cli/build.rs` and `cli/migrations` diff against main is empty +- [x] AC5: DB paths use `run_with_retry_elapsed`; Test C passes (2 tests) +- [x] AC6: Test C passes on independent connections; no lost or duplicate rows +- [x] AC7: `run_credential_operation` has zero matches; Test D passes +- [x] AC8: Test B passes (`abandoned_transaction_commits_nothing_and_rolls_back_on_next_use_of_its_connection`) +- [x] AC9: remaining `spawn_blocking` sites enumerated (encryption key, git snapshot, protected worktree, worktree lock, guarded-shell supervisor, adapter state/lock); zero `block_in_place`, zero `SyncStorageGuard`, no token-storage `spawn_blocking`; per-site reasons recorded in T04 +- [x] AC10: scope reconciled in T05; no OpenTelemetry/OTLP matches +- [x] AC11: T05 context sync recorded as done; not re-audited line by line beyond task evidence +- [x] AC12: all Test A–G filters confirmed to exist and pass; flake check and ci-checks pass +- [x] AC13: only remaining dyn match is `parse/command_runtime.rs:10` (`Option<&dyn LoggerTrait>`), unchanged on this branch (unrelated untouched legacy seam); zero BoxFuture/async_trait/dyn Db/dyn CredentialStore +- [x] AC14: `AuthenticatedControlPlaneClient` with native async `CredentialStore`; Test D passes +- [x] AC15: lifecycle async propagation verified by compilation and T02 evidence +- [x] AC16: `tokio::sync::OnceCell` with Result-valued `get_or_init`; no `get_or_try_init` +- [x] AC17: no executor-bridge matches; DB work not in `spawn_blocking` +- [x] AC18: Tests A–G present and passing (A documented compile-time) +- [x] AC19: Test F passes +- [x] AC20: Test G passes +- [x] AC21: Test E passes +- [x] AC22: Deferred cleanup ledger present in the plan (line 418) + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- `nix flake check` reported 0 checks run because results were already cached at HEAD; relies on Nix store caching being valid for this exact tree. +- AC11 and AC22 rest on recorded task evidence plus the ledger's presence, not a full re-read of every context file. +- Turso 0.8.1 deferred rollback and WorkOS remote-rotation ambiguity remain documented limitations. diff --git a/context/plans/mutation-cursor-ref-reconciliation-wiring.md b/context/plans/mutation-cursor-ref-reconciliation-wiring.md new file mode 100644 index 000000000..0483ce975 --- /dev/null +++ b/context/plans/mutation-cursor-ref-reconciliation-wiring.md @@ -0,0 +1,537 @@ +# Plan: mutation-cursor-ref-reconciliation-wiring + +## Change summary + +Make the existing mutation-cursor snapshot-ref reconciler (`runtime/ref_reconciliation.rs`, async `reconcile_worktree`) operational. Today it is "retained and unwired": nothing in production calls it, so pins left by interrupted `coordinate()` paths accumulate forever. + +This plan wires reconciliation in two deliberately different ways: + +- **Actual reconciliation (Git and DB I/O) runs only through `sce doctor --fix`.** A new runtime maintenance entrypoint performs exactly one `WorktreeLock` acquisition, verifies and opens the existing repository Agent Trace DB without side effects, runs the existing reconciliation algorithm and atomic deletion transaction, and records a diagnostic result. +- **The mutation-hook completion path (`Close`/`Flush`) performs only a lightweight advisory check.** It never lists refs, opens the DB, or runs reconciliation. It records that reconciliation is recommended in a small durable state file at most once per advisory window for time-based staleness, and may record again earlier only for a newly recorded explicit-reconciliation failure (see AC5); `sce doctor` (read-only) reports the recommendation and `sce doctor --fix` acts on it. + +**Fully automatic reclamation is not delivered by this PR.** No lifecycle-owned, noncritical-path execution point exists today (see D2), and inline reconciliation on the tool-completion path cannot be bounded. Automatic out-of-band reconciliation is a documented follow-up PR with stated prerequisites. Nothing in this plan or in the synchronized context may claim that orphan pins are reclaimed automatically. + +The change extends existing behavior and preserves the reconciliation algorithm, `GitSnapshotService`, `MutationTraceStore`, `WorktreeLock`, the atomic SHA-conditional ref deletion transaction, the protocol, the Quint model, and mutation-attribution rules. It must never delete a tree required by any repository-wide durable root and must never change the success/failure outcome of a mutation boundary. + +Review history (all resolutions are binding on T01–T06): + +*Round 1 (six findings)* + +1. **Single lock acquisition (P1).** `reconcile_worktree_inner` already acquires `WorktreeLock` (`ref_reconciliation.rs:172`). The reconciler is split into a lock-acquiring wrapper and a shared lock-held implementation. Each maintenance operation acquires the lock exactly once. See D1. +2. **Hook latency (P1).** Resolved definitively in round 2: no reconciliation on the hook path. See D2. +3. **Failure state (P2).** Failures are durably recorded as diagnostics with a failure streak. Round 2 removed automatic retry scheduling because no automatic full pass exists. See D3. +4. **Corrected AC3 (P2).** Two invariants are claimed, not "every durable root is pinned": local consistency (fail closed, delete nothing) and repository-wide deletion safety (never remove another worktree's protecting pin). Reconciliation never repairs another worktree's missing pins. +5. **PR metadata (P2).** The real PR is #305, head branch `ref-recon`, base `fix-logging`. +6. **Database-opening contract (P2).** The existing hook opener is not read-only. Reconciliation uses a dedicated verified-existing-DB opener with no create/migrate/repair/initialize side effects. See D4. + +*Round 2 (three findings)* + +7. **Unbounded inline latency (P1).** A cooperative budget checked between phases cannot bound `git for-each-ref`, DB open, repository-wide root queries, `git update-ref`, or Tokio blocking-worker shutdown, and reconciliation holds the lock that the coordinator waits on for 10 s. Selected: advisory-only hook trigger, reconciliation through `sce doctor --fix`, automatic execution deferred to a follow-up. The 250 ms budget, size/cost gates, and deferral machinery are removed. See D2. +8. **Repeated scans after state-persistence failure (P2).** The reservation-before-expensive-work requirement applies to any *automatic* full pass. Because this PR has none, no reservation infrastructure is built; the requirement is recorded as a binding prerequisite of the follow-up. The advisory path emits a recommendation only after the state write succeeds, and `Completed` vs `CompletedStatePersistFailed` stay distinct for explicit passes. See D2/D3. +9. **Far-future timestamps (P2).** The state file stores only past-event timestamps and never a deadline; eligibility is computed as `now − timestamp ≥ window`, so rereading cannot extend a deadline. A timestamp more than `FUTURE_SKEW_TOLERANCE` ahead of `now` is invalid and is normalized once under `WorktreeLock` and persisted; a timestamp within the tolerance is valid and treated as age 0. See D3. + +*Round 3 (four findings; plan-only corrections, D1–D4 and the T01–T06 structure unchanged)* + +10. **External-taint marker assertion (P2).** `CoordinateError::MarkerClearAfterCommit` means the boundary committed but clearing the marker failed, so the marker may legitimately remain armed. The advisory must never create, clear, or modify the marker; it is not required to leave it unarmed. Normal completion and `MarkerClearAfterCommit` are asserted separately (AC4, S18a, S18b). `coordinator.rs`, `ProtectedWorktree` and external-taint recovery semantics are not changed. +11. **Advisory contention vs coordinator contention (P2).** The advisory runs only after `coordinate()` has returned, so an explicit pass holding `WorktreeLock` can make `coordinate()` itself fail (pre-commit lock timeout) before the advisory is ever reached. Contention is now tested as three distinct scenarios: direct advisory contention (S15), coordinator contention (S16), and the post-coordinate handoff race (S33). See AC12. +12. **Git-ref assertions (P2).** `coordinate()` legitimately captures and pins trees. Tests no longer expect refs to be unchanged across `Close`/`Flush`; they assert that the advisory itself performs no ref creation, update, deletion, or inventory, by comparing against a coordinate-only baseline or an injected advisory seam. See AC2, AC5, S14. +13. **Future-skew contradiction (P3).** D3 accepts timestamps up to `FUTURE_SKEW_TOLERANCE` (5 min) ahead of the injected `now` as age 0, so "no stored timestamp is ever in the future" was wrong. Replaced with the exact tolerance invariant and seven explicit clock cases (S24). See AC13. + +Stacking: + +- Repository: `crocoder-dev/shared-context-engineering` +- Parent: PR #304, branch `fix-logging`. PR #304 is not modified by this plan. +- This work: PR #305, head branch `ref-recon`, base `fix-logging`. No other branch or PR is created. +- Verified at authoring time: PR #304 head `0be74aad5bea5070828846d3277041a4649c9842`; PR #305 head is whatever `ref-recon` points to (it advances with every plan or implementation commit, so T01 re-records the then-current heads rather than relying on a value written here). +- If #304 advances, rebase `ref-recon` onto its latest head and record the exact SHA in the T01 audit findings. + +## Authoring-time observations (to be re-verified by T01) + +Facts the design relies on. T01 must re-read each cited location and either confirm or flag it as invalidating a later task. + +- `ref_reconciliation.rs:159-243` `reconcile_worktree_inner` acquires `WorktreeLock` via `acquire_inner_async` with a fixed 10 s timeout (`:172`), resolves the worktree ID, calls `open_db`, lists pins, loads local and repository-wide durable roots, and deletes through `delete_pins(lock.lease(), …)` (`:233`). The `lock` local is dropped on return. +- `worktree_lock.rs`: the lock is a non-reentrant advisory file lock (`File::try_lock`) on `/sce/mutation-cursor.lock`; `acquire_inner` polls every 100 ms until the deadline, so a zero timeout is exactly one `try_lock` attempt. `WorktreeLockLease` is an `Arc` clone of the lock inner; the file unlocks only when the last clone drops. +- `git_snapshot.rs:281-300` `run_ref_mutation_inner` moves the `WorktreeLockLease` into the `spawn_blocking` closure (`let _lease = lease;`). A dropped caller future therefore cannot release the file lock while the blocking `git update-ref` worker runs. `delete_pins_inner` has a post-preflight seam but `run_ref_mutation` passes a no-op `on_worker_entered`. +- `protected_worktree.rs:14` `WORKTREE_LOCK_TIMEOUT` is 10 s: a competing `coordinate()` fails its boundary (before durable completion, with the existing lock-timeout error) if any holder keeps the lock longer than that. +- `protected_worktree.rs:178-184` `abandon_after_spawn_without_unlock` leaves the lock held for an external-guard child; other lock users must skip or wait per their own policy. +- `hooks/mutation_scope.rs:356-378`: `classify_coordinate` maps `Ok(_)` and `MarkerClearAfterCommit` to `Ok(String::new())` and every other `CoordinateError` to a boundary failure. `drive_mutation_scope` returns after `coordinate_boundary(...).await`, so the coordinate-held lock is released before any code placed after it runs. +- `hooks/runtime.rs:53-69` `open_agent_trace_db_for_hook_runtime` → `agent_trace_storage/mod.rs:66-74, 195-207` `open_repository_db_for_hook_runtime`: (a) `TursoDb::open_without_migrations_at` (`db/mod.rs:856-860`) calls `ensure_db_parent_dir` and `turso::Builder::new_local(...).build()`, which can create parent directories and an empty DB file; (b) on schema-not-ready it calls `repair_missing_repository_schema_migration_metadata` (`agent_trace_db/repository.rs:136`), which inserts the baseline migration row; (c) it then calls `verify_or_initialize_repository_metadata` (`repository.rs:171`), which inserts repository metadata when absent. It avoids schema migrations but is **not** read-only. +- Pin count is not bounded by orphan count: every retained historical tree root keeps its pin (`A → B → C → D` retains all four), so inventory and durable-root load cost grow with history. A full pass is O(history), not O(orphans). +- Tokio runtime shutdown joins in-flight `spawn_blocking` workers, so dropping a future (for example through `tokio::time::timeout`) does not shorten process latency for work already in a blocking worker. +- `resolve_git_dir` and `resolve_worktree_id` spawn `git` subprocesses (`runtime/git_snapshot.rs`). Any hook-path advisory that called them would add a new subprocess to the completion path. +- Candidate execution points for out-of-band work found in the tree: the Git `post-commit` subcommand (`hooks/commit_hooks.rs:114`), per-harness session-end/stop events (for example `hooks/claude_mutation_scope/lifecycle.rs`, `hooks/codex/stop.rs`, Codex session-end fixtures), `sce setup` lifecycle, and the doctor command (`doctor/mod.rs`, `doctor/fixes.rs`). None is a lifecycle-owned, noncritical-path executor that is shared across harnesses; see D2. + +## Design decisions + +- **D1 — Single lock acquisition and ownership (round 1).** + - `ref_reconciliation.rs` is split into: (a) `reconcile_worktree` / `reconcile_worktree_inner`, kept with unchanged signatures and behavior, which acquire `WorktreeLock` once and delegate; and (b) a new lock-held implementation `reconcile_with_held_lock` that borrows the held `WorktreeLock`, never calls `acquire_inner_async`, and derives deletion leases from the held lock via `lock.lease()`. + - There are exactly two maintenance operations, each with its own single lock acquisition and neither calling the other: + - `reconcile_explicit` — acquires the lock once with the existing bounded 10 s wait, then runs (b), then records the diagnostic result in the state file while still holding the same lock. It never calls the lock-acquiring wrapper (a). + - `advise_if_due` — hook-path advisory; acquires the lock once with a zero timeout (single `try_lock`), reads/normalizes/writes the state file, releases. It never calls (a) or (b) and has no DB or Git parameters. + - State reads that decide a write and the write itself happen under the same single acquisition. An unlocked read is permitted only as an advisory fast exit (and for read-only `sce doctor` reporting) and is always rechecked under the lock before any write. + - The `WorktreeLock` stays owned by the explicit pass for its whole duration. Mutating Git work receives `lock.lease()` and, as today, moves it into the blocking worker, so cancellation of the caller future cannot release the lock while a Git ref mutation worker is still running. + - The hook ingress invokes `advise_if_due` only after `coordinate_boundary(...)` has returned, so the mutation coordinator's own lock is never held during the advisory. + - No reentrant lock, no lock-passing between processes, no global lock registry. + - Documented concurrency contract: an explicit pass holds the lock for its whole duration because the pin-then-root ordering that makes deletion safe requires it (inventory, root reads, and deletion must not interleave with `coordinate()`'s pin-then-CAS). A mutation boundary arriving meanwhile waits up to `WORKTREE_LOCK_TIMEOUT` (10 s) and then fails *before durable completion* with the existing lock-timeout error. This is an operator-initiated, documented trade-off of `sce doctor --fix`; the hook-path advisory never waits on the lock. +- **D2 — Hook-path trigger is advisory-only; reconciliation is explicit (round 2, resolves P1).** + - *Why inline reconciliation is rejected.* The hook completion path cannot bound `git for-each-ref`, DB open, repository-wide durable-root queries (they scale with historical evidence), or a started `git update-ref` (which must complete safely), and Tokio shutdown waits for blocking workers. A size cap, a historical-cost gate, or a cooperative between-phase budget only predicts cost; none gives an upper bound. Wrapping any step in a timeout would either abort unsafe Git work or fail to shorten latency. Additionally, reconciliation holds `WorktreeLock`, so a slow pass can push an unrelated boundary past its 10 s lock timeout. Cooperative budgets, caps, cost ceilings, deferral windows, and any 250 ms promise are therefore **removed from the design**, and no replacement timeout is introduced. + - *Execution-point investigation.* Alternatives considered and rejected (T01 re-verifies each claim): + 1. Git `post-commit` subcommand — synchronous inside `git commit`, which an agent tool can invoke, so it is the same critical path with the same unbounded work. + 2. Per-harness session-end/stop events — not part of the shared mutation-scope ingress, differ per harness (adapter changes are out of scope), still synchronous hook processes that block session shutdown, and have no uniform lifecycle owner. + 3. `sce setup` / lifecycle service — install-time, not recurring. + 4. Detached Tokio task, fire-and-forget subprocess, background daemon — prohibited (a short-lived hook process cannot own them; no supervision, ownership, or failure recovery exists). + 5. `sce doctor --fix` — operator-owned, foreground, can report its true result. **Selected.** + No suitable lifecycle-owned noncritical-path mechanism exists in the current architecture. + - *Selected design.* + - `Close`/`Flush` (durably completed, including the `MarkerClearAfterCommit` case, after `coordinate_boundary` returned) calls `advise_if_due`, a synchronous function with this complete I/O budget: one capped read (≤ `MAINTENANCE_STATE_MAX_BYTES`) of the state file, at most one zero-wait `try_lock`, at most one atomic write of that small file, all local. It performs no `git` subprocess, no DB open or query, no ref listing, no `spawn_blocking`, no await point after the lock is taken. It is a bounded set of small local file operations of the same class the hook path already performs for the lock and taint marker; it is not claimed to be hard real-time, and any unexpected failure is swallowed into a logged outcome. + - The advisory must not add a new Git subprocess. It obtains the worktree's Git dir without spawning Git (T01 determines whether it can be derived from the filesystem — `.git` directory, or `gitdir:` file for linked worktrees — and proves equality with `resolve_git_dir` for the main and a linked worktree, or whether `coordinate`'s already-resolved value can be reused without protocol changes). **Decision rule:** if neither is possible without a new Git subprocess, the hook trigger is dropped entirely and `sce doctor` computes the recommendation from the state file alone; AC1/AC5 are then satisfied with zero hook callers and T03 reduces to documenting that outcome. This fallback needs no user confirmation because it only removes a convenience signal. + - `advise_if_due` outcomes: absent/invalid-version/oversized/corrupt state → write an anchor and return `Anchored` (no advice). Reference time (`last_success`, else `anchor`) older than `RECONCILIATION_ADVISORY_AFTER` and no valid advice within the same window → persist `last_advised`, then (only after the write succeeded and its persistence was confirmed) return `Advised`. `AdvisedDurabilityUncertain`: the atomic replacement happened (the advisory state may be visible) but crash durability could not be confirmed; an actionable diagnostic is emitted and durability is never claimed. `StateWriteFailed` means no confirmed successful advisory-state update and preserves the `PersistFailure`: with `NotApplied` the replacement did not occur and the previous bytes are intact; with `DurabilityUncertain` (only for an `Anchored` or `NoAction` write, i.e. an anchor or normalization) the replacement occurred, the new state may be visible, crash durability is unconfirmed, and no advice is emitted. A due `Advised` decision with `NotApplied` is `StateWriteFailed`; with `DurabilityUncertain` it is `AdvisedDurabilityUncertain` (an actual recommendation was due and the advice-state replacement occurred). `StateWriteFailed` never emits a recommendation and no scan runs. `StateUnavailable`: the maintenance state cannot be accessed or evaluated (unresolvable Git dir, unreadable state, lock I/O error); it is not a write outcome, emits no advice, and has no retry loop. `NoAction` (nothing due, no write) and `Busy` (zero-wait lock contention, skipped, not an error) emit no actionable advisory. + - *Cadence.* The normal time-based reminder for an unresolved condition repeats at most once per `RECONCILIATION_ADVISORY_AFTER` (24 h) window per worktree. The one exception is a newly recorded explicit reconciliation failure (`last_attempt` newer than `last_advised`): it may trigger one immediate advisory inside an otherwise active 24 h window, is suppressed once `last_advised` records it, and then follows the normal 24 h cadence until a successful explicit pass clears it. The advisory is therefore not unconditionally one-per-window. + - `sce doctor` (read-only) reports whether reconciliation is recommended (last success age, last attempt/failure streak, last advice) computed from the state file as a pure function; `sce doctor --fix` runs `reconcile_explicit` regardless of any state. + - *Deferred to a follow-up PR (explicitly not in this plan).* Fully automatic out-of-band reconciliation. Binding prerequisites for that PR: (1) a lifecycle-owned execution point with defined ownership, lifetime, scheduling and failure recovery (for example an explicitly invoked `sce` maintenance command run by a user-managed scheduler, or a future shared session-end mechanism) — never a detached task, daemon, or timeout-cancelled Git operation; (2) a **durable next-attempt reservation persisted under `WorktreeLock` before opening the DB, listing pins, or loading durable roots**, bounded, never interpreted as success, superseded by the success cadence after success or exponential failure backoff after failure, and bypassed by explicit repair; if the reservation cannot be persisted the automatic pass is skipped and the maintenance-state failure is reported; (3) distinct outcomes for completed, completed-but-state-not-persisted, failed, and skipped/deferred; (4) bounded or split lock-hold analysis against the 10 s coordinator timeout; (5) the same invalid-timestamp normalization as D3. T06 records this follow-up in context. + - *Trade-offs.* (+) hook completion path has no unbounded work and no new failure mode; correct under every ordering with `coordinate()`; no new infrastructure without a consumer. (−) orphan pins are reclaimed only when an operator runs `sce doctor --fix`; the recommendation is time-based (staleness since last success), not evidence-based, because detecting orphans needs the Git/DB scan this design keeps off the hook path. +- **D3 — Durable maintenance state: diagnostics, no stored deadlines (rounds 1–2).** + - One small JSON file per worktree at `/sce/ref-maintenance.json` (versioned, capped at `MAINTENANCE_STATE_MAX_BYTES`, 4 KiB), beside the lock and taint marker. Fields (all optional past-event unix-ms timestamps unless noted): `anchor`, `last_success`, `last_attempt`, `last_advised`; `last_attempt_outcome` (`completed` | `failed`); `last_report` (retained/deleted/local-required counts); `consecutive_failures` (integer); `last_failure` (short classification plus bounded message). + - **No field is ever used as a deadline.** Eligibility is computed at read time as `now − reference ≥ window` (saturating at 0). Re-reading therefore cannot move a deadline. The stored-timestamp invariant is scoped to **successful state-writing operations**: after a successful write, every stored timestamp `t` satisfies `t ≤ now + FUTURE_SKEW_TOLERANCE`. It is not claimed after read-only or unsuccessful operations (see the two operation classes below). A value in `(now, now + FUTURE_SKEW_TOLERANCE]` is valid, is preserved byte-for-byte (not rewritten), and is evaluated as age 0; it can delay eligibility by at most the tolerance and cannot slide, because it is never refreshed by reads. + - **Two operation classes.** + - *Read-only or unsuccessful operations* — plain `sce doctor`; a lock-busy advisory or explicit pass; a failed state read; a failed state write; a process interrupted before atomic replacement. These never mutate maintenance state (a failed-before-rename write, `NotApplied`, or an interrupted write leaves the previous bytes intact; unreadable state is `StateUnavailable`, not a failed write), so they may observe preexisting invalid timestamps and cannot guarantee normalization. They must evaluate invalid timestamps conservatively (treated as eligible/recommended, never as age 0 and never as a reason to defer), must not postpone an advisory or explicit repair indefinitely, and must not report invalid state as successfully normalized. + - *Successful state-writing operations* — a successful `advise_if_due` write and a successful `reconcile_explicit` state record. While holding `WorktreeLock` they normalize invalid timestamps and persist the normalized state atomically, so that every stored timestamp satisfies the tolerance after the successful write; valid near-future timestamps are preserved as specified above; repeated reads never move eligibility forward; subsequent invocations, including after a process restart, observe the persisted normalization. + - Writes happen only under `WorktreeLock`, via atomic staging-and-swap (temp file in the same directory, then rename; pattern identified in T01) so a crash leaves either the old or the new file. No in-memory registry or process-global state. + - Transitions: advisory — anchor and `last_advised` only. Explicit success — `last_attempt` = `last_success` = now, `last_report` set, `consecutive_failures` = 0, failure fields cleared. Explicit failure (DB unavailable/invalid, malformed ref, missing required pins, durable-root read error, delete-transaction error) — `last_attempt` = now, outcome `failed`, `consecutive_failures` += 1, `last_failure` set, `last_success` untouched. Lock contention (including explicit lock timeout) — **no state change**; never classified as a failure. + - **Invalid timestamp normalization (deterministic).** A timestamp is invalid when it is strictly more than `FUTURE_SKEW_TOLERANCE` (5 min) after the injected `now` (far-future values, or clock rollback beyond the tolerance); a timestamp exactly equal to `now + FUTURE_SKEW_TOLERANCE` is valid. A valid timestamp ahead of `now` (within the tolerance) is treated as age 0, so a small clock rollback never makes an advisory eligible by itself. Under `WorktreeLock`, `advise_if_due` and `reconcile_explicit` normalize once, as part of their successful state write: invalid `last_success`/`last_attempt`/`last_advised`/`anchor` values are cleared; if `last_success` and `anchor` are both unusable the state is **immediately eligible** for advice, so the advisory persists `last_advised = now` and returns `Advised` in the same locked operation, and a fresh `anchor = now` is written. The normalized state is persisted before returning, so a second read sees valid timestamps, performs no rewrite, and does not shift or postpone eligibility; once the injected clock advances, ordinary window-based eligibility resumes. If the lock is busy or the normalizing write fails before the rename (`NotApplied`), nothing is persisted and the invalid bytes stay as they were; if it fails after the rename (`DurabilityUncertain`) the normalized state may already be visible but is never claimed durable. In every case the in-memory evaluation still treats invalid values conservatively (eligible), so a later invocation can normalize or re-read them safely. Plain `sce doctor` never writes; it evaluates invalid timestamps as "recommended" with an "invalid maintenance timestamps" note and leaves the file byte-identical. `sce doctor --fix` runs regardless of timestamp validity and rewrites valid state afterwards. + - Recovery: a missing, unparsable, wrong-version, or oversized file is treated as default and rewritten by the next recorded operation; it never fails an explicit pass. + - Result distinction for explicit passes (typed outcome `ExplicitOutcome`): `Completed(report)`; `CompletedStatePersistFailed { report, warning }` — the cleanup stands and is reported as done, with a typed `StatePersistWarning`; `Failed { error, state_warning: Option }` — a failed pass keeps both the reconciliation error and any persistence warning; `Skipped(Busy)`. Never merged. `StatePersistWarning` is `PreviousStateUnreadable(io::Error)` (the update was not attempted) or `WriteFailed(PersistFailure)`, where `PersistFailure::NotApplied` means the atomic replacement did not occur and the previous state is preserved, and `PersistFailure::DurabilityUncertain` means the replacement occurred (the new state may be visible) but crash durability could not be confirmed. The doctor facade preserves this as `StateWarning::{NotApplied, DurabilityUncertain, PreviousStateUnreadable}` with a stable kind, an optional phase, and a cause; it reports "Maintenance-state update was not applied." / "Maintenance-state durability could not be confirmed." and never claims that diagnostics were not recorded when they may have been. Advisory outcomes: `Anchored | NoAction | Advised | AdvisedDurabilityUncertain | Busy | StateWriteFailed | StateUnavailable`. + - Time comes from an injected wall clock (`Fn() -> i64` unix ms) passed as a generic parameter for deterministic tests. +- **D4 — Verified-existing Agent Trace DB opener (round 1).** + - Side-effect guarantees are stated per database state. **Missing DB:** Turso is invoked in non-creating read-only mode (`read_only(true)` with multiprocess WAL), so that open cannot create the database; no DB file, parent directory, WAL, or shared-memory file is created; no Git refs are listed or deleted. **Existing but invalid DB:** opening the existing file is permitted and incidental Turso WAL/shared-memory bookkeeping is permitted if unavoidable (documented); logical immutability is required — no schema migration, no migration-metadata repair, no repository-metadata initialization, no application-table INSERT/UPDATE/DELETE, no Git-ref deletion — and verification fails with the matching typed error. **Existing valid DB:** verify repository identity and schema readiness before reconciliation, preserve the existing multiprocess-WAL configuration, allow only the database access reconciliation needs, and run no initialization, repair, or migration. + - Contract for reconciliation: (1) never create a DB file or any parent directory (the non-creating read-only Turso open fails for a missing DB, including one removed between a preliminary check and the open); (2) never run migrations; (3) never call `repair_missing_repository_schema_migration_metadata`; (4) never call `verify_or_initialize_repository_metadata` or otherwise insert/update repository metadata; (5) verify repository identity (stored `repository_id` equals the identity resolved for this checkout, `source_instance_id` valid) and schema readiness (baseline plus the migration providing durable mutation roots) **before** any ref listing or deletion; (6) any failure returns an error with refs untouched, as a maintenance error that never arms `ExternalTaintMarker` and never becomes `CoordinateError::AgentTraceDbUnavailable`. + - The existing hook opener cannot satisfy (1), (3), or (4), and changing it would alter every hook caller. The smallest satisfying change is a dedicated opener in `agent_trace_storage/mod.rs` (for example `resolve_existing_agent_trace_storage_for_maintenance`) plus one read-only verification method on `RepositoryAgentTraceDb` (for example `verify_existing_repository_metadata(repository_id)`) wrapping the existing `select_repository_metadata_row` without insert. The opener relies on no separate existence check: the Turso open itself is non-creating and read-only (`TursoDb::open_existing_without_migrations_at`), and a failed open is classified `Missing` only when the path is then absent. Plain `sce doctor` shares that open through `RepositoryAgentTraceDb::open_existing_schema_ready_at`, which inspects schema readiness without reading or requiring repository metadata. Incidental WAL/shared-memory bookkeeping on an existing DB is an accepted limitation: the guarantee is logical immutability (schema objects and migration/metadata rows unchanged) and, for non-database files, byte identity; full filesystem byte-identity for valid Turso databases is not claimed. + - Typed failure kinds (small enum, converted to `anyhow` at the existing `open_db` seam): `Missing`, `Unreadable`, `IncompatibleSchema`, `MissingMetadata`, `RepositoryMismatch`. The existing hook opener and all its callers are unchanged. + - Considered and rejected: opening the DB before taking the lock to shorten lock-hold. It would require a second lock acquisition just to record an open failure, violating the one-acquisition-per-operation rule in D1; the lock-first order of the existing reconciler is kept and the D1 concurrency contract documents the consequence. + +## Acceptance criteria + +How this plan is proven complete. `/validate` runs these checks; no task performs final validation. + +- [x] AC1: Reconciliation has one real non-test **explicit** caller (`sce doctor --fix`) and no automatic reconciliation caller; the mutation-scope ingress has at most an advisory caller that cannot reach reconciliation. + - Validate: `nix shell nixpkgs#ripgrep -c rg "reconcile|advise_if_due" cli/src --glob '!*test*'` shows the reconciliation entrypoint referenced only from `doctor/` (and its definition module), and `advise_if_due` (if present per the D2 decision rule) referenced from `hooks/mutation_scope.rs`; the hook module imports no reconciliation, `GitSnapshotService`, store, or DB-opener symbol for this purpose. +- [x] AC2: `sce doctor --fix` deletes an orphan snapshot ref; `Close`/`Flush` never deletes. + - Validate: T05 S1 drives the doctor fix path against a real Git repo and asserts the orphan `refs/sce/mutation-cursor//` is gone; S14 asserts the advisory path creates, updates, deletes, and inventories no refs (ref changes made by `coordinate()` itself, such as snapshot pins, are legitimate and are not attributed to maintenance). This plan does **not** claim automatic orphan reclamation. +- [x] AC3: A successful pass establishes two distinct invariants, and claims nothing stronger. + - **Local consistency:** `durable_roots(W) ⊆ pinned_trees(W)` is checked before any deletion for the target worktree `W`. If it fails, the pass deletes nothing and returns `MissingRequiredPins`. + - **Repository-wide deletion safety:** `delete(W, T) ⇒ T ∉ durable_roots(repository)`. A pin is removed only when its tree is outside every worktree's durable roots, so reconciliation never removes an existing SCE pin that protects another worktree's required tree. + - Reconciliation does **not** repair, recreate, or verify missing pins belonging to other worktrees, and does not guarantee that every repository-wide durable root is pinned. + - Validate: T05 S2, S3, S4, S5 and S13 pass. S2/S3 prove preservation; S4 proves local fail-closed; S13 proves another worktree's missing pin is neither repaired nor treated as a failure of `W`'s pass. +- [x] AC4: Hook-path maintenance (advisory) cannot change the outcome of an already durably committed mutation boundary. + - Validate: T05 S17, S18a, S18b and S33 pass: every advisory outcome (`Anchored`, `NoAction`, `Advised`, `AdvisedDurabilityUncertain`, `Busy`, `StateWriteFailed`, `StateUnavailable`, corrupt/invalid state, seam error) and a failing diagnostic-output writer leave the hook result identical to the coordinate-only baseline. The advisory never creates, clears, or modifies the external-taint marker. After normal completion the marker state is whatever the existing coordinator leaves (cleared) and is unchanged by the advisory (S18a). `MarkerClearAfterCommit` remains a durably completed boundary with a successful hook result; the marker state resulting from the failed clear is preserved by the advisory, and the next real runtime invocation still takes the existing conservative inherited-taint recovery path (S18b). `MarkerClearAfterCommit` is never treated as equivalent to a successful clear. +- [x] AC5: The mutation-hook completion path performs no reconciliation, ref listing, DB I/O, or new Git subprocess, and its maintenance-state writes follow a precise cadence. Time-based reminders: a stale state becomes eligible after `RECONCILIATION_ADVISORY_AFTER` (24 h) and further stale-state reminders are suppressed for 24 h after `last_advised`. Failure-triggered reminders: an outstanding failed explicit reconciliation is recommended immediately; a newly recorded failure (`last_attempt` newer than `last_advised`) may be advised even inside the previous 24 h window and is suppressed once recorded; repeated reminders for the same unresolved failure follow the normal 24 h cadence; a successful explicit reconciliation clears the failure-based recommendation. Anchoring and clock-skew normalization are the only other writes. Actionable reminders reach an explicit diagnostic channel (stderr) in addition to structured logging; delivery is best-effort and neither exactly-once nor guaranteed to be seen by a human (T03 remediation, context/cli/mutation-scope-ref-advisory.md). + - Validate: T05 S14, S15, S19 and S20 pass (S14: the advisory performs no Git-ref creation, update, deletion, or inventory, no reconciliation, and no DB I/O; coordinator ref effects are legitimate and distinguished from it); `rg` over the advisory module shows no import of `GitSnapshotService`, `MutationTraceStore`, DB openers, `resolve_git_dir`, `resolve_worktree_id`, `spawn_blocking`, or `Command`. +- [x] AC6: Contention, cancellation, and failed maintenance never violate the pin-to-CAS safety boundary. + - Validate: T05 S6, S7 and S12 pass using deterministic synchronization (no sleep-based proof): concurrent `coordinate()` cannot race pin deletion; dropping the explicit-pass future during a ref-deletion worker does not release the lock until the worker finishes; each operation completes without self-deadlock. +- [x] AC7: No new unbounded in-memory registry, dynamic dispatch, detached background worker, timeout-cancelled Git work, or nested runtime. + - Validate: `git diff fix-logging...HEAD` inspection plus `nix shell nixpkgs#ripgrep -c rg "dyn Future|BoxFuture|tokio::spawn|Runtime::new|block_on|tokio::time::timeout|JoinHandle|abort\(|static .*(Mutex|RwLock|OnceLock|HashMap)" ` returns no new hits. +- [x] AC8: Mutation-cursor and Quint model-based tests remain valid without protocol changes. + - Validate: `git diff fix-logging...HEAD --stat` shows no change to `protocol.rs` or `spec/mutation_cursor.qnt`; the existing `mbt` tests pass. +- [x] AC9: `sce doctor` stays non-mutating — it never creates or logically modifies the Agent Trace DB (its schema inspection uses the non-creating read-only open of D4) and never writes maintenance state or refs; `sce doctor --fix` reports its actual cleanup result, exposes it as a typed `ref_reconciliation_fix` JSON object (outcome, counts, failure kind, state-warning kind/phase) alongside the unchanged `fix_results`, distinguishes `NotApplied` from `DurabilityUncertain` persistence warnings, and recovers from every failure, stale, and invalid-state condition. + - Validate: T05 S9, S25, S26 and S27 pass: counts for deleted/retained/local-required; distinct output for completed, completed-with-state-persist-failure, failed (unavailable/invalid DB, malformed refs, missing pins), and busy; plain `sce doctor` writes no ref and no state; a recommendation is shown when stale, when the latest explicit pass failed (regardless of `last_advised`), or when state is invalid. The doctor recommendation is independent of whether a hook reminder was ever seen, which is why lossy hook delivery is acceptable. +- [x] AC10: Context matches the implementation, the "retained and unwired" status is gone, and context states that automatic reclamation is deferred to a follow-up with its prerequisites. + - Validate: `nix shell nixpkgs#ripgrep -c rg -i "retained and unwired|unwired" context/cli` returns no stale claims; `rg -i "automatic.*(reclam|reconcil)" context` hits only the deferral/follow-up statements; documented APIs match code. +- [x] AC11: Exactly one `WorktreeLock` acquisition per maintenance operation, with no self-deadlock. + - Validate: T02 and T05 S12; `rg "acquire_inner_async|acquire_async|acquire_inner" cli/src/services/mutation_trace/runtime` shows each entrypoint acquires once and `reconcile_with_held_lock` never acquires. +- [x] AC12: The hook completion path contains no unbounded maintenance work, and the documented concurrency contract for explicit reconciliation holds. + - Validate: T05 S14, S15, S16 and S33 pass, as three distinct contention scenarios. S15 (direct advisory contention): with an explicit pass parked mid-inventory or mid-DB holding the lock, `advise_if_due` called directly returns `Busy` immediately and changes no state or refs. S16 (coordinator contention): with the lock held, `coordinate()` keeps its existing bounded lock acquisition; a short injected timeout fails with the existing lock-timeout error before durable completion, the advisory does not run after the failed coordinate, and the same boundary succeeds once the pass is released. S33 (post-coordinate handoff race): after a successful `coordinate()` released its lock, a competing explicit pass takes the lock before the paused advisory resumes; the advisory returns `Busy` and the committed boundary still returns success. The advisory takes no Git/DB parameters. No timeout, budget, or size cap is claimed to bound hook latency. +- [x] AC13: Maintenance state is durable, crash-safe, and cannot postpone eligibility indefinitely. + - Two guarantee classes plus a distinct uncertain-write outcome (matching D3). **Read-only or unsuccessful operations** (plain `sce doctor`, lock-busy advisory/explicit pass, state that cannot be read or evaluated (`StateUnavailable`, which is not a failed write), a write that fails before the atomic rename (`NotApplied`), a write interrupted before atomic replacement) never mutate maintenance state, may observe invalid timestamps, evaluate them conservatively, never indefinitely postpone an advisory or explicit repair, and never report invalid state as normalized. **`NotApplied`** means the failure happened before the rename: the previous state remains intact and byte-identical, the advisory reports `StateWriteFailed`, and no successful advisory is claimed. **`DurabilityUncertain`** means the failure happened after the rename: the new state may already be visible, crash durability is unconfirmed, the advisory reports `AdvisedDurabilityUncertain` when the recommendation was being recorded (an actionable diagnostic remains available) and `StateWriteFailed` carrying `DurabilityUncertain` when only an anchor or normalization was being written (no recommendation is emitted), and a normalization written this way is never claimed to be durably persisted; a later invocation recovers safely in either case. Explicit reconciliation retains `CompletedStatePersistFailed` and the typed persistence-warning phase. **Successful state-writing operations** normalize invalid timestamps under `WorktreeLock`, persist atomically, and afterwards every stored timestamp satisfies `t ≤ now + FUTURE_SKEW_TOLERANCE`; valid near-future timestamps are preserved; repeated reads never move eligibility forward; later invocations observe the persisted normalization. + - Validate: T05 S19–S25 and S34–S39 pass: single time-based advice per 24 h window with persistence before emission (a newly recorded failure may be advised inside that window, once, per AC5); concurrent processes cannot double-advise; state-persistence failure causes no advice and no scan; interrupted atomic write preserves the previous state; process restart preserves eligibility; timestamps within `FUTURE_SKEW_TOLERANCE` are valid (age 0, preserved, never advise by themselves) while timestamps beyond it and clock rollback beyond it are conservatively eligible and, on a successful write, normalized once, persisted, and recovered (S24 cases 1–7); the exact post-condition `t ≤ now + FUTURE_SKEW_TOLERANCE` is asserted only after successful state-writing operations (S24, S36, S37), while S34, S35 and S36's pre-rename leg assert byte-identical preexisting state instead, and S36's post-rename leg asserts only that the new state may be visible with durability reported as uncertain; corrupt/missing state is recovered. Deterministic regression tests (injected clocks and deterministic synchronization, no timing-based sleeps): S34 read-only doctor observing an invalid future timestamp; S35 lock-busy advisory leaving invalid state byte-identical; S36 failed state write during normalization, with both a pre-rename leg (previous bytes unchanged, `StateWriteFailed`) and a post-rename leg (visibility may have changed, `AdvisedDurabilityUncertain`, no durable-normalization claim), a later invocation recovering safely in both; S37 successful normalization persisting valid timestamps; S38 process restart after normalization; S39 eventual advisory eligibility after time progresses. +- [x] AC14: Reconciliation opens only a verified existing DB through a non-creating read-only open that production callers cannot replace, and never initializes, repairs, or migrates it; failure cases assert logical database immutability (incidental WAL bookkeeping on existing databases is permitted and documented), and only the missing-DB case asserts no newly created database-related files or directories, including when the file is removed between a preliminary check and the open. Plain `sce doctor` shares the same non-creating read-only open. + - **Missing DB:** the non-creating read-only Turso open fails without creating the DB file, its parent directory, or database sidecars (WAL/shared-memory); no Git ref is listed or deleted. A separate `sce doctor --fix` repair (global-state bootstrap) may create the parent directory independently of reconciliation; reconciliation itself creates nothing. + - **Existing but invalid DB:** opening is permitted; incidental Turso WAL/shared-memory bookkeeping is permitted if unavoidable and documented; no schema migration, migration-metadata repair, repository-metadata initialization, or application-table INSERT/UPDATE/DELETE; no Git-ref deletion; verification fails with the matching typed error. Tests do not assert filesystem immutability for this case. + - **Existing valid DB:** repository identity and schema readiness are verified before reconciliation; the existing multiprocess-WAL configuration is preserved; only the access reconciliation requires is performed; no initialization, repair, or migration. + - Validate: T05 S28–S32 pass: missing DB (no new files or directories, non-creating open fails, refs not listed), incompatible schema, missing metadata, repository identity mismatch (refs untouched, logical DB contents byte-for-byte equivalent by table/row comparison, typed error), and normal existing-DB operation. +- [x] AC15: Deferred automatic reclamation is explicit and not implied. + - Validate: the plan's D2 follow-up prerequisites are copied into the synchronized context by T06; no code path or doc claims automatic reclamation; T05 S14 proves hook completion never reconciles. + +### Review-finding traceability + +| Finding | Design resolution | Implementing task | Regression scenarios | +| --- | --- | --- | --- | +| R1 P1 double lock | D1 | T02 (T04 consumes) | S6, S7, S12, S17 | +| R1 P1 latency → R2 P1 unbounded inline latency | D2 (advisory only; reconciliation via doctor; automatic deferred) | T01 (investigation), T02, T03, T04 | S14, S15, S16, S17, S18a, S18b, S33 | +| R1 P2 failure state | D3 (diagnostic streak, no stored deadlines) | T02, T04 | S11, S26, S27 | +| R2 P2 persistence failure / repeated scans | D2/D3 (no automatic full pass; advice only after successful persist; reservation is a follow-up prerequisite) | T02, T03 | S20, S22, S23 | +| R2 P2 far-future timestamps → R3 P3 future-skew contradiction → R4 P2 normalization scope | D3 (past-only fields, `FUTURE_SKEW_TOLERANCE` valid as age 0, normalize beyond it once under lock on successful writes; read-only/unsuccessful operations never mutate and evaluate conservatively) | T02 | S24 (cases 1–7), S25, S34–S39 | +| R4 P2 database side-effect assertions | D4/AC14 (per-DB-state guarantees: missing = no files; existing invalid = logical immutability, incidental WAL allowed) | T02 | S28–S32 | +| R3 P2 external-taint marker assertion | AC4 (advisory never touches the marker; `MarkerClearAfterCommit` preserved) | T03, T05 | S18a, S18b | +| R3 P2 advisory vs coordinator contention | AC12 (three scenarios: direct, coordinator, post-coordinate handoff) | T05 | S15, S16, S33 | +| R3 P2 Git-ref assertions | AC2, AC5 (advisory adds no ref delta; coordinator pins legitimate) | T05 | S14 | +| R1 P2 AC3 durability claim | AC3 invariants | T02 (no behavior change), T05, T06 | S2, S3, S4, S5, S13 | +| R1 P2 PR metadata | Stacking section | T01 (re-verify) | none (plan metadata) | +| R1 P2 DB opening side effects | D4 | T01 (audit), T02 | S28–S32 | +| Concurrency / duplicate passes | D1, D3 | T02, T03 | S12, S21 | + +Previous acceptance criteria that changed in round 2: AC1 (explicit caller only; no automatic reconciliation caller), AC2 (doctor deletes; hook never does), AC4 (advisory, not automatic maintenance), AC5 (hook path does no I/O beyond state), AC9 (adds recovery conditions), AC10 (adds deferral statement), AC12 (budget/deferral language removed; concurrency contract added), AC13 (backoff replaced by durable-diagnostic and timestamp-recovery guarantees). New: AC15. Round 3 changed AC2 (ref assertion scoped to the advisory), AC4 (marker semantics, `MarkerClearAfterCommit` preserved), AC5 (S14/S15 references), AC12 (three contention scenarios, new S33) and AC13 (exact skew-tolerance invariant, S24 cases 1–7); scenarios S15, S16, S18 (now S18a/S18b), S24 were rewritten and S33 was added. + +*Round 4 (two findings; plan-only corrections, D1–D4 architecture and the T01–T06 structure unchanged)* + +14. **Timestamp-normalization scope (P2).** The `t ≤ now + FUTURE_SKEW_TOLERANCE` post-condition was asserted after every operation, which read-only (plain `sce doctor`), lock-busy, failed-read/write, and interrupted operations cannot satisfy. D3, AC13, T02 and T05 S24–S25 now separate read-only/unsuccessful operations (never mutate, evaluate conservatively, never postpone repair, never report normalization) from successful state-writing operations (normalize under `WorktreeLock`, persist atomically, post-condition holds). Added S34–S39. +15. **Database side-effect assertions (P2).** AC14 and S28–S32 asserted that failed verification never creates files, contradicting D4's accepted incidental WAL bookkeeping for existing databases. D4, AC14 and S28–S32 now specify per-state guarantees: missing DB (stronger: no Turso call, no new files/directories, no ref listing or deletion); existing invalid DB (logical immutability, no migration/repair/initialization/application writes, incidental WAL bookkeeping permitted); existing valid DB (verify before reconciliation, preserve WAL configuration, no initialization/repair/migration). + +### Full validation + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` +- `nix flake check` +- `nix build .#ci-checks` +- `nix run .#pkl-check-generated` (only if generated config inputs are touched) +- `git diff --check` +- `git diff fix-logging...HEAD --stat` (confirms no `protocol.rs` / Quint changes and only intended files) +- `nix shell nixpkgs#ripgrep -c rg "dyn Future|BoxFuture|tokio::spawn|Runtime::new|block_on|tokio::time::timeout|JoinHandle|abort\(" ` (AC7) +- Focused runs with non-zero match counts, repeated for race scenarios: `… test --manifest-path cli/Cargo.toml ref_reconciliation`, `… maintenance`, `… mutation_scope`, `… doctor`, `… agent_trace_storage` + +Record actual outcomes in the validation report; compilation alone does not validate the plan. + +### Context sync + +- `context/cli/mutation-trace-ref-reconciliation.md` +- `context/cli/mutation-trace-runtime-coordinator.md` +- `context/cli/mutation-trace-snapshot-service.md` +- `context/cli/mutation-trace-store.md` (if its usage contract changed) +- `context/cli/mutation-scope-runtime.md` +- `context/cli/mutation-scope-hook-ingress.md` +- `context/context-map.md` +- Relevant doctor documentation (`context/sce/agent-trace-hook-doctor.md`, `context/sce/doctor-human-text-contract.md`) +- Agent Trace storage documentation covering the new verified-existing-DB opener (located by T06) + +## Task context synchronization lifecycle + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/services/mutation_trace/runtime/` (new maintenance module with `reconcile_explicit`, `advise_if_due`, and the state module; `ref_reconciliation.rs` split; `git_snapshot.rs` and `worktree_lock.rs` only for test seams as needed), `cli/src/services/agent_trace_storage/mod.rs` and `cli/src/services/agent_trace_db/repository.rs` (the dedicated verified-existing-DB opener and its read-only metadata check only), `cli/src/services/hooks/mutation_scope.rs` (advisory call only), `cli/src/services/doctor/`, the checkout-local state file `/sce/ref-maintenance.json`, tests, and the context files listed above. +- **Out of scope:** `protocol.rs`, `spec/mutation_cursor.qnt`, mutation-attribution rules, Agent Trace schema migrations, the existing `open_agent_trace_db_for_hook_runtime` behavior for other callers, per-harness (Claude/Codex/OpenCode/Pi) adapter changes, adapter recovery state, any automatic or out-of-band reconciliation execution. +- **Constraints:** + 1. Reuse async reconciliation logic, `GitSnapshotService`, `MutationTraceStore`, `WorktreeLock`; the lock-held implementation is the single shared algorithm. + 2. Static dispatch and existing generic `AsyncFn` patterns only; no `dyn Future`, `BoxFuture`, additional Tokio runtime. + 3. Never delete a ref whose tree is in any repository-wide durable-root set. + 4. Never run any maintenance operation while holding a lock from `coordinate()`; maintenance acquires its own lock after `coordinate()` returned. + 5. Never convert a successful mutation-boundary completion into failure because the advisory was busy, failed, errored, or hit invalid state. + 6. No detached cleanup task, background worker, daemon, fire-and-forget subprocess, or timeout-cancelled Git/DB work; no reconciliation on the hook path. + 7. Do not modify `protocol.rs`, the Quint state machine, or attribution rules. + 8. Never invoke `git gc` or remove Git objects directly; ref deletion goes only through the existing atomic SHA-conditional transaction (`delete_pins`). No independent ref-deletion implementation. + 9. Do not bootstrap, migrate, repair, or initialize an Agent Trace DB to run cleanup (D4). + 10. Exactly one `WorktreeLock` acquisition per maintenance operation; never reacquire inside the lock-held implementation. + 11. Maintenance failures never arm `ExternalTaintMarker`. + 12. The maintenance state file never stores a deadline; every successful state write leaves no timestamp more than `FUTURE_SKEW_TOLERANCE` ahead of the injected `now`; read-only and unsuccessful operations (including a `NotApplied` write failure) never mutate it; a post-rename `DurabilityUncertain` failure may leave the new state visible and is reported as uncertain, never as durably persisted; all writes are atomic and under `WorktreeLock`. + 13. No schema migration, new attribution state, or new protocol action. + 14. Follow repository rules: no code comments in touched code, Cargo only via Nix, `nix flake check` for verification. +- **Non-goal (automatic reclamation):** see D2. Follow-up PR with the listed prerequisites, including the durable next-attempt reservation requirement. +- **Non-goal (retired-worktree cleanup):** the reconciler only inventories the namespace of a currently resolvable worktree, so refs owned by removed linked worktrees survive. No global namespace sweep here. Record a follow-up for repository-scoped cleanup with active-worktree inventory, repository-wide durable-root retention, and protection against worktree removal/recreation races. +- **Non-goal (repairing other worktrees' pins):** reconciliation of worktree `W` does not recreate or verify pins that only another worktree requires. +- **Non-goal (evidence-based advice):** the advisory is time-based; detecting actual orphans requires the Git/DB scan kept off the hook path. +- **Non-goal (historical event retention):** trees referenced by retained mutation events are never deleted; bounding history needs a separate event-retention and snapshot-compaction policy. +- **Non-goal (protocol changes):** no new action, state field, attribution category, Quint transition, or schema migration. + +## Assumptions + +- Constants (T03 records final values): `RECONCILIATION_ADVISORY_AFTER` 24 h; `MAINTENANCE_STATE_MAX_BYTES` 4096; `FUTURE_SKEW_TOLERANCE` 5 min; explicit lock wait 10 s (existing). +- The state file is per-worktree under `/sce/`, matching the existing lock and taint-marker layout; absent or unparsable means default state. +- Worktree identity is already Git-topology-derived (`resolve_worktree_id`: `main` and `worktrees/`); the checkout-ID wording in the context docs is stale, and `SkippedNoCheckoutIdentity` appears unreachable in code (declared, never constructed). T01 confirms and T06 removes or justifies it. +- Doctor output additions are additive fields/rows only. + +## Task stack + +- [x] T01: `Audit reconciliation and mutation-scope lifecycle integration points` (status:done) + - Task ID: T01 + - Scope: In — read-only audit against PR #304/#305 code of `runtime/ref_reconciliation.rs`, `git_snapshot.rs`, `worktree_lock.rs`, `coordinator.rs`, `protected_worktree.rs`, `hooks/mutation_scope.rs`, `hooks/commit_hooks.rs`, `hooks/` harness lifecycle modules, `doctor/mod.rs`, `doctor/fixes.rs`, `doctor/inspect.rs`, `hooks/runtime.rs`, `agent_trace_storage/mod.rs`, `agent_trace_db/repository.rs`, `db/mod.rs`, and existing reconciliation tests; record findings in this plan under an `Audit findings` subsection. Required findings: + - production call graph for `Start`/`Advance`/`Close`/`Flush`/`Abandon`; which `CoordinateOutcome`/`CoordinateError` variants are durably completed (including `MarkerClearAfterCommit`); when each lock is released, including the `abandon_after_spawn_without_unlock` path; + - **lock ownership (D1):** confirm `reconcile_worktree_inner` acquires `WorktreeLock` at its start and the lock is non-reentrant; specify the exact wrapper / lock-held split and the parameter carrying lock ownership; confirm `delete_pins` moves the lease into the `spawn_blocking` worker so dropping the caller future cannot release the file lock early; confirm the 10 s coordinator lock timeout; + - **execution-point investigation (D2):** re-verify each rejected alternative (post-commit, per-harness session-end/stop, setup lifecycle, detached/daemon) with file references, confirm that no lifecycle-owned noncritical-path executor exists, and confirm runtime shutdown joins blocking workers; confirm the selected design (advisory-only hook trigger plus `sce doctor --fix`) or escalate if a suitable mechanism is found; + - **advisory Git-dir resolution (D2 decision rule):** determine whether the worktree Git dir can be obtained for the advisory without a new Git subprocess (filesystem derivation proven equal to `resolve_git_dir` for the main and a linked worktree, or reuse of a value `coordinate` already resolves without protocol changes) and record the outcome: advisory kept, or hook trigger dropped; + - **state (D3):** identify the existing atomic staging-and-swap pattern to reuse for the state file; confirm placement beside the lock and marker; confirm no in-memory registry is needed; confirm the existing clock source and how to inject a wall clock; + - **DB opening (D4):** inspect `open_agent_trace_db_for_hook_runtime` and its resolver; record exactly which steps can create directories/files, repair migration metadata, or initialize repository metadata; determine whether Turso supports a non-creating/read-only open; define the smallest dedicated verified-existing-DB opener and exact verification queries (schema readiness including the durable-roots migration, metadata row presence, `repository_id` match, valid `source_instance_id`); + - current worktree identity forms; stale checkout-ID docs and unreachable branches including `SkippedNoCheckoutIdentity`; + - confirm PR #304 head and PR #305 head/branch (`ref-recon`, base `fix-logging`) and record the SHAs. + Out — any code or context change. + - Dependencies: none + - Done when: the plan records verified call sites with file/line references, lock ownership and the wrapper/lock-held split, error classification, the D2 execution-point conclusion and advisory Git-dir decision, the DB-opening side effects and chosen opener contract, and flags any finding that invalidates a later task's scope. + - Verify: re-read cited lines to confirm each recorded fact; `gh pr view 304 --json headRefOid` and `gh pr view 305 --json headRefName,baseRefName,headRefOid` match the recorded values. + - Completed: 2026-10-09 + - Files changed: `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Added the `Audit findings` section. Confirmed lock ownership and the wrapper/lock-held split, the call graph and completion classes, the D2 conclusion (advisory plus `sce doctor --fix`), a filesystem Git-dir derivation that avoids a new Git subprocess (hook trigger kept), the D3 staging-and-swap pattern and clock source, and the D4 non-read-only opener findings and contract. Flags recorded: the `sync/auto_sync.rs` detached-spawn precedent, the `ReconcileError` "003" vs `004` migration wording, and the `doctor/inspect.rs:305` repair-seam caller. No finding invalidates a later task's scope. + - Verify: cited lines re-read during the audit and line references corrected; `gh pr view 304` → head `0be74aad5bea5070828846d3277041a4649c9842`, `gh pr view 305` → `ref-recon`/base `fix-logging`/head `fb7e78fbfa41f9f054803eef2dddb180f5d5289a` (both match the recorded values); passed. + - Context impact: none (plan-only audit; no behavior or durable context change). + - Context synchronization: synced + +- [x] T02: `Add the reusable maintenance entrypoints (explicit reconciliation and hook advisory)` (status:done) + - Task ID: T02 + - Scope: In — + - **Reconciler split (D1):** extract the body of `reconcile_worktree_inner` after lock acquisition into `reconcile_with_held_lock`, which borrows the held `WorktreeLock` and contains no acquire call; keep `reconcile_worktree` and `reconcile_worktree_inner` behavior and signatures, now acquiring once and delegating. + - **`reconcile_explicit(repository_root)`:** takes no database opener; resolves the worktree and Git dir, acquires the lock **once** (existing bounded 10 s wait), resolves the repository identity and canonical Agent Trace path itself and opens the DB only through the D4 verified read-only opener, runs `reconcile_with_held_lock`, then records the outcome in the state file while still holding the same lock, then drops the lock. Typed outcome `Completed(report) | CompletedStatePersistFailed { report, warning } | Failed { error, state_warning } | Skipped(Busy)` (a failed pass keeps both the reconciliation error and any state-persistence warning); lock contention leaves the state untouched and is not a failure; it never calls the lock-acquiring wrapper; it ignores all prior state (stale, failed streak, invalid or corrupt) except to rewrite it. + - **`advise_if_due`:** synchronous function with no DB, Git, store, or snapshot parameters; zero-wait single `try_lock`; reads, normalizes, and writes the state under that one acquisition; returns `Anchored | NoAction | Advised | AdvisedDurabilityUncertain { warning } | Busy | StateWriteFailed { warning } | StateUnavailable` (see the second remediation); persists `last_advised` before returning `Advised`; takes injected wall clock and Git dir. Subject to the T01 decision rule. + - **State module (D3):** `ref-maintenance.json` read (capped, default-on-corrupt), invalid-timestamp normalization under lock, atomic write, past-only fields, success/failure transitions, and a pure `evaluate_recommendation(state, now)` function shared by the advisory and read-only doctor. + - **D4 opener:** the dedicated verified-existing-DB opener and the read-only repository metadata verification method. + - **Test seams only as required:** a worker-entered seam threaded through `delete_pins` to `run_ref_mutation_inner`'s existing `on_worker_entered`; phase-boundary hooks (post-lock, post-DB-open, post-inventory) used to park a pass deterministically; injectable write-failure and rename-interruption hooks for the atomic writer; injected clock. + Out — hook and doctor wiring, any new deletion code, any automatic/reservation/backoff scheduling, protocol/store schema changes, changes to the existing hook DB opener, capped inventory or budget logic. + - Dependencies: T01 + - Done when: both entrypoints are callable from tests; the explicit pass and the advisory each complete without self-deadlock; state never contains a deadline; after every successful state write no stored timestamp exceeds `now + FUTURE_SKEW_TOLERANCE`; timestamps within the tolerance are accepted as age 0 and preserved; on a successful write invalid timestamps normalize once and persist; read-only, lock-busy, unreadable-state (`StateUnavailable`) and pre-rename-failed (`NotApplied`) operations never mutate state (a post-rename `DurabilityUncertain` failure may leave the new state visible and is reported as uncertain), evaluate invalid timestamps conservatively, and never postpone eligibility indefinitely; failures leave refs untouched; the DB opener fails a missing DB at its single non-creating read-only open without creating files or directories, and for an existing invalid DB performs no migration, repair, initialization, or application-table write (incidental WAL bookkeeping permitted and documented); no second deletion path exists. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml ref_reconciliation`; focused maintenance-state and `agent_trace_storage` unit tests (deterministic, no sleeps); `rg` confirms the only ref-deletion call is the existing `delete_pins`, each entrypoint acquires `WorktreeLock` once, `reconcile_with_held_lock` contains no acquire call, and the advisory module imports no Git/DB/store symbol. + - Completed: 2026-10-09 + - Files changed: `cli/src/services/agent_trace_db/repository.rs`, `cli/src/services/agent_trace_storage/mod.rs`, `cli/src/services/db/mod.rs`, `cli/src/services/mutation_trace/runtime/mod.rs`, `cli/src/services/mutation_trace/runtime/ref_reconciliation.rs`, `cli/src/services/mutation_trace/runtime/maintenance_state.rs`, `cli/src/services/mutation_trace/runtime/ref_advisory.rs`, `cli/src/services/mutation_trace/runtime/ref_maintenance.rs`, `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Split `reconcile_worktree_inner` into lock acquisition plus `reconcile_with_held_lock` (borrows `&WorktreeLock`, phase-hook seam `ReconcilePhase`); added `ref_maintenance::reconcile_explicit[_with]` (single lock acquisition, typed `Completed | CompletedStatePersistFailed | Failed | Skipped(Busy)`), `ref_advisory::advise_if_due[_with]` (sync, zero-wait `try_lock`, filesystem Git-dir derivation, no Git/DB/store imports), and `maintenance_state` (capped versioned `ref-maintenance.json`, past-only timestamps, tolerance-based normalization under lock, staging-and-swap writes with injectable rename, pure `evaluate_recommendation`). Added the D4 opener `RepositoryAgentTraceDb::open_verified_existing_at` with `ExistingRepositoryDbError` (`Missing | Unreadable | IncompatibleSchema | MissingMetadata | RepositoryMismatch`), read-only `verify_existing_repository_metadata`, `TursoDb::open_existing_without_migrations_at` (no parent-dir creation), and `resolve_existing_agent_trace_storage_for_maintenance`. New items carry dead-code allowances until T03/T04 wire them. The existing hook opener and its callers are unchanged; `delete_pins` remains the only deletion path. + - Verify: `cargo clippy --all-targets --all-features -D warnings` clean; `test … ref_reconciliation` 5 passed; `test … maintenance` 8 passed; `test … mutation_trace` 123 passed; `rg` shows `reconcile_with_held_lock` has no acquire call and `maintenance_state.rs` has no Git/DB/store symbol (`ref_advisory.rs` matches only `Command` inside its test module); `git diff --check` clean. The `agent_trace_storage` filter matches zero tests, so opener coverage lives in the `ref_reconciliation` tests. Original T02 verification used an existence pre-check; superseded by the remediation below. + - Context impact: important (new runtime maintenance entrypoints, state file, and verified-existing-DB opener; T06 owns the durable context update). + - Context synchronization: synced + - Remediation (2026-10-09, PR #305 review of `d699b26`; T02 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01 unchanged, T03–T06 untouched): + - P1 authoritative DB: the production entrypoint is now `reconcile_explicit(repository_root: &Path) -> ExplicitOutcome`. It resolves the repository identity and canonical path itself (`resolve_agent_trace_storage_runtime_config` → `resolve_existing_agent_trace_storage_for_maintenance`) and opens only through `open_verified_existing_at`, which checks schema/all migrations, metadata presence, stored-vs-resolved repository identity, and `source_instance_id` before any pin inventory. `reconcile_explicit_with` (the opener-injection seam) is private to `ref_maintenance` and used only by its child test module; T04 must call `reconcile_explicit` and must not add an opener. The single lock-held algorithm and `reconcile_worktree` wrapper are unchanged. Typed `ExistingRepositoryDbError` stays downcastable from `ReconcileError::AgentTraceDbUnavailable`. + - P2 persistence diagnostics: `ExplicitOutcome::Failed { error, state_warning }` retains the reconciliation error and an optional `StatePersistWarning` (`PreviousStateUnreadable | WriteFailed(PersistFailure)`); `CompletedStatePersistFailed` carries the same warning type. Lock contention returns `Skipped(Busy)` before any state read or write. A completed ref deletion is never rolled back. + - P2 durability: `write_state_atomically` returns `PersistFailure::NotApplied { phase, source, staging_cleanup }` (previous state intact, staging removed, cleanup failure reported) for failures up to and including rename, and `PersistFailure::DurabilityUncertain { phase, source }` for parent-directory open/sync failures after rename (new state may already be visible; previous state is not claimed preserved). The filesystem seam is the statically dispatched `StateFilesystem` trait with `RealStateFilesystem`; `FaultInjectingFilesystem` is test-only. `advise_if_due` maps any `PersistFailure` to `StateUnavailable`, so `Advised` is never reported without a confirmed write. + - P2 existing-DB race: Turso 0.8.1 has no create-disabled builder flag, but `Builder::read_only(true)` opens the main file with `OpenFlags::ReadOnly`; the Unix/generic IO layers then open it without `create` and without write access, so existence and open are one OS call (missing → not-found, no file or directory created). `open_existing_without_migrations_at` now uses it with `experimental_multiprocess_wal(true)` unchanged; the `is_file()` pre-check is removed and a failed open is classified `Missing` only by a post-failure `symlink_metadata` NotFound. All verification queries run on the connection that open produced. + - Limitations (not claimed solved): (1) Turso opens the `-wal`/`-tshm` sidecars by path, so a replacement that swaps only the main file leaves a torn main+stale-WAL pairing whose contents are whatever Turso assembles; only a coherent replacement (main file and sidecars) is meaningful, and the check proves the connection's data matches the resolved repository, not that the path was not swapped later. (2) Turso's process-wide registry is keyed by file identity: if the same process already holds that DB open, the open shares that instance and the read-only flag is not enforced for it (the file necessarily exists, so nothing is created). (3) A DB replaced after verification but before the delete transaction is not detected; the guard is that roots are read from the verified connection under the held `WorktreeLock`. (4) Cross-process multiprocess-WAL coexistence is covered only by the builder configuration and an in-process live-writer test, not by a second OS process. + - Evidence: `test … ref_reconciliation` 20 passed (all in `ref_maintenance/tests.rs`, including: signature-only production API, other-repository DB cannot authorize deletion, mismatched/missing metadata and missing/incompatible schema fail before inventory (`phases` empty, pins retained), missing DB creates nothing, removed/replaced-before-open, removed-after-open never recreated, read-only connection, live-writer coexistence, lock-held path never reacquires, wrapper acquires once and releases, failure+persist-failure keeps both errors for missing DB and missing required pin, durability-uncertain completion keeps deletion, lock contention never increments the streak); `test … maintenance` 38 passed (`maintenance_state` tests cover staging create/write/sync, rename, parent open/sync failures, success, interrupted pre-rename recovery, post-rename visibility vs durability); `test … mutation_trace` 148 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; no new `allow`, `dyn`, `let _ =`, or comments; `acquire_inner_async` appears once in `ref_maintenance.rs` and once in `reconcile_worktree_inner`. + - Remediation 2 (2026-10-09, follow-up review of `de055d70`; T02 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01 unchanged and still done/synced, T03–T06 untouched). Architecture unchanged: reconciliation runs only through `reconcile_explicit`; the advisory stays lightweight (no Git, DB, or reconciliation), takes one zero-wait lock acquisition, uses static dispatch, and no mutation protocol, attribution, Quint, or ref-deletion code changed. + - P2 failure-based recommendation: `evaluate_recommendation` now treats an outstanding explicit-pass failure (`last_attempt_outcome == Failed` or `consecutive_failures > 0`) as an independent reason to recommend repair, in addition to staleness and invalid timestamps. A first-ever failed pass (no anchor, no `last_success`) is therefore recommended immediately. A never-attempted default state is not a failure. Multiple consecutive failures stay recommended; `record_success` clears the streak, `last_failure`, and the recommendation; a failure after a recent success is recommended again. Emitting an advisory never resets `consecutive_failures` or `last_failure`. + - Recommendation status vs reminder cadence: `Recommendation.recommended` answers "should `sce doctor` tell the operator to run `sce doctor --fix`" and ignores `last_advised`. `Evaluation.advise` answers "may the hook emit another reminder": staleness reminders are suppressed for 24 h after `last_advised`; a failure reminder is suppressed only when `last_advised` is within the window and not older than `last_attempt`, so a newly recorded failure can be advised immediately even if an earlier staleness reminder is recent, while a reminder issued after the failure is not repeated inside the window. A failure reminder repeats every 24 h until a successful explicit pass. Clock-skew normalization and the existing cadence are unchanged. + - P2 persistence outcomes: `AdvisoryOutcome` is now `Anchored | NoAction | Advised | AdvisedDurabilityUncertain { warning: PersistFailure } | Busy | StateWriteFailed { warning: PersistFailure } | StateUnavailable` (no longer `Copy`/`Eq` because it carries `PersistFailure`). `Advised` means the reminder state was persisted with confirmed durability. `PersistFailure::NotApplied` (up to and including rename) on an advisory yields `StateWriteFailed`: the previous state is intact, nothing is claimed persisted, `recommendation()` is `None`, and the next eligible call advises again. `PersistFailure::DurabilityUncertain` (parent-directory open or sync after rename) yields `AdvisedDurabilityUncertain`: the rename completed, `last_advised` may be visible, and the caller must still surface the recommendation. No rollback, second lock acquisition, or automatic reconciliation. A write failure while only anchoring or normalizing yields `StateWriteFailed` carrying whichever `PersistFailure` variant occurred; there is no recommendation to surface. With `NotApplied` the previous state is intact; with `DurabilityUncertain` the rename completed, the anchor or normalized state may be visible, and only crash durability is unconfirmed. The diagnostic text follows the variant ("could not be written" vs "durability could not be confirmed"). + - Caller contract for T03: `AdvisoryOutcome::recommendation()` returns `Reconciliation is recommended. Run sce doctor --fix.` for `Advised` and `AdvisedDurabilityUncertain`, and `None` otherwise; `persistence_warning()` returns the `PersistFailure` separately for logging. Outcomes are diagnostic values, not errors: T03 must log them and never convert them into a mutation-boundary failure. Suggested levels: `Advised` and `AdvisedDurabilityUncertain` (with the persistence warning as a distinct field) at warn; `StateWriteFailed` and `StateUnavailable` at warn; `Anchored`/`NoAction`/`Busy` at debug. + - Delivery semantics and limitations: a durable state update is not the same as an operator-visible advisory. The state is written before the caller surfaces the message, so a process crash between the write and the log leaves `last_advised` set and the reminder unseen for up to 24 h; exactly-once delivery is not promised. `sce doctor` is the backstop because it evaluates the recommendation from the failure/staleness fields independently of `last_advised`, so an unresolved failure or stale state is never hidden from plain `sce doctor`. After `AdvisedDurabilityUncertain`, a crash can still roll the directory entry back, in which case the next eligible call simply advises again. + - Regression scenarios and acceptance criteria (all deterministic, injected clocks and `FaultInjectingFilesystem`, no sleeps): first-ever failed pass with missing DB is recommended (unit and end-to-end through `reconcile_explicit`); failure advisory emits once, persists `last_advised`, keeps the failure diagnostics, and a second advisory inside the window is `NoAction`; plain evaluation stays recommended despite the recent reminder; the reminder repeats after the window; repeated failures stay recommended; success clears; a failure after a recent success and an older reminder is recommended and advised; rename failure preserves prior state and the next call advises; parent-open and parent-sync failures yield `AdvisedDurabilityUncertain` with the new state visible, the recommendation surfaced, a following call `NoAction`, and `evaluate_recommendation` still true; an anchor write failure is `StateWriteFailed`; confirmed advisories keep the ordinary duplicate-suppression tests. + - Evidence: `test … maintenance` 48 passed; `test … ref_reconciliation` 21 passed; `test … mutation_trace` 158 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; no added comments, `dyn`, `let _ =`, or new `allow`; no new `acquire` call sites. + +- [x] T03: `Trigger the advisory check from the shared mutation-scope ingress` (status:done) + - Task ID: T03 + - Scope: In — in `hooks/mutation_scope.rs`, after `classify_coordinate` yields a durably completed `Close` or `Flush` (including the `MarkerClearAfterCommit` case) and `coordinate_boundary(...)` has fully returned, call `advise_if_due` through an injected seam; the outcome is logged as structured events and, for actionable outcomes, written to an injectable diagnostic writer (stderr in production) and is never returned as a boundary error (`Anchored`/`NoAction`/`Busy` debug, no diagnostic; `Advised`/`AdvisedDurabilityUncertain`/`StateWriteFailed`/`StateUnavailable` warn plus diagnostic); no trigger from `Start`, `Advance`, `Abandon`, or failed coordinates; a seam error or unexpected condition is contained and logged; the hook module gains no reconciliation, Git, or DB dependency. Apply the T01 decision rule: if the advisory cannot avoid a new Git subprocess, implement no hook call and document that outcome. Record the final constants in the plan and document the deferred follow-up (D2) in the plan's audit findings. Out — per-harness adapter changes, reconciliation on the hook path, detached tasks, background workers, timeouts. + - Dependencies: T02 + - Done when: a completed `Close`/`Flush` can run the advisory and no other path does; hook output and exit status are identical whether the advisory returns any outcome or errors; the advisory runs only after the coordinator's lock has been released; `MarkerClearAfterCommit` handling is unchanged and the advisory never creates, clears, or modifies the external-taint marker; no reconciliation, ref inventory, or DB I/O is reachable from the hook module's new call. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml mutation_scope`; trigger-policy unit tests over every boundary and outcome with a fake advisory seam; a seam that tries to take the real `WorktreeLock` and asserts it is free at invocation; `rg` over the hook module shows no reconciliation/DB-opener/snapshot import added. + - Completed: 2026-10-09 + - Files changed: `cli/src/services/hooks/mutation_scope.rs`, `cli/src/services/mutation_trace/runtime/mod.rs`, `cli/src/services/mutation_trace/runtime/ref_advisory.rs`, `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: `drive_mutation_scope` takes an injected advisory seam and calls it once, only for `Close`/`Flush` whose `coordinate_boundary` result is `Ok` or `MarkerClearAfterCommit`, after the await (coordinator lock already released) and after `classify_coordinate`; the report is only logged (`sce.hooks.mutation_scope.ref_reconciliation_advisory`; warn for `advised*`/`state_write_failed`/`state_unavailable`, debug for `anchored`/`no_action`/`busy`) and never changes the hook result. Production passes `advise_after_completed_boundary` (system clock, filesystem Git-dir derivation, no Git subprocess), which maps `AdvisoryOutcome` to the new `AdvisoryReport`/`AdvisorySeverity`. The hook gains no reconciliation, snapshot, store, or DB-opener import for this purpose. Final constants unchanged from the plan assumptions (`RECONCILIATION_ADVISORY_AFTER` 24 h, `MAINTENANCE_STATE_MAX_BYTES` 4096, `FUTURE_SKEW_TOLERANCE` 5 min). Deferred automatic reclamation (D2 prerequisites) is already recorded in this plan; T06 carries it into context. + - Verify: `test … mutation_scope` 18 passed (5 new `advisory_trigger_tests`: runs after completed Close/Flush, runs after `MarkerClearAfterCommit` with unchanged success result, never for Start/Advance, never after a failed coordinate, never for Abandon); `test … hooks` 27 passed; `test … mutation_trace` 158 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; `rg` shows no new reconciliation/DB/snapshot import in the hook module and no added comments. Not covered here (T05 S17/S18/S33): a real-lock "acquirable at invocation" assertion, because the seam tests use a fake coordinate that never holds the lock; ordering after `coordinate_boundary` returns is structural. + - Context impact: important (hook ingress now emits the advisory; T06 owns the durable context update). + - Context synchronization: synced + - Remediation (2026-10-09, PR #305 review of `91e9ac2`; T03 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01/T02 unchanged, T04–T06 untouched). Architecture unchanged: advisory only on durably completed `Close`/`Flush`, no reconciliation/Git/DB work, single lock design, static dispatch. + - Delivery (P2): `Logger::warn` is not an operator channel (observability routes only ERROR to stderr, and only with file logging off), and `last_advised` is persisted before logging, so a reminder could be suppressed for 24 h without ever being seen. `drive_mutation_scope` now takes a generic `E: Write` diagnostic sink (production: `std::io::stderr()`, created in `run_mutation_scope_from_payload`; tests: `Vec`/failing writer). `AdvisoryReport.diagnostic` carries the text; `Advised` → `SCE: Reconciliation is recommended. Run sce doctor --fix.`; `AdvisedDurabilityUncertain` → that line plus `SCE: Advisory-state durability could not be confirmed: .`; `StateWriteFailed`/`StateUnavailable` → a state-write-failed / state-unavailable line plus `Run sce doctor to check reconciliation status.` (never claiming advice was persisted); `Anchored`/`NoAction`/`Busy` → none. Structured logging (event ID, `outcome`, `persistence_warning`) is retained; the diagnostic write is independent of the logger (works with `None`), ignores write/flush errors, and never alters the hook result. `Logger::warn` and observability defaults are unchanged. Stdout stays empty. + - Harness reality: Claude Code and Codex run the hook as a command, so stderr is available to the harness (shown only in the harness's verbose/hook output, not guaranteed in the transcript). The OpenCode plugin (`spawnSync`, captured output) and Pi extension (`stdio` `ignore`/captured and dropped) discard the child's stderr, so for those harnesses the stderr line is not operator-visible; the structured log file (default `log_to_file`) and `sce doctor` (which reports the stale/failed recommendation independent of hook delivery) are the supported surfaces. Plugin changes are out of T03 scope. + - Docs: AC4, AC5, AC9, AC13, the T03 scope and S20 now state the time-based vs failure-triggered cadence and the full seven-outcome enum. + - Tests: `mutation_scope` advisory tests (recording logger + injected sink) cover all seven report shapes (severity, event ID, outcome, persistence warning, diagnostic), missing logger, failing writer (result stays `Ok("")`), `MarkerClearAfterCommit`, failed coordinate (no advisory, no output), and single coordinate/advise invocation; `ref_advisory` adds `maintenance_report_maps_every_outcome_to_severity_and_diagnostic` over real outcomes. Existing `ref_advisory` tests already cover the 24 h cadence and an in-window new-failure reminder. Removed the now-unneeded file-level `#![allow(dead_code)]` from `ref_advisory.rs`. + - Verify: `test … mutation_scope` 25 passed; `test … hooks` 34 passed; `test … maintenance` 49 passed; `test … mutation_trace` 159 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean. + +- [x] T04: `Run explicit reconciliation from sce doctor --fix and report recommendation` (status:done) + - Task ID: T04 + - Scope: In — extend the existing doctor `--fix` flow to call `reconcile_explicit` (one lock acquisition inside the entrypoint; doctor never holds the worktree lock itself); additive report fields/rows for retained, deleted, and locally-required counts; distinct rows for completed, completed-with-state-persist-warning, failed (unavailable/invalid DB with the D4 failure kind, malformed refs, missing required pins, durable-root errors, delete-transaction error), and skipped-busy; plain `sce doctor` stays read-only but reads the state file (never writes it or any ref) and reports whether reconciliation is recommended (last success age, last attempt and failure streak, last advice, invalid-timestamp note) via the pure `evaluate_recommendation`; skipped or failed cleanup is never reported as success; `--fix` runs regardless of recommendation, failure streak, or timestamp validity; no DB bootstrap/migration/repair; no change to adapter recovery state; existing output contracts backward compatible; the documented concurrency note (a mutation boundary may wait up to 10 s while `--fix` reconciles) appears in help or fix output wording only if existing output contracts allow it additively. Out — new doctor subcommands, changes to mutation-scope repair logic. + - Dependencies: T02 + - Done when: `sce doctor --fix` always runs a pass and reports its true typed result; a successful pass resets the failure streak and sets `last_success`; `sce doctor` performs no ref or state mutation; existing doctor tests still pass unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml doctor` + - Completed: 2026-10-09 + - Files changed: `cli/src/services/doctor/mod.rs`, `cli/src/services/doctor/render.rs`, `cli/src/services/mutation_trace/runtime/mod.rs`, `cli/src/services/mutation_trace/runtime/ref_doctor.rs`, `cli/src/services/mutation_trace/runtime/ref_advisory.rs` (rustfmt-only: removed a leading blank line), `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Added the `pub(crate)` facade `runtime/ref_doctor.rs` (`run_explicit_reconciliation`, `inspect_reconciliation_recommendation`, plain-data `ReconciliationFix` / `ReconciliationCounts` / `ReconciliationRecommendation`) over `reconcile_explicit` and `evaluate_recommendation`. `sce doctor --fix` now always runs one explicit pass after the existing fixes and appends a fix-result row (category `mutation_scope_health`): completed → `fixed` with deleted/retained/locally-required counts; completed with state-persist failure → `fixed` plus a diagnostics warning; failed → `failed` with a typed kind (`agent_trace_db_missing`, `agent_trace_db_incompatible_schema`, `agent_trace_db_missing_metadata`, `agent_trace_db_repository_mismatch`, `malformed_ref`, `missing_required_pins`, `durable_roots_unavailable`, `delete_transaction_failed`, …) and any state warning; busy → `skipped`. Plain `sce doctor` reads the state file through a filesystem-derived Git dir (no lock, no write, no ref access) and, only when reconciliation is recommended or the state is unusable, adds a `Snapshot ref reconciliation` text section and a JSON `ref_reconciliation` object (`null` otherwise). Doctor receives the two calls as injected seams. No DB bootstrap/migration/repair, no adapter recovery state, and no mutation-scope repair logic changed. + - Verify: `test … doctor` 9 passed (4 new fix-result mapping tests, 4 new `ref_doctor` tests covering absent/recent state, failure recommendation with byte-identical state, unusable/invalid-timestamp state left unrewritten, outcome mapping); `test … mutation_trace` 163 passed; full `cargo test` 437 passed; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; no comments added and the pre-existing comment in the edited `execute_doctor_with_lifecycle_providers` was removed per instruction. + - Context impact: important (doctor now runs explicit reconciliation and reports the recommendation; T06 owns the durable context update). + - Remediation (2026-10-09, PR #305 review of `bab36fe`; T04 reopened `done`→`in_progress`, returned to `done`/`synced` after the verification below; T01–T03 unchanged, T05–T06 untouched). Architecture unchanged: single `WorktreeLock`, static dispatch, one reconciliation pass per `--fix`. + - Plain doctor DB inspection now opens through `RepositoryAgentTraceDb::open_existing_schema_ready_at` (non-creating read-only Turso open, multiprocess WAL preserved, no migrations, no metadata requirement); `Missing` yields no extra problem, `IncompatibleSchema`/`Unreadable` keep their existing diagnostics. The write-capable hook opener is untouched. + - `ReconciliationFix` keeps a typed `StateWarning` (`NotApplied` | `DurabilityUncertain` | `PreviousStateUnreadable`, with kind, phase, cause) instead of a string; human text reads "Maintenance-state update was not applied." / "Maintenance-state durability could not be confirmed.". + - JSON adds the optional top-level `ref_reconciliation_fix` object (`outcome` ∈ `completed | completed_state_persist_failed | failed | busy`, `counts`, `failure {kind,message}`, `state_warning {kind,phase,message,cause}`); `null` for plain doctor; `fix_results` unchanged. + - Human recommendation text adds "Last reconciliation attempt Nm ago." when an attempt timestamp exists. + - D3/D4/AC9/AC14 corrected above. + - Context synchronization: synced + +- [x] T05: `Add production-path and concurrency regression tests` (status:done; reopened `in_progress` 2026-10-09 for the verification-gap remediation below, returned to `done`/`synced` after it) + - Task ID: T05 + - Scope: In — real Git repos, linked worktrees, and repository Agent Trace DBs. Every race, ordering and timing scenario uses deterministic coordination (barriers, channels, existing `on_lock_contention`/`on_worker_entered`-style seams, parked phase hooks, injected clock); **no timing-based sleeps as correctness proofs**. T02 and T03 add unit-level coverage for their own code; T05 owns the production-path matrix below and each scenario must fail when the guarded behavior is deliberately broken. + - *Reconciliation core (explicit path):* (S1) `sce doctor --fix` reclaims an orphan pin left by an interrupted coordinate; (S2) `A → B → C → D` retains every historical tree; (S3) worktree A's pin is retained while worktree B needs the tree (**preservation**); (S4) missing required local pin fails closed with zero deletion; (S5) malformed/symbolic refs cause zero deletion; (S6) concurrent `coordinate()` and reconcile cannot race pin deletion — a coordinate attempt parked on the lock via barrier proceeds only after the pass releases it, and a pin it creates is never deleted by the in-flight pass; (S7) dropping the explicit-pass future while the deletion worker is parked at the worker-entered seam leaves the lock held until the worker is released, then free; (S8) idempotent reconciliation; (S9) plain `sce doctor` mutates neither refs nor state; `--fix` reports the true result for every outcome kind (completed, completed-with-persist-warning, failed kinds, busy); (S10) explicit pass against an external holder returns `Skipped(Busy)`, leaves state byte-identical, and does not count as a failure; (S11) explicit pass with DB failure leaves refs untouched, records a failure streak, never arms the external-taint marker; (S12) the explicit pass and the advisory each complete with one lock acquisition and no self-deadlock, and each returns busy (never deadlocks) against an externally held lock; (S13) with worktree B's required pin deliberately missing, reconciling A deletes only A's eligible pins, does not create or repair B's pin, does not fail, and never deletes any pin protecting B's tree. + - *Hook path and latency (P1):* (S14) the advisory is invoked only after a durably completed `Close`/`Flush` (including `MarkerClearAfterCommit`), never from `Start`/`Advance`/`Abandon`/failed coordinates, and never reaches reconciliation, ref inventory, or DB open — its signature takes none (originally proposed as spy/panic seams on the reconciler, inventory and opener; replaced, because such seams would add production dependencies to a deliberately path-only interface, by two complementary forms of evidence: (A) static inspection of the complete production call graph reachable from `advise_after_completed_boundary()` showing no reconciliation, `GitSnapshotService`, ref listing/mutation, Agent Trace DB access, Git subprocess or unbounded work, and (B) the real-hook behavioral tests named in the coverage map); `coordinate()` keeps all of its existing snapshot and pinning semantics (required pins are still created and intact — protecting pinning and durable evidence is not weakened), and the advisory itself performs no Git-ref creation, update, or deletion: the test compares the ref set produced by a full hook run against an equivalent coordinate-only baseline (or injects a fake advisory seam and asserts the real advisory adds no ref delta), so any ref change from the coordinator is legitimate and is never attributed to maintenance; (S15) direct advisory contention (scenario A): an explicit pass parked at the post-inventory seam (and, in a second parameterization, at the post-DB-open seam) holds `WorktreeLock`; `advise_if_due` is invoked directly, returns `Busy` through a channel handshake without waiting for the pass, and leaves the state file byte-identical and the ref set unchanged; (S16) coordinator contention (scenario B): with the explicit pass parked holding the lock, a `coordinate()` Close/Flush with a short injected lock timeout fails with the existing lock-timeout error before durable completion and mutates nothing, the existing bounded-acquisition semantics are unchanged, the failed coordinate is a legitimate pre-commit failure, the advisory seam is never invoked after it, and once the pass is released the same boundary succeeds (documents the D1 concurrency contract); (S17) the advisory runs only after the coordinator's lock is released — a seam asserts the worktree lock is acquirable at invocation; (S18a) normal completion: every advisory outcome, a state-write failure, a corrupt state file, and a seam error leave the hook result and `Ok` status identical to the coordinate-only baseline; the marker state is snapshotted immediately before the advisory runs and asserted byte-identical (existence) afterwards, so the advisory never creates, clears, or modifies it, and the normal coordinator-cleared state is unchanged; (S18b) `MarkerClearAfterCommit`: reusing the coordinator's existing marker-clear failure fixture (no change to `coordinator.rs`, `ProtectedWorktree`, or recovery semantics; if unreachable from the hook layer, a fake coordinate seam returns that variant while a real armed marker file exists), the boundary is classified as durably completed, the hook returns the same successful empty-stdout result with the advisory allowed to run, the marker the failed clear left behind is asserted still armed and unmodified after the advisory, and a subsequent real `coordinate()` invocation observes the inherited taint and takes the existing conservative recovery path; (S33) post-coordinate handoff race (scenario C): `coordinate()` completes successfully and releases its lock; a seam pauses the hook immediately before the advisory call; a competing explicit reconciliation acquires `WorktreeLock` through a barrier; the advisory resumes and returns `Busy`; the already committed boundary still returns success, and the maintenance state and refs are untouched by the advisory. + - *State, crashes, invalid timestamps (P2):* (S19) absent state → anchor written once, no advice; a second call writes nothing (file byte-identical); (S20) due state → exactly one time-based advice per window with `last_advised` persisted *before* the advice is returned; repeated calls inside the window do not write or advise; after the injected clock passes the window the advice recurs once; a failure recorded after `last_advised` is advised once inside the window, then suppressed, then repeats on the normal cadence until a successful explicit pass clears it; (S21) concurrent processes: one advisory parked inside its locked section, a second returns `Busy`, a third after release returns `NoAction` — exactly one advice, and an explicit pass concurrent with an advisory serializes (no duplicate explicit passes, second returns busy/waits per policy); (S22) state-persistence failure (injected write error): the advisory emits no advice and never scans; an explicit pass whose final write fails returns `CompletedStatePersistFailed` — distinct from `Completed` and `Failed` — with its deletions intact; (S23) crash durability: an interrupted atomic write (temp file left, rename not performed) leaves the previous state readable and eligibility preserved, a stale temp file is ignored/overwritten, and a fresh call (simulating process restart) re-reads state and advises at the correct time; (S24) clock-skew tolerance, with an injected clock for every case (`FUTURE_SKEW_TOLERANCE` = 5 min): (1) stored `last_success` equal to `now` — valid, age 0, no advice, file byte-identical; (2) one minute ahead of `now` — valid, age 0, preserved, no advice, no rewrite; (3) exactly five minutes ahead — valid (boundary is inclusive), age 0, preserved, no advice, no rewrite; (4) more than five minutes ahead (just beyond, and far-future) for `last_success`, `last_advised` and `anchor` — invalid, normalized once under the lock and persisted per D3 (invalid fields cleared; with `last_success` and `anchor` both unusable the advisory fires immediately once, persists `last_advised = now` and a fresh `anchor = now`); (5) clock rollback beyond tolerance (stored values valid at write time, injected `now` moved back more than five minutes) — treated as case 4; a rollback within tolerance is case 2/3 and never makes an advisory eligible by itself; (6) repeated reads after normalization — the file is byte-identical across repeated calls, no rewrite occurs, and eligibility is neither postponed nor slid; (7) process restart (fresh call re-reading the file) followed by clock advance past `RECONCILIATION_ADVISORY_AFTER` — advice recurs exactly once, proving eventual eligibility. After every **successful state-writing** operation an assertion checks the exact invariant `t ≤ now + FUTURE_SKEW_TOLERANCE` for every stored timestamp (not "no timestamp is in the future"); after read-only, lock-busy, or failed operations the assertion is instead that the state file is byte-identical to its prior content, with no normalization claimed; (S25) missing, unparsable, wrong-version, and oversized state files are recovered without failing any operation and are rewritten valid by the next successful recorded operation (until then, read-only and failed operations leave them untouched and treat them as default state). + - *Timestamp-normalization regressions (P2; injected clocks and deterministic synchronization, no sleeps):* (S34) read-only doctor observing an invalid future timestamp — plain `sce doctor` reports "recommended" with the invalid-timestamp note, the state file is byte-identical afterwards, and it is not reported as normalized; (S35) lock-busy advisory — with the lock held through a barrier and an invalid future timestamp stored, `advise_if_due` returns `Busy` and the state file is byte-identical; the same holds for a lock-busy explicit pass (`Skipped(Busy)`); (S36) failed state write during a normalizing operation, in two legs. *Pre-rename leg* (`NotApplied`, e.g. a rename failure): the previous state is byte-identical and readable, the advisory reports `StateWriteFailed` with no advice and no claimed normalization, the explicit pass reports `CompletedStatePersistFailed`, and a later successful operation still normalizes it. *Post-rename leg* (`DurabilityUncertain`, e.g. a parent-directory sync failure): the new state may already be visible, so byte identity is not asserted; the advisory reports `AdvisedDurabilityUncertain` with an actionable diagnostic, the normalization is not claimed durable, and a later invocation recovers safely (re-reads the visible state, performs no rewrite). `StateUnavailable` is reserved for state that cannot be read or evaluated and is not a write outcome; (S37) successful normalization — a normalizing advisory and a normalizing explicit pass each persist state in which every timestamp satisfies `t ≤ now + FUTURE_SKEW_TOLERANCE`, valid near-future values (within tolerance) are preserved byte-for-byte, and a repeated read performs no rewrite and does not move eligibility forward; (S38) process restart after normalization — a fresh call re-reading the persisted file observes the normalized timestamps, performs no rewrite, and does not re-fire the immediate advice; (S39) eventual eligibility — with invalid timestamps unnormalized (S34/S35/S36 states) repeated read-only evaluations never push eligibility out, and after the injected clock advances past `RECONCILIATION_ADVISORY_AFTER` a successful advisory fires exactly once and persists normalized state. + - *Explicit recovery:* (S26) `sce doctor --fix` runs and reports its true result from each of: failed streak, stale advice, invalid timestamps, corrupt state, unwritable state (→ `CompletedStatePersistFailed`); (S27) explicit success resets the failure streak and sets `last_success`; explicit failure increments the streak without touching `last_success`; lock-busy never changes state. + - *DB opening (D4):* (S28) missing DB — fails with the `Missing` typed error from the single non-creating read-only open (no existence pre-check); no database file, parent directory, WAL, or shared-memory file is created (directory listing before and after is identical, including when the parent directory is absent); Git refs are neither listed nor deleted (spy on `list_pins`/`delete_pins` never called); (S29) existing DB with incompatible schema — opening is permitted; fails with `IncompatibleSchema`; no migration or repair row written (migration table rows identical before and after); no ref deleted; (S30) existing DB with missing repository metadata — fails with `MissingMetadata`; no `repository_metadata` row inserted; (S31) existing DB with repository identity mismatch — fails with `RepositoryMismatch`; no row changed; (S32) existing valid DB — identity and schema readiness are verified before any ref listing (ordering asserted via seam), the multiprocess-WAL configuration is preserved, the pass completes, and no initialization, repair, or migration runs. For S29–S31 the assertions are logical: a before/after dump of every table's rows is equal (no migration, repair, initialization, or application-table INSERT/UPDATE/DELETE), refs are unchanged, and the external-taint marker is unarmed. Filesystem immutability is **not** asserted for existing-DB failure cases, because incidental Turso WAL/shared-memory bookkeeping is permitted when unavoidable; only S28 asserts that no database-related file or directory is created. + Out — production behavior changes beyond test seams strictly required. + - Dependencies: T03, T04 + - Done when: all scenarios pass reliably, each fails when the guarded behavior is deliberately broken, no test depends on timing or sleeps, and every review finding in the traceability table maps to at least one passing scenario. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` focused on the new tests, repeated runs for the race cases; zero-match filters are not accepted as passes; `rg "sleep"` over new test code returns no hits. + - Reopened: 2026-10-09 (the earlier `done`/`synced` marking was unsupported: S6, S7, S16, S17, S18b and S33 were listed as "not added" or covered only through fake closures, helper tests or enum mappings). + - Files changed (reopening pass): `cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs` and new `ref_maintenance/tests/{support,coordination,worktrees,cross_process,fail_closed,state_recovery}.rs`; `cli/src/app/doctor_reconciliation_cli_tests.rs` (new, registered in `cli/src/app.rs`); `ref_advisory.rs` (tests); `git_snapshot.rs` (deletion-cancellation test); production test seams only — `coordinator.rs` (`coordinate_inner` takes `lock_timeout`; `coordinate` passes `WORKTREE_LOCK_TIMEOUT`, so production behaviour is unchanged), `protected_worktree.rs` (`acquire_with_timeout` is `pub(super)`), `git_snapshot.rs` (`delete_pins_inner` forwards `on_worker_entered`; the dead `run_ref_mutation` wrapper was removed), `hooks/mutation_scope.rs` (`drive_mutation_scope` is `pub(crate)`). No protocol, Quint, dispatch, runtime or timeout changes. + - Findings that shaped the tests: (1) a parked pass must run on its own thread and runtime — parking inside `tokio::spawn` blocks a worker and starves the process/IO driver the coordinator needs, which made the coordinator stall before it ever tried the lock; (2) `sce doctor --fix` runs a separate global-state repair that creates the Agent Trace DB parent directory before reconciliation, so the CLI-level missing-DB guarantee is "no DB file or sidecars", not "no parent directory"; (3) doctor failure kinds are `agent_trace_db_missing` and `agent_trace_db_repository_mismatch`, not a generic `agent_trace_db_unavailable`. + - Test-suite simplification (2026-10-09, PR #305 audit; no production behavior changed; scenario statuses unchanged): 19 registrations removed (498 → 479). Deleted: `ref_reconciliation_production_entrypoint_accepts_only_a_repository_root` (signature-only), `ref_reconciliation_worktree_wrapper_acquires_once_and_releases` (with the then-unused `#[cfg(test)]` `reconcile_worktree`, `reconcile_worktree_inner` and `RECONCILIATION_LOCK_TIMEOUT`), `ref_reconciliation_advisory_is_busy_while_an_explicit_pass_holds_the_lock` (superseded by the two independent-task S15 tests), `advisory_runs_after_marker_clear_after_commit_with_success_result` (subset of `marker_clear_after_commit_logs_taint_warning_and_still_advises_once`). Consolidated into tables: five `maintenance_state` pre-rename tests → `maintenance_state_pre_rename_failures_preserve_previous_state`; two post-rename tests → `maintenance_state_post_rename_failures_report_uncertain_durability`; two timestamp tests → `maintenance_advisory_accepts_valid_timestamp_offsets_without_rewrite`; two unusable-state tests → `maintenance_advisory_reanchors_unusable_state_variants`; four doctor JSON tests → `json_ref_reconciliation_fix_results_match_contract` + `json_completed_state_persist_failed_results_match_contract` (whole-object equality); six doctor text tests → `reconciliation_fix_results_render_expected_text`. The duplicate `unreadable` leg was dropped from the S26 table. Close/Flush S16 tests share `assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary)` and keep their exact names. Strengthened: malformed ref compares the full `refs/sce/` inventory (name, object id, symref) and the `MalformedPin` ref name; unreadable previous state asserts `AgentTraceDbUnavailable` → typed `ExistingRepositoryDbError::Missing`, untouched state directory and pins; the verified opener test proves the statement valid on a writable connection first and asserts `database is readonly`; `Sandbox::run` now asserts the real `ExitCode` (typed `==`) for every doctor invocation; the non-database lifecycle test asserts `AgentTraceDbConnectionFailed`; the pre-rename table also checks byte identity, absent-file and recovery. The guard-removal records below remain historical: test names they cite that no longer exist are covered by the consolidated equivalents named in the map above. Guard re-check: forcing `read_only(false)` in `db/mod.rs` makes `ref_reconciliation_verified_open_is_read_only` fail (restored). Verification: full CLI suite 479 passed, 0 failed; focused `maintenance_state` 6, `ref_advisory` 24, `ref_reconciliation` 48, `doctor` 19, `mutation_scope` 24, `lifecycle` 9 (all non-zero); 8 consecutive passes of the 62 `doctor_reconciliation_cli`/`ref_reconciliation`/cancellation/lock-timeout tests; clippy `-D warnings` clean; `nix flake check` passed; `git diff --check` clean. + - Verification, remediation pass (2026-10-09): full CLI suite 495 passed, 0 failed (two new doctor CLI tests); focused `doctor_reconciliation_cli` 7, `ref_reconciliation` 50, `mutation_scope` 25, `maintenance` 85 (all non-zero); new doctor CLI tests 5 consecutive passes; every `ref_maintenance::tests` test and the cancellation tests 20 consecutive passes; `clippy --all-targets --all-features -D warnings` clean; `git diff --check` clean; `cargo fmt` made no change; `nix flake check` passed (clippy, fmt, CLI tests, Quint connect). + - Verification (earlier pass, 2026-10-09): full CLI suite 493 passed, 0 failed; focused `ref_reconciliation` 50, `mutation_scope` 25, `doctor` 25, `maintenance` 83 (all non-zero); `clippy --all-targets --all-features -D warnings` clean; `nix flake check` passed (clippy, fmt, CLI tests, Quint connect); `git diff --check` clean; 20 consecutive passes of every `ref_maintenance::tests::*` and cancellation test, 5 consecutive passes of the doctor CLI tests; no added comments, sleeps, `allow(dead_code)`, `dyn`, `BoxFuture`, detached tasks or extra production runtimes. + - Guard-removal checks actually performed: repository-wide roots replaced by worktree-local roots → S3 and S13 tests fail; deletion worker drops its lease → the new deletion-cancellation test and the existing pin-cancellation test fail; advisory ignores lock contention → S15 (both phases) and S33 fail. The remaining guards (S6, S16, S17, S18b, S21) were checked in the remediation pass; see the results table below. + - Remediation (2026-10-09, PR #305 review of `982976f4`, advisory persistence semantics; T01–T06 statuses unchanged): `AdvisoryOutcome::diagnostic()` now follows the carried `PersistFailure`. `StateWriteFailed` + `NotApplied` keeps `SCE: Reconciliation advisory state could not be written: . Run sce doctor to check reconciliation status.`; `StateWriteFailed` + `DurabilityUncertain` (post-rename anchor or normalization write) reads `SCE: Reconciliation advisory-state durability could not be confirmed: . Run sce doctor to check reconciliation status.` and never claims the update was not written or that advice was emitted. Outcome mapping, scheduling, locking, and Git-ref reconciliation are unchanged; no new variants or seams. Added tests in `ref_advisory::tests`: `maintenance_anchor_post_rename_failure_is_state_write_failed_with_visible_anchor` (absent state, `SyncParentDirectory` fault, anchor visible, next call `NoAction` with byte-identical state), `maintenance_no_action_normalization_post_rename_failure_is_state_write_failed` (fresh anchor plus far-future `last_advised`, no advice due; invalid timestamp removed; next call `NoAction`, no rewrite); extended `maintenance_report_maps_every_outcome_to_severity_and_diagnostic` (uncertain `StateWriteFailed`), `maintenance_advisory_rename_failure_preserves_state_and_next_call_can_advise` (pre-rename wording) and `assert_durability_uncertain_advisory` (recommendation retained, durability wording). Verification: see the final verification record for this remediation. + - Remediation (2026-10-09, PR #305 review of `39f78941`; no gap was waived and no production code changed): (1) S9 persistence-warning outcome and S26 recovery table now run through the real `sce doctor --fix` (`doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable`, `doctor_fix_recovers_from_every_initial_maintenance_state_condition`), using a naturally unreadable state (a directory at `.git/sce/ref-maintenance.json`) rather than a fault seam; `NotApplied` and `DurabilityUncertain` stay at the persistence layer (`ref_advisory::tests`, `rm::state_recovery`), with no CLI fault seam. (2) The doctor CLI child now captures the `ExitCode` returned by `run_with_dependency_check_and_streams`, writes `success`/`failure` to an `exit` file beside `stdout`/`stderr`, `CliRun` exposes it, and every doctor invocation asserts it. The CLI contract (unchanged): `render_report` always returns `Ok`, so plain doctor and `--fix` exit successfully for completed, failed-fix, busy and persistence-warning outcomes; an individual failed fix is reported in the payload, not in the exit status. (3) S14 is verified by static call-graph inspection plus real-hook ref-delta assertions (see the S14 row); no spy/panic seams were added. (4) The five remaining guard-removal checks were performed in a throwaway `git worktree` at `39f78941` (shared `CARGO_TARGET_DIR`, removed afterwards); results are in the table below. (5) D2 now distinguishes `StateUnavailable`, `StateWriteFailed`, `AdvisedDurabilityUncertain`, `Advised`, `NoAction` and `Busy`, and documents the single immediate advisory for a newly recorded failure; D3, AC5, AC13, S20 and S36 already agreed and were not changed. + - Guard-removal results for the five remaining guards (actual outcomes; every mutation lived only in the throwaway worktree and none is in the diff): + + | Scenario | Guard temporarily changed | Targeted test | Expected failure observed | Restored test passes | + |---|---|---|---|---| + | S6 | `ProtectedWorktree::acquire_with_timeout` took its lock from a diverted Git dir (`git_dir.join("bypass")`), so the coordinator no longer contends with the pass | `ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_db_open` and `…_after_pin_inventory` | Yes, both: the coordinator ran to completion without ever signalling lock contention (`signal sender dropped before signalling`) | Yes (2 passed) | + | S16 | `classify_coordinate` in `hooks/mutation_scope.rs` swallowed `CoordinateError::LockAcquisition` as a successful empty result | `ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries` and `ref_reconciliation_flush_lock_timeout_under_maintenance_commits_nothing_and_retries` | Yes, both: `expect_err` on the contended boundary failed (`the lock cannot be acquired while the pass holds it: ""`), i.e. the lock timeout was no longer reported as a pre-commit failure | Yes (2 passed) | + | S17 | `Close` leaked the coordinator's `WorktreeLock` past completion (flagged in `coordinate_inner`, `std::mem::forget` of the lock in `complete_blocking`), the observable equivalent of running the advisory before the release | `ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock` | Yes: the zero-wait acquisition inside the advisory callback failed (`the coordinator lock is released before the advisory runs: TimedOut { timeout: 0ns }`). A first attempt that leaked the lock on every boundary failed during test setup and was discarded as not evidence | Yes (1 passed) | + | S18b | `coordinate_inner` passed `false` instead of `protected.inherited_external_taint()` to `coordinate_protected` | `ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate` | Yes: `inherited taint conservatively abandons the live scope` failed with `left: Some(Active)` | Yes (1 passed) | + | S21 | `advise_if_due_with` acquired the lock and dropped it immediately, so no lock covered the state-writing section | `ref_reconciliation_advisory_lock_serializes_independent_processes` | Yes: the second independent process returned `Advised` where `Busy` was required (duplicate advice). A first attempt failed to compile (unused import) and was discarded as not evidence | Yes (1 passed) | + + No sleeps were used to force interleavings; the tests use their existing barriers, parked-phase seams and process rendezvous. + - Context impact: none (tests and test seams only). + - Context synchronization: synced + + - **Coverage map (S1–S39; 40 rows because S18 is split into S18a/S18b; all 40 Verified after the remediation pass).** Status is `Verified`, `Partially verified` or `Not verified`; a related unit test never upgrades a production-path scenario. Test names are exact; paths are relative to `cli/src/services/mutation_trace/runtime/` unless they start with `app::` or `services::`. `rm::` = `ref_maintenance::tests::`. + + | Scenario | Exact test | Test level | Verified invariant | Status | + |---|---|---|---|---| + | S1 | `app::doctor_reconciliation_cli_tests::doctor_inspection_is_read_only_and_fix_reconciles_the_orphan_pin_once`; `rm::ref_reconciliation_verified_database_reconciles_orphan_pin_and_records_success` | CLI end-to-end; pass-level | `sce doctor --fix` deletes an orphan SCE pin and records success (the orphan is created directly with `git update-ref`, not by an interrupted coordinate) | Verified | + | S2 | `rm::worktrees::ref_reconciliation_historical_roots_of_a_real_coordinator_sequence_stay_protected` | Production integration (real coordinator, real Git) | A→B→C→D from real `Start`/`Flush` boundaries: all four trees stay pinned although only D is the cursor; one orphan is reclaimed | Verified | + | S3 | `rm::worktrees::ref_reconciliation_pin_of_one_real_worktree_is_retained_while_another_needs_the_tree`; `rm::ref_reconciliation_retains_every_repository_wide_durable_root_and_never_repairs_other_worktrees` | Production integration (real linked worktree); synthetic roots | Main's pin of a tree that only the linked worktree durably requires is retained | Verified | + | S4 | `rm::fail_closed::ref_reconciliation_missing_local_pin_fails_closed_while_other_orphans_exist` | Pass-level | Missing local required pin returns `MissingRequiredPins` and deletes nothing, even with an eligible orphan present | Verified | + | S5 | `rm::ref_reconciliation_malformed_namespace_ref_deletes_nothing_and_records_failure` | Pass-level | Symbolic ref in the namespace yields `MalformedPin` naming that ref; the full `refs/sce/` inventory (ref name, object id, symref target) is identical before and after, so a delete-and-replace cannot pass; failure recorded | Verified | + | S6 | `rm::coordination::ref_reconciliation_real_coordinate_waits_for_a_pass_parked_after_db_open`; `…_after_pin_inventory` | Production integration (real `coordinate_inner`, real lock) | Coordinator hits contention, opens no DB and arms no marker while the pass holds the lock; the pin created after release survives; only the orphan is deleted; new pin precedes the durable-root commit; lock free afterwards | Verified | + | S7 | `services::mutation_trace::runtime::git_snapshot::tests::cancelled_delete_keeps_worktree_lock_until_git_ref_deletion_completes` | Cancellation integration (real `delete_pins`) | Dropping the caller and outer lock owner leaves the lock held by the parked worker; after release the pin is gone, no `.lock` files remain, a new pin proceeds. Existing `cancelled_pin_keeps_worktree_lock_until_git_ref_mutation_completes` covers the pin side | Verified | + | S8 | `rm::ref_reconciliation_retains_every_repository_wide_durable_root_and_never_repairs_other_worktrees` (second pass); CLI test second `--fix` run | Pass-level; CLI | Second pass deletes nothing | Verified | + | S9 | `app::doctor_reconciliation_cli_tests::doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable` (persist-warning); `…::doctor_inspection_is_read_only_and_fix_reconciles_the_orphan_pin_once` (plain doctor, completed); `…::doctor_fix_reports_missing_database_without_creating_it_or_deleting_refs` and `…::doctor_fix_reports_repository_mismatch_without_touching_the_database_or_refs` (failed); `…::doctor_fix_reports_busy_when_the_worktree_lock_is_held_and_keeps_other_repairs` (busy); `services::doctor::render::reconciliation_render_tests::{json_ref_reconciliation_fix_results_match_contract (cases completed, failed without warning, failed with warning, busy), json_completed_state_persist_failed_results_match_contract (cases not_applied, durability_uncertain, previous_state_unreadable)}` (whole-object JSON equality per case) and `services::doctor::reconciliation_fix_tests::reconciliation_fix_results_render_expected_text` (table: completed, pre-rename failure, post-rename failure, unreadable previous state, failed, busy; required and forbidden text per case) (mapping tests for `NotApplied`/`DurabilityUncertain`); `rm::ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome`; `ref_advisory::tests::{maintenance_normalization_pre_rename_failure_keeps_previous_bytes_and_recovers_later, maintenance_normalization_post_rename_failure_is_visible_but_durability_is_uncertain}` | CLI end-to-end for completed, failed, busy and `completed_state_persist_failed` (kind `previous_state_unreadable`, via a directory at the state path: orphan deleted, counts 1/0/0, `failure` null, detail reports the warning without "failed", directory untouched, exit success); persistence layer and render/mapping unit tests for `NotApplied` and `DurabilityUncertain` | Plain doctor mutates neither refs nor state; `--fix` reports the true result and the CLI exit status is asserted. At CLI level only `previous_state_unreadable` is exercised for the persist-warning outcome; `not_applied` and `durability_uncertain` are covered at the persistence and rendering layers by design (no CLI fault seam) | Verified | + | S10 | `rm::ref_reconciliation_failure_streak_counts_but_lock_contention_never_does`; `rm::fail_closed::ref_reconciliation_second_explicit_pass_is_busy_and_records_exactly_one_pass`; CLI busy test | Pass-level; CLI | Lock contention returns `Skipped(Busy)`, leaves state byte-identical and is not a failure | Verified | + | S11 | `rm::fail_closed::ref_reconciliation_database_failure_never_arms_the_external_taint_marker`; `rm::ref_reconciliation_missing_database_creates_nothing_and_deletes_nothing` | Pass-level | DB failure leaves refs untouched, records a streak, never arms the marker | Verified | + | S12 | `rm::ref_reconciliation_lock_held_implementation_never_reacquires_the_lock`; S6/S15 tests (the removed test-only `reconcile_worktree` wrapper test and synchronous advisory-busy test are superseded by these and by the production `reconcile_explicit` lock-once path exercised in S6/S15/S16/S21) | Pass-level; production integration | Single acquisition per operation, shown by bounded completion under the existing self-contending lock (no acquisition counter) | Verified | + | S13 | `rm::worktrees::ref_reconciliation_missing_pin_in_another_real_worktree_neither_fails_nor_is_repaired` | Production integration (real linked worktree) | Reconciling main reclaims its orphan, keeps the pin protecting the linked tree, never recreates the linked worktree's missing pin, does not fail; the linked worktree's own pass fails closed | Verified | + | S14 | Behavioral: `services::hooks::mutation_scope::advisory_trigger_tests::{advisory_runs_once_after_completed_close_and_flush, marker_clear_after_commit_logs_taint_warning_and_still_advises_once, advisory_never_runs_for_start_or_advance, advisory_never_runs_after_failed_coordinate, advisory_never_runs_for_abandon}` (when it runs); `rm::coordination::{ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries, ref_reconciliation_flush_lock_timeout_under_maintenance_commits_nothing_and_retries}` (real failed coordinate: advisory count 0); `rm::coordination::ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock` (real `Close`: refs identical before/after the advisory, marker snapshot identical, commit unchanged, zero-wait lock acquirable); `rm::coordination::ref_reconciliation_advisory_failures_after_a_real_commit_never_change_the_hook_result` (errors do not change the result); `rm::coordination::ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate` (marker fingerprint identical across the advisory); `rm::coordination::ref_reconciliation_advisory_after_a_real_commit_is_busy_when_a_pass_wins_the_handoff` (refs and state untouched by the advisory). Structural: call graph below | Hook policy (fake coordinate); production integration (real coordinator, real Git) for ref delta; static inspection | Advisory fires only after a durably completed `Close`/`Flush` and never after a failed coordinate, `Start`, `Advance` or `Abandon`; it adds no ref delta, never modifies the external-taint marker, its errors never change a committed result, and the coordinator's lock is already released. Structural, from `advise_after_completed_boundary(&Path)`: it calls only `advise_if_due(repository_root, system_unix_time_ms)` → `derive_git_dir` (filesystem walk, `std::fs`, no Git subprocess), `read_state`/`plan_advisory`/`normalize`/`evaluate` (pure or `std::fs`/`std::io`, capped read), the synchronous `worktree_lock::acquire_inner` with `Duration::ZERO` (one `try_lock`; returns `TimedOut` before any sleep), `write_state_atomically` (staged file, rename, directory sync) and `outcome_after_write` (pure). Production code in `ref_advisory.rs` and `maintenance_state.rs` has no `crate::` import, no `async`/`await`/`tokio`/`spawn_blocking`, no `Command`, no Git invocation (the only `git` token is the `.git` path component in `derive_git_dir`), and no reference to `reconcile_explicit`, `reconcile_with_held_lock`, `GitSnapshotService`, `list_pins`, `delete_pins`, `RepositoryAgentTraceDb` or a store. The `async`/`spawn_blocking` hits in `worktree_lock.rs` are only `acquire_async`/`acquire_inner_async`, which the advisory does not call. The only production caller is the injected seam in `hooks/mutation_scope.rs`. Verified through static production call-graph inspection and real-hook ref-delta assertions. No spy/panic seams added. | Verified | + | S15 | `rm::coordination::ref_reconciliation_independent_advisory_task_is_busy_while_a_pass_is_parked_after_db_open`; `…_after_pin_inventory`; | Concurrency integration (independent blocking task) | Advisory returns `Busy` without waiting, state and refs unchanged, pass continues, later advisory follows the resulting state (`NoAction`, then `Advised` after the window). The earlier synchronous in-callback test was deleted: it asserted nothing these two tests do not (`Busy`, no state file while parked, refs intact, `NoAction` afterwards) and was not independent concurrency | Verified | + | S16 | `rm::coordination::ref_reconciliation_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries` (`Close`); `…_flush_lock_timeout_under_maintenance_commits_nothing_and_retries` (`Flush`); both are thin `#[tokio::test]` wrappers over `assert_coordinate_lock_timeout_under_maintenance_commits_nothing_and_retries(Boundary)`, so each stays separately runnable | Production integration (real coordinator, injected 300 ms timeout) | Existing lock-timeout error; no durable transition, no pins, no marker, no advisory after the failure; the same boundary succeeds after release | Verified | + | S17 | `rm::coordination::ref_reconciliation_advisory_runs_after_the_real_coordinator_released_its_lock` | Hook/coordinator integration | Zero-wait `WorktreeLock` acquisition succeeds inside the advisory callback; the commit is already durable and unchanged afterwards; refs unchanged; empty stdout | Verified | + | S18a | `rm::coordination::ref_reconciliation_advisory_failures_after_a_real_commit_never_change_the_hook_result`; `advisory_trigger_tests::{diagnostic_write_failure_does_not_change_mutation_result, state_failures_log_warn_and_write_diagnostic_without_claiming_advice, quiet_outcomes_log_debug_and_emit_no_diagnostic}` | Hook/coordinator integration; hook policy | Write-failed, corrupt and unreadable-state advisories leave the hook result, committed state, refs and marker unchanged | Verified | + | S18b | `rm::coordination::ref_reconciliation_marker_clear_failure_after_commit_keeps_taint_for_the_next_coordinate` | Recovery integration (real coordinator; unlink fault injected inside the protected section) | Failed marker clear is a durable success; marker fingerprint identical across the advisory; after the fault is lifted the next real `coordinate()` takes the inherited-taint path (live scope `Abandoned`, health restored, revision advanced ≥2, marker cleared) whereas an untainted control flush left it `Active` | Verified | + | S19 | `ref_advisory::tests::maintenance_advisory_anchors_once_then_leaves_state_untouched` | Unit (advisory) | Anchor written once, second call byte-identical | Verified | + | S20 | `ref_advisory::tests::{maintenance_advisory_advises_once_per_window_and_persists_before_returning, maintenance_failure_advisory_emits_once_and_doctor_still_recommends_repair, maintenance_failure_advisory_repeats_after_the_cadence_window, maintenance_failure_after_recent_success_is_recommended_and_advised_despite_older_reminder}` | Unit (advisory) | One time-based advice per window, persisted first; failure advice once then cadence | Verified | + | S21 | `rm::cross_process::ref_reconciliation_advisory_lock_serializes_independent_processes`; `rm::fail_closed::ref_reconciliation_second_explicit_pass_is_busy_and_records_exactly_one_pass` | Independent OS processes (self re-exec, stdin/stdout rendezvous); in-process | Parked advisory in one process, `Busy` in a second, `NoAction` in a third; exactly one advice; no state write while parked. Explicit-vs-explicit is in-process only | Verified | + | S22 | `ref_advisory::tests::maintenance_advisory_anchor_write_failure_reports_state_write_failed_without_advice`; `rm::ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome` | Unit; pass-level | Persistence failure: no advice; explicit pass reports `CompletedStatePersistFailed` with deletions intact | Verified | + | S23 | `maintenance_state::tests::maintenance_state_recovers_after_interrupted_pre_rename_write`; `maintenance_state::tests::maintenance_state_pre_rename_failures_preserve_previous_state` (table: `CreateStaging`/`WriteStaging`/`SyncStaging`/`Rename` with previous state, `Rename` without; byte-identical or absent, no staging leftovers, later write recovers); `ref_advisory::tests::maintenance_advisory_rename_failure_preserves_state_and_next_call_can_advise` | Unit | Interrupted write preserves previous state; fresh call advises at the right time | Verified | + | S24 | `ref_advisory::tests::{maintenance_advisory_accepts_valid_timestamp_offsets_without_rewrite (table: offsets `0`, `+60 s`, `FUTURE_SKEW_TOLERANCE_MS`; `NoAction`, byte-identical state, no recommendation), maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately, maintenance_advisory_clock_rollback_beyond_tolerance_normalizes_while_within_it_does_not}`; `rm::state_recovery::ref_reconciliation_invalid_last_advised_does_not_suppress_a_due_advisory`; `maintenance_advisory_advises_once_per_window_and_persists_before_returning` | Unit (injected clocks) | Cases 1–7: equal and near-future valid; exactly +5 min valid; beyond tolerance (anchor, last_success, last_advised) normalized once; rollback beyond vs within tolerance; byte-identical repeats; window recurrence | Verified | + | S25 | `ref_advisory::tests::{maintenance_advisory_anchors_once_then_leaves_state_untouched, maintenance_advisory_reanchors_unusable_state_variants (table: wrong schema version, oversized file, corrupt JSON; `StateRead::Unusable`, `Anchored`, valid state with `anchor == NOW`, no recommendation, repeat call does not rewrite)}`; `rm::state_recovery::ref_reconciliation_explicit_pass_runs_and_rewrites_valid_state_from_every_stored_condition` | Unit; pass-level | Missing, corrupt, wrong-version and oversized state recover without failure and are rewritten valid | Verified | + | S26 | `app::doctor_reconciliation_cli_tests::doctor_fix_recovers_from_every_initial_maintenance_state_condition` (failed streak, expired advisory, invalid far-future timestamps, corrupt JSON, oversized file → real `sce doctor --fix`); `app::doctor_reconciliation_cli_tests::doctor_fix_reports_state_persist_failure_when_maintenance_state_is_unreadable` (the table no longer repeats the unreadable case; this dedicated test is the single end-to-end check for `CompletedStatePersistFailed` / `previous_state_unreadable`); pass-level: `rm::state_recovery::ref_reconciliation_explicit_pass_runs_and_rewrites_valid_state_from_every_stored_condition`, `…::ref_reconciliation_explicit_success_resets_the_streak_and_failure_keeps_last_success`, `rm::ref_reconciliation_unreadable_previous_state_is_reported_without_overwriting` (primary error `AgentTraceDbUnavailable` with typed `ExistingRepositoryDbError::Missing`, `PreviousStateUnreadable` warning, state directory and pin inventory unchanged), `rm::ref_reconciliation_completed_pass_with_unwritten_state_reports_distinct_outcome` | CLI end-to-end (real repo, real Agent Trace DB, real `sce doctor --fix --format json`); pass-level | From each readable condition the explicit pass runs regardless of advisory eligibility, reclaims the orphan pin, reports `completed` (deleted 1), clears the failure streak and failure fields, sets `last_success`, normalizes far-future timestamps (all `≤ now + FUTURE_SKEW_TOLERANCE`), leaves a state a second `--fix` reads as valid, and exits successfully. From an unreadable state path (a directory) the pass still deletes the orphan, reports `completed_state_persist_failed` with `previous_state_unreadable`, makes no false persistence claim, leaves the directory untouched, and exits successfully | Verified | + | S27 | `rm::state_recovery::ref_reconciliation_explicit_success_resets_the_streak_and_failure_keeps_last_success`; `rm::ref_reconciliation_failure_streak_counts_but_lock_contention_never_does` | Pass-level | Success resets streak and sets `last_success`; failure increments without touching `last_success`; busy changes nothing | Verified | + | S28 | `rm::ref_reconciliation_missing_database_creates_nothing_and_deletes_nothing`; `rm::ref_reconciliation_verified_open_never_creates_missing_database_or_directories`; `rm::ref_reconciliation_verified_open_rejects_database_removed_before_open`; `app::doctor_reconciliation_cli_tests::doctor_fix_reports_missing_database_without_creating_it_or_deleting_refs` | Pass-level; opener; CLI | Typed `Missing`; no DB file, directory or sidecar created by the open; no refs listed or deleted. The CLI shows the parent directory comes from the separate global-state repair | Verified | + | S29 | `rm::state_recovery::ref_reconciliation_verification_failures_leave_the_logical_database_unchanged` (case `incompatible`); `rm::ref_reconciliation_missing_metadata_or_schema_fails_before_pin_inventory` | Pass-level | `IncompatibleSchema`; full-row dump of every table equal before and after; no pin inventory; refs and marker untouched | Verified | + | S30 | same test (case `missing_metadata`) | Pass-level | `MissingMetadata`; no `repository_metadata` row inserted (full-row dump equal) | Verified | + | S31 | same test (case `mismatch`); `app::doctor_reconciliation_cli_tests::doctor_fix_reports_repository_mismatch_without_touching_the_database_or_refs` | Pass-level; CLI | `RepositoryMismatch`; foreign metadata row intact; refs untouched | Verified | + | S32 | `rm::ref_reconciliation_verified_database_reconciles_orphan_pin_and_records_success` (phase order `DbOpened` → `PinsInventoried`); `rm::ref_reconciliation_verified_open_coexists_with_a_live_multiprocess_wal_writer`; `rm::ref_reconciliation_verified_open_is_read_only` | Pass-level; opener | Identity and schema verified before any ref listing; multiprocess-WAL coexistence; the opened handle rejects a statement first proven valid on a writable connection (rolled-back transaction) with `database is readonly`, and the rows stay empty. Breaking the guard (`read_only(false)` in `db/mod.rs`) makes this test fail | Verified | + | S33 | `rm::coordination::ref_reconciliation_advisory_after_a_real_commit_is_busy_when_a_pass_wins_the_handoff` | Concurrency integration (real coordinator; hook parked between coordinate and advisory; pass on its own thread) | Pass takes the lock in the handoff window; advisory returns `Busy`; state and refs unchanged by the advisory; committed `Close` still reports success; only the orphan is deleted by the pass | Verified | + | S34 | `ref_doctor::tests::recommendation_flags_unusable_and_invalid_state_without_rewriting` | Unit (doctor inspection) | Read-only doctor flags invalid timestamps and leaves bytes untouched | Verified | + | S35 | `ref_advisory::tests::maintenance_advisory_reports_busy_and_leaves_invalid_state_byte_identical`; `rm::state_recovery::ref_reconciliation_busy_explicit_pass_leaves_invalid_state_byte_identical` | Unit; pass-level | Lock-busy advisory and explicit pass leave invalid state byte-identical | Verified | + | S36 | `ref_advisory::tests::{maintenance_normalization_pre_rename_failure_keeps_previous_bytes_and_recovers_later, maintenance_normalization_post_rename_failure_is_visible_but_durability_is_uncertain}`; `maintenance_state::tests::{maintenance_state_pre_rename_failures_preserve_previous_state, maintenance_state_post_rename_failures_report_uncertain_durability (table: `OpenParentDirectory`, `SyncParentDirectory`; exact phase, `DurabilityUncertain`, new state visible, no `previous state preserved` claim, later write recovers)}`; `rm::state_recovery::ref_reconciliation_failed_state_write_during_normalization_has_two_distinct_legs` | Unit; pass-level | Pre-rename: previous bytes identical, `StateWriteFailed`/`NotApplied`. Post-rename: new state visible, `AdvisedDurabilityUncertain`/`DurabilityUncertain` when the normalization also recorded advice, or `StateWriteFailed`/`DurabilityUncertain` with no recommendation for an anchor or `NoAction` normalization (`ref_advisory::tests::{maintenance_anchor_post_rename_failure_is_state_write_failed_with_visible_anchor, maintenance_no_action_normalization_post_rename_failure_is_state_write_failed}`); no durable-normalization claim; later invocation recovers | Verified | + | S37 | `ref_advisory::tests::maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately`; `rm::state_recovery::ref_reconciliation_successful_pass_normalizes_invalid_state_and_a_later_read_does_not_rewrite` | Unit; pass-level | Normalizing advisory and explicit pass persist `t ≤ now + tolerance`; near-future value preserved; repeated read does not rewrite | Verified | + | S38 | `ref_advisory::tests::maintenance_advisory_normalizes_far_future_timestamps_and_advises_immediately` (fresh call re-reads the file); `rm::cross_process::ref_reconciliation_advisory_lock_serializes_independent_processes` | Unit; independent processes | A fresh call observes normalized state, performs no rewrite and does not re-fire | Verified | + | S39 | `rm::state_recovery::ref_reconciliation_unnormalized_invalid_state_stays_eligible_until_a_successful_advisory` | Pass-level (injected clocks, held lock) | Repeated read-only evaluation and busy advisories never move eligibility or bytes; the next successful advisory fires exactly once and persists normalized state | Verified | + +- [x] T06: `Remove dead-code exemptions and synchronize context` (status:done) + - Task ID: T06 + - Scope: In — remove declaration-level `#[allow(dead_code)]` from reconciliation declarations (`ref_reconciliation.rs`, `list_pins`/`delete_pins`/`PinnedRef`/`PinInventoryError` in `git_snapshot.rs`, durable-root readers in `store.rs`) that now have production consumers; separately review genuinely unreachable states (e.g. `SkippedNoCheckoutIdentity`, stale checkout-ID wording) and remove or justify each; update the context files in Context sync with the real async APIs, the wrapper/lock-held split and single-acquisition rule, the documented D1 concurrency contract, Git-derived worktree identities, the two maintenance entrypoints, the advisory trigger and why reconciliation is kept off the hook path, the maintenance state file (past-only fields, normalization, atomic writes), the distinct explicit outcomes, the verified-existing-DB opener contract, doctor reporting, and the corrected two-invariant durability statement (no claim that all repository-wide roots are pinned). Record in context, and not as implemented behavior: the deferred automatic out-of-band reclamation follow-up with its binding prerequisites from D2 (lifecycle-owned execution point, durable next-attempt reservation before expensive work, distinct outcomes, lock-hold analysis, timestamp normalization), the retired-worktree cleanup follow-up, and known storage limitations (historical retention, no repair of other worktrees' pins). Out — behavior changes. + - Dependencies: T05 + - Done when: no reconciliation-related `allow(dead_code)` remains without a stated reason; clippy with `-D warnings` is clean; context no longer says "retained and unwired", describes checkout-ID identity, or claims automatic reclamation; `context/context-map.md` entries match. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings`; `nix shell nixpkgs#ripgrep -c rg -n "allow\(dead_code" cli/src/services/mutation_trace`; `rg -i "retained and unwired" context`; `rg -i "automatic.*(reclam|reconcil)" context` hits only deferral statements. + - Completed: 2026-10-09 + - Files changed: `cli/src/services/mutation_trace/runtime/{git_snapshot,maintenance_state,ref_maintenance,ref_reconciliation}.rs`, `cli/src/services/mutation_trace/runtime/ref_maintenance/tests.rs`, `cli/src/services/mutation_trace/store.rs`, `context/cli/{agent-trace-storage,async-persistence-guarantees,mutation-scope-runtime,mutation-trace-protocol,mutation-trace-ref-reconciliation,mutation-trace-runtime-coordinator,mutation-trace-snapshot-service,mutation-trace-store}.md`, `context/{context-map,overview}.md`, `context/sce/{agent-trace-hook-doctor,doctor-human-text-contract}.md`, `context/plans/mutation-cursor-ref-reconciliation-wiring.md` + - Result: Removed every "retained and unwired" and "wired by later tasks" dead-code allow from the reconciliation cluster (`ref_reconciliation.rs`, `git_snapshot.rs`, `store.rs` durable-root readers, `ref_maintenance.rs`, `maintenance_state.rs`). The lock-acquiring `reconcile_worktree`/`reconcile_worktree_inner`, `RECONCILIATION_LOCK_TIMEOUT` and their imports are now `#[cfg(test)]` because production uses `reconcile_explicit`. The single-event loaders in `store.rs` (`load_mutation_event`, `load_mutation_event_active_scopes`, `decode_boundary_kind`, their SQL constants and row mappers) have no production reader even in tests; they keep declaration-level allows with an accurate reason (attribution history uses `load_mutation_event_page`), not deleted to avoid scope expansion. Removed the unreachable `ReconciliationOutcome::SkippedNoCheckoutIdentity` and its `ref_maintenance` match arm; `ReconciliationOutcome` now has the single variant `Reconciled`. Removed comments in the touched and adjacent code per the user's instruction. Rewrote `context/cli/mutation-trace-ref-reconciliation.md` and updated the other context files listed in Context sync (no checkout-identity wording, no "unwired" claims, automatic reclamation stated only as a deferred follow-up with its D2 prerequisites, retired-worktree cleanup and storage limitations recorded). + - Verify: `clippy --all-targets --all-features -D warnings` clean; `nix flake check` passed (clippy, fmt, CLI tests, Quint connect); `git diff --check` clean; `rg "retained and unwired|unwired" context` (excluding plans) leaves no stale claim; `rg -i "automatic.*(reclam|reconcil)" context` hits only deferral statements; remaining `dead_code` allows in `mutation_trace` are the single-event loaders, coordinator/scope-runtime outcome payloads and test-model items, each with a stated reason. + - Context impact: important (this task is the context synchronization for the whole plan). + - Context synchronization: synced + +## Audit findings + +Recorded by T01 (read-only audit, 2026-10-09). Line numbers refer to the tree at `fb7e78fb`. + +**PR metadata (confirmed via `gh pr view`).** PR #304: head branch `fix-logging`, head `0be74aad5bea5070828846d3277041a4649c9842`, base `dead-code-blues`. PR #305: head branch `ref-recon`, base `fix-logging`, head `fb7e78fbfa41f9f054803eef2dddb180f5d5289a`. No rebase needed. + +**Lock ownership (D1) — confirmed.** +- `ref_reconciliation.rs:159-243` `reconcile_worktree_inner` resolves `git_dir` (`:168`), then acquires `WorktreeLock` via `acquire_inner_async(&git_dir, RECONCILIATION_LOCK_TIMEOUT /*10 s, :19*/, on_lock_contention)` (`:172`), then `resolve_worktree_id` (`:176`), `open_db` (`:180`), `GitSnapshotService::new` (`:184`), `list_pins` (`:190`), `load_tree_roots` (`:207`), `load_all_tree_roots` (`:222`), `delete_pins(lock.lease(), &stale)` (`:233`). `lock` drops on return. +- `worktree_lock.rs`: non-reentrant (`File::try_lock`, `:114`); `acquire_inner` loops `try_lock` and, on `WouldBlock`, checks `now >= deadline` before sleeping (`:123-135`), so `Duration::ZERO` is exactly one attempt then `TimedOut`. `WorktreeLockLease` is an `Arc` clone (`:147-151`); the file unlocks only when the last clone drops (`WorktreeLockInner::drop`, `:158-164`). `acquire_inner` (sync, `pub(super)`, `:79`) is available for the synchronous advisory; `acquire_inner_async` wraps it in `spawn_blocking` (`:65-77`). +- `git_snapshot.rs:281-298` `run_ref_mutation_inner` moves the lease into `spawn_blocking` (`let _lease = lease;`, `:292`) and calls `on_worker_entered()` (`:293`); `run_ref_mutation` passes `|| {}` (`:278`). `delete_pins` is `:214`, `delete_pins_inner` `:231`. Dropping the caller future therefore cannot release the file lock while the worker runs. +- `protected_worktree.rs:14` `WORKTREE_LOCK_TIMEOUT` = 10 s; `ProtectedWorktree::acquire_with_timeout` (`:109-138`) = `resolve_git_dir` → lock → `arm_marker_with_lock` → `resolve_worktree_id`. The lock is released when `coordinate_inner` (`coordinator.rs:215-252`) returns (`protected.complete()` consumes `self`; `complete_blocking` clears the marker, then the `lock` field drops). `abandon_after_spawn_without_unlock` (`:182-185`) calls `close_without_unlock` (`worktree_lock.rs:153`) for the external-guard child path only. +- **Split**: `reconcile_with_held_lock

(repository_root, &WorktreeLock, worktree_id/open_db…)` takes everything from `:176` onward; the lock is passed as `&WorktreeLock` (so it can call `lock.lease()` at the single `delete_pins` call). `reconcile_worktree_inner` becomes: resolve git dir, acquire, delegate. `reconcile_explicit` resolves git dir itself and acquires once; it never calls `reconcile_worktree[_inner]`. +- Plan-text correction: `ReconcileError::DurableRoots` doc says "migration `003`"; the durable-roots/tree-root queries belong to the mutation-trace protocol migration `004_mutation_trace_protocol.sql` (current set `001`–`006` in `cli/migrations/agent-trace-repository/`; `003` is `claude_model_state`). T02 should not hard-code a single migration id; D4 readiness must use the existing "all expected migrations applied" check (`db/mod.rs:1189 migration_metadata_problems`, used by `ensure_schema_ready` `:1251`). + +**Production call graph and completion classes.** +- Every harness adapter funnels into `hooks::mutation_scope::run_mutation_scope_from_payload` (`hooks/{claude,codex,opencode,pi}_mutation_scope/lifecycle.rs`; `hooks/mod.rs:108` direct subcommand; `doctor/inspect.rs:305` repair seam). → `run_mutation_scope_from_payload_with` (`mutation_scope.rs:269`) → `drive_mutation_scope` (`:303`): `Start`/`Advance`/`Close`/`Flush` → `coordinate_boundary(...).await` → `classify_coordinate` (`:362-376`); `Abandon` → `classify_abandon` (`:378-392`). `external_mutation_guard` is a separate subcommand (`hooks/mod.rs:123`) using `coordinate_on_held_worktree`. +- Durably completed: `Ok(_)` and `Err(CoordinateError::MarkerClearAfterCommit)` (`:366-371`); all other `CoordinateError` variants (`SnapshotFailure`, `CasConflictExhausted`, `RevisionExhausted`, `ExternalTaintMarker`, `AgentTraceDbUnavailable`, `LockAcquisition`, `ScopeIdentityConflict`, `ScopeProvenanceRegistration`, `Other`) are pre-completion failures. +- Hook-ingress seam: `drive_mutation_scope` takes injected `coordinate_boundary`/`abandon` closures; `classify_coordinate(coordinate_boundary(...).await, logger)` at `:356-359` is the exact insertion point (after the await, so the coordinator lock is already dropped). Requires `boundary` to be `Close`/`Flush`, and the result to be `Ok` or `MarkerClearAfterCommit`. `run_mutation_scope_from_payload_with` also gets `doctor/inspect.rs:305`'s repair path, so an advisory placed in `drive_mutation_scope` fires there too; harmless (it still never reconciles) but T03 should test it is unaffected. + +**Execution-point investigation (D2) — conclusion confirmed, with one flag.** +- No lifecycle-owned, noncritical-path, cross-harness executor exists. `post-commit` (`hooks/commit_hooks.rs:114`) is synchronous inside `git commit`; per-harness stop/session-end handlers (for example `hooks/codex/stop.rs`) are separate, per-harness, synchronous hook processes that persist their own DB rows and are not part of the shared mutation-scope ingress; `sce setup` is install-time; `doctor/mod.rs` is the operator-owned foreground path. +- **Flag (does not invalidate the plan):** `post-commit` already launches a detached child via `sync/auto_sync.rs:39` `launch` (spawns the current executable with null stdio and drops the `Child`). That is a fire-and-forget subprocess precedent. D2/constraint 6 prohibit fire-and-forget subprocesses for reconciliation, and a short-lived hook process cannot supervise one, so the selected design stands; the follow-up PR may cite this precedent when evaluating an out-of-band executor. +- Tokio shutdown joining blocking workers and the unbounded cost of `git for-each-ref`/durable-root queries are as stated in the authoring-time observations; nothing found contradicts them. +- Decision: advisory-only hook trigger plus `sce doctor --fix`. + +**Advisory Git-dir resolution (D2 decision rule) — advisory kept, filesystem derivation.** +- `coordinate` resolves the Git dir internally (`resolve_git_dir`, a `git rev-parse --absolute-git-dir` subprocess, `git_snapshot.rs:602-613`, via `run_rev_parse` `:699`) and does not return it (`CoordinateOutcome` carries `worktree_id`, `observed_tree`, …, not the Git dir), so reuse would need a protocol/API change. Not chosen. +- Filesystem derivation: the hook's `repository_root` is `std::env::current_dir()` (`hooks/mod.rs:63`), which may be a subdirectory. Starting there, walk ancestors to the nearest entry named `.git`; if it is a directory it is the Git dir; if it is a file containing `gitdir: `, the Git dir is that path (resolved relative to the file's parent when relative). Verified empirically in a scratch repo: from a subdirectory of the main worktree `git rev-parse --absolute-git-dir` equals `/.git`; in a linked worktree the `.git` file contains `gitdir: /.git/worktrees/`, which equals `--absolute-git-dir`. The lock file and state file are addressed by inode, so path spelling differences (symlinks) do not change behavior, but T02 should canonicalize both anyway and treat any mismatch as `StateUnavailable`. +- Guardrails T02 must implement: if `GIT_DIR`, `GIT_COMMON_DIR`, or `GIT_WORK_TREE` is set, or no `.git` is found, or the `gitdir:` target does not exist, return `StateUnavailable` (no advice, no Git subprocess). Because `coordinate` just succeeded from the same directory, the nearest-`.git` walk cannot find a different repository than Git did. +- Outcome: **hook trigger kept; no new Git subprocess.** The derivation function takes only `&Path`, performs only local `std::fs` reads, and lives in the advisory/state module (no `resolve_git_dir` import). + +**State (D3) — confirmed.** +- Placement: `/sce/` beside `mutation-cursor.lock` (`worktree_lock.rs:9-11`) and `mutation-cursor-tainted` (`external_taint.rs:26-29`). No in-memory registry is needed. +- No atomic-write helper exists in the runtime module: `ExternalTaintMarker::persist` (`external_taint.rs:82-112`) is create+`sync_data` of an empty file with best-effort parent-dir `sync_all` (`:141-145`). The repository's staging-and-swap pattern is `setup/install.rs` (`create_*_staging_path` with `{epoch_nanos}-{pid}-{attempt}` names at `:224-256`, `fs::write` to staging, `rename_fn` injected for test interruption at `:40,185`, `cleanup_path_if_exists` on failure). T02 should mirror this with an injected rename/write seam rather than invent a new pattern, plus the marker's best-effort parent-dir sync. +- Clock: there is no injectable wall clock in the runtime; `hooks/runtime.rs` has `current_unix_time_ms()` (`SystemTime`/`UNIX_EPOCH`, i64 ms). T02 takes `now: impl Fn() -> i64` and production passes `current_unix_time_ms`-equivalent logic. `sync/sync.rs:78` defines a `now()` trait for another subsystem; not reused. + +**DB opening (D4) — confirmed not read-only; opener contract.** +- `open_agent_trace_db_for_hook_runtime` (`hooks/runtime.rs:53-69`) resolves config → `resolve_agent_trace_storage_for_hook_runtime` → `open_storage_for_hook_runtime` → `open_repository_db_for_hook_runtime` (`agent_trace_storage/mod.rs:195-207`): (a) `TursoDb::open_without_migrations_at` calls `ensure_db_parent_dir` and `turso::Builder::new_local(path).experimental_multiprocess_wal(true).build()` (`db/mod.rs:856-880`), which can create parent directories and an empty DB file; (b) on `ensure_schema_ready_for_hooks` failure it calls `repair_missing_repository_schema_migration_metadata` (`agent_trace_db/repository.rs:136-150`), which inserts the baseline migration row; (c) `verify_or_initialize_repository_metadata` (`repository.rs:171-230`) inserts the `repository_metadata` row and claims `source_instance_id` when absent/invalid. +- Turso: crate `turso = 0.8.1` (`cli/Cargo.toml:32`); no read-only open or open flag is used anywhere in `cli/src`. T02 first confirms whether the 0.8.1 builder exposes one; if not, accept the documented limitation (existence pre-check, `SELECT`-only afterward, tests assert logical contents unchanged). The multiprocess-WAL builder option must be kept so the open coordinates with live hook writers. +- Smallest opener: a new function beside `open_repository_db_for_hook_runtime` that (1) resolves identity and path exactly as the hook runtime does (`resolve_identity`, `agent_trace_db_path_for_repository`), (2) returns `Missing` if the path is not an existing file before touching Turso, (3) opens without creating directories (a new `open_existing_without_migrations_at` on `TursoDb` that skips `ensure_db_parent_dir`, or an equivalent guard), (4) runs `ensure_schema_ready_for_hooks` (all expected migrations `001`–`006` applied) and on failure returns `IncompatibleSchema` without repair, (5) calls a new read-only `verify_existing_repository_metadata(repository_id)` built on `select_repository_metadata_row` (`repository.rs:232`) returning `MissingMetadata` when the row is absent or `source_instance_id` fails `is_valid_source_instance_id` (`:74`), and `RepositoryMismatch` via the existing `ensure_repository_id_matches`. The hook opener and its callers are unchanged. + +**Identity and stale-doc findings.** +- `resolve_worktree_id` (`git_snapshot.rs:666-685`): `git_dir == git_common_dir` → `main`; otherwise `worktrees/`. It is Git-topology-derived and failure is `Err`. +- `ReconciliationOutcome::SkippedNoCheckoutIdentity` (`ref_reconciliation.rs:59`) is declared and documented (`:36`, `:50-51`) but never constructed anywhere in `cli/src`; confirmed unreachable. `context/cli/mutation-trace-ref-reconciliation.md` still describes `read_checkout_id → Ok(None)` and "checkout identity" namespaces (stale). T06 removes the variant (callers must then match one variant) and rewrites the docs. +- `#[allow(dead_code, reason = "ref reconciliation retained and unwired; …")]` appears on `ref_reconciliation.rs` declarations, `git_snapshot.rs` (`run_ref_mutation`, `assert_pins_are_unchanged_direct_refs`, others) and ten places in `store.rs` (`:199, :225, :249, :259, :272, :713, :844, :874, :1022, :1195`); T06 removes only those that gain production consumers. + +**Findings affecting later tasks.** None invalidates a task's scope. T02 must: use `acquire_inner` (sync, zero timeout) for the advisory; implement filesystem Git-dir derivation with the stated guardrails; add the `Missing`/`IncompatibleSchema`/`MissingMetadata`/`RepositoryMismatch` opener and confirm Turso read-only support; reuse the `setup/install.rs` staging pattern. T03 must also account for the `doctor/inspect.rs:305` caller. T06 must also fix the stale "003" wording in the `ReconcileError` docs. + +## Open questions + +- Is a 24-hour `RECONCILIATION_ADVISORY_AFTER` the right window for the time-based recommendation? It is only a starting value recorded by T03. +- If T01 finds the advisory cannot get the Git dir without a new Git subprocess, the plan drops the hook trigger and relies on `sce doctor` alone (decision rule in D2, no further confirmation needed). Confirm you are happy with that outcome rather than accepting one extra `git rev-parse` on the `Close`/`Flush` path. +- Do you want a follow-up PR scoped now for automatic out-of-band reclamation (D2 prerequisites), or only the documented deferral? +- `SkippedNoCheckoutIdentity` looks unreachable in the current code (the identity is Git-topology-derived and its failure is an `Err`). The plan assumes T06 removes it; confirm you do not want it kept for compatibility. +- D4 adds a dedicated opener and one read-only verification method in `agent_trace_storage` / `agent_trace_db`. T01 may find that Turso cannot open an existing DB without incidental WAL bookkeeping; the plan treats that as an accepted limitation (logical contents unchanged) unless you want a stricter guarantee. + +## Validation Report + +**Status:** validated + +**Date:** 2026-10-09 + +### Commands run + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` -> exit 0 (498 passed, 0 failed before the test-suite simplification; 479 passed, 0 failed after it) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` -> exit 0 (clean) +- `nix flake check` -> exit 0 (all checks passed; non-host systems omitted) +- `nix build .#ci-checks` -> exit 0 +- `nix run .#pkl-check-generated` -> not run (no generated config inputs touched) +- `git diff --check` -> exit 0 +- `git diff fix-logging...HEAD --stat` -> exit 0 (no `protocol.rs` or `.qnt` change) +- AC7 `rg` over changed files -> every added hit is in test code; no production hit +- focused `ref_reconciliation` 50, `maintenance` 88, `mutation_scope` 25, `doctor` 27, `agent_trace_storage` 1 passed (all non-zero) + +### Success-criteria verification + +- [x] AC1: only caller chain is `doctor/mod.rs` -> `ref_doctor` -> `reconcile_explicit`; the hook module references only `advise_after_completed_boundary` +- [x] AC2: doctor fix and advisory ref tests pass +- [x] AC3: worktree and fail-closed tests pass +- [x] AC4: advisory outcome and marker tests pass (`mutation_scope` 25 passed) +- [x] AC5: `ref_advisory.rs` non-test code imports no Git, store or DB symbol +- [x] AC6: coordination tests pass in the full suite +- [x] AC7: no production-code hits for the forbidden patterns +- [x] AC8: no `protocol.rs` or `spec/mutation_cursor.qnt` change; `mbt` tests pass +- [x] AC9: doctor tests pass +- [x] AC10: no "unwired" claims in `context/cli`; automatic-reclamation hits are deferral statements +- [x] AC11: each entrypoint acquires `WorktreeLock` once; `reconcile_with_held_lock` does not acquire +- [x] AC12: contention and handoff tests pass +- [x] AC13: maintenance-state and recovery tests pass +- [x] AC14: verified-open and logical-immutability tests pass, including the new `agent_trace_storage` missing-DB test +- [x] AC15: deferral and prerequisites recorded in `context/cli/mutation-trace-ref-reconciliation.md` + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- Cross-process multiprocess-WAL coexistence is covered only in-process (T02 limitation 4). +- The new `agent_trace_storage` test is uncommitted in the working tree at validation time. diff --git a/context/plans/observability-log-lock-striping.md b/context/plans/observability-log-lock-striping.md new file mode 100644 index 000000000..cfaefa010 --- /dev/null +++ b/context/plans/observability-log-lock-striping.md @@ -0,0 +1,177 @@ +# Plan: observability-log-lock-striping + +## Change summary + +`cli/src/services/observability.rs::file_log_lock` keeps a process-global `OnceLock>>>>`. Every distinct log path ever written adds an `Arc>` that is never evicted, so synchronization memory grows with the number of historical paths. This plan replaces the registry with a fixed `static LOG_LOCKS: [Mutex<()>; LOG_LOCK_STRIPES]` (64 stripes) selected by a deterministic hash of the path modulo the stripe count. It is a replacement of the lock-selection mechanism only; append, fallback, retention, permission, redaction, and formatting behavior are preserved. Same-process serialization only: no cross-process guarantee exists today and none is added. + +Investigation findings (recorded from the code before selecting the design; T01 re-confirms them while implementing): + +- Production callers: `append_log_line_with_cleanup` is reached only through `append_log_line`, which is reached only from `Logger::write_log_line` (`observability.rs`). `append_log_line_once_with_cleanup` is reached only from the v2 fallback closure inside `append_log_line_with_cleanup`. `file_log_lock` has exactly those two call sites. +- Ownership: `Logger` is a `Clone + Debug` plain value holding `ObservabilityConfig`, `log_dir`, and retention limit; it owns no shared state. App startup builds one in `app.rs` (`Logger::from_resolved_config`) and stores it in the app context, but `Clone` and `pub` construction let independent instances target the same path, so per-instance mutexes (Option A) would be unsound and would need new shared ownership. Option B (static striping) is selected as the smallest correct design. +- Concurrency: logging is synchronous `std::fs` I/O invoked from the main thread, `tokio::task::spawn_blocking` paths, and plain `std::thread` users, so the critical section stays a synchronous `std::sync::Mutex`; no async mutex. +- Deadlock hazard: today the v2 fallback runs while the primary path's guard is still held, and takes the v2 path's own lock. With striping the primary and v2 paths can hash to the same stripe, and `std::sync::Mutex` is not reentrant, so the fallback must run after the primary guard is released. This is the main correctness risk of the change. A regression here deadlocks instead of failing, so the same-stripe test must run in a bounded child process (T02), never in-process and never via a detached thread plus `recv_timeout` (the deadlocked thread would outlive the failed test). +- Poisoning failure domain: `lock()` is called only at the two sites in `append_log_line_with_cleanup` / `append_log_line_once_with_cleanup`. The guarded section is `persist_log_line` (returns `Result`, uses `?`, no panicking calls expected) plus the retention `cleanup` callback, which is the only caller-supplied code run under the guard and therefore the only realistic poisoning source (a panic inside cleanup). T01 re-confirms this by inspection. Today a poisoned lock affects one path; with 64 stripes it affects every path hashing to that stripe, including unrelated ones. This enlarged failure domain is a documented, accepted consequence; existing poisoned-lock handling is preserved unchanged (no `into_inner` recovery, no `clear_poison`, no new fallback). +- Test-process infrastructure: `cli/src/app.rs::app_output_boundary_uses_isolated_process` already re-executes `std::env::current_exe()` with `--exact --nocapture` and an env marker to run an isolated child. T02 reuses this pattern (std only, no new dependency). +- Behavior to preserve: lock-acquisition failure (poisoned stripe) is returned as `failed to lock log file '': ...` and does not trigger the fallback (the fallback is only reached from the `persist_log_line` error arm, after the lock was acquired); fallback is a single non-recursive `-v2` attempt whose combined error text is unchanged; retention cleanup runs only after a write that created the file and runs inside the guard of the path that was written; files are opened with owner-only `0o600` on Unix; redaction happens before `append_log_line`. +- Cross-process: no filesystem lock or cross-process guarantee exists or is documented; `context/sce/cli-observability-contract.md` only says "serializes writes independently per path", which becomes inaccurate once paths can share a stripe. + +## Acceptance criteria + +- [x] AC1: No path-keyed lock registry remains in `observability.rs`; synchronization storage is a fixed-size `[Mutex<()>; N]` static independent of the number of distinct log paths, with no per-path `Arc`/`Mutex` allocation, no `dyn`, no boxed callbacks, no `unsafe`, and no new dependency. + - Validate: `nix shell nixpkgs#ripgrep -c rg -n "BTreeMap|OnceLock|Arc': ...` and does not attempt the fallback. + - Validate: code inspection of the guard scope in `append_log_line_with_cleanup` (guard dropped before `attempt_v2_log_fallback` is called, no guard live across it); the same-stripe fallback child test (steps 3-5); the existing combined-error test (step 6); the poisoned-lock test (step 7). +- [x] AC4b: Same-stripe fallback regression is deterministic and bounded: the test builds a primary path whose `v2_log_path` maps to the same stripe using the production `log_lock_stripe` selection, forces primary failure, and runs in an isolated child process with a finite parent-enforced timeout; the child is killed and reaped on timeout; timeout, unexpected termination, or wrong output fails the test; success requires the fallback file to exist with the expected content. + - Validate: the test passes under `observability`; temporarily holding the primary guard across the fallback makes it fail by timeout (not hang) and leaves no orphan child. +- [x] AC4c: Poisoning semantics are preserved and documented: a poisoned stripe still yields `failed to lock log file '': ...` with no fallback and no silent recovery; no new recovery or fallback code is introduced; the enlarged (per-stripe rather than per-path) failure domain is stated in the plan assumptions and durable context. + - Validate: focused poisoned-stripe test (if practical, see T02); `rg -n "into_inner|clear_poison|PoisonError" cli/src/services/observability.rs` shows no new recovery logic; inspection of `context/sce/cli-observability-contract.md`. +- [x] AC5: Retention-after-creation, owner-only Unix permissions, and redaction-before-persistence behave as before under the new locking. + - Validate: `observability` tests covering retention, `0o600` permissions, and redaction pass; no existing test is removed or weakened (`git diff main` shows only additions to the test module). +- [x] AC6: Durable context states the ownership model, constant memory bound, same-process guarantee, absence of cross-process coordination, and stripe-collision contention trade-off. + - Validate: inspection of `context/sce/cli-observability-contract.md` (and `context/overview.md`/`context/glossary.md` only if they already describe per-path locking). +- [x] AC7: The concurrency tests are not flaky. + - Validate: the targeted logging concurrency tests pass on 20 consecutive runs. + +### Full validation + +- All existing observability tests remain unchanged and pass (`git diff main` shows only additions to the test module). +- The same-stripe fallback regression executes in a subprocess with a bounded timeout (AC4b); the targeted concurrency tests are repeated 20 times (AC7). +- Review the diff for new dynamic dispatch, heap allocations on the lock-selection path, `unsafe`, dependencies, or unnecessary abstractions: `nix shell nixpkgs#ripgrep -c rg -n "dyn |Box<|Arc<|unsafe|Vec<" ` over the changed lock-selection code, and `git diff main -- cli/Cargo.toml cli/Cargo.lock` empty. +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` +- `nix flake check` +- `nix build .#ci-checks` +- `git diff --check` + +### Context sync + +- `context/sce/cli-observability-contract.md` must replace "serializes writes independently per path" with the striped same-process model and the explicit no-cross-process statement. +- `context/context-map.md` annotation for that file is updated only if its summary text becomes inaccurate. + +## Task context synchronization lifecycle + +Persist this field in every plan; this is durable plan state, not chat state: + +- **Task context synchronization:** every task carries `pending | synced | blocked`. + A completed task must be `synced` before another task can start or the plan can + finish. +- For `blocked`, record **Blocker**, **Required action**, and **Retry condition** + beside the status. Never infer `synced` from conversation history; write every + lifecycle transition to the plan file. + +## Constraints and non-goals + +- **In scope:** `cli/src/services/observability.rs` (lock selection, the two append functions, inline tests) and the durable context named under Context sync. +- **Out of scope:** OpenTelemetry integration, other observability refactors, log formats/filenames/schema, retention policy, public configuration, logging queues, background workers, new runtimes, filesystem/cross-process locks. +- **Constraints:** std-only; static dispatch; keep generic `F: FnOnce(&Path) -> Result<()>` cleanup callbacks; no `dyn`, boxed callbacks, `unsafe`, new dependencies, or lint suppressions; the hash must be deterministic and independent of mutable global state (e.g. `std::hash::DefaultHasher::new()` with its fixed keys, not `RandomState`); `Cargo` runs go through `nix develop -c ./scripts/run-cli-cargo.sh`; tests stay a minimal focused set using the real implementation and no sleeps or timing assertions. +- **Non-goal:** guaranteeing cross-process serialization, per-instance lock ownership, or eviction caches. + +## Assumptions + +- 64 stripes (`LOG_LOCK_STRIPES`) is adequate; the exact count only affects collision contention, not correctness. +- Poisoned-lock behavior stays as today (error `failed to lock log file '': ...`, no fallback, no recovery). The failure domain changes from one path to every path on the poisoned stripe (~1/64 of paths per poisoned stripe). Poisoning requires a panic while the guard is held; the only caller-supplied code under the guard is the retention `cleanup` callback, so poisoning is not expected in normal logging. This is accepted, not mitigated, in this plan. +- Releasing the primary guard before the v2 fallback is acceptable: the fallback writes a different file, and the only previously-held-lock effect was ordering against other primary writers, which the fallback never touched. The fallback attempt is no longer atomic with the primary failure with respect to other writers; this is not observable beyond line ordering across two different files. +- Two stripes are never held at once by one call, so there is no lock-order inversion between stripes. +- The child-process timeout is bounded (a few seconds, generous for CI) and is a failure bound only, never a correctness assertion on timing. +- The existing contract doc, not a new ADR, is the right durable home for the model; no decision record is required. + +## Task stack + +- [x] T01: `Replace the path registry with fixed lock striping and release the guard before fallback` (status:done) + - Completed: 2026-10-08 + - Files changed: `cli/src/services/observability.rs` + - Result: Replaced `file_log_lock` registry with `LOG_LOCK_STRIPES = 64` / `static LOG_LOCKS` and `log_lock_stripe_index`/`log_lock_stripe` (deterministic `DefaultHasher::new()`); `append_log_line_with_cleanup` scopes the primary guard in an inner block that returns on success and yields the persist error, so `attempt_v2_log_fallback` runs only after the guard is dropped and never after lock-acquisition failure; `append_log_line_once_with_cleanup` uses the stripe directly. Poisoned-lock error text unchanged; only the retention `cleanup` callback runs caller code under the guard. + - Verify: `observability` tests passed (8/8); `rg "BTreeMap|OnceLock|Arc; LOG_LOCK_STRIPES] = [const { Mutex::new(()) }; LOG_LOCK_STRIPES]`; a small `log_lock_stripe(path) -> &'static Mutex<()>` using a deterministic std hash modulo the stripe count; remove `file_log_lock`, the `BTreeMap`/`Arc`/`OnceLock` imports, and the registry; update `append_log_line_with_cleanup` so the primary guard is dropped before the v2 fallback is attempted on both success and failure paths (scope the guard in an inner block/helper covering lock + persist + retention, returning the persist result; call `attempt_v2_log_fallback` only after that scope ends, and never when lock acquisition itself failed) and `append_log_line_once_with_cleanup` takes the stripe directly; keep error strings, retention-inside-guard, and lock-failure-skips-fallback behavior; keep existing poisoned-lock error handling unchanged (no recovery, no new fallback) and confirm by inspection that only the cleanup callback can panic under the guard; add a doc comment stating same-process-only synchronization, no cross-process guarantee, that stripe collisions only add contention, and that a poisoned stripe now affects all paths on that stripe. Out — tests beyond keeping existing ones green, context docs, any other observability change. + - Dependencies: none + - Done when: the registry is gone, both append functions use the static stripes, the fallback cannot nest a lock on the same stripe (AC4a ordering holds), poisoned-lock handling is unchanged, no `dyn`/`Box`/`unsafe`/new dependency is introduced, and all existing `observability` tests pass unchanged. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml observability`; `nix shell nixpkgs#ripgrep -c rg -n "BTreeMap|OnceLock|Arc': ...`, no v2 file, no recovery), retention-callback-runs-once under concurrent creation, `0o600` on Unix, redaction before persistence. No existing test changed. Also fixed a pedantic-clippy `cast_possible_truncation` in T01's `log_lock_stripe_index` (`usize::try_from(...).unwrap_or_default()`, same behavior on 64-bit). + - Verify: `observability` tests passed (20/20); 20 consecutive runs all passed; mutation holding the primary guard across the fallback made the same-stripe child test fail by 30s bounded timeout with the child killed and reaped (no orphan), then reverted; `clippy --all-targets --all-features -D warnings` passed; `git diff --check` clean. + - Context impact: none beyond T03 (tests only; the small stripe-index cast fix does not change documented behavior). + - Deviation: poisoned-stripe case runs in an isolated child so the shared static cannot be poisoned for other tests; the T01 cast fix was required for the clippy gate and is behavior-neutral. + - Task ID: T02 + - Scope: In — inline tests in `observability.rs` that call the real `append_log_line_with_cleanup`/`Logger` paths: (a) same-path multi-thread writes with exact complete-line count and per-line integrity; (b) multiple independent `Logger` clones/instances writing one path; (c) many (thousands of) distinct paths asserting stripe selection is stable, always in range, and the stripe array length is the constant; (d) fallback coverage: (d1) in-process, primary failure forcing the v2 fallback (e.g. a directory occupying the primary file name) with paths on different stripes, plus the combined error text when both fail; (d2) **same-stripe fallback regression in an isolated child process** (see below); (d2a) a focused poisoned-stripe test, if practical: poison the stripe of a temp path by panicking inside the `cleanup` callback of a write that creates the file (caught with `std::panic::catch_unwind` or a joined thread), then assert the next write to that path returns `failed to lock log file '': ...`, no `-v2` file is created, and nothing recovers the lock; use a path/stripe unique to the test so other tests are unaffected, and skip with a recorded rationale in the plan only if it cannot be isolated from the shared static; (e) retention under concurrent creation and `0o600` permissions on Unix (`#[cfg(unix)]`); (f) a secret in a logged record is redacted in the persisted file. Out — production behavior changes, correctness assertions on elapsed time, test-only copies of the locking logic, new dependencies, detached-thread-plus-`recv_timeout` harnesses, background workers. + - Same-stripe child-process test (d2) requirements: + 1. Build a primary path in a temp dir whose `v2_log_path(primary)` maps to the same stripe, using the production `log_lock_stripe` (compare by index or `std::ptr::eq` on the returned stripe); search deterministically over candidate file names (bounded loop) until one collides, and assert the collision before running. + 2. Force primary persistence to fail (e.g. a directory occupying the primary file name) so the fallback is triggered. + 3. Run the scenario in a child: parent re-executes `std::env::current_exe()` with `--exact --nocapture` and an env marker, mirroring `cli/src/app.rs::app_output_boundary_uses_isolated_process`; without the marker the child test is a no-op that passes. The child performs the real `append_log_line_with_cleanup` call, then prints a fixed success sentinel. + 4. Parent uses `Command::spawn` + a bounded `try_wait` polling loop against a finite deadline (std only; sleeps are only poll intervals, never assertions). + 5. On deadline expiry the parent calls `kill()` then `wait()` so the child is reaped, then fails the test with captured output. + 6. Timeout, non-success exit status or signal termination, or missing/incorrect sentinel output fails the test. + 7. On success the parent asserts the `-v2` fallback file exists with the exact expected line content and that the primary path is still the blocking directory. + 8. Child state (temp dir, env such as `HOME`/`XDG_*`) is isolated like the existing pattern; stdout/stderr are captured via pipes and drained so a full pipe cannot itself cause a hang. + - Dependencies: T01 + - Done when: each listed case has a deterministic test against the real implementation, all pass, the same-stripe child test fails by bounded timeout (not by hanging) if the guard were held across the fallback, no orphan child remains, and no existing test is removed or weakened. + - Verify: `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml observability`; repeat the targeted concurrency and fallback tests 20 times in a loop to detect flakiness; temporarily mutate T01 to hold the primary guard across the fallback and confirm the child test fails by timeout, then revert. + - Context synchronization: synced + +- [x] T03: `Document the striped log-lock ownership model` (status:done) + - Completed: 2026-10-08 + - Files changed: `context/sce/cli-observability-contract.md` + - Result: Replaced "serializes writes independently per path" with a pointer to a new lock-model bullet (64 static `Mutex<()>` stripes, deterministic hash, constant memory, same-process serialization across threads and `Logger` instances, collision contention only, one stripe held at a time with the primary guard released before the v2 fallback, enlarged poisoning domain with unchanged error text/no fallback/no recovery, no cross-process coordination); v2 sentence now says "striped serialization". `context/context-map.md` annotation does not mention per-path locking, so left unchanged. + - Verify: inspected changed lines; `git diff --check` clean. + - Context impact: none beyond this task (this task is the context update). + - Task ID: T03 + - Scope: In — update `context/sce/cli-observability-contract.md` (and its `context/context-map.md` annotation only if inaccurate) to state: static 64-stripe `Mutex<()>` ownership, constant memory independent of path count, same-process serialization of identical paths across threads and independent `Logger` instances, stripe-collision contention trade-off, guard released before the v2 fallback (lock-ordering contract), the enlarged poisoning failure domain (a poisoned stripe fails every path on it, unchanged error text, no recovery), and the explicit absence of cross-process coordination. Out — broad documentation refactoring, other context files. + - Dependencies: T01 + - Done when: the contract no longer says writes are serialized "independently per path" and contains the accurate model, the bound, the guarantee, the non-guarantee, and the trade-off, in current-state wording. + - Verify: inspect the changed lines in `context/sce/cli-observability-contract.md`; `git diff --check`. + - Context synchronization: synced + +## Open questions + +None. The request fully specifies scope, design, and constraints; the one design hazard (same-stripe fallback deadlock) is handled inside T01 and proven by the bounded child-process test in T02. + +## Validation Report + +**Status:** validated +**Date:** 2026-10-08 + +### Commands run + +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml observability` -> exit 0 (20/20 passed, incl. stripe selection, many-paths, same-stripe child, poisoned-stripe child, retention, 0o600, redaction) +- `observability` test run repeated 20 consecutive times -> 0 failures +- `nix develop -c ./scripts/run-cli-cargo.sh test --manifest-path cli/Cargo.toml` -> exit 0 (368 passed, 0 failed) +- `nix develop -c ./scripts/run-cli-cargo.sh clippy --manifest-path cli/Cargo.toml --all-targets --all-features -- -D warnings` -> exit 0 +- `nix flake check` -> exit 0 (all checks passed) +- `nix build .#ci-checks` -> exit 0 +- `git diff --check` -> exit 0 +- `rg -n "BTreeMap|OnceLock|Arc no hits +- `git diff d90415e0 -- cli/Cargo.toml cli/Cargo.lock` -> empty (plan commits only touch observability.rs and context) + +### Success-criteria verification + +- [x] AC1 -> registry removed; `static LOG_LOCKS: [Mutex<()>; 64]`; rg shows no BTreeMap/OnceLock/Arc/dyn/unsafe/Box hits; Cargo files unchanged by plan commits +- [x] AC2 -> concurrent same-path and independent-Logger tests pass +- [x] AC3 -> stripe-selection test (5000 paths, constant array length, stable index) passes +- [x] AC4 -> same-stripe bounded child test, different-stripe fallback, and combined-error tests pass +- [x] AC4a -> inspection of `append_log_line_with_cleanup` (observability.rs:343-363): guard scoped in an inner block, dropped before `attempt_v2_log_fallback`; lock failure returns via `?` without fallback; combined error text unchanged +- [x] AC4b -> child test with finite deadline and kill+wait passes; T02 recorded the mutation check (guard held across fallback fails by bounded timeout, no orphan); not re-run in this validation +- [x] AC4c -> poisoned-stripe child test passes; no into_inner/clear_poison/PoisonError in file; contract doc states the enlarged failure domain +- [x] AC5 -> retention, 0o600, and redaction tests pass; diff removals contain no test code +- [x] AC6 -> `context/sce/cli-observability-contract.md` lines 36-38 state ownership, constant bound, same-process guarantee, no cross-process coordination, collision contention +- [x] AC7 -> 20 consecutive runs, 0 failures + +### Failed checks and follow-ups + +- None. + +### Residual risks + +- A poisoned stripe fails all paths hashing to it (accepted and documented). +- The guard-held-across-fallback mutation check relies on T02's recorded evidence rather than a fresh run. diff --git a/context/sce/agent-trace-db.md b/context/sce/agent-trace-db.md index 4811232f8..c2d431dc2 100644 --- a/context/sce/agent-trace-db.md +++ b/context/sce/agent-trace-db.md @@ -23,12 +23,10 @@ - `MessageRole` enum: `User` / `Assistant` — maps to `messages.role` DB constraint. - `InsertMessageInsert`: owned payload struct with insertable parent `messages` columns (`session_id`, `message_id`, `role`, `generated_at_unix_ms`); message body text belongs to `parts.text`, not the parent message row. - `INSERT_MESSAGE_SQL`: parameterized single-row SQL using `INSERT ... ON CONFLICT (session_id, message_id) DO NOTHING` — leverages the unique index `idx_messages_session_message` so duplicate parent-message events remain non-failing without mutating the existing row. -- `insert_message(input)`: typed single-row helper that executes the duplicate-ignore parent-message insert; retained as part of the adapter surface. - `insert_messages(inputs)`: typed batch helper that generates and executes one parameterized multi-row `messages` insert for valid conversation-trace `message` batches while preserving duplicate-ignore semantics. - `PartType` enum: `Text` / `Reasoning` / `Patch` / `Question` — serializes known conversation part kinds for typed inserts. `parts.type` is stored as `TEXT NOT NULL` without a database-level enum `CHECK` constraint. - `InsertPartInsert`: owned payload struct with `part_type`, `text`, `session_id`, `message_id`, and `generated_at_unix_ms`. - `INSERT_PART_SQL`: parameterized single-row append-only INSERT into `parts` (no upsert; multiple rows per `(session_id, message_id)` allowed). -- `insert_part(input)`: typed single-row helper that inserts a part row without requiring a matching `messages` row (supports out-of-order writes); retained as part of the adapter surface. - `insert_parts(inputs)`: typed batch helper that generates and executes one parameterized multi-row append-only `parts` insert for valid conversation-trace `message.part` batches. - `insert_conversation_text_event(message, part)`: atomic one-message-plus-one-part write for exactly-once conversation text events (currently used only by `sce hooks codex`'s `UserPromptSubmit`/`Stop` arms; see [codex-integration-runtime.md](codex-integration-runtime.md)). Delegates to `TursoDb::execute_transactional_insert_pair_if_absent` (`cli/src/services/db/mod.rs`), a generic, schema-agnostic primitive: inside one `BEGIN IMMEDIATE` transaction it checks whether `(session_id, message_id)` already exists and, only if absent, inserts the message row then the part row and commits; an existing row causes no insert statements to run, and the no-write transaction commits, returning `Ok(false)`. `BEGIN IMMEDIATE` serializes concurrent callers against the same database file, so a replayed or concurrent duplicate delivery leaves exactly one message row and one part row — stronger than `insert_messages`/`insert_parts`' own guarantee, which dedups only the parent message row via `ON CONFLICT DO NOTHING` and leaves `parts` unguarded. `insert_messages`/`insert_parts` remain unchanged and are still what OpenCode/Claude/Pi conversation-trace intake uses. - `lifecycle.rs`: service lifecycle provider for setup/doctor integration. @@ -41,9 +39,11 @@ pub type RepositoryAgentTraceDb = TursoDb; ``` +Repository persistence, readiness checks, transactions, and migration operations are native async methods inherited from `TursoDb`. Hook, lifecycle, mutation-trace, and sync callers await them on the application runtime before returning or advancing their cursor. The async migration changes no SQL, schema, WAL, encryption, or retry policy. + This adapter has no canonical `DbSpec::db_path()`; callers must resolve `/sce/repos//agent-trace.db` first and use explicit-path `TursoDb` constructors. Its migration list is `generated_migrations::AGENT_TRACE_REPOSITORY_MIGRATIONS`: the fresh multi-statement baseline `cli/migrations/agent-trace-repository/001_repository_schema.sql` plus the additive `002_repository_source_instance_id.sql` (adds `repository_metadata.source_instance_id`), `003_claude_model_state.sql` (adds the non-exported Claude model-state register), `004_mutation_trace_protocol.sql`, and `005_mutation_scope_provenance.sql` (the mutation-cursor protocol tables and the scope-provenance table; see [mutation-trace store](../cli/mutation-trace-store.md) and [mutation-scope provenance](../cli/mutation-scope-provenance.md)). The baseline schema includes `repository_metadata` plus the existing repository-level Agent Trace tables, indexes, and triggers, and intentionally has no `checkout_id` columns on trace tables. `RepositoryAgentTraceDb::verify_or_initialize_repository_metadata(repository_id) -> Result` inserts the singleton metadata row on first initialization, errors if an existing DB stores a different repository ID, and atomically claims `source_instance_id` for this physical database via `UPDATE ... WHERE source_instance_id = ''` (a losing racer's generated candidate is discarded and an already-valid stored value is never overwritten), returning the typed `RepositoryMetadata { repository_id, source_instance_id }`. `source_instance_id` is generated by application code (`generate_source_instance_id()`, UUID v4) and validated with `is_valid_source_instance_id()` (non-empty once trimmed); it is never derived from `repository_id`, remote URL, checkout ID, filesystem path, hostname, or user/workspace identity, and stays stable across reopen and repeated `sce setup` runs. `RepositoryAgentTraceDb::repair_missing_repository_schema_migration_metadata()` is a narrow concurrent-first-open repair seam: it never creates trace tables, but if every required repository schema table already exists and only the one-file baseline migration record is missing, it records `001_repository_schema` and rechecks readiness. -`RepositoryAgentTraceDb` exposes repository-level write helpers for the current row families by delegating to the same typed insert payloads and parameterized SQL used by the checkout-scoped adapter: `insert_diff_trace`, `insert_post_commit_patch_intersection`, `insert_agent_trace`, `insert_message`, `insert_messages`, `insert_part`, `insert_parts`, and `insert_conversation_text_event`. It also exposes `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` by delegating to the shared recent diff-trace query/parser helper, so repository-scoped attribution reads use the same chronological inclusive window semantics without a checkout filter. These methods preserve the existing row shapes and do not add checkout provenance columns or checkout-scoped write/query APIs. +`RepositoryAgentTraceDb` exposes repository-level write helpers for the current row families by delegating to the same typed insert payloads and parameterized SQL used by the checkout-scoped adapter: `insert_diff_trace`, `insert_post_commit_patch_intersection`, `insert_agent_trace`, `insert_messages`, `insert_parts`, and `insert_conversation_text_event`. It also exposes `recent_diff_trace_patches(cutoff_time_ms, end_time_ms)` by delegating to the shared recent diff-trace query/parser helper, so repository-scoped attribution reads use the same chronological inclusive window semantics without a checkout filter. These methods preserve the existing row shapes and do not add checkout provenance columns or checkout-scoped write/query APIs. The repository-scoped adapter is consumed by `agent_trace_storage`, active hook runtime opening, Agent Trace setup/doctor lifecycle, and `sce sync`. Hook writers/readers resolve the current repository storage context before using `RepositoryAgentTraceDb`. The migration-running `new_at(path)` constructor is used by setup/lifecycle; hook runtime uses the no-migration constructor and fails open with `Run 'sce setup'.` guidance when schema readiness is not met. There is no longer a checkout-scoped adapter or trace database inspection service. @@ -55,7 +55,7 @@ to `diff_traces` or the direct intersection table. ## Non-goals -- No read/query helper for loading messages with their joined parts exists in the current runtime; the typed write helpers (`insert_message`, `insert_messages`, `insert_part`, `insert_parts`, `insert_conversation_text_event`) are the only exposed message/part API surface. Message/part query helpers are deferred to a future task. +- No read/query helper for loading messages with their joined parts exists in the current runtime; the typed write helpers (`insert_messages`, `insert_parts`, `insert_conversation_text_event`) are the only exposed message/part API surface. Message/part query helpers are deferred to a future task. - No part upsert/deduplication; `parts` uses only the internal integer `id` for row identity (append-only per the `INSERT_PART_SQL` contract). ## Database path @@ -88,6 +88,7 @@ SCE creates one Agent Trace DB per logical Git repository on demand through setu - `001_repository_schema.sql` (migration ID `001_repository_schema`) — creates `repository_metadata`, `diff_traces` (including `payload_type TEXT NOT NULL DEFAULT 'patch'`), `post_commit_patch_intersections`, `agent_traces`, `messages`, and `parts`, plus the lookup indexes and `updated_at` triggers, in one `execute_batch` statement recorded as a single migration ID. - `002_repository_source_instance_id.sql` (migration ID `002_repository_source_instance_id`) — additive `ALTER TABLE repository_metadata ADD COLUMN source_instance_id TEXT NOT NULL DEFAULT ''`; existing/placeholder rows default to an empty string and application code (never SQL) replaces it through the atomic claim in `verify_or_initialize_repository_metadata`. This migration does not rewrite `001_repository_schema.sql`. See [context/decisions/2026-08-10-agent-trace-source-instance-id.md](../decisions/2026-08-10-agent-trace-source-instance-id.md) for why `source_instance_id` exists as a second identity independent of `repository_id`. - `003_claude_model_state.sql` (migration ID `003_claude_model_state`) — additive `claude_model_state` table keyed by `(session_id, agent_id)`, storing `model_id`, constrained `observation_kind`, `source`, and non-negative `observed_at_ms`. The repository adapter's guarded upsert treats it as a best-effort latest-locally-observed register: older observations do not replace newer ones, equal-time `PostModelSwitch` wins over `SessionStart`, and same-kind ties use deterministic model/source ordering. This table is local attribution state outside the four Agent Trace export streams and has no sync cursor. Repositories created before this migration must rerun `sce setup` so the setup lifecycle applies migration `003_claude_model_state`; hook paths do not migrate. +- `004_mutation_trace_protocol`, `005_mutation_scope_provenance` (insert-once `mutation_trace_scope_provenance`) and `006_mutation_trace_health_invariant` (rebuilds the worktree/event tables with the `tainted`/`failure_kind` `CHECK`) — mutation-trace tables; see [mutation-trace-store.md](../cli/mutation-trace-store.md) and [mutation-scope-provenance.md](../cli/mutation-scope-provenance.md). The former checkout-scoped `AGENT_TRACE_MIGRATIONS` constant and its 15-file `cli/migrations/agent-trace/` chain (`001_create_diff_traces` … `015_add_diff_traces_payload_type`) were removed by the `retire-legacy-agent-trace-db` plan; `build.rs` auto-discovers migration directories, so deleting the directory dropped the constant on regeneration. The repository schema captures the same tables/columns/indexes/triggers that the old incremental chain produced. @@ -202,7 +203,7 @@ Both triggers compare `OLD.*` vs `NEW.*` for all mutable columns (excluding `upd - AgentTraceDb open failures are logged at error level through `sce.hooks.diff_trace.agent_trace_db_open_failed`; later conversion/insert failures retain `sce.hooks.diff_trace.agent_trace_db_write_failed`. Both failure classes preserve deterministic failed-persistence success text and create no artifact fallback. Open failures use the producer-native unprefixed session and do not also emit the write-failure event. - Existing artifact files are not backfilled into the database. -Post-commit intersection rows are written by the active `post-commit` hook flow through repository-scoped Agent Trace DB access, and the same flow inserts built Agent Trace payloads into `agent_traces` via `RepositoryAgentTraceDb::insert_agent_trace()` (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). The persisted `trace_json` is the schema-validated `build_agent_trace(...)` output and includes top-level `metadata.sce.version` from the compiled `sce` CLI package version, always-emitted `metadata.sce.line_changes` touched-line attribution counts, plus `content_hash` on every emitted range. Range `content_hash` values are computed from the touched-line kind/content of the post-commit hunk that produced the persisted range, not from DB IDs, paths, line positions, or runtime metadata. +Post-commit intersection rows are written by the active `post-commit` hook flow through repository-scoped Agent Trace DB access, and the same flow inserts built Agent Trace payloads into `agent_traces` via `RepositoryAgentTraceDb::insert_agent_trace()` (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). The persisted `trace_json` is the schema-validated `build_agent_trace_from_evidence(...)` output and includes top-level `metadata.sce.version` from the compiled `sce` CLI package version, always-emitted `metadata.sce.line_changes` touched-line attribution counts, plus `content_hash` on every emitted range. Range `content_hash` values are computed from the touched-line kind/content of the post-commit hunk that produced the persisted range, not from DB IDs, paths, line positions, or runtime metadata. `sce hooks conversation-trace` is the current runtime writer for `messages` and `parts`. diff --git a/context/sce/agent-trace-embedded-schema-validation.md b/context/sce/agent-trace-embedded-schema-validation.md index 81cbfa6d8..516d993e8 100644 --- a/context/sce/agent-trace-embedded-schema-validation.md +++ b/context/sce/agent-trace-embedded-schema-validation.md @@ -13,7 +13,6 @@ Current internal validation seam for Agent Trace JSON in the Rust CLI. - The canonical schema remains at `config/schema/agent-trace.schema.json`. For repository builds, `cli/build.rs` copies it to `OUT_DIR/static/schema/agent-trace.schema.json`, and the CLI embeds that staged file at compile time via `include_str!`. Validation does not read the schema from disk at runtime; channel-specific package fallback preparation remains a distribution concern. - `agent_trace_schema_validator()` compiles the embedded schema once and caches the `jsonschema::Validator` in a `OnceLock`. - `validate_agent_trace_value(&serde_json::Value)` validates already-parsed JSON values against the embedded schema. -- `validate_agent_trace_json(&str)` parses a JSON string and then validates it against the embedded schema. - Top-level `version` must match strict numeric `x.y.z` (`^[0-9]+\.[0-9]+\.[0-9]+$`); two-part values like `x.y` are rejected. - Top-level `vcs` is optional at schema level; payloads without `vcs` validate, and payloads that include `vcs` must still provide both `type` and `revision`. - Validation failures use `AgentTraceValidationError` to distinguish: @@ -24,7 +23,7 @@ Current internal validation seam for Agent Trace JSON in the Rust CLI. ## Boundaries - This seam is internal-only; no user-facing `sce` command currently exposes Agent Trace validation. -- The existing minimal `build_agent_trace(...)` output remains unchanged by this validation seam. +- The `build_agent_trace_from_evidence(...)` output remains unchanged by this validation seam. - File-path loading and validation entrypoints are not part of the current implemented slice. ## Verification diff --git a/context/sce/agent-trace-export-readers.md b/context/sce/agent-trace-export-readers.md index 0d71c541d..72fb87c96 100644 --- a/context/sce/agent-trace-export-readers.md +++ b/context/sce/agent-trace-export-readers.md @@ -19,9 +19,9 @@ flowchart LR `AgentTraceExportReader::new(&db)` takes a `&RepositoryAgentTraceDb` directly. The reader does not resolve storage, open a database, or generate identity itself. The existing storage resolver composes cleanly with it: ```rust -let storage = resolve_agent_trace_storage_at_state_root(&context, &state_root)?; +let storage = resolve_agent_trace_storage(&context).await?; let reader = AgentTraceExportReader::new(&storage.db); -let rows = reader.read_messages_after(cursor, limit)?; +let rows = reader.read_messages_after(cursor, limit).await?; ``` `storage.metadata` (`RepositoryMetadata { repository_id, source_instance_id }`, see [agent-trace-db.md](agent-trace-db.md#repository-scoped-adapter-seam)) identifies *which* physical database produced the rows; the reader never reads, generates, or accepts a source identity of its own. Identity and progress tracking are deliberately separate concerns: @@ -29,11 +29,9 @@ let rows = reader.read_messages_after(cursor, limit)?; - `repository_id` / `source_instance_id` identify the *source* (the physical database). - Each stream's `table.id` (returned as `sourceRowId`) is the per-stream *progress marker* — the caller's cursor. It has no relationship to `source_instance_id` and is never used to derive or validate it. -Test coverage: `cli/src/services/agent_trace_export/mod.rs::tests::source_instance_integration` resolves storage through `resolve_agent_trace_storage_at_state_root`, asserts both metadata fields are populated, and reads one stream through a reader built from `storage.db`. - ## Reader contract -Four methods, one per capture stream, sharing one shape: `(cursor: i64, limit: usize) -> Result>`, running +Four async methods, one per capture stream, sharing one shape: `(cursor: i64, limit: usize) -> Result>`, awaiting the DB query and row materialization, running ```sql SELECT ... FROM WHERE id > ?1 ORDER BY id ASC LIMIT ?2 diff --git a/context/sce/agent-trace-hook-doctor.md b/context/sce/agent-trace-hook-doctor.md index 26fa399b5..ad2be4b91 100644 --- a/context/sce/agent-trace-hook-doctor.md +++ b/context/sce/agent-trace-hook-doctor.md @@ -132,6 +132,10 @@ The broadened contract for `sce doctor` must cover the following problem invento - Agent Trace DB file exists but cannot be opened (connection failure) or has incomplete schema (missing/unapplied migrations) — reported as `AgentTraceDbConnectionFailed` / `AgentTraceDbSchemaNotReady` with manual-only remediation directing to `sce setup` - Repository-scoped DB path/health remains in the complete report and JSON; human text summarizes it under `Environment` → `Repository identity` without exposing healthy identity metadata. Repository DB records include repository ID, identity source, safe canonical identity, configured remote name, and never raw remote URLs. Checkout identity was removed by the `remove-checkout-id` plan and is no longer reported (see `context/cli/checkout-identity.md`) +### Snapshot-ref reconciliation + +- `sce doctor --fix` always runs one explicit snapshot-ref reconciliation pass and reports its true typed result; plain `sce doctor` stays read-only (no ref or maintenance-state write) and reports a recommendation only when one is due. Details in [../cli/doctor-ref-reconciliation.md](../cli/doctor-ref-reconciliation.md). Reconciliation is never automatic. + ### Repository targeting and git readiness - repo-scoped checks are required but `sce doctor` is run outside a git repository diff --git a/context/sce/agent-trace-minimal-generator.md b/context/sce/agent-trace-minimal-generator.md index 8f6631781..5c5928010 100644 --- a/context/sce/agent-trace-minimal-generator.md +++ b/context/sce/agent-trace-minimal-generator.md @@ -11,21 +11,20 @@ Given a `constructed_patch` (AI candidate) and a `post_commit_patch` (canonical - **`ai`** — every touched line in the `post_commit_patch` hunk is covered by direct evidence, mutation-derived AI evidence, or both. - **`mixed`** — a non-empty proper subset of the hunk's touched lines is covered. - **`unknown`** — no touched line in the hunk is covered. -3. With no mutation evidence this is equivalent to the direct-only slot rule, since a direct-intersection hunk always holds an ordered sub-multiset of its `post_commit_patch` hunk's touched lines: `ai` when the `intersection_patch` hunk has identical touched lines (same count, kind, `line_number`, content, order), `mixed` when that hunk exists but is a proper subset, `unknown` when no `intersection_patch` hunk shares the `old_start`. The direct-only path `build_agent_trace(...)` produces exactly this classification. +3. With no mutation evidence this is equivalent to the direct-only slot rule, since a direct-intersection hunk always holds an ordered sub-multiset of its `post_commit_patch` hunk's touched lines: `ai` when the `intersection_patch` hunk has identical touched lines (same count, kind, `line_number`, content, order), `mixed` when that hunk exists but is a proper subset, `unknown` when no `intersection_patch` hunk shares the `old_start`. `build_agent_trace_from_evidence(...)` with an empty `mutation_ai_patch` produces exactly this classification. 4. Map `Conversation.contributor.model_id` from the matched direct and mutation hunks when contributor type is `ai` or `mixed`: an absent evidence source does not count as unknown, one present model passes through, and both present models must be known and equal. Omit `model_id` when both sources are present but either model is missing or they disagree. A hunk classified `ai`/`mixed` purely through mutation-derived coverage may therefore emit the mutation model. 5. For each emitted conversation, derive optional `conversation.related` entries from non-empty `session_id` values on touched lines in both matched evidence hunks; emit related entries as `{ "type": "session", "url": "https://sce.crocoder.dev/sessions/" }`, deduplicated by session ID with deterministic ordering, and omit `related` when no included lines provide `session_id`. Structured diff-trace reconstruction supplies the persisted canonical `cc_...` row session on direct lines, while mutation reconstruction supplies scope provenance on mutation-attributed lines. 6. Emit one `Conversation` per `post_commit_patch` hunk, each carrying the trace lookup `url`, one `TraceFile` per `post_commit_patch` file, and one range per hunk with a deterministic `content_hash` computed from that hunk's touched-line kind/content. ## Separated direct/mutation evidence -`build_agent_trace_from_evidence(evidence, post_commit_patch, metadata)` is the seam that classifies from two independent AI-evidence sources: +`build_agent_trace_from_evidence(evidence, post_commit_patch, metadata)` is the only supported generator entrypoint. It handles direct-only evidence (empty `mutation_ai_patch`), mutation-only evidence, and combined evidence through one path, classifying from two independent AI-evidence sources: - `evidence.direct_patch` — the reconstructed direct patch, intersected against `post_commit_patch` internally exactly as before. It supplies direct model/session provenance and remains the sole source of the top-level `tool` object (still omitted when the direct intersection is empty). - `evidence.mutation_ai_patch` — a target-shaped set of committed touched lines with resolved scope provenance that causal mutation-lineage replay attributed to AI (an AI event's line that survived every later observed tree transition into the committed tree), produced by [../cli/mutation-trace-agent-attribution.md](../cli/mutation-trace-agent-attribution.md). It widens combined AI coverage for hunk classification and supplies mutation model/session metadata when available. Per hunk, a `post_commit_patch` touched line is covered when it pairs one-to-one on `(kind, line_number, content)` with a line in the direct intersection hunk or the mutation-AI hunk at the same `old_start`; the covered fraction drives `ai` / `mixed` / `unknown` as in the Contract above. `line_changes` buckets follow that combined classification. -`build_agent_trace(constructed_patch, post_commit_patch, metadata)` is retained unchanged and delegates to `build_agent_trace_from_evidence` with an empty `mutation_ai_patch`. Wiring the mutation-AI patch in at post-commit is owned by [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md). ## Domain types @@ -58,7 +57,7 @@ the boundary that mutation protocol attribution is not itself scope provenance. Current output includes top-level metadata fields with this contract: - `version` is fixed to `"0.1.0"` and remains the Agent Trace payload/schema version -- `id` is generated per `build_agent_trace(...)` call as a UUIDv7 string derived from the same commit-time moment used for `timestamp` +- `id` is generated per `build_agent_trace_from_evidence(...)` call as a UUIDv7 string derived from the same commit-time moment used for `timestamp` - `timestamp` is sourced from explicit commit metadata input (`AgentTraceMetadataInput.commit_timestamp`) and must be RFC 3339 - `vcs` is emitted only when explicit commit metadata input includes `AgentTraceMetadataInput.vcs_type` - when `vcs` is emitted, `vcs.type` is sourced from the schema-aligned enum (`git | jj | hg | svn`) and `vcs.revision` is sourced from `AgentTraceMetadataInput.commit_revision` @@ -109,15 +108,13 @@ Current output includes top-level metadata fields with this contract: ## Public API -- `classify_hunk(post_commit_hunk, intersection_hunks) -> HunkContributor` — the direct-only slot rule, retained as a primitive; the builder itself now classifies through the internal combined direct+mutation line-coverage rule. - `range_content_hash(hunk) -> String` — internal helper that computes the serialized range-level `murmur3:` content fingerprint from `PatchHunk.lines` using versioned, length-delimited touched-line serialization in patch order. The hash input includes touched-line kind and content, and excludes hunk positions, line numbers, file paths, trace metadata, contributor/model metadata, VCS metadata, tool metadata, and database IDs. -- `build_agent_trace(constructed_patch, post_commit_patch, metadata) -> Result` — direct-only entrypoint, retained unchanged; delegates to `build_agent_trace_from_evidence` with an empty `mutation_ai_patch`. It validates `metadata.commit_timestamp` as RFC 3339, uses it as top-level `timestamp`, derives a UUIDv7 `id` from that same commit-time moment, derives one conversation URL from that `id`, conditionally emits `vcs` only when `metadata.vcs_type` is present (mapping `vcs.type` from metadata and `vcs.revision` from `metadata.commit_revision`), carries optional tool metadata inputs (`metadata.tool_name`, `metadata.tool_version`) for top-level `tool` mapping, and always emits `metadata.sce.version` from the compiled package version. When the direct `intersection_patch.files` is empty, `tool` is always `None` regardless of metadata values. - `build_agent_trace_from_evidence(evidence: AgentTraceEvidence, post_commit_patch, metadata) -> Result` — separated-evidence entrypoint (see [Separated direct/mutation evidence](#separated-directmutation-evidence)): identical top-level metadata behavior, classifies each hunk from the union of direct and mutation-derived AI coverage, unions direct/mutation related sessions, and emits a model only when the present evidence sources agree under the combined rule; top-level `tool` remains bound to the direct intersection. ## Test fixture contract - Golden fixtures under `cli/src/services/agent_trace/fixtures/**/golden.json` pin deterministic literal values for top-level `id`, `timestamp`, optional `vcs`, `metadata.sce.version`, `metadata.sce.line_changes`, per-conversation `url`, range-level `content_hash`, and expected file/conversation shapes. -- Reconstruction fixtures pair `incremental_*.patch` inputs with a `post_commit.patch` and drive `build_agent_trace`. Evidence fixtures (`direct_only`, `exclusive_without_direct`, `direct_plus_mutation`, `partial_combined`, `newer_nonexclusive_blocks`, `mutation_only_no_provenance`) instead pair `direct.patch` + `mutation_ai.patch` + `post_commit.patch` and drive `build_agent_trace_from_evidence`, pinning mutation-only coverage with and without resolved provenance, the combined model-agreement and session-union rules, the absence of fabricated metadata when provenance is unknown, and the empty-mutation path's byte identity with `build_agent_trace`. +- Evidence fixtures (`direct_only`, `exclusive_without_direct`, `direct_plus_mutation`, `partial_combined`, `newer_nonexclusive_blocks`, `mutation_only_no_provenance`) pair `direct.patch` + `mutation_ai.patch` + `post_commit.patch` and drive `build_agent_trace_from_evidence`, pinning mutation-only coverage with and without resolved provenance, the combined model-agreement and session-union rules, the absence of fabricated metadata when provenance is unknown, and the direct-only case (empty `mutation_ai_patch`), whose classification and provenance match the direct-intersection rule. - Tests validate golden fixtures and built payloads against the embedded schema, assert core runtime metadata directly (`version`, `timestamp`, optional `vcs`, and `metadata.sce.version`), and compare `vcs`, optional `tool`, `metadata.sce.line_changes`, and normalized `files` against fixture truth. Expected fixture URLs are normalized to the runtime `AgentTrace.id` before the existing file-shape comparison because UUIDv7 generation includes non-deterministic bits. ## Relationship to existing patch service diff --git a/context/sce/auth-db.md b/context/sce/auth-db.md index 227c0d953..3e688d889 100644 --- a/context/sce/auth-db.md +++ b/context/sce/auth-db.md @@ -44,8 +44,8 @@ Current migration baseline: ## Token storage integration -- `cli/src/services/token_storage.rs` now uses `AuthDb` for all persistence operations (`save_tokens`, `load_tokens`, `delete_tokens`) via a `OnceLock>` lazy singleton. -- Token-storage operations are synchronous because they wrap the encrypted local DB and OS credential store. Async consumers such as the Agent Trace control-plane client must call them through a blocking-task boundary; the sync command's rule is documented in [agent-trace-sync-command.md](../cli/agent-trace-sync-command.md). +- `cli/src/services/token_storage.rs` uses `AuthDb` through async `save_tokens`, `load_tokens`, and `delete_tokens` operations and a `tokio::sync::OnceCell>`. Completed initialization success or failure is cached; cancellation before initialization completes leaves the cell uninitialized. +- Auth commands and the generic async control-plane credential store directly await token persistence on the application runtime. Database operations do not cross a `spawn_blocking` boundary; the OS keyring adapter retains its platform-specific synchronous behavior where required. - `token_file_path()` returns the auth DB path from `auth_db_path()` instead of a JSON file path. - `TokenStorageError` exposes `PathResolution` and `Database` variants; former `Io`, `Serialization`, `CorruptedTokenFile`, and `Permission` variants have been removed. - No JSON file I/O remains in `token_storage.rs`. diff --git a/context/sce/cli-observability-contract.md b/context/sce/cli-observability-contract.md index ddbcbbc82..12692b805 100644 --- a/context/sce/cli-observability-contract.md +++ b/context/sce/cli-observability-contract.md @@ -33,8 +33,9 @@ Runtime observability consumes the shared resolved observability config from `cl - Sessionless file logs route to `/sce-.log`; session-aware file logs route to `/sce--.log`. - Session filename sanitization preserves ASCII letters, digits, `-`, and `_`; percent-encodes every other UTF-8 byte as uppercase `%HH`; and represents an explicitly empty `Some("")` session ID with the reserved `%EMPTY` token. - `sce hooks diff-trace` and `conversation-trace` pass producer-native session context into this existing routing argument when available. Diff-trace logging never uses the AgentTraceDb-only `oc_`/`cc_`/`pi_` prefix; skipped conversation items use their own session; batch-wide conversation insert failures use the first valid insert's session. Agent Trace DB open failures use hook-specific error events (`sce.hooks.diff_trace.agent_trace_db_open_failed` and `sce.hooks.conversation_trace.agent_trace_db_open_failed`) and do not also emit their broader write/intake events for the same failure. Session IDs remain absent from rendered record fields unless separately supplied as fields. -- File routing creates the configured directory when needed, uses owner-only create permissions on Unix, and serializes writes independently per path. -- If file logging is enabled and the selected primary file cannot be opened, appended to, or flushed, the logger retries the complete rendered record exactly once at a sibling path with `-v2` inserted before `.log`: `sce--v2.log` or `sce---v2.log`. Existing v2 files use the same create-or-append and per-path serialization behavior. +- File routing creates the configured directory when needed, uses owner-only create permissions on Unix, and serializes writes through a fixed set of in-process lock stripes (see the next bullet). +- Log-file write synchronization is a static array of 64 `std::sync::Mutex<()>` stripes in `cli/src/services/observability.rs`; a path selects its stripe via a deterministic `DefaultHasher::new()` hash modulo the stripe count. Synchronization memory is constant and independent of the number of distinct log paths (no per-path allocation or eviction). Writes to the same path are serialized within one process across threads and independent `Logger` instances; distinct paths that hash to the same stripe only contend, they never fail. No cross-process coordination exists or is guaranteed. A call holds at most one stripe at a time: the primary path's guard covers persistence and retention cleanup and is released on success and failure before the v2 fallback acquires its own stripe, so primary and v2 paths sharing a stripe cannot deadlock. A poisoned stripe (a panic under the guard, realistically only inside the retention cleanup callback) fails every path on that stripe with the unchanged `failed to lock log file '': ...` error, does not trigger the fallback, and is never recovered. +- If file logging is enabled and the selected primary file cannot be opened, appended to, or flushed, the logger retries the complete rendered record exactly once at a sibling path with `-v2` inserted before `.log`: `sce--v2.log` or `sce---v2.log`. Existing v2 files use the same create-or-append and striped serialization behavior. - Successful v2 persistence suppresses the terminal `Failed to write SCE log file` diagnostic and leaves the CLI command result unchanged. If both persistence attempts fail, the logger emits one redacted terminal file-write diagnostic to stderr and continues fail-open; a partial primary append may therefore coexist with the complete fallback record. - Directory creation, primary lock acquisition, and retention cleanup failures do not trigger alternate-name generation. The fallback attempt is non-recursive: no v3, timestamped, random, or unbounded variants are tried. - After a successful write to a newly created primary or v2 SCE log file, the logger runs one best-effort retention pass over direct regular `*.log` children of `log_dir`. Existing-file appends do not scan or delete files. Cleanup keeps the resolved `log_file_retention_limit` newest files (default `10`). Files are ordered newest-first by filesystem modification time with path/name ordering as the deterministic tie-break, and older `.log` files are removed regardless of whether their names are SCE-owned. diff --git a/context/sce/cli-shared-output-format-contract.md b/context/sce/cli-shared-output-format-contract.md index aa7b5ec3a..a6d13dccb 100644 --- a/context/sce/cli-shared-output-format-contract.md +++ b/context/sce/cli-shared-output-format-contract.md @@ -6,7 +6,7 @@ - Canonical type lives at `cli/src/services/output_format.rs` as `OutputFormat`. - Allowed values are `text` and `json`. -- Parsing is command-context aware via `OutputFormat::parse(raw, help_command)` so invalid values include command-specific help guidance. +- `OutputFormat` derives clap `ValueEnum` for parsing; the former `OutputFormat::parse(raw, help_command)` helper was removed as unused. ## Current command integration diff --git a/context/sce/doctor-human-text-contract.md b/context/sce/doctor-human-text-contract.md index bf00c59d4..e4116634e 100644 --- a/context/sce/doctor-human-text-contract.md +++ b/context/sce/doctor-human-text-contract.md @@ -16,6 +16,12 @@ The `Environment` domain contains `State`, `Configuration`, and `Repository identity`. The `Repository` domain contains `Git repository`, `Post-commit Agent Trace auto-sync`, and `Git hooks`. +When snapshot-ref reconciliation is recommended (or its maintenance state is +unusable), plain `sce doctor` additionally renders an additive `Snapshot ref +reconciliation` section; it is omitted otherwise. `sce doctor --fix` appends one +`mutation_scope_health` fix-result row for its reconciliation pass. Details in +[../cli/doctor-ref-reconciliation.md](../cli/doctor-ref-reconciliation.md). + ## Header and status vocabulary - Diagnose mode renders `SCE doctor`. diff --git a/context/sce/flake-build-performance.md b/context/sce/flake-build-performance.md index 2e5b94ab9..7af52bf02 100644 --- a/context/sce/flake-build-performance.md +++ b/context/sce/flake-build-performance.md @@ -33,6 +33,31 @@ rebuilding dependency artifacts or formatting. The `cli-generated-input` flake check verifies the payload roots and both inventories independently of Cargo compilation. +## Rust test profile and focused Quint MBT + +Crane defaults to the release profile, including for `cargoTest` and +`cargoClippy`. To avoid repeatedly optimizing the test binary, the flake now +has two isolated sets of dependency artifacts: + +- `cargoArtifacts` remains release-profile, reused by native and release + package builds where applicable. Release optimization and embedding are + unchanged. +- `cargoArtifactsTest` uses `CARGO_PROFILE = "test"` and + `CARGO_PROFILE_TEST_DEBUG = "line-tables-only"`. The same artifacts are + passed to `cli-tests`, `cli-clippy`, and the optional focused Quint MBT + runner, preventing accidental test/release profile cache mixing. + +The existing `checks.cli-tests` already includes `mutation_trace::mbt`. +Therefore `nix flake check` runs this suite once rather than also building a +second focused MBT check. The focused runner is available independently at +`nix build .#mutation-trace-quint-connect -L`, and the Quint workflow calls +that package when targeted testing is useful. + +**Cold-cache trade-off:** test and release dependency profiles produce +different Cargo artifacts, so the first build may compile both sets. Subsequent +test and Clippy runs reuse the test-profile cache. Benchmark cold and warm runs +on both Linux and macOS before claiming a measured speedup. + ## Benchmark method Measurements were taken on x86_64-linux with 8 logical cores, Nix 2.34.8, and a diff --git a/context/sce/local-db.md b/context/sce/local-db.md index 8a4875845..c7167d765 100644 --- a/context/sce/local-db.md +++ b/context/sce/local-db.md @@ -10,7 +10,7 @@ pub type LocalDb = TursoDb; - `LocalDbSpec`: `DbSpec` implementation for the canonical per-user local database. - `LocalDb`: type alias for `TursoDb`. -- `LocalDb::new()`, `execute()`, `query()`, and `query_map()`: inherited from `TursoDb` with current hardcoded constructor and operation retry policies. +- `LocalDb::new()`, `execute()`, `query()`, and `query_map()`: async operations inherited from `TursoDb` with the configured constructor and operation retry policies. - `local_db/lifecycle.rs`: owns local DB health, parent-directory bootstrap, and setup initialization through `LocalDb::new()`. ## Database path @@ -33,13 +33,13 @@ Agent Trace-specific persistence is split out of the neutral local DB baseline a ```rust use crate::services::local_db::LocalDb; -let db = LocalDb::new()?; -let affected = db.execute("VACUUM", ())?; -let mut rows = db.query("PRAGMA database_list", ())?; +let db = LocalDb::new().await?; +let affected = db.execute("VACUUM", ()).await?; +let mut rows = db.query("PRAGMA database_list", ()).await?; ``` ## Error handling -The shared `TursoDb` adapter returns `anyhow::Result` with service-name-qualified diagnostics for path resolution, parent-directory creation, runtime creation, database open/connect, migration execution, and SQL execution/query failures. Open/connect and `execute`/`query`/`query_map` transient failures are retry-wrapped by the shared adapter; migration execution and row-mapping failures are not retried. +The shared `TursoDb` adapter returns `anyhow::Result` with service-name-qualified diagnostics for path resolution, parent-directory creation, database open/connect, migration execution, and SQL execution/query failures. Open/connect and `execute`/`query`/`query_map` transient failures are retry-wrapped by the shared async adapter; migration execution and row-mapping failures are not retried. See also: [shared-turso-db.md](shared-turso-db.md), [agent-trace-db.md](agent-trace-db.md), [overview.md](../overview.md), [glossary.md](../glossary.md), [context-map.md](../context-map.md) diff --git a/context/sce/mutation-scope-health-status.md b/context/sce/mutation-scope-health-status.md index 12e4021e6..6b657ee25 100644 --- a/context/sce/mutation-scope-health-status.md +++ b/context/sce/mutation-scope-health-status.md @@ -62,7 +62,7 @@ four-way status each adapter module already computed. Doctor is a read-only snapshot: a currently executing hook process can, in a narrow window, hold a state shape mid-transition (for example Claude's `recovery_pending = true` with non-empty `attempts` between -`mark_recovery_pending()` and `remove_attempt()` inside one still-running +`mark_recovery_pending_and_pending_abandon()` and `remove_attempt()` inside one still-running hook call). Health classifies the durable state observed at inspection time against whether normal future adapter lifecycle events can recover it, assuming the operation that produced it has stopped progressing; it is not a diff --git a/context/sce/setup-githooks-hook-asset-packaging.md b/context/sce/setup-githooks-hook-asset-packaging.md index 8a5a8e3e5..a7d12ea67 100644 --- a/context/sce/setup-githooks-hook-asset-packaging.md +++ b/context/sce/setup-githooks-hook-asset-packaging.md @@ -24,16 +24,7 @@ Available-CLI behavior remains hook-specific: ## Setup-service accessor surface -`cli/src/services/setup/mod.rs` exposes hook-template access through: - -- `iter_required_hook_assets()` for deterministic full-set iteration -- `get_required_hook_asset(RequiredHookAsset)` for stable per-hook lookup - -`RequiredHookAsset` is the canonical hook mapping enum for this packaging layer: - -- `PreCommit` -- `CommitMsg` -- `PostCommit` +`cli/src/services/setup/mod.rs` exposes hook-template access through `iter_required_hook_assets()` for deterministic full-set iteration; the per-hook lookup helper and `RequiredHookAsset` enum were removed as unused. ## Determinism and validation diff --git a/context/sce/shared-turso-db.md b/context/sce/shared-turso-db.md index 3eb41d477..8b5d07143 100644 --- a/context/sce/shared-turso-db.md +++ b/context/sce/shared-turso-db.md @@ -10,45 +10,61 @@ - `migrations()` returns ordered embedded migration `(id, sql)` pairs. - `cli/build.rs` scans immediate `cli/migrations//*.sql` directories at compile time, copies them into `OUT_DIR/static/migrations`, and writes `OUT_DIR/generated_migrations.rs` with database-named migration constants (`AGENT_TRACE_REPOSITORY_MIGRATIONS`, `AUTH_MIGRATIONS`, etc.). Generated entries use the filename stem as the migration ID, embed the staged SQL with `include_str!`, and are sorted by the numeric prefix before the first `_`. - `TursoDb`: generic unencrypted adapter that owns: - - tokio current-thread runtime creation + - no Tokio runtime ownership; all Turso futures are awaited by the application runtime - Turso local database open/connect flow using `turso::Builder::new_local()` with `experimental_multiprocess_wal(true)`, which gives concurrent `sce` processes cross-process correctness and locking on the same local database; it does not make a contended writer wait - per-`DbSpec` Turso busy timeout, resolved by `resolve_busy_timeout::()` and installed with `Connection::busy_timeout(...)` on the connection returned by `connect()` in `open_without_migrations_at` (the open path behind `new`, `new_at`, and `open_without_migrations`). It is Turso's wait policy for `Busy`: Turso sleeps in short phases until the lock clears or the timeout elapses, and only then returns `Busy`. It is connection-wide, so it also covers `BEGIN IMMEDIATE` issued by `Transaction::new_unchecked(.., Immediate)` on the same connection. The Agent Trace DB (`db_config_key() == "agent_trace_db"`) uses `policies.database_retry.agent_trace_db.busy_timeout_ms` when configured (`0` leaves the handler unset), falling back to `AGENT_TRACE_DB_BUSY_TIMEOUT_MS = 1_000`; every other database resolves to zero, which leaves the busy handler unset, so `local_db` behavior is unchanged. `EncryptedTursoDb` (auth DB) never installs a busy handler. A single attempt may therefore spend up to about 1000 ms waiting inside Turso before returning `Busy`; the Agent Trace write-contention retry below decides whether SCE tries once more - Agent Trace write-contention deadline, resolved by `resolve_contention_deadline::()` from `policies.database_retry.agent_trace_db.contention_deadline_ms` with fallback to `AGENT_TRACE_DB_CONTENTION_DEADLINE_MS = 2_250` (zero for every other database). It is the cutoff for starting another outer write-contention retry, not a hard operation timeout - Agent Trace write-contention retry (`write_contention_policy::()`, `run_with_write_contention_retry`), returned only for `agent_trace_db` and applied only to write units whose whole replay is safe: the two transactional primitives below (each attempt is a complete `BEGIN IMMEDIATE` → `COMMIT` unit) and `execute_idempotent_write(sql, params)`, the opt-in single-statement entrypoint used only by the repository-metadata `INSERT … ON CONFLICT DO NOTHING`, the guarded source-instance claim `UPDATE … WHERE source_instance_id = ''`, and the mutation-trace worktree/scope/scope-provenance `…_IF_ABSENT` inserts. It retries only typed `Busy`/`BusySnapshot` (classified before conversion to `anyhow`), allows at most `2` attempts, sleeps a full-jitter backoff drawn from `0..=100ms`, and admits the second attempt at two points: before the backoff sleep it requires `remaining contention deadline >= backoff + busy_timeout`, and after the sleep actually returns it re-checks `remaining >= busy_timeout`, so scheduler oversleep can never start an attempt outside the admission contract (time is measured on a monotonic clock from the operation start, so Turso's busy wait counts; nothing starts after the deadline expires). Deterministic failures return after one attempt without sleeping. On every other database these entrypoints keep the generic query retry, and `execute_idempotent_write` is plain `execute()` - - config-driven connection-open retry around only the `build().await.connect()` block using `run_with_retry_sync` (resolved from `policies.database_retry..connection_open` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults `3` attempts, `1s` timeout, `25ms..200ms` backoff) - - config-driven operation retry for `execute()`, `query()`, and `query_map()` using `run_with_retry_sync` (resolved from `policies.database_retry..query` via the same `OnceLock` with fallback to hardcoded defaults `5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). This is the outer retry for every operation outside the Agent Trace write-contention set, including Agent Trace reads (`query`/`query_values`/`query_map`), `passive_checkpoint`, generic `execute()`, and migrations. Its retry-policy budget is not a wall-clock bound; see the retry-budget note below + - config-driven connection-open retry around only the `build().await.connect()` block using async `run_with_retry` (resolved from `policies.database_retry..connection_open` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults `3` attempts, `1s` timeout, `25ms..200ms` backoff) + - config-driven operation retry for `execute()` and `query_map()` using async `run_with_retry` (resolved from `policies.database_retry..query` via the same `OnceLock` with fallback to hardcoded defaults `5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). This is the outer retry for every operation outside the Agent Trace write-contention set, including Agent Trace reads (`query_map`), `passive_checkpoint`, and generic `execute()`; migration execution is deliberately outside every retry. Its retry-policy budget is not a wall-clock bound; see the retry-budget note below - parent-directory creation - - retry-backed synchronous `execute()`, `query()`, raw-value `query_values()`, and row-mapping `query_map()` wrappers via the public adapter methods, with config-driven query retry resolved from `policies.database_retry..query` - - migration-running initialization through `new()` and generic embedded migration execution through `run_migrations()` delegated to the shared internal `TursoConnectionCore` with per-database `__sce_migrations` metadata + - retry-backed async `execute()` and row-mapping `query_map()` operations via the public adapter methods, with config-driven query retry resolved from `policies.database_retry..query` + - migration-running initialization through `new()` and generic embedded migration execution through `run_migrations()` using shared migration helpers with per-database `__sce_migrations` metadata - explicit-path migration-running initialization through `new_at(path)`, preserving the same service-specific retry/migration behavior while letting callers supply a database path outside `DbSpec::db_path()` - no-migration opening through `open_without_migrations()`, which preserves parent-directory creation and connection-open retry but does not create `__sce_migrations` or apply embedded migrations - explicit-path no-migration opening through `open_without_migrations_at(path)` for path-resolved hot runtime callers - `migration_metadata_problems(&self) -> Result>`: non-mutating readiness check that queries `__sce_migrations` metadata and compares applied migration IDs against `M::migrations()`; returns a list of problems (missing metadata table, incomplete applied migrations, unexpected extra migrations) or an empty list when the schema is ready - `ensure_schema_ready(&self, setup_guidance: &str) -> Result<()>`: non-mutating hook-readiness gate that calls `migration_metadata_problems()` and bails with a formatted error including `M::db_name()` and the caller-provided guidance string when problems are found; returns `Ok(())` when the schema is ready - - `passive_checkpoint(&self) -> Result<()>`: runs `PRAGMA wal_checkpoint(PASSIVE)` through the same query/runtime/retry path as `execute()`/`query()` (config-driven query retry, `block_on_isolated`), draining the checkpoint result row without exposing its busy/log/checkpointed statistics. PASSIVE checkpoints only what is currently safe to move from the WAL into the main database file and never blocks on active readers or writers, so it does not guarantee WAL truncation; safe to call repeatedly; not a durability boundary on its own. Routine maintenance only, not exposed on `EncryptedTursoDb`. `sce hooks post-commit` is the sole current caller: it runs `RepositoryAgentTraceDb::passive_checkpoint()` exactly once after post-commit Agent Trace persistence succeeds; a failing checkpoint is logged as a warning and never fails the hook or affects already-persisted data (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). + - `passive_checkpoint(&self) -> Result<()>`: runs `PRAGMA wal_checkpoint(PASSIVE)` through the same async query/retry path as `execute()` (config-driven query retry), draining the checkpoint result row without exposing its busy/log/checkpointed statistics. PASSIVE checkpoints only what is currently safe to move from the WAL into the main database file and never blocks on active readers or writers, so it does not guarantee WAL truncation; safe to call repeatedly; not a durability boundary on its own. Routine maintenance only, not exposed on `EncryptedTursoDb`. `sce hooks post-commit` is the sole current caller: it runs `RepositoryAgentTraceDb::passive_checkpoint()` exactly once after post-commit Agent Trace persistence succeeds; a failing checkpoint is logged as a warning and never fails the hook or affects already-persisted data (see [agent-trace-hooks-command-routing.md](agent-trace-hooks-command-routing.md)). - `count_write_contention(body) -> (T, WriteContentionCounts)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports the write-contention `attempts`, `outer_retries`, and `exhaustions` it caused on the current thread. Test instrumentation only; not production telemetry and not shared across threads or processes. - `record_write_contention_timeline(body) -> (T, Vec<(Instant, WriteContentionTimelineEvent)>)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and returns the `Instant`-stamped attempt start/end, backoff requested and backoff slept events of the write-contention retry on the current thread. Test-only measurement instrumentation used by the lock-contention suite; never logged in production. -- `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query`/`query_values`/`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. -- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same structural shape (connection, runtime bridge, and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. -- `EncryptedTursoDb` exposes the same public synchronous `execute()`, `query()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. -- `TursoConnectionCore` is internal to `cli/src/services/db/mod.rs` and owns the shared Turso connection plus tokio current-thread runtime bridging used by the public adapter methods; generic embedded migration execution with per-database `__sce_migrations` metadata is delegated to `run_embedded_migrations` helpers; encryption vs unencrypted behavior remains constructor-only at the public adapter layer. +- `count_read_statements(body) -> (T, usize)` (`#[cfg(test)]`, `pub(crate)`): runs `body` and reports how many `TursoDb` read statements (`query_map`) it issued on the current thread, each counted once in its synchronous prelude before the retry wrapper. A deterministic seam for tests that must prove an operation reads from a single DB snapshot — one statement — rather than several independent `SELECT`s a concurrent commit could tear across (first used by `MutationTraceStore::load_tree_roots` / `load_all_tree_roots`). Not compiled into production builds. +- `EncryptedTursoDb`: encrypted-adapter seam parallel to `TursoDb` with the same static shape (connection and spec marker). `EncryptedTursoDb::new()` resolves the encryption key via `encryption_key::get_or_create_encryption_key()` (environment variable `SCE_AUTH_DB_ENCRYPTION_KEY` with OS credential-store fallback), enables Turso experimental local encryption, applies strict `aegis256` cipher selection through `turso::EncryptionOpts` during local DB open/connect, wraps that open/connect block in the same connection-open retry policy resolved from `policies.database_retry..connection_open`, and runs embedded migrations after connect. +- `EncryptedTursoDb` exposes async `execute()`, `query_map()`, and `run_migrations()` methods; operation methods use the same config-driven query retry policy as `TursoDb`. +- Both public adapters keep concrete Turso connections and marker types; there is no shared runtime-owning core or synchronous executor bridge. - Shared lifecycle helpers: - `collect_db_path_health()` emits common parent/path health problems for DB-backed services. - `bootstrap_db_parent()` creates the resolved DB parent directory for repair/setup flows. -## Application runtime compatibility +## Application runtime -The adapter remains synchronous and owns a current-thread Tokio runtime built by `build_current_thread_runtime`. When Tokio is already entered, `block_on_isolated` polls that runtime on a fresh scoped thread to protect reentry; this does not protect runtime destruction inside async execution. DB-capable dispatch and sync storage guards establish blocking construction/destruction scopes at the caller without changing DB APIs, retry/WAL/migration policies or credential storage. See the [dispatch lifetime inventory](../cli/cli-command-surface.md#application-runtime-dispatch) and [application execution runtime](../architecture.md#application-execution-runtime). PR2 is future work converting Turso to async and removing DB-owned runtimes, `build_current_thread_runtime`, `block_on_isolated`, and the DB-related `block_in_place` compatibility scopes together. +Both adapters expose native async operations and own no Tokio runtime. Construction, migration, query, transaction, checkpoint, and encrypted operations are awaited by their callers on the application runtime; no `build_current_thread_runtime`, `block_on_isolated`, or DB-related `block_in_place` scope remains in production. See the [application execution runtime](../architecture.md#application-execution-runtime) and the [directly awaited Turso persistence decision](../decisions/2026-10-07-directly-awaited-turso-persistence.md). ## Transactional primitives `TursoDb` offers two generic multi-statement transaction primitives beyond -the single-statement `execute()`/`query()`/`query_map()` wrappers. Both are +the single-statement `execute()`/`query_map()` wrappers. Both are public on `TursoDb` only (not `EncryptedTursoDb`). Each attempt runs the whole `BEGIN IMMEDIATE` → `COMMIT` unit and classifies every `turso::Error` as retryable (`Busy`/`BusySnapshot`) or deterministic. On the Agent Trace DB the unit runs under the write-contention retry above; on other databases it -runs under the generic `run_with_retry_sync` query retry: +runs under the generic async `run_with_retry` query retry. + +Both primitives take `&mut self` and begin the transaction with the checked +`Transaction::new(&mut Connection, Immediate)`, so the exclusive borrow spans +the whole transaction and no other operation on the same adapter can run +inside it (a compile-time guarantee; `Transaction::new_unchecked` is not used). +Retry helpers accept the narrow native-async `RetryOperation` / `WriteAttempt` +traits (blanket-implemented for plain closures) because an `FnMut` closure +cannot return a future that borrows its own captured `&mut`. Turso 0.8.1 +limits cancellation guarantees: dropping a transaction uncommitted only +schedules a rollback that runs on the next use of that connection, and the +writer lock stays held until then, so other writers get `Busy` meanwhile; a +commit that already completed may be durable when its caller is cancelled. +Local Turso futures complete without yielding, so no mid-transaction await +point exists today. Tests: `db::transaction_cancellation_tests` (abandoned +transaction) and `agent_trace_db::transaction_tests` (whole-transaction retry +and bounded contention exhaustion on independent connections). - `execute_transactional_insert_pair_if_absent(operation_name, retry_hint, exists_sql, exists_params, first_sql, first_params, second_sql, @@ -117,9 +133,9 @@ All three database areas (local DB, auth DB, Agent Trace DB) have lifecycle prov Migrations are deliberately outside the connection-open retry block. The generic constructors retry only local Turso open/connect; schema changes are not generically retried because migration SQL must not be replayed after partial execution. Service-specific resolvers may add bounded recovery around their own initialization contracts when they can prove safety; the repository-scoped Agent Trace resolver does this only for its one-file fresh schema and only repairs missing migration metadata after all required repository schema tables already exist. -`TursoDb` and `EncryptedTursoDb` operation methods use the same config-driven query retry policy, resolved from `policies.database_retry..query` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). `execute()`, `query()`, and `query_values()` convert caller parameters to owned Turso params before retry so each attempt can clone the same values. `query_values()` returns fully fetched column names plus raw `turso::Value` rows for deterministic rendering by operator-facing services. `query_map()` retries the initial query and full row-fetch loop, then runs caller-provided row mapping after retry completion so mapping failures are surfaced as logic errors and are not retried. +`TursoDb` and `EncryptedTursoDb` operation methods use the same config-driven query retry policy, resolved from `policies.database_retry..query` via `DATABASE_RETRY_CONFIG` `OnceLock` with fallback to hardcoded defaults (`5` attempts, `200ms` `timeout_ms`, `25ms..100ms` backoff). `execute()` converts caller parameters to owned Turso params before retry so each attempt can clone the same values. `query_map()` retries the initial query and full row-fetch loop, then runs caller-provided row mapping after retry completion so mapping failures are surfaced as logic errors and are not retried. -Retry-policy budget versus wall-clock time: the generic query retry policy keeps its historical `5` attempts, `200ms` `timeout_ms`, and `25..100ms` backoff configuration, and its earlier `<= 2_000ms` figure describes that configured retry/budget calculation, not a hard wall-clock operation timeout. `run_with_retry_sync` cannot interrupt a synchronous operation already in progress: it only compares an attempt's elapsed time with `timeout_ms` after the attempt returns, to label the error. Actual failure latency is the sum of each attempt's real duration plus backoff. For the Agent Trace write-contention set, each attempt can include up to `busy_timeout_ms` (default 1000 ms) of Turso busy waiting and at most one outer retry starts, so a write blocked by a long-held lock fails after about 2–2.1 s with the defaults (about 1–1.1 s under the earlier 500 ms / 1250 ms defaults). None of `query.timeout_ms`, the `<= 2_000ms` policy figure, or `contention_deadline_ms` is a hard wall-clock deadline. +Retry-policy budget versus wall-clock time: the generic query retry policy keeps its historical `5` attempts, `200ms` `timeout_ms`, and `25..100ms` backoff configuration, and its earlier `<= 2_000ms` figure describes that configured retry/budget calculation, not a hard wall-clock operation timeout. `run_with_retry` preserves post-attempt diagnostic timeout semantics and does not cancel an operation merely because its configured attempt budget elapsed: it only compares an attempt's elapsed time with `timeout_ms` after the attempt returns, to label the error. Actual failure latency is the sum of each attempt's real duration plus backoff. For the Agent Trace write-contention set, each attempt can include up to `busy_timeout_ms` (default 1000 ms) of Turso busy waiting and at most one outer retry starts, so a write blocked by a long-held lock fails after about 2–2.1 s with the defaults (about 1–1.1 s under the earlier 500 ms / 1250 ms defaults). None of `query.timeout_ms`, the `<= 2_000ms` policy figure, or `contention_deadline_ms` is a hard wall-clock deadline. | Layer | Role | | --- | --- | diff --git a/flake.nix b/flake.nix index dde943a68..2e46ea6fa 100644 --- a/flake.nix +++ b/flake.nix @@ -388,7 +388,75 @@ src = craneLib.cleanCargoSource ./cli; }; - cargoArtifacts = craneLib.buildDepsOnly cargoDepsArgs; + # Release consumers only need compiled dependencies, not check artifacts. + cargoArtifacts = craneLib.buildDepsOnly ( + cargoDepsArgs + // { + buildPhaseCargoCommand = "cargoWithProfile build --locked"; + } + ); + + # Keep production artifacts on Crane's release profile. The test and + # Clippy derivations use the unoptimized Cargo test profile with a + # separate dependency cache, so changing CLI sources does not require + # repeatedly optimizing the entire Rust test binary. + testProfileArgs = { + CARGO_PROFILE = "test"; + # Smaller dependency artifacts for test/Clippy derivations. + # This uses the measured J variant (debug info disabled for all + # test-profile crates); Rust panic locations remain available. + CARGO_PROFILE_TEST_DEBUG = "0"; + }; + + # The deps-only derivation must also prepare test/dev-dependency artifacts. + # The usual cargo build is redundant here: the checked targets are followed + # by cargo test --no-run during the check phase. + # Only test/Clippy vendoring removes the Turso kits' C ABI outputs. + # Native and release Cargo vendoring remains untouched. + testCargoVendorDir = craneLib.vendorCargoDeps { + cargoLock = ./cli/Cargo.lock; + overrideVendorCargoPackage = package: drv: + if builtins.elem package.name [ "turso_sdk_kit" "turso_sync_sdk_kit" ] then + assert package.version == "0.8.1"; + drv.overrideAttrs (old: { + postPatch = (old.postPatch or "") + '' + ${pkgs.python3}/bin/python3 - <<'PY' + from pathlib import Path + import re + + manifest = Path("Cargo.toml") + source = manifest.read_text() + pattern = re.compile(r"(?m)^crate-type[ \t]*=[ \t]*\[[^\]]*\]") + matches = list(pattern.finditer(source)) + if len(matches) != 1: + raise SystemExit("Expected exactly one Turso crate-type declaration") + match = matches[0] + kinds = re.findall(r'"([^"]+)"', match.group()) + if sorted(kinds) != ["cdylib", "lib", "staticlib"]: + raise SystemExit(f"Unexpected Turso crate types: {kinds}") + manifest.write_text(source[:match.start()] + 'crate-type = ["lib"]' + source[match.end():]) + PY + ''; + }) + else + drv; + }; + + cargoArtifactsTest = craneLib.buildDepsOnly ( + cargoDepsArgs + // testProfileArgs + // { + pname = "sce-test-deps"; + cargoVendorDir = testCargoVendorDir; + doCheck = true; + buildPhaseCargoCommand = "cargoWithProfile check --locked --all-targets"; + } + ); + + testCargoArgs = commonCargoArgs // testProfileArgs // { + cargoVendorDir = testCargoVendorDir; + cargoArtifacts = cargoArtifactsTest; + }; scePackage = craneLib.buildPackage ( commonCargoArgs @@ -419,12 +487,19 @@ CARGO_BUILD_TARGET = muslTarget; }; - cargoArtifactsMusl = craneLibMusl.buildDepsOnly cargoDepsArgsMusl; + cargoArtifactsMusl = craneLibMusl.buildDepsOnly ( + cargoDepsArgsMusl + // { + buildPhaseCargoCommand = "cargoWithProfile build --locked"; + } + ); scePackageMusl = craneLibMusl.buildPackage ( (commonCargoArgs // muslEnvVars // releaseCommitArgs) // { - inherit cargoArtifactsMusl; + # Crane expects cargoArtifacts rather than cargoArtifactsMusl. + # Reuse the separately prepared musl dependency artifacts. + cargoArtifacts = cargoArtifactsMusl; nativeBuildInputs = [ rustToolchainMusl pkgs.cmake @@ -1529,6 +1604,24 @@ ''} ''; + # The dedicated MBT runner is opt-in as a package, not a flake check: + # cli-tests already executes the same mutation_trace::mbt test module. + # The Quint workflow tests the spec separately, without rebuilding Rust MBT. + mutationTraceQuintConnectCheck = craneLib.cargoTest ( + testCargoArgs + // { + pname = "sce-mutation-trace-quint-connect"; + doCheck = true; + cargoTestExtraArgs = "mutation_trace::mbt"; + nativeCheckInputs = [ pkgs.git pkgs.quint ]; + preCheck = '' + rustc --version + cargo --version + quint --version + ''; + } + ); + sceApp = { type = "app"; program = "${scePackage}/bin/sce"; @@ -1552,6 +1645,7 @@ ci-checks = ciChecks; bun = bunPackage; quint = pkgs.quint; + mutation-trace-quint-connect = mutationTraceQuintConnectCheck; quint-language-server = quintLanguageServerPackage; turso = tursoPackage; default = scePackage; @@ -1562,23 +1656,21 @@ # `mutation_trace::mbt` (the Quint Connect model-based-testing # harness) is an ordinary `#[cfg(test)]` module reached by the # full `cargo test` this check runs, so the pinned Quint binary - # must be on PATH here too, not only in the dedicated - # `mutation-trace-quint-connect` check below. + # must be on PATH for this check. The focused MBT runner is + # available separately as .#mutation-trace-quint-connect. cli-tests = craneLib.cargoTest ( - commonCargoArgs + testCargoArgs // { pname = "sce-cli-tests"; - inherit cargoArtifacts; doCheck = true; nativeCheckInputs = [ pkgs.git pkgs.quint ]; } ); cli-clippy = craneLib.cargoClippy ( - commonCargoArgs + testCargoArgs // { pname = "sce-cli-clippy"; - inherit cargoArtifacts; cargoClippyExtraArgs = "--all-targets --all-features"; } ); @@ -1590,27 +1682,6 @@ } ); - # Focused Quint Connect model-based-testing check: runs only - # `mutation_trace::mbt` under the repository's pinned Rust - # toolchain and pinned Quint binary, reusing the same - # `cargoArtifacts`/`commonCargoArgs` pipeline as `cli-tests` - # rather than a bespoke Rust+Quint environment. - mutation-trace-quint-connect = craneLib.cargoTest ( - commonCargoArgs - // { - pname = "sce-mutation-trace-quint-connect"; - inherit cargoArtifacts; - doCheck = true; - cargoTestExtraArgs = "mutation_trace::mbt"; - nativeCheckInputs = [ pkgs.git pkgs.quint ]; - preCheck = '' - rustc --version - cargo --version - quint --version - ''; - } - ); - cli-generated-input = cliGeneratedInputCheck; pkl-generated = pklGeneratedCheck; codex-hook-command = codexHookCommandCheck;