diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a014c2..40bc42a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,270 +1,61 @@ -name: ci +name: qualification on: push: branches: [main] pull_request: -# A new push to the same PR (or to main) supersedes the run in flight; -# stop paying for the old one. release.yml deliberately does not cancel. concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} + group: qualify-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - FUSE_T_VERSION: 1.2.7 - CARGO_TERM_COLOR: always - # acyclic-fs is fetched from its own git repo (see Cargo.toml); cargo's - # built-in libgit2 fetcher can't resolve a pinned commit SHA that isn't a - # branch tip on that host, so delegate to the system git CLI instead. - CARGO_NET_GIT_FETCH_WITH_CLI: true +permissions: + contents: read jobs: - # Three cheap jobs (changes, deny, lint) run in parallel; the expensive - # test matrix (FUSE-T install, release build, acceptance suite on two - # OSes) only starts once deny and lint pass, so a formatting slip or a - # banned crate is reported in about a minute instead of after a full - # matrix run — and it is skipped outright when nothing that could change - # a test result was touched. - - # Which parts of the tree a change touches. A job-level filter rather - # than a workflow-level `paths-ignore` because branch protection on main - # requires the `test (...)` checks: a workflow that never runs leaves - # them "expected" forever and blocks the merge, whereas a job skipped by - # `if:` reports as skipped, which counts as passing. - changes: - runs-on: ubuntu-24.04 - # paths-filter lists a PR's files through the API; the default token - # here is read-only on contents and nothing else. - permissions: - contents: read - pull-requests: read - outputs: - code: ${{ steps.filter.outputs.code }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - # On a push to main the filter diffs against the pre-push commit, - # which a depth-1 checkout does not have. - fetch-depth: 0 - - id: filter - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 - with: - # "some file changed that is not docs/prose": a negation-only list - # matches nothing, so the positive `**` is required, and - # some-with-excludes makes the negations apply to it. - predicate-quantifier: some-with-excludes - filters: | - code: - - '**' - - '!**/*.md' - - '!docs/**' - - '!LICENSE' - - '!.github/CODEOWNERS' - - # Licenses, advisories, and sources per deny.toml. Keeps the published - # SBOM inside the permissive allowlist and fails on known-vulnerable or - # yanked crates. Always runs: the secrets scan applies to docs too. - deny: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Product name is single-sourced (product.toml) - run: bash scripts/check-product-name.sh - - name: No forbidden files or credential patterns - run: bash scripts/check-no-secrets.sh - - name: Code quality (line width, TODO format, comment blocks, duplication) - run: bash scripts/check-code-quality.sh - - uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2.1.1 - with: - command: check - arguments: --locked - log-level: warn - - # Formatting and lint. Fast and toolchain-only (no FUSE-T needed). - lint: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Install toolchain - run: | - rustup toolchain install stable --profile minimal --component rustfmt --component clippy - rustup default stable - - name: Cache cargo - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: lint-cargo-${{ hashFiles('Cargo.lock') }} - - name: cargo fmt --check - run: cargo fmt --all --check - - name: cargo clippy - run: cargo clippy --workspace --all-targets --all-features -- -D warnings - - # Line coverage of the unit and integration tests, as a job summary and - # an lcov artifact, with a floor so a change can't quietly delete tests. - # The daemon, client, and MCP server are exercised by the acceptance - # scripts rather than `cargo test`, so they read as 0% here; raise the - # floor as unit coverage of those grows, not by counting the scripts. - coverage: - needs: [changes, deny, lint] - if: needs.changes.outputs.code == 'true' - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Install toolchain - run: | - rustup toolchain install stable --profile minimal --component llvm-tools-preview - rustup default stable - - uses: taiki-e/install-action@3f74d7c16a4242f1c95561e98edc25d36adb4375 # v2.87.12 - with: - tool: cargo-llvm-cov - - name: Cache cargo - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: coverage-cargo-${{ hashFiles('Cargo.lock') }} - - name: cargo llvm-cov - run: | - cargo llvm-cov --workspace --all-features --lcov --output-path lcov.info --fail-under-lines 48 - { - echo '## Test coverage (lines)' - echo '```' - cargo llvm-cov report --summary-only - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: lcov - path: lcov.info - - # Windows, which nothing else in this file covers. The lint job runs on - # Linux, so every `#[cfg(windows)]` block in the tree is invisible to it — - # a Windows-only arm can stop compiling and no other check notices. This - # job builds and lints those arms, runs the workspace tests, and drives - # the parts of the product whose behaviour genuinely differs there - # (named-pipe transport, UTF-16LE names, renaming the repo root, copy - # forks). The POSIX acceptance suite is not run: it assumes mount tooling, - # symlinks and modes that Windows does not have. - windows: - needs: [changes, deny, lint] - if: >- - always() - && needs.deny.result == 'success' - && needs.lint.result == 'success' - && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') - runs-on: windows-2022 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - - name: Install toolchain - run: | - rustup toolchain install stable --profile minimal --component rustfmt --component clippy - rustup default stable - - - name: Cache cargo - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: windows-cargo-${{ hashFiles('Cargo.lock') }} - - - name: cargo fmt --check - run: cargo fmt --all --check - - # The point of the job: lint the cfg(windows) arms the Linux lint job - # never compiles. - - name: cargo clippy - run: cargo clippy --workspace --all-targets --all-features -- -D warnings - - - name: Unit and integration tests - run: cargo test --workspace - - - name: Release build - run: cargo build --release --locked -p acyclic - - - name: Windows end-to-end smoke - shell: bash - env: - ACYCLIC_BIN: ${{ github.workspace }}/target/release/acyclic.exe - run: bash tests/acceptance/windows-smoke.sh - - test: - needs: [changes, deny, lint] - # Fail closed: run when the change filter says code moved, and also - # when the filter job itself failed (an empty output must not read as - # "nothing to test", since a skipped required check counts as passing). - # deny and lint failing still skip this job; those are required checks - # in their own right, so the PR stays red. - if: >- - always() - && needs.deny.result == 'success' - && needs.lint.result == 'success' - && (needs.changes.result != 'success' || needs.changes.outputs.code == 'true') + qualify: strategy: fail-fast: false matrix: - os: [macos-14, ubuntu-24.04] + os: [ubuntu-24.04, windows-2022, macos-14] runs-on: ${{ matrix.os }} + timeout-minutes: 20 steps: - name: Checkout plugin uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - path: graphcoder-plugin - - - name: Install FUSE-T (macOS) + path: worktrees/graphcoder/plugin-sdk-lab + persist-credentials: false + - name: Checkout SDK + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: acyclic-labs/sdk + ref: codex/generation-reader + path: sdk + persist-credentials: false + - name: Install FUSE-T on macOS if: runner.os == 'macOS' + shell: bash run: | - set -euo pipefail curl --fail --location --retry 5 \ - "https://github.com/macos-fuse-t/fuse-t/releases/download/${FUSE_T_VERSION}/fuse-t-macos-installer-${FUSE_T_VERSION}.pkg" \ + https://github.com/macos-fuse-t/fuse-t/releases/download/1.2.7/fuse-t-macos-installer-1.2.7.pkg \ --output /tmp/fuse-t.pkg sudo installer -pkg /tmp/fuse-t.pkg -target / - test -d /usr/local/include/fuse3 - - - name: Install toolchain - run: rustup toolchain install stable --profile minimal && rustup default stable - - - name: Cache cargo + - name: Install Bun + uses: oven-sh/setup-bun@735343b667d3e6f658f44d0eca948eb6282f2b76 # v2.0.2 + with: + bun-version: 1.3.14 + - name: Cache immutable dependencies and build outputs uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cargo/registry ~/.cargo/git - graphcoder-plugin/target - key: ${{ matrix.os }}-cargo-${{ hashFiles('graphcoder-plugin/Cargo.lock') }} - - - name: Unit and integration tests - working-directory: graphcoder-plugin - run: cargo test --workspace - - - name: Release build (acceptance + latency gate run against it) - working-directory: graphcoder-plugin - run: cargo build --release - - - name: Acceptance suite - working-directory: graphcoder-plugin - env: - ACYCLIC_BIN: ${{ github.workspace }}/graphcoder-plugin/target/release/acyclic - ACYCLIC_QUAL: ${{ github.workspace }}/graphcoder-plugin/target/release/acyclic-qual - # Smaller corpus keeps CI wall-clock sane; the budget is unchanged. - ACYCLIC_LAT_FILES: "5000" - ACYCLIC_LAT_MB: "64" - ACYCLIC_SOAK_ROUNDS: "30" - run: bash tests/acceptance/run-all.sh + ~/.bun/install/cache + sdk/target-* + key: qualify-${{ matrix.os }}-${{ hashFiles('sdk/Cargo.lock', 'sdk/bun.lock', 'sdk/rust/**/*.rs', 'sdk/typescript/**/*.ts', 'worktrees/graphcoder/plugin-sdk-lab/Cargo.lock', 'worktrees/graphcoder/plugin-sdk-lab/crates/**/*.rs') }} + restore-keys: | + qualify-${{ matrix.os }}- + - name: Run the shared bounded gate + shell: pwsh + run: python sdk/scripts/qualify-local.py --plugin-root worktrees/graphcoder/plugin-sdk-lab diff --git a/CHANGELOG.md b/CHANGELOG.md index d6bdff2..e4e77de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ Pre-1.0; `main` is the only supported line (see `SECURITY.md`). ### Changed +- **The unfinished session-shadowing feature was removed.** It depended on + replacing the live repository with a native mount, was not wired into host + approval flows, and could not work consistently across platforms. Existing + `dry_run` configuration is now rejected; explicit forks remain available. + Before replacing the old binary, use that binary to stop every daemon with + an active shadow mount. A new protocol-v2 CLI cannot stop a protocol-v1 + daemon. If the old daemon already crashed, recover the real tree with + `fusermount3 -u ` (or `fusermount -u `) on Linux, or + `umount -f ` (then `diskutil unmount force ` if needed) on macOS. - **A cold daemon no longer delays the agent's first turn.** The session-start hook waits at most 300ms for the daemon; past that it prints a one-line notice and returns while the first snapshot builds in the background (251s @@ -19,8 +28,8 @@ Pre-1.0; `main` is the only supported line (see `SECURITY.md`). O(tree) (7s per fork and 6s per promote on 5,000 files); it now runs on the existing idle and every-N timers, like every other checkpoint. A fork is cut at its exact base generation, published or not. -- **The daemon exits after an hour idle** with no session, fork, or Safe Mode - session (`daemon_idle_exit_ms`, 0 disables). Twenty daemons were found alive +- **The daemon exits after an hour idle** with no session or fork + (`daemon_idle_exit_ms`, 0 disables). Twenty daemons were found alive on one machine, eleven for repos that no longer existed. - `acyclic status` reports the store size from a cache refreshed in the background instead of walking the object directory on every call. @@ -79,8 +88,6 @@ were 0.0.1. Host coverage, Speculation and Windows are what this release adds. the conversation, not just by commit. - **Forks** (Launch 3) — copy-on-write materialization for running parallel attempts, with promotion back via three-way merge. -- **Safe Mode** (Launch 4) — session redirection and interposition so agent mistakes land in a - redirected session rather than the working tree; needs the native mount layer. - **Speculation** — the daemon computes what the agent is about to ask for while nobody is waiting: the previous-session brief when a session ends, and (optionally, with a model command the developer names) a summary of each turn at the turn boundary. Results are keyed @@ -91,10 +98,10 @@ were 0.0.1. Host coverage, Speculation and Windows are what this release adds. - Published to npm as `@acyclic-labs/plugin`. - **Windows x64 support** — the daemon transport gains a named-pipe implementation alongside the Unix domain socket, and host names are encoded per platform (UTF-16LE on Windows) so capture, - diff, exclusions and the Safe Mode guard agree with the filesystem. Verified on Windows 11 and - covered by a `windows-2022` CI job. Two caveats: forks are always full copies there (ProjFS - projects a fork but does not carry writes back, so a mounted fork would silently lose work) and - Safe Mode needs a real mount, so it is unavailable. See `docs/windows-verification.md`. + diff, exclusions and guarded fork paths agree with the filesystem. Verified on Windows 11 and + covered by a `windows-2022` CI job. Forks use full copies there because ProjFS projects a fork + but does not carry writes back, so a mounted fork would silently lose work. See + `docs/windows-verification.md`. ### Fixed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c9fbc54..1d4f9e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,7 +13,7 @@ Acceptance suites (end-to-end, run against a real repo) live in `tests/acceptanc tests/acceptance/run-all.sh ``` -Individual suites (`journey.sh`, `timeline.sh`, `forks.sh`, `merge.sh`, `safe-mode.sh`, etc.) can +Individual suites (`journey.sh`, `timeline.sh`, `forks.sh`, `merge.sh`, etc.) can be run with `bash tests/acceptance/.sh` if you're iterating on one feature. The `*-e2e.sh` suites drive the real host CLIs (Claude Code, Codex, cursor-agent, OpenCode) and @@ -76,7 +76,7 @@ guards what those can't express. The rules, and why each exists: - **No lossy `as` casts** between integer widths or signs (`cast_possible_truncation`, `cast_sign_loss`, `cast_possible_wrap`, `cast_precision_loss`, `cast_lossless`). Use `u64::from`, `i64::try_from(x).unwrap_or(i64::MAX)`, or an `allow` that says why the value - is in range. `acyclic_engine::unix_now()` and `short_hex()` exist so the two most common + is in range. `acyclic::unix_now()` and `short_hex()` exist so the two most common cases are written once. - **Closed sets are enums, not strings.** Anything the CLI parses or the wire carries with a fixed vocabulary — checkpoint kinds, hook events, host names, restore actions, diff change @@ -100,13 +100,13 @@ guards what those can't express. The rules, and why each exists: The public product name is defined once, in `product.toml`, and threaded through everywhere else: `product::NAME` in Rust, `scripts/product.sh` in shell. Never hardcode the name as a literal string or path in source — `scripts/check-product-name.sh` fails CI if it drifts. Crate names -(`acyclic`, `acyclic-engine`, `acyclic-proto`, `acyclic-qual`) are internal identifiers and are +(`acyclic`, `acyclic-qual`) are internal identifiers and are exempt from this check. ## Design context `docs/design/` has the design docs and implementation notes behind the bigger features (Rewind, -Timeline, Forks, Safe Mode). Worth a skim before working on any of them — they capture the +Timeline, Forks). Worth a skim before working on any of them — they capture the tradeoffs and constraints that shaped the current architecture, including a few (like snapshot GC) that are intentionally deferred. diff --git a/Cargo.lock b/Cargo.lock index 426e5bc..da452c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,62 +5,51 @@ version = 4 [[package]] name = "acyclic" version = "0.0.2" -dependencies = [ - "acyclic-engine", - "acyclic-fs", - "acyclic-proto", - "clap", - "libc", - "rmcp", - "schemars", - "serde", - "serde_json", - "tempfile", - "tokio", - "windows-sys 0.61.2", -] - -[[package]] -name = "acyclic-engine" -version = "0.0.2" dependencies = [ "acyclic-fs", "blake3", "bytes", - "diffy", + "clap", "hex", "libc", + "rmcp", "rusqlite", + "schemars", "serde", "serde_json", "tempfile", "thiserror", "tokio", "toml", + "windows-sys 0.61.2", ] [[package]] name = "acyclic-fs" -version = "0.2.0-rc.1" -source = "git+https://github.com/acyclic-labs/sdk.git?rev=22b4e752f46d8f6db6e024b3137fe520ef64bcc1#22b4e752f46d8f6db6e024b3137fe520ef64bcc1" +version = "0.2.0-rc.5" dependencies = [ + "acyclic-native-runtime", "acyclic-objects", "acyclic-stream", "async-trait", "blake3", "bytes", + "cap-fs-ext", "cap-primitives", "cap-std", "cc", + "diffy", "fs2", "fuser", "futures", "hex", + "io-uring", "libc", "notify", "prost", "prost-types", "serde", + "serde_json", "thiserror", "tokio", "tonic", @@ -71,16 +60,31 @@ dependencies = [ "windows", ] +[[package]] +name = "acyclic-native-runtime" +version = "0.1.0" +dependencies = [ + "block2", + "bytes", + "dispatch2", + "io-uring", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "acyclic-objects" -version = "1.0.0-rc.1" -source = "git+https://github.com/acyclic-labs/sdk.git?rev=22b4e752f46d8f6db6e024b3137fe520ef64bcc1#22b4e752f46d8f6db6e024b3137fe520ef64bcc1" +version = "1.0.0-rc.4" dependencies = [ + "acyclic-native-runtime", "async-trait", "blake3", "bytes", "fs2", "futures", + "getrandom 0.3.4", + "hex", + "imbl", "libc", "prost", "prost-types", @@ -88,28 +92,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "acyclic-proto" -version = "0.0.2" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "acyclic-qual" -version = "0.0.2" -dependencies = [ - "acyclic-engine", - "acyclic-fs", - "tokio", -] - [[package]] name = "acyclic-stream" -version = "1.0.0-rc.3" -source = "git+https://github.com/acyclic-labs/sdk.git?rev=22b4e752f46d8f6db6e024b3137fe520ef64bcc1#22b4e752f46d8f6db6e024b3137fe520ef64bcc1" +version = "1.0.0-rc.7" dependencies = [ + "acyclic-native-runtime", "async-trait", "bytes", "fs2", @@ -204,6 +191,12 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "archery" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca55ee147b1926dbea904f50fe4902494e97bc742205abbbf10c709e43815f" + [[package]] name = "arrayvec" version = "0.7.8" @@ -316,18 +309,45 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cap-fs-ext" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ff379b70af8e08307a8f65e7040c7301cb4a572538ade16b4984f0da77847f" +dependencies = [ + "cap-primitives", + "cap-std", + "io-lifetimes 3.0.1", + "windows-sys 0.61.2", +] + [[package]] name = "cap-primitives" version = "4.0.3" @@ -531,6 +551,18 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -761,6 +793,20 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -769,7 +815,7 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", ] [[package]] @@ -969,6 +1015,27 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "imbl" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bad832b9b463ed9398b8506488cc2e3b897a9d44f13af115f382aac71f4fec" +dependencies = [ + "archery", + "equivalent", + "imbl-sized-chunks", + "rand_core", + "rand_xoshiro", + "version_check", + "wide", +] + +[[package]] +name = "imbl-sized-chunks" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0813be332553f857953298749fa19549e8b61b80589757c29b4e2a804fa9c6" + [[package]] name = "indexmap" version = "2.14.1" @@ -1023,6 +1090,17 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" +[[package]] +name = "io-uring" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.12.1" @@ -1209,6 +1287,21 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + [[package]] name = "once_cell" version = "1.21.4" @@ -1451,12 +1544,33 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core", +] + [[package]] name = "ref-cast" version = "1.0.27" @@ -1645,6 +1759,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2221,6 +2344,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -2275,6 +2407,16 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2600,6 +2742,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "zerocopy" version = "0.8.56" diff --git a/Cargo.toml b/Cargo.toml index 80afe71..3c796f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,9 +2,6 @@ resolver = "2" members = [ "crates/acyclic", - "crates/acyclic-engine", - "crates/acyclic-proto", - "crates/acyclic-qual", ] [workspace.package] @@ -15,7 +12,7 @@ repository = "https://github.com/acyclic-labs/graphcoder-plugin" homepage = "https://acyclic.dev" [workspace.dependencies] -acyclic-fs = { git = "https://github.com/acyclic-labs/sdk.git", rev = "22b4e752f46d8f6db6e024b3137fe520ef64bcc1", features = ["local", "native-watch", "native-mount"] } +acyclic-fs = { path = "../../../sdk/rust/crates/filesystem", features = ["local", "native-watch", "native-mount"] } tokio = { version = "1.48", features = ["rt-multi-thread", "macros"] } # Code-quality guards beyond clippy's defaults. CI runs clippy with @@ -67,4 +64,3 @@ cast_lossless = "warn" # function naming the invariant, so a reviewer can find them all. unsafe_code = "warn" unsafe_op_in_unsafe_fn = "warn" - diff --git a/README.md b/README.md index 7c729fb..c3cdacf 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Checkpoint every agent action, rewind exactly, see the blast radius. The store c **A local product with plugin distribution.** The product is an agent-native state engine that runs on your machine — snapshots, forks, and indexing over your working tree. The plugins are thin adapters that deliver it through Claude Code, Codex, OpenCode, any agent that can run a shell command, and Claude Desktop over MCP. The engine is the moat; the plugins are the channel. -> Status: Launches 1–4 built (Rewind, Timeline, Forks, Safe Mode), acceptance suites green on macOS and Linux, published to npm as `@acyclic-labs/plugin`. Launch 1's release gate is met: snapshot exclusions, a store-growth proof, license scanning, an attested SBOM per binary, `scripts/install.sh`, and a clean-machine install test. What v1 deliberately does not do is prune or purge history; see [Retention and purge](#retention-and-purge). Launch 5 (Monorepo) is spec. The spec lives on the [Acyclic plugins docs page](https://acyclic.dev/docs/plugins). +> Status: Launches 1–3 built (Rewind, Timeline, Forks), acceptance suites green on macOS, Linux, and Windows, published to npm as `@acyclic-labs/plugin`. Launch 1's release gate is met: snapshot exclusions, a store-growth proof, license scanning, an attested SBOM per binary, `scripts/install.sh`, and a clean-machine install test. What v1 deliberately does not do is prune or purge history; see [Retention and purge](#retention-and-purge). Launch 5 (Monorepo) is spec. The spec lives on the [Acyclic plugins docs page](https://acyclic.dev/docs/plugins). ## Install @@ -55,7 +55,7 @@ Not yet covered: an `install` writer for Codex's MCP config (TOML), Kimi Code CL ## The public name -`product.toml` at the repo root holds the public name once. The CLI command, `./config.toml`, the state and config directories, hook commands, skill names, message prefixes, the `_TRACE` and `_HOOK` variables, release asset names, and the npm bin all derive from it at build or packaging time (`crates/acyclic-engine/build.rs`, `scripts/product.sh`, the workflows). Crate names stay `acyclic*` because they are internal. `scripts/install.sh` is fetched standalone and mirrors the name, repo, and npm package; `scripts/check-product-name.sh` fails CI if any of them drifts or if any user-facing Rust string spells the name out. Renaming is: change `product.toml`, update the three mirror lines in `install.sh`, rebuild. +`product.toml` at the repo root holds the public name once. The CLI command, `./config.toml`, the state and config directories, hook commands, skill names, message prefixes, the `_TRACE` and `_HOOK` variables, release asset names, and the npm bin all derive from it at build or packaging time (`crates/acyclic/build.rs`, `scripts/product.sh`, the workflows). Crate names stay `acyclic*` because they are internal. `scripts/install.sh` is fetched standalone and mirrors the name, repo, and npm package; `scripts/check-product-name.sh` fails CI if any of them drifts or if any user-facing Rust string spells the name out. Renaming is: change `product.toml`, update the three mirror lines in `install.sh`, rebuild. ## Configuration @@ -68,7 +68,7 @@ Not yet covered: an `install` writer for Codex's MCP config (TOML), Kimi Code CL | `commit_every` / `commit_idle_ms` | `25` / `60000` | How often per-tool-call checkpoints are published to the durable store. | | `auto_checkpoint_idle_ms` | `5000` | Idle-timer safety net: checkpoints changes on its own once the watcher has been quiet this long, for hosts with no lifecycle-hook API (Claude Desktop). `0` disables it. Cheap no-op for hooked hosts, which already drain the watcher themselves. | | `quiesce_ms` / `quiesce_cap_ms` | `50` / `500` | Watcher quiet window before a capture. | -| `dry_run` / `guarded_paths` | `false` / `[]` | Safe Mode (Launch 4). | +| `guarded_paths` | `[]` | Repo-relative prefixes that mounted forks cannot write. Copy-mode forks do not enforce this mount-layer policy. | | `[decompose]` / `[merge]` | | Fork decomposition policy and merge limits (Launch 3). | | `store_dir` | `~/.local/share/acyclic/stores` | Where stores live. Never inside the repo. | @@ -111,7 +111,7 @@ Median lead is the number to watch: it is how far ahead of the request a claimed `acyclic status` reports the store size; trash is pruned by TTL. The store itself is never garbage-collected in v1, on purpose. At the pinned `acyclic-fs` revision a generation stays reachable only while it is a workspace head or carries a retention fact (checkpoint label, pin, fork base), retention facts cannot be released, and closure proofs do not follow generation parents. So the fs collector would either destroy every checkpoint but the head or, if every checkpoint were pinned first, never free anything again. Purge-through-history has the same dependency: content cannot be physically removed from a retained generation. Both land when the fs grows a retention-release fact; until then, keep secrets out of the store with `exclude`, which is the compliance control that ships. Details and the upstream ask are in `docs/design/implementation-rewind.md`, Phase 4. -Known caveats: mtimes are not restored on rewind, a rewind warrants an editor reload, forks and Safe Mode sessions do not see excluded paths, and baseline capture runs at roughly 230 s/GiB on first `init`. +Known caveats: mtimes are not restored on rewind, a rewind warrants an editor reload, forks do not see excluded paths, and baseline capture runs at roughly 230 s/GiB on first `init`. ## Thesis @@ -124,7 +124,7 @@ V1 is entirely local: no sandboxes, no managed sessions, no cloud sync. It ships One engine, thin adapters: - **`acyclic` CLI + daemon** — watcher, Merkle-DAG snapshot store, index. Host-agnostic. -- **Per-host adapters** — hook-based for CLIs with a lifecycle-hook API (Claude Code, Codex, Cursor), MCP-based for desktop apps and IDEs without one (Claude Desktop, VS Code; Cursor gets both). Every adapter is a `HostAdapter` in `crates/acyclic/src/install.rs`; the MCP server itself is `crates/acyclic/src/mcp.rs`, a thin translation of each tool call into the same `acyclic-proto::Op` the hooks send. The table under [Install](#per-host) says what each one writes and how far it has been verified; `docs/design/06-installation.md` has the design and the ship decision for the MCP path. +- **Per-host adapters** — hook-based for CLIs with a lifecycle-hook API (Claude Code, Codex, Cursor), MCP-based for desktop apps and IDEs without one (Claude Desktop, VS Code; Cursor gets both). Every adapter is a `HostAdapter` in `crates/acyclic/src/install.rs`; the MCP server itself is `crates/acyclic/src/mcp.rs`, a thin translation of each tool call into the same private `proto::Op` the hooks send. The table under [Install](#per-host) says what each one writes and how far it has been verified; `docs/design/06-installation.md` has the design and the ship decision for the MCP path. ## Launch plan @@ -133,7 +133,6 @@ One engine, thin adapters: | 1 | Rewind | Merkle snapshot store + host hooks | Never fear letting the agent loose | built (`tests/acceptance/journey.sh`, `crash.sh`, `soak.sh`, `latency.sh`, `claude-e2e.sh`) | | 2 | Timeline | Turn-linked metadata index | The repo at any point in the conversation | built (`timeline.sh`) | | 3 | Forks | Copy-on-write materialization | N parallel attempts, pick the winner | built: mounted forks, promote with three-way merge (`forks.sh`, `merge.sh`, `claude-merge-e2e.sh`) | -| 4 | Safe Mode | Session redirection + interposition | Agents on the codebase, not agents' mistakes in it | built, needs the native mount layer (`safe-mode.sh`) | | 5 | Monorepo | Merkle-aware content + symbol index | The repo that finally works with agents | not started | Run everything with `tests/acceptance/run-all.sh`; the live Claude Code scenarios are gated by `ACYCLIC_E2E=1`. diff --git a/SECURITY.md b/SECURITY.md index 326903c..c4a3500 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,7 @@ Please do not open public issues for security vulnerabilities. Email **security@ The engine (`acyclic` CLI and daemon), the host-tool adapters (Claude Code, Codex, OpenCode), and the release pipeline (signed artifacts, SBOM, provenance). -Of particular interest: snapshot-store data exposure (secrets retained in checkpoints), interposition bypasses (guarded paths, dry-run escapes), and supply-chain issues in the adapters. +Of particular interest: snapshot-store data exposure (secrets retained in checkpoints), guarded fork interposition bypasses, and supply-chain issues in the adapters. ## Supported versions diff --git a/crates/acyclic-engine/Cargo.toml b/crates/acyclic-engine/Cargo.toml deleted file mode 100644 index 5bc2900..0000000 --- a/crates/acyclic-engine/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "acyclic-engine" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false -build = "build.rs" -description = "Rewind engine: capture pipeline, checkpoint index, rewind, and diff over acyclic-fs" - -[dependencies] -acyclic-fs.workspace = true -tokio = { version = "1.48", features = ["rt-multi-thread", "macros", "sync", "time"] } -rusqlite = { version = "0.40", features = ["bundled"] } -serde = { version = "1", features = ["derive"] } -serde_json = "1" -toml = "0.8" -thiserror = "2" -hex = "0.4" -blake3 = "1" -bytes = "1" -diffy = "0.4" - -[target.'cfg(unix)'.dependencies] -libc = "0.2" - -[build-dependencies] -toml = "0.8" - -[dev-dependencies] -tempfile = "3" - -[lints] -workspace = true diff --git a/crates/acyclic-engine/src/diff.rs b/crates/acyclic-engine/src/diff.rs deleted file mode 100644 index 8d5d978..0000000 --- a/crates/acyclic-engine/src/diff.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! Blast-radius diff between two generations, keyed by path. -//! -//! `Volume::diff_generations` returns FileId-keyed changes with no path -//! strings, so v1 walks both generations' directory records and compares -//! records per path. Content addressing makes the comparison exact: equal -//! payload object ids mean equal content. (Merkle-guided walking that skips -//! identical subtrees needs an upstream cursor/path API — tracked.) - -use std::collections::BTreeMap; -use std::path::PathBuf; - -use acyclic_fs::kernel::{FileKind, NamespacePath}; -use acyclic_fs::{CancellationToken, GenerationId, ObjectId, WorkCounters}; - -use crate::store::{LocalCheckout, Store}; -use crate::{EngineError, Result}; - -const PAGE_ENTRIES: u32 = 1_024; - -/// One changed path between two generations. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct FileChange { - pub path: PathBuf, - pub change: ChangeKind, - pub file_kind: FileKind, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ChangeKind { - Added, - Removed, - Modified, - MetadataOnly, -} - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) struct RecordSummary { - pub(crate) kind: FileKind, - /// Full payload for content comparison (inline bytes included). - /// `None` for directories: their payload changes with any descendant. - pub(crate) payload: Option, - pub(crate) metadata: ObjectId, -} - -impl RecordSummary { - /// Same kind and same content; metadata (mode, times) is ignored. - pub(crate) fn same_content(&self, other: &RecordSummary) -> bool { - self.kind == other.kind && self.payload == other.payload - } -} - -/// Computes the path-keyed diff `before → after`. -pub async fn diff( - store: &Store, - before: GenerationId, - after: GenerationId, -) -> Result> { - if before == after { - return Ok(Vec::new()); - } - let mut before_checkout = store.checkout_exact(before).await?; - let mut after_checkout = store.checkout_exact(after).await?; - let before_map = walk(&mut before_checkout).await?; - let after_map = walk(&mut after_checkout).await?; - - let mut changes = Vec::new(); - for (path, summary) in &before_map { - match after_map.get(path) { - None => changes.push(FileChange { - path: path.clone(), - change: ChangeKind::Removed, - file_kind: summary.kind, - }), - Some(other) if other == summary => {} - Some(other) => { - let change = if other.kind == summary.kind && other.payload == summary.payload { - ChangeKind::MetadataOnly - } else { - ChangeKind::Modified - }; - changes.push(FileChange { - path: path.clone(), - change, - file_kind: other.kind, - }); - } - } - } - for (path, summary) in &after_map { - if !before_map.contains_key(path) { - changes.push(FileChange { - path: path.clone(), - change: ChangeKind::Added, - file_kind: summary.kind, - }); - } - } - // Snapshots carry `.git` so rewind restores it, but a blast-radius - // report is about the working tree: object and ref churn from ordinary - // git commands would otherwise swamp the real changes. - changes.retain(|change| !is_git_internal(&change.path)); - changes.sort_by(|left, right| left.path.cmp(&right.path)); - Ok(changes) -} - -/// `.git` itself or anything beneath it, at the repo root only. -pub(crate) fn is_git_internal(path: &std::path::Path) -> bool { - path.components() - .next() - .is_some_and(|first| first.as_os_str() == ".git") -} - -/// Path → record summary of every entry in `generation` (directories included). -pub(crate) async fn summaries( - store: &Store, - generation: GenerationId, -) -> Result> { - let mut checkout = store.checkout_exact(generation).await?; - walk(&mut checkout).await -} - -/// Walks every directory record in a generation into path → record summary. -/// Directories themselves are included (metadata-only changes are visible). -async fn walk(checkout: &mut LocalCheckout) -> Result> { - let cancel = CancellationToken::new(); - // The volume's own limits, not the defaults: a store raises the - // per-component byte budget on hosts whose names cost more than one - // byte per character (see `names::maximum_component_bytes`), and a walk - // built on the default would refuse paths the store happily holds. - let limits = checkout.volume_config().limits; - let mut result = BTreeMap::new(); - // (namespace components, os path) work queue, starting at the root. - let mut queue: Vec<(Vec, PathBuf)> = - vec![(Vec::new(), PathBuf::new())]; - - while let Some((components, os_path)) = queue.pop() { - let directory = NamespacePath::new(components.clone(), limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?; - let mut after = None; - loop { - let page = checkout - .list_directory_records( - &directory, - after.as_ref(), - PAGE_ENTRIES, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("list directory"))? - .value; - for entry in &page.entries { - let child_os = os_path.join(logical_to_os(&entry.name)); - let payload = if entry.record.kind == FileKind::Directory { - None - } else { - Some(entry.record.payload) - }; - result.insert( - child_os.clone(), - RecordSummary { - kind: entry.record.kind, - payload, - metadata: entry.record.metadata, - }, - ); - if entry.record.kind == FileKind::Directory { - let mut child_components = components.clone(); - child_components.push(entry.name.clone()); - queue.push((child_components, child_os)); - } - } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, - } - } - } - Ok(result) -} - -fn logical_to_os(name: &acyclic_fs::kernel::LogicalName) -> std::ffi::OsString { - crate::names::bytes_to_os(name.as_bytes()) -} - -#[cfg(test)] -mod tests { - use super::is_git_internal; - use std::path::Path; - - #[test] - fn only_root_git_dir_is_internal() { - assert!(is_git_internal(Path::new(".git"))); - assert!(is_git_internal(Path::new(".git/HEAD"))); - assert!(is_git_internal(Path::new(".git/objects/ab/cd"))); - assert!(!is_git_internal(Path::new(".gitignore"))); - assert!(!is_git_internal(Path::new("src/.git/config"))); - assert!(!is_git_internal(Path::new("vendor/.gitkeep"))); - assert!(!is_git_internal(Path::new("a.txt"))); - } -} diff --git a/crates/acyclic-engine/src/exclude.rs b/crates/acyclic-engine/src/exclude.rs deleted file mode 100644 index bd22a7f..0000000 --- a/crates/acyclic-engine/src/exclude.rs +++ /dev/null @@ -1,414 +0,0 @@ -//! Snapshot exclusions: paths that never enter a checkpoint. -//! -//! The store deliberately captures what git ignores, so a declared secret -//! path (`.env`, `secrets/`) would otherwise live in history for longer than -//! it lives in the working tree. `exclude` in `.acyclic/config.toml` keeps -//! such paths out, enforced in three places: -//! -//! 1. Watcher hints at or under an excluded prefix are dropped before the -//! capture runs, so the ordinary per-tool-call path never reads them. -//! 2. After any capture that may have re-walked an excluded path (the full -//! baseline, or a hint on one of its ancestors), the path is scrubbed from -//! the checkout before the generation is checkpointed. -//! 3. A full rewind carries the live excluded paths into the restored tree: -//! no checkpoint holds them, so the working copy is the only copy. -//! -//! Exclusion is not purge. A generation captured before a path was excluded -//! still holds it, and at the pinned sdk revision the fs has no way to -//! release a retained generation, so nothing can be physically removed from -//! history. That gap is documented in docs/design/implementation-rewind.md. - -use std::ffi::OsString; -use std::path::{Component, Path, PathBuf}; - -use acyclic_fs::kernel::{FileKind, LogicalName, NamespacePath}; -use acyclic_fs::model::VolumeLimits; -use acyclic_fs::{CancellationToken, WatchBatch, WatchChange, WorkCounters}; - -use crate::store::LocalCheckout; -use crate::{EngineError, Result}; - -const PAGE_ENTRIES: u32 = 1_024; - -/// Parsed `exclude` rules: repo-relative path prefixes. A rule matches the -/// path itself and everything under it; `secrets` and `secrets/` are the -/// same rule. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Exclusions { - prefixes: Vec>>, -} - -impl Exclusions { - /// Parses the config list. Rules must be relative and stay inside the - /// repo; an empty rule or one naming the repo root is refused, since - /// excluding everything is the same as not running the engine. - pub fn parse(patterns: &[String]) -> Result { - let mut prefixes: Vec>> = Vec::new(); - for pattern in patterns { - let trimmed = pattern.trim().trim_end_matches('/'); - let mut components = Vec::new(); - for component in Path::new(trimmed).components() { - match component { - Component::Normal(name) => components.push(os_to_bytes(name)), - Component::CurDir => {} - _ => { - return Err(EngineError::Config(format!( - "exclude rule {pattern:?} must be a relative path inside the repo" - ))) - } - } - } - if components.is_empty() { - return Err(EngineError::Config(format!( - "exclude rule {pattern:?} would exclude the whole repo" - ))); - } - if !prefixes.contains(&components) { - prefixes.push(components); - } - } - Ok(Self { prefixes }) - } - - pub fn is_empty(&self) -> bool { - self.prefixes.is_empty() - } - - /// Every rule as a repo-relative host path. - pub fn host_paths(&self) -> Vec { - self.prefixes - .iter() - .map(|prefix| host_path(prefix)) - .collect() - } - - /// True when `relative` is an excluded path or lies under one. - pub fn covers_host(&self, relative: &Path) -> bool { - let mut components = Vec::new(); - for component in relative.components() { - match component { - Component::Normal(name) => components.push(os_to_bytes(name)), - Component::CurDir => {} - _ => return false, - } - } - self.covers_bytes(&components) - } - - /// True when `path` is an excluded path or lies under one. - pub fn covers(&self, path: &NamespacePath) -> bool { - let components: Vec> = path - .components() - .iter() - .map(|name| name.as_bytes().to_vec()) - .collect(); - self.covers_bytes(&components) - } - - fn covers_bytes(&self, components: &[Vec]) -> bool { - self.prefixes - .iter() - .any(|prefix| components.starts_with(prefix)) - } - - /// True when `path` is a strict ancestor of an excluded path (the repo - /// root included). A capture hinted at such a path may re-walk the - /// excluded subtree, so the checkout needs a scrub afterwards. - fn is_ancestor(&self, path: &NamespacePath) -> bool { - let components = path.components(); - self.prefixes.iter().any(|prefix| { - components.len() < prefix.len() - && components - .iter() - .zip(prefix.iter()) - .all(|(name, want)| name.as_bytes() == want.as_slice()) - }) - } - - /// Drops hints the capture must not read and rewrites renames that - /// cross the exclusion boundary so the uncovered side is re-examined. - /// Returns the batch and whether a scrub is needed after capturing it. - pub fn filter_batch(&self, batch: WatchBatch) -> (WatchBatch, bool) { - if self.is_empty() { - return (batch, false); - } - let WatchBatch::Changes { - epoch, - first_sequence, - next_sequence, - changes, - } = batch - else { - return (batch, false); - }; - let mut scrub = false; - let mut kept = Vec::with_capacity(changes.len()); - for change in changes { - match change { - WatchChange::Created(path) - | WatchChange::Modified(path) - | WatchChange::MetadataChanged(path) - | WatchChange::Removed(path) => { - if self.covers(&path) { - continue; - } - scrub |= self.is_ancestor(&path); - kept.push(WatchChange::Modified(path)); - } - WatchChange::Renamed { from, to } => match (self.covers(&from), self.covers(&to)) { - (true, true) => {} - (true, false) => { - scrub |= self.is_ancestor(&to); - kept.push(WatchChange::Modified(to)); - } - (false, true) => { - scrub |= self.is_ancestor(&from); - kept.push(WatchChange::Modified(from)); - } - (false, false) => { - scrub |= self.is_ancestor(&from) || self.is_ancestor(&to); - kept.push(WatchChange::Renamed { from, to }); - } - }, - } - } - ( - WatchBatch::Changes { - epoch, - first_sequence, - next_sequence, - changes: kept, - }, - scrub, - ) - } - - /// Removes every excluded path that is present in the checkout. Returns - /// how many rules had something to remove. - pub async fn scrub(&self, checkout: &mut LocalCheckout) -> Result { - if self.is_empty() { - return Ok(0); - } - let limits = checkout.volume_config().limits; - let cancel = CancellationToken::new(); - let mut removed = 0; - for prefix in &self.prefixes { - let names = logical_names(prefix, limits)?; - let path = namespace(names.clone(), limits)?; - let lookup = checkout - .lookup_no_follow(&path, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("exclusion lookup"))? - .value; - let Some(record) = lookup.record else { - continue; - }; - if record.kind == FileKind::Directory { - remove_subtree(checkout, names, limits, &cancel).await?; - } - checkout - .remove(path, None, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("exclusion remove"))?; - removed += 1; - } - Ok(removed) - } -} - -/// Post-order removal of a directory's contents (the fs refuses to unbind a -/// non-empty directory). The directory binding itself is left to the caller. -fn remove_subtree<'a>( - checkout: &'a mut LocalCheckout, - directory: Vec, - limits: VolumeLimits, - cancel: &'a CancellationToken, -) -> std::pin::Pin> + 'a>> { - Box::pin(async move { - let path = namespace(directory.clone(), limits)?; - let mut entries = Vec::new(); - let mut after = None; - loop { - let page = checkout - .list_directory_records( - &path, - after.as_ref(), - PAGE_ENTRIES, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("exclusion list"))? - .value; - for entry in &page.entries { - entries.push((entry.name.clone(), entry.record.kind)); - } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, - } - } - for (name, kind) in entries { - let mut child = directory.clone(); - child.push(name); - if kind == FileKind::Directory { - remove_subtree(checkout, child.clone(), limits, cancel).await?; - } - let child_path = namespace(child, limits)?; - checkout - .remove(child_path, None, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("exclusion remove"))?; - } - Ok(()) - }) -} - -fn logical_names(components: &[Vec], limits: VolumeLimits) -> Result> { - components - .iter() - .map(|bytes| { - LogicalName::new( - crate::names::encoding(), - bytes.clone(), - limits.maximum_component_bytes, - ) - .map_err(|error| EngineError::Config(format!("exclude rule component: {error:?}"))) - }) - .collect() -} - -fn namespace(names: Vec, limits: VolumeLimits) -> Result { - NamespacePath::new(names, limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) -} - -fn host_path(components: &[Vec]) -> PathBuf { - let mut path = PathBuf::new(); - for component in components { - path.push(bytes_to_os(component)); - } - path -} - -fn os_to_bytes(name: &std::ffi::OsStr) -> Vec { - crate::names::os_to_bytes(name) -} - -fn bytes_to_os(bytes: &[u8]) -> OsString { - crate::names::bytes_to_os(bytes) -} - -#[cfg(test)] -mod tests { - use super::*; - use acyclic_fs::{WatchEpoch, WatchSequence}; - - fn rules(list: &[&str]) -> Exclusions { - Exclusions::parse(&list.iter().map(|s| s.to_string()).collect::>()).expect("parse") - } - - fn ns(path: &str) -> NamespacePath { - let limits = VolumeLimits::default(); - let names = path - .split('/') - .filter(|part| !part.is_empty()) - .map(|part| { - LogicalName::new( - crate::names::encoding(), - crate::names::str_to_bytes(part), - limits.maximum_component_bytes, - ) - .expect("name") - }) - .collect(); - NamespacePath::new(names, limits).expect("path") - } - - fn batch(changes: Vec) -> WatchBatch { - WatchBatch::Changes { - epoch: WatchEpoch::from_u64(1), - first_sequence: WatchSequence::from_u64(1), - next_sequence: WatchSequence::from_u64(2), - changes, - } - } - - fn changes(batch: &WatchBatch) -> &[WatchChange] { - match batch { - WatchBatch::Changes { changes, .. } => changes, - WatchBatch::RescanRequired { .. } => panic!("rescan"), - } - } - - #[test] - fn rules_normalize_and_reject_escapes() { - let parsed = rules(&[".env", "secrets/", "./build/out/"]); - assert!(parsed.covers_host(Path::new(".env"))); - assert!(parsed.covers_host(Path::new("secrets/key.pem"))); - assert!(parsed.covers_host(Path::new("build/out"))); - assert!(!parsed.covers_host(Path::new("build"))); - assert!(!parsed.covers_host(Path::new(".env.example"))); - assert!(Exclusions::parse(&["../x".into()]).is_err()); - assert!(Exclusions::parse(&["/etc".into()]).is_err()); - assert!(Exclusions::parse(&["".into()]).is_err()); - assert!(Exclusions::parse(&["./".into()]).is_err()); - assert_eq!( - parsed.host_paths(), - vec![ - PathBuf::from(".env"), - PathBuf::from("secrets"), - PathBuf::from("build/out") - ] - ); - } - - #[test] - fn covered_hints_are_dropped_and_ancestors_demand_a_scrub() { - let parsed = rules(&["secrets"]); - let (filtered, scrub) = parsed.filter_batch(batch(vec![ - WatchChange::Created(ns("secrets/key.pem")), - WatchChange::Modified(ns("src/main.rs")), - ])); - assert_eq!( - changes(&filtered), - &[WatchChange::Modified(ns("src/main.rs"))] - ); - assert!(!scrub); - - let (filtered, scrub) = parsed.filter_batch(batch(vec![WatchChange::Modified(ns(""))])); - assert_eq!(changes(&filtered).len(), 1); - assert!(scrub, "a root hint may re-walk the excluded subtree"); - } - - #[test] - fn renames_across_the_boundary_reexamine_the_uncovered_side() { - let parsed = rules(&["secrets"]); - let (filtered, _) = parsed.filter_batch(batch(vec![WatchChange::Renamed { - from: ns("staging"), - to: ns("secrets"), - }])); - assert_eq!(changes(&filtered), &[WatchChange::Modified(ns("staging"))]); - - let (filtered, _) = parsed.filter_batch(batch(vec![WatchChange::Renamed { - from: ns("secrets"), - to: ns("public"), - }])); - assert_eq!(changes(&filtered), &[WatchChange::Modified(ns("public"))]); - - let (filtered, _) = parsed.filter_batch(batch(vec![WatchChange::Renamed { - from: ns("secrets/a"), - to: ns("secrets/b"), - }])); - assert!(changes(&filtered).is_empty()); - } - - #[test] - fn empty_rules_pass_batches_through_untouched() { - let parsed = rules(&[]); - let original = batch(vec![WatchChange::Created(ns("anything"))]); - let (filtered, scrub) = parsed.filter_batch(original.clone()); - assert_eq!(filtered, original); - assert!(!scrub); - } -} diff --git a/crates/acyclic-engine/src/fork.rs b/crates/acyclic-engine/src/fork.rs deleted file mode 100644 index a3df150..0000000 --- a/crates/acyclic-engine/src/fork.rs +++ /dev/null @@ -1,397 +0,0 @@ -//! Fork engine primitives (Launch 3). -//! -//! A fork is a writable overlay mount of a Head checkout: reads hydrate -//! lazily from the store (O(1) creation, `node_modules` included), writes -//! accumulate in that checkout's private overlay — invisible to the real -//! tree and to every other fork. The pipeline mints fork checkouts (it owns -//! the volume); the daemon owns the mount sessions (they must live in the -//! long-lived process). -//! -//! When the host has no mount provider (no usable `/dev/fuse` on Linux, or -//! loopback NFS blocked on macOS) a fork degrades to a *copy*: the base -//! generation is materialized into a real directory, and at promote time -//! that directory is captured back into the fork's overlay so the same -//! commit, conflict check, and swap run unchanged. The promise a fork makes -//! — a writable tree that never touches the real one until promoted — holds -//! either way; only the O(1) creation cost is lost. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use acyclic_fs::model::VolumeConfig; -use acyclic_fs::SharedCheckout; -use acyclic_fs::{ - capture_baseline, capture_root_identity, probe_native_mount, CancellationToken, CaptureOptions, - GenerationId, LocalAuthorityBackend, LocalObjectBackend, NativeMountKind, VolumeId, - WorkCounters, -}; - -use crate::{EngineError, Result}; - -/// How a fork is realized on this host. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum ForkMode { - /// Routed native mount: O(1) creation, lazy hydration. - Mount, - /// Materialized directory: full copy up front, captured back at promote. - Copy, -} - -impl ForkMode { - pub fn as_str(self) -> &'static str { - match self { - ForkMode::Mount => "mount", - ForkMode::Copy => "copy", - } - } -} - -/// What the host can do for fork and Safe Mode mounts, probed once at -/// daemon start and reported by `status`/`init`. -#[derive(Clone, Debug)] -pub struct MountCapability { - /// Human name of the provider this build would use ("fuse", "nfs loopback"). - pub provider: &'static str, - pub available: bool, - /// Why not, when unavailable. - pub reason: Option, -} - -impl MountCapability { - pub fn fork_mode(&self) -> ForkMode { - if self.available { - ForkMode::Mount - } else { - ForkMode::Copy - } - } -} - -/// Live probe of the native mount provider. -/// -/// Windows is held to copy forks whatever the probe says. `ProjFS` mounts -/// and projects correctly there — a fork's tree appears and reads back fine — -/// but writes into the projection stop at the `ProjFS` local cache and never -/// reach the overlay checkout, so `fork-diff` reports no changes and -/// `promote` lands nothing. Silently discarding a fork's work is far worse -/// than copying it, and copy forks are verified on Windows: write, diff and -/// promote all behave. -/// -/// Revisit when the sdk's `ProjFS` provider carries writes back. The probe -/// still runs so `status` can name the provider it found. -pub fn mount_capability() -> MountCapability { - let probe = probe_native_mount(); - let provider = match probe.kind { - Some(NativeMountKind::LinuxFuse) => "fuse", - Some(NativeMountKind::MacOsNfs) => "nfs loopback", - Some(NativeMountKind::WindowsProjFs) => "projfs", - None => "none", - }; - #[cfg(windows)] - { - let _ = probe.available; - MountCapability { - provider, - available: false, - reason: Some( - "ProjFS projects a fork but does not carry writes back to the store, \ - so forks use full copies (promote works the same)" - .to_owned(), - ), - } - } - #[cfg(not(windows))] - MountCapability { - provider, - available: probe.available, - reason: probe.unavailable_reason, - } -} - -/// Platform-specific instructions for making mounts available. Printed by -/// `init` and `install` when the probe fails, so a user learns on day one -/// rather than the day they first try a fork. -pub fn mount_setup_hint() -> &'static str { - if cfg!(target_os = "linux") { - concat!( - "forks will use full copies until /dev/fuse is usable. To enable mounts:\n", - " sudo modprobe fuse # load the kernel module\n", - " sudo usermod -aG fuse \"$USER\" # if /dev/fuse is group-restricted; log in again\n", - " docker run --device /dev/fuse --cap-add SYS_ADMIN ... # inside a container\n", - "Safe Mode (dry_run) needs mounts and refuses to start without them.", - ) - } else if cfg!(target_os = "macos") { - concat!( - "forks will use full copies: the built-in NFS mount tools (/sbin/mount_nfs, /sbin/umount)\n", - "are missing or blocked by policy. No extra software is needed on macOS;\n", - "ask your administrator to allow loopback NFS mounts.\n", - "Safe Mode (dry_run) needs mounts and refuses to start without them.", - ) - } else if cfg!(windows) { - concat!( - "forks use full copies on Windows. ProjFS can project a fork, but it does\n", - "not carry writes back to the store, so a mounted fork would silently lose\n", - "your work; copies land correctly through `promote`. Nothing to install.\n", - "Safe Mode (dry_run) needs mounts, so it is unavailable on Windows for the\n", - "same reason.", - ) - } else { - "native mounts are not supported on this platform; forks use full copies \ - and Safe Mode (dry_run) is unavailable." - } -} - -/// Root for copy-mode fork directories (a sibling of the mount root). -pub fn forks_copy_root(repo_root: &Path) -> Option { - Some(forks_root(repo_root)?.join("copy")) -} - -/// Captures the current content of `root` into a fork's overlay, so that -/// promote/resolve see it exactly as they would see writes through a mount. -/// The overlay must be pristine (a fresh fork seed): capture is a full-tree -/// baseline against the checkout, so only paths that actually differ from -/// the base become pending mutations. -pub async fn capture_copy(shared: &SharedLocalCheckout, root: &Path) -> Result<()> { - let options = CaptureOptions { - source_root: root.to_path_buf(), - expected_root_identity: capture_root_identity(root) - .map_err(EngineError::fs("copy root identity"))?, - maximum_paths: 4_000_000, - maximum_extent_spans: 65_536, - }; - let cancel = CancellationToken::new(); - let mut guard = shared.lock().await; - capture_baseline(&mut guard, &options, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("capture copy fork"))?; - Ok(()) -} - -/// The mount-safe checkout wrapper for the local backend. -pub type SharedLocalCheckout = SharedCheckout; - -/// What the pipeline hands the daemon for one new fork. -pub struct ForkSeed { - pub shared: Arc, - pub config: VolumeConfig, - pub volume_id: VolumeId, - /// Published head the fork was cut from; promote conflicts are judged - /// against movement past this point. - pub base: GenerationId, -} - -/// Result of a promote request. -#[derive(Clone, Debug)] -pub enum PromoteOutcome { - Promoted { - generation: GenerationId, - /// None when the fork had no writes (nothing to land). - old_tree: Option, - }, - /// The mainline moved past the fork's base — v1 surfaces the conflict - /// legibly instead of merging. - Conflict { message: String }, -} - -/// Result of the commit half of a Safe Mode session resolve (see -/// [`crate::pipeline::PipelineHandle::resolve_session`]) — the swap itself -/// is deferred to a separate `apply_session` call so the caller can show an -/// approval-gated diff in between. -#[derive(Clone, Debug)] -pub enum SessionResolveOutcome { - /// The overlay committed cleanly; `generation` is ready for - /// `apply_session`, or can simply be left unswapped (Safe Mode reject). - Resolved { generation: GenerationId }, - /// Nothing was written; there is nothing to diff or apply. - NoChanges, - /// The mainline moved past the fork's base — same v1 stance as promote. - Conflict { message: String }, -} - -/// Where a repo's fork workspaces live: a sibling of the repo, outside the -/// working tree so capture never sees them. -pub fn forks_root(repo_root: &Path) -> Option { - let parent = repo_root.parent()?; - let name = repo_root.file_name()?.to_string_lossy(); - Some(parent.join(format!(".{name}.forks"))) -} - -/// The single mountpoint projecting every fork as a routed subdirectory. -pub fn forks_mount_root(repo_root: &Path) -> Option { - Some(forks_root(repo_root)?.join("mnt")) -} - -/// Best-effort cleanup of fork dirs left by a dead daemon: unmount anything -/// still attached, then remove the directories. Mount sessions do not -/// survive the daemon in v1. -pub fn sweep_stale_forks(repo_root: &Path) { - let Some(root) = forks_root(repo_root) else { - return; - }; - let Ok(entries) = std::fs::read_dir(&root) else { - return; - }; - // FUSE-T's go-nfsv4 helpers outlive a killed daemon and wedge the - // vendor's tiny shared NFS port pool for every future mount on the - // host — reap any helper serving one of OUR workspaces first. - #[cfg(target_os = "macos")] - { - let _ = std::process::Command::new("pkill") - .arg("-f") - .arg(format!("go-nfsv4.*{}", root.display())) - .status(); - } - for entry in entries.flatten() { - let path = entry.path(); - #[cfg(target_os = "macos")] - let _ = std::process::Command::new("umount") - .arg("-f") - .arg(&path) - .status(); - #[cfg(target_os = "linux")] - { - let _ = std::process::Command::new("fusermount") - .arg("-u") - .arg(&path) - .status(); - } - let _ = std::fs::remove_dir_all(&path); - } - let _ = std::fs::remove_dir(&root); -} - -/// Best-effort cleanup of a Safe Mode shadow mount a dead daemon left -/// directly on `repo_root` itself. Unlike [`sweep_stale_forks`], the -/// directory is never removed here -- it IS the real repo -- only -/// force-unmounted so its real content reappears. A no-op if nothing is -/// mounted there. -pub fn sweep_stale_dry_session(repo_root: &Path) { - #[cfg(target_os = "macos")] - { - let _ = std::process::Command::new("umount") - .arg("-f") - .arg(repo_root) - .status(); - } - #[cfg(target_os = "linux")] - { - let _ = std::process::Command::new("fusermount") - .arg("-u") - .arg(repo_root) - .status(); - } - // A ProjFS virtualization root stops with the process that owned it, so - // a dead daemon leaves nothing mounted over the repo to reap. - #[cfg(windows)] - { - let _ = repo_root; - } -} - -/// Reaps a Safe Mode shadow left by a *crashed* daemon before the caller -/// touches `repo`. A dead daemon's shadow is a loopback-NFS/FUSE mountpoint -/// whose server is gone, so every `stat`/`open` under it (config load, path -/// canonicalization) blocks indefinitely — the CLI would hang before it -/// could even spawn a fresh daemon to clean up. -/// -/// Distinguishing a dead shadow from a live session (whose shadow is fine) -/// without a store/config lookup — which would itself stat `repo` — is done -/// by probing: a live server answers a `stat` immediately, while a dead one -/// either blocks or fails (the NFS layer surfaces `ETIMEDOUT`/`ENOTCONN`). -/// So this force-unmounts whenever a bounded probe does not cleanly succeed; -/// a healthy repo always stats OK and is never disturbed, and a `umount` of a -/// path that is not actually a mount is a harmless no-op. -pub fn reap_dead_shadow(repo: &Path) { - #[cfg(any(target_os = "macos", target_os = "linux"))] - { - use std::time::Duration; - // Resolve to an absolute mountpoint via the (always-live) parent, so - // the probe/unmount target is stable without stat-ing `repo` itself. - let target = match (repo.parent(), repo.file_name()) { - (Some(parent), Some(name)) => match parent.canonicalize() { - Ok(parent) => parent.join(name), - Err(_) => repo.to_path_buf(), - }, - _ => repo.to_path_buf(), - }; - let probe = target.clone(); - let (tx, rx) = std::sync::mpsc::channel(); - // Detached: if the stat is truly wedged it never returns, but the - // force-unmount below releases it and this short-lived CLI exits. - std::thread::spawn(move || { - let _ = tx.send(std::fs::metadata(&probe).is_ok()); - }); - let healthy = matches!(rx.recv_timeout(Duration::from_secs(5)), Ok(true)); - if !healthy { - #[cfg(target_os = "macos")] - let _ = std::process::Command::new("umount") - .arg("-f") - .arg(&target) - .status(); - #[cfg(target_os = "linux")] - let _ = std::process::Command::new("fusermount") - .arg("-u") - .arg(&target) - .status(); - } - } - // A ProjFS shadow dies with the daemon that projected it, so there is no - // wedged mountpoint to probe for and nothing to force-unmount. - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - let _ = repo; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn probe_names_a_provider_and_picks_a_mode() { - let capability = mount_capability(); - assert_ne!(capability.provider, ""); - assert_eq!(capability.available, capability.reason.is_none()); - assert_eq!( - capability.fork_mode(), - if capability.available { - ForkMode::Mount - } else { - ForkMode::Copy - } - ); - assert!(!mount_setup_hint().is_empty()); - } - - #[test] - fn copy_root_sits_beside_mount_root() { - let repo = Path::new("/work/my-repo"); - assert_eq!( - forks_copy_root(repo).expect("copy"), - Path::new("/work/.my-repo.forks/copy") - ); - assert_eq!( - forks_mount_root(repo).expect("mnt"), - Path::new("/work/.my-repo.forks/mnt") - ); - } - - #[test] - fn forks_root_is_a_hidden_sibling() { - let root = forks_root(Path::new("/work/my-repo")).expect("root"); - assert_eq!(root, Path::new("/work/.my-repo.forks")); - } - - #[test] - fn sweep_removes_stale_directories() { - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - std::fs::create_dir(&repo).expect("repo"); - let root = forks_root(&repo).expect("root"); - std::fs::create_dir_all(root.join("dead-fork")).expect("stale"); - std::fs::write(root.join("dead-fork/leftover"), b"x").expect("file"); - - sweep_stale_forks(&repo); - assert!(!root.exists()); - } -} diff --git a/crates/acyclic-engine/src/names.rs b/crates/acyclic-engine/src/names.rs deleted file mode 100644 index 8467f9f..0000000 --- a/crates/acyclic-engine/src/names.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! Host names ↔ engine names, in this platform's volume profile encoding. -//! -//! [`crate::store::volume_config`] picks a [`FilesystemProfile`] per platform, -//! and every layer that names a file has to agree with it: capture and -//! restore, the exclusion rules, the diff walk, and the fork mount router. -//! The mount router is the unforgiving one — the `ProjFS` provider decodes -//! every entry name it is handed as UTF-16LE, so a name that reaches it in -//! any other encoding is *projected as mojibake* rather than rejected. -//! -//! The byte form of a name is therefore platform-defined, and this module is -//! the only place that defines it: raw bytes on Unix, UTF-16LE on Windows. -//! Everywhere else in the engine a name is opaque bytes, compared and stored -//! but never interpreted. Anything that mints name bytes from host paths or -//! from configured text must come through here, or it will disagree with the -//! volume on the first non-ASCII name — and on Windows, on every name. - -use std::ffi::{OsStr, OsString}; - -use acyclic_fs::kernel::NameEncoding; -use acyclic_fs::model::FilesystemProfile; - -/// The volume profile for this host. -#[must_use] -pub const fn profile() -> FilesystemProfile { - #[cfg(windows)] - { - FilesystemProfile::Windows - } - #[cfg(unix)] - { - FilesystemProfile::Posix - } - #[cfg(not(any(unix, windows)))] - { - FilesystemProfile::Portable - } -} - -/// The name encoding that matches [`profile`]. -#[must_use] -pub const fn encoding() -> NameEncoding { - #[cfg(windows)] - { - NameEncoding::WindowsUtf16Le - } - #[cfg(unix)] - { - NameEncoding::PosixBytes - } - #[cfg(not(any(unix, windows)))] - { - NameEncoding::Utf8 - } -} - -/// Upper bound on one name component, in the bytes [`encoding`] produces. -/// -/// Both families cap a component at 255 *characters*; UTF-16LE spends two -/// bytes per unit, so the byte budget has to double on Windows or a legal -/// 200-character filename is refused as too long. -#[must_use] -pub const fn maximum_component_bytes() -> u32 { - #[cfg(windows)] - { - 510 - } - #[cfg(not(windows))] - { - 255 - } -} - -/// A host name in this platform's engine byte form. -/// -/// Lossless in both directions: Unix names are arbitrary bytes and Windows -/// names are arbitrary UTF-16, and neither is forced through UTF-8 on the -/// way past. [`bytes_to_os`] is the exact inverse. -#[cfg(unix)] -#[must_use] -pub fn os_to_bytes(name: &OsStr) -> Vec { - use std::os::unix::ffi::OsStrExt; - name.as_bytes().to_vec() -} - -#[cfg(windows)] -#[must_use] -pub fn os_to_bytes(name: &OsStr) -> Vec { - use std::os::windows::ffi::OsStrExt; - name.encode_wide().flat_map(u16::to_le_bytes).collect() -} - -#[cfg(not(any(unix, windows)))] -#[must_use] -pub fn os_to_bytes(name: &OsStr) -> Vec { - name.to_string_lossy().into_owned().into_bytes() -} - -/// Inverse of [`os_to_bytes`]. -#[cfg(unix)] -#[must_use] -pub fn bytes_to_os(bytes: &[u8]) -> OsString { - use std::os::unix::ffi::OsStringExt; - OsString::from_vec(bytes.to_vec()) -} - -/// Inverse of [`os_to_bytes`]. A trailing odd byte cannot occur in a name -/// this module produced, and is dropped rather than failing the whole walk. -#[cfg(windows)] -#[must_use] -pub fn bytes_to_os(bytes: &[u8]) -> OsString { - use std::os::windows::ffi::OsStringExt; - let (pairs, _odd_trailing_byte) = bytes.as_chunks::<2>(); - let units: Vec = pairs.iter().copied().map(u16::from_le_bytes).collect(); - OsString::from_wide(&units) -} - -#[cfg(not(any(unix, windows)))] -#[must_use] -pub fn bytes_to_os(bytes: &[u8]) -> OsString { - String::from_utf8_lossy(bytes).into_owned().into() -} - -/// UTF-8 text as engine name bytes: configured exclude rules and the fork -/// route ids the mount router projects as directory names. -#[must_use] -pub fn str_to_bytes(text: &str) -> Vec { - os_to_bytes(OsStr::new(text)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn os_bytes_round_trip_through_the_host_encoding() { - for name in ["a.txt", "sub", "ünïcøde", "日本語", ".git"] { - let bytes = os_to_bytes(OsStr::new(name)); - assert_eq!(bytes_to_os(&bytes), OsString::from(name), "{name}"); - } - } - - #[test] - fn str_bytes_agree_with_os_bytes() { - assert_eq!(str_to_bytes("fork-id"), os_to_bytes(OsStr::new("fork-id"))); - } - - /// The encoding and the profile are one decision; a host that reported - /// mismatched halves would capture names the mount layer cannot project. - #[test] - fn encoding_matches_profile() { - let expected = match profile() { - FilesystemProfile::Posix => NameEncoding::PosixBytes, - FilesystemProfile::Windows => NameEncoding::WindowsUtf16Le, - FilesystemProfile::Portable | FilesystemProfile::Browser => NameEncoding::Utf8, - }; - assert_eq!(encoding(), expected); - } - - /// An ASCII name costs one byte per character on Unix and two on - /// Windows; the budget has to cover 255 characters either way. - #[test] - fn component_budget_covers_a_maximal_host_name() { - let longest = "x".repeat(255); - let bytes = os_to_bytes(OsStr::new(&longest)); - assert!( - u32::try_from(bytes.len()).is_ok_and(|len| len <= maximum_component_bytes()), - "255-character name needs {} bytes, budget is {}", - bytes.len(), - maximum_component_bytes() - ); - } -} diff --git a/crates/acyclic-engine/src/rewind.rs b/crates/acyclic-engine/src/rewind.rs deleted file mode 100644 index c00550b..0000000 --- a/crates/acyclic-engine/src/rewind.rs +++ /dev/null @@ -1,1061 +0,0 @@ -//! Full-tree rewind: materialize the target generation into a sibling temp -//! directory, atomically exchange it with the working tree, keep the old tree -//! in trash, and journal every phase so kill -9 leaves the repo fully-old or -//! fully-new — never mixed. - -use std::path::{Component, Path, PathBuf}; - -#[cfg(unix)] -use acyclic_fs::kernel::MetadataField; -use acyclic_fs::kernel::{FileKind, FilePayload, LogicalName, NamespacePath}; -use acyclic_fs::{materialize_checkout, ByteRange, MaterializeOptions}; -use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; -use serde::{Deserialize, Serialize}; - -use crate::exclude::Exclusions; -use crate::store::{LocalCheckout, Store}; -use crate::{EngineError, Result}; - -const MAXIMUM_DIRECTORY_ENTRIES: u32 = 1_024; -const MAXIMUM_EXTENT_SPANS: u32 = 65_536; -const TRANSFER_BYTES: u64 = 8 * 1024 * 1024; - -/// What a completed rewind reports back. -#[derive(Clone, Debug)] -pub struct RewindOutcome { - pub restored: GenerationId, - /// Where the replaced tree went (trash, TTL-pruned). - pub old_tree: PathBuf, - /// User-facing caveat: open editors keep inodes from the old tree. - pub warning: &'static str, -} - -/// What a single-path restore did to the working tree. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum RestoreAction { - /// The path now matches the checkpoint's content. - Restored, - /// The path was absent at the checkpoint and has been removed. - Removed, -} - -/// Result of [`restore_path`]. -#[derive(Clone, Debug)] -pub struct RestoreOutcome { - pub path: PathBuf, - pub action: RestoreAction, -} - -/// Restores ONE path (file, symlink, or directory subtree) from `target` -/// into the working tree, leaving every other path untouched. The content -/// is staged in a hidden sibling and swapped in atomically (directory -/// subtrees use the same exchange as a full rewind), so a crash leaves the -/// path either old or new. Called from the pipeline, which brackets it with -/// checkpoints so the timeline records the restore. -pub async fn restore_path( - store: &Store, - target: GenerationId, - relative: &Path, -) -> Result { - let root = store.repo_root.clone(); - restore_path_into(store, target, &root, relative).await -} - -/// [`restore_path`] against an arbitrary root directory instead of the -/// working tree: a copy-mode fork's directory during a rebase. -pub async fn restore_path_into( - store: &Store, - target: GenerationId, - root: &Path, - relative: &Path, -) -> Result { - let components = validate_relative(relative)?; - let destination = root.join(relative); - let parent = destination - .parent() - .ok_or_else(|| EngineError::Restore("path has no parent".into()))?; - let name = destination - .file_name() - .ok_or_else(|| EngineError::Restore("path has no name".into()))? - .to_string_lossy() - .into_owned(); - - let mut checkout = store.checkout_exact(target).await?; - let limits = checkout.volume_config().limits; - let cancel = CancellationToken::new(); - let namespace = namespace_path(&components, limits)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - - let Some(record) = lookup.record else { - // Faithful restore of an absent path: remove it if it exists now. - return match std::fs::symlink_metadata(&destination) { - Ok(metadata) => { - if metadata.is_dir() { - std::fs::remove_dir_all(&destination)?; - } else { - std::fs::remove_file(&destination)?; - } - Ok(RestoreOutcome { - path: relative.to_path_buf(), - action: RestoreAction::Removed, - }) - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - Err(EngineError::Restore(format!( - "{} does not exist at that checkpoint or in the tree", - relative.display() - ))) - } - Err(error) => Err(error.into()), - }; - }; - - // Stage next to the destination so the final rename never crosses a - // filesystem; the parent must exist (it did at the checkpoint, but the - // tree may have lost it since). - std::fs::create_dir_all(parent)?; - let staged = parent.join(format!( - ".{name}.{}-restore-{}", - crate::product::NAME, - std::process::id() - )); - let _ = remove_any(&staged); - let written = write_node( - &mut checkout, - &namespace, - record.kind, - &record.payload, - &staged, - limits, - &cancel, - ) - .await; - if let Err(error) = written { - let _ = remove_any(&staged); - return Err(error); - } - - // Swap in. An existing destination is exchanged atomically and the old - // node discarded; an absent one is a plain rename. - match std::fs::symlink_metadata(&destination) { - Ok(_) => { - if let Err(error) = atomic_exchange(&destination, &staged) { - let _ = remove_any(&staged); - return Err(error); - } - let _ = remove_any(&staged); - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - if let Err(error) = std::fs::rename(&staged, &destination) { - let _ = remove_any(&staged); - return Err(error.into()); - } - } - Err(error) => { - let _ = remove_any(&staged); - return Err(error.into()); - } - } - Ok(RestoreOutcome { - path: relative.to_path_buf(), - action: RestoreAction::Restored, - }) -} - -/// Writes one checkpoint node (recursively for directories) to a fresh host -/// path. Modes are applied; mtimes are not (same contract as a full rewind). -pub(crate) fn write_node<'a>( - checkout: &'a mut LocalCheckout, - namespace: &'a NamespacePath, - kind: FileKind, - payload: &'a FilePayload, - host: &'a Path, - limits: acyclic_fs::model::VolumeLimits, - cancel: &'a CancellationToken, -) -> std::pin::Pin> + 'a>> { - Box::pin(async move { - match kind { - FileKind::Regular => { - let length = match payload { - FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, - FilePayload::Regular { logical_bytes, .. } => *logical_bytes, - _ => { - return Err(EngineError::Restore( - "regular file with foreign payload".into(), - )) - } - }; - let mut file = std::fs::File::create(host)?; - let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); - let mut offset = 0; - while offset < length { - let take = chunk.min(length - offset); - let read = checkout - .read_file_range( - namespace, - ByteRange { - offset, - length: take, - }, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("read file range"))? - .value; - use std::io::Write; - file.write_all(&read.bytes)?; - offset += take; - } - file.sync_all()?; - drop(file); - apply_mode(checkout, namespace, host, cancel).await - } - FileKind::SymbolicLink => { - let target = checkout - .read_symbolic_link(namespace, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("read symlink"))? - .value; - create_symlink(&target, host) - } - FileKind::Directory => { - std::fs::create_dir(host)?; - let mut entries = Vec::new(); - let mut after = None; - loop { - let page = checkout - .list_directory_records( - namespace, - after.as_ref(), - MAXIMUM_DIRECTORY_ENTRIES, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("list directory"))? - .value; - for entry in &page.entries { - entries.push((entry.name.clone(), entry.record.kind, entry.record.payload)); - } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, - } - } - for (name, kind, payload) in entries { - let mut components = namespace.components().to_vec(); - components.push(name.clone()); - let child = NamespacePath::new(components, limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?; - let child_host = host.join(logical_to_os(&name)); - write_node( - checkout, - &child, - kind, - &payload, - &child_host, - limits, - cancel, - ) - .await?; - } - apply_mode(checkout, namespace, host, cancel).await - } - other => Err(EngineError::Restore(format!( - "cannot restore a {other:?} node (only files, symlinks, and directories)" - ))), - } - }) -} - -async fn apply_mode( - checkout: &mut LocalCheckout, - namespace: &NamespacePath, - host: &Path, - cancel: &CancellationToken, -) -> Result<()> { - let metadata = checkout - .read_metadata(namespace, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("read metadata"))? - .value; - #[cfg(unix)] - if let MetadataField::Value(mode) = metadata.posix_mode { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(host, std::fs::Permissions::from_mode(mode & 0o7777))?; - } - #[cfg(not(unix))] - let _ = (metadata, host); - Ok(()) -} - -pub(crate) fn validate_relative(relative: &Path) -> Result>> { - let mut components = Vec::new(); - for component in relative.components() { - match component { - Component::Normal(name) => components.push(os_to_bytes(name)), - Component::CurDir => {} - _ => { - return Err(EngineError::Restore(format!( - "{}: path must be relative to the repo root and stay inside it", - relative.display() - ))) - } - } - } - if components.is_empty() { - return Err(EngineError::Restore(format!( - "restoring the whole tree is `{} rewind`, not a path restore", - crate::product::NAME - ))); - } - Ok(components) -} - -pub(crate) fn namespace_path( - components: &[Vec], - limits: acyclic_fs::model::VolumeLimits, -) -> Result { - let names = components - .iter() - .map(|bytes| { - LogicalName::new( - crate::names::encoding(), - bytes.clone(), - limits.maximum_component_bytes, - ) - .map_err(|error| EngineError::Restore(format!("bad path component: {error:?}"))) - }) - .collect::>>()?; - NamespacePath::new(names, limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) -} - -pub(crate) fn remove_any(path: &Path) -> std::io::Result<()> { - match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path), - Ok(_) => std::fs::remove_file(path), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error), - } -} - -fn os_to_bytes(name: &std::ffi::OsStr) -> Vec { - crate::names::os_to_bytes(name) -} - -fn logical_to_os(name: &LogicalName) -> std::ffi::OsString { - crate::names::bytes_to_os(name.as_bytes()) -} - -#[cfg(unix)] -fn create_symlink(target: &[u8], host: &Path) -> Result<()> { - use std::os::unix::ffi::OsStrExt; - std::os::unix::fs::symlink(std::ffi::OsStr::from_bytes(target), host)?; - Ok(()) -} - -#[cfg(not(unix))] -fn create_symlink(_target: &[u8], _host: &Path) -> Result<()> { - Err(EngineError::Restore("symlink restore is unix-only".into())) -} - -/// Crash-recovery journal. Present on disk only while a swap is in flight. -#[derive(Debug, Serialize, Deserialize)] -pub struct Journal { - pub target_generation: String, - pub repo_root: PathBuf, - pub tmp: PathBuf, - pub phase: Phase, - /// Excluded paths (repo-relative) moved from the live tree into `tmp` - /// before the swap. Absent in journals written before exclusions. - #[serde(default)] - pub carried: Vec, -} - -/// Moves each `relative` path from `from` to `into`, replacing whatever the -/// materialized tree had there (the live copy wins). Stops at the first -/// failure; the caller unwinds with [`move_back`]. -fn carry(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { - for path in relative { - let source = from.join(path); - let destination = into.join(path); - if std::fs::symlink_metadata(&source).is_err() { - continue; - } - if let Some(parent) = destination.parent() { - std::fs::create_dir_all(parent)?; - } - let _ = remove_any(&destination); - std::fs::rename(&source, &destination).map_err(|error| { - EngineError::Restore(format!("carry excluded path {}: {error}", path.display())) - })?; - } - Ok(()) -} - -/// Best-effort inverse of [`carry`]: every listed path present in `from` -/// and absent in `into` goes back. Used by crash recovery, where the only -/// wrong answer is losing a live copy. -fn move_back(from: &Path, into: &Path, relative: &[PathBuf]) { - for path in relative { - let source = from.join(path); - let destination = into.join(path); - if std::fs::symlink_metadata(&source).is_err() - || std::fs::symlink_metadata(&destination).is_ok() - { - continue; - } - if let Some(parent) = destination.parent() { - let _ = std::fs::create_dir_all(parent); - } - let _ = std::fs::rename(&source, &destination); - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub enum Phase { - /// Materializing into tmp; repo untouched. Recovery: delete tmp. - Materializing, - /// Excluded paths moving from repo into tmp. Recovery: move back any - /// that already moved, then delete tmp. - Carrying, - /// Atomic exchange in flight (or two-step: repo moved aside to tmp2). - /// Recovery: if repo missing, move tmp into place; else delete tmp. - Swapping, -} - -/// Materializes `generation` into `destination`, which must not exist yet. -/// Used for copy-mode forks; a rewind does the same into its temp sibling. -pub async fn materialize_into( - store: &Store, - generation: GenerationId, - destination: &Path, -) -> Result<()> { - std::fs::create_dir(destination)?; - let mut checkout = store.checkout_exact(generation).await?; - let cancel = CancellationToken::new(); - materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: destination.to_path_buf(), - maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, - maximum_extent_spans: MAXIMUM_EXTENT_SPANS, - transfer_bytes: TRANSFER_BYTES, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(|error| { - let _ = std::fs::remove_dir_all(destination); - EngineError::Restore(format!("materialize: {error:?}")) - })?; - Ok(()) -} - -/// Executes a rewind against the store's repo. Called from the pipeline with -/// captures paused; the caller re-baselines afterwards. Excluded paths are -/// carried from the live tree into the restored one: no checkpoint holds -/// them, so the working copy is the only copy. -pub async fn execute( - store: &Store, - target: GenerationId, - trash_ttl_days: u32, - exclusions: &Exclusions, -) -> Result { - let repo = &store.repo_root; - let parent = repo - .parent() - .ok_or_else(|| EngineError::Restore("repo root has no parent".into()))?; - let name = repo - .file_name() - .ok_or_else(|| EngineError::Restore("repo root has no name".into()))? - .to_string_lossy() - .into_owned(); - let nonce = std::process::id(); - let tmp = parent.join(format!(".{name}.{}-tmp-{nonce}", crate::product::NAME)); - let journal_path = store.paths.rewind_journal(); - - // 1. Materialize the target into an empty sibling directory. A tmp left - // by an earlier attempt that failed before the swap is stale by - // construction -- the name carries this daemon's pid, and a rewind that - // got as far as the swap removes it -- so clear it rather than refusing - // every later rewind with "already exists". - let _ = remove_any(&tmp); - std::fs::create_dir(&tmp).map_err(|error| { - EngineError::Restore(format!("rewind: stage {}: {error}", tmp.display())) - })?; - write_journal( - &journal_path, - &Journal { - target_generation: hex::encode(target.digest().as_bytes()), - repo_root: repo.clone(), - tmp: tmp.clone(), - phase: Phase::Materializing, - carried: Vec::new(), - }, - )?; - let mut checkout = store.checkout_exact(target).await?; - let cancel = CancellationToken::new(); - materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: tmp.clone(), - maximum_directory_entries: MAXIMUM_DIRECTORY_ENTRIES, - maximum_extent_spans: MAXIMUM_EXTENT_SPANS, - transfer_bytes: TRANSFER_BYTES, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(|error| { - let _ = std::fs::remove_dir_all(&tmp); - let _ = std::fs::remove_file(&journal_path); - EngineError::Restore(format!("materialize: {error:?}")) - })?; - - // 2. Carry the live excluded paths into the new tree. Journaled first, - // so a crash mid-carry can move them back. - let carried: Vec = exclusions - .host_paths() - .into_iter() - .filter(|relative| std::fs::symlink_metadata(repo.join(relative)).is_ok()) - .collect(); - if !carried.is_empty() { - write_journal( - &journal_path, - &Journal { - target_generation: hex::encode(target.digest().as_bytes()), - repo_root: repo.clone(), - tmp: tmp.clone(), - phase: Phase::Carrying, - carried: carried.clone(), - }, - )?; - if let Err(error) = carry(repo, &tmp, &carried) { - // Undo what moved, then abandon the rewind with the repo whole. - move_back(&tmp, repo, &carried); - let _ = std::fs::remove_dir_all(&tmp); - let _ = std::fs::remove_file(&journal_path); - return Err(error); - } - } - - // 3. Atomic exchange: repo <-> tmp. After this the old tree is at `tmp`. - write_journal( - &journal_path, - &Journal { - target_generation: hex::encode(target.digest().as_bytes()), - repo_root: repo.clone(), - tmp: tmp.clone(), - phase: Phase::Swapping, - carried, - }, - )?; - atomic_exchange(repo, &tmp)?; - - // 4. Old tree to trash (best effort: EXDEV falls back to a sibling path). - let trash_root = store.paths.trash(); - let old_tree = park_replaced_tree(&tmp, &trash_root, parent, &name)?; - std::fs::remove_file(&journal_path) - .map_err(|error| EngineError::Restore(format!("rewind: clear journal: {error}")))?; - prune_trash(&trash_root, trash_ttl_days); - - Ok(RewindOutcome { - restored: target, - old_tree, - warning: "reload your editor: open files still point at the replaced tree", - }) -} - -/// Moves the replaced tree out of the way and returns where it landed. -/// -/// The trash lives in the store, which can be on another volume than the -/// repo; a cross-device rename fails rather than copying, so a sibling of -/// the repo is the fallback. Either way the tree is off the repo path. -fn park_replaced_tree(tmp: &Path, trash_root: &Path, parent: &Path, name: &str) -> Result { - let stamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |duration| duration.as_secs()); - let trashed = trash_root.join(format!("{name}-{stamp}")); - if std::fs::rename(tmp, &trashed).is_ok() { - return Ok(trashed); - } - let sibling = parent.join(format!(".{name}.{}-trash-{stamp}", crate::product::NAME)); - std::fs::rename(tmp, &sibling).map_err(|error| { - EngineError::Restore(format!( - "rewind: park the replaced tree at {}: {error}", - sibling.display() - )) - })?; - Ok(sibling) -} - -/// Startup crash recovery. Reads the journal (if any) and finishes or unwinds -/// the interrupted rewind so the repo is whole before the pipeline baselines. -pub fn recover(journal_path: &Path) -> Result> { - let text = match std::fs::read_to_string(journal_path) { - Ok(text) => text, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(error) => return Err(error.into()), - }; - let journal: Journal = serde_json::from_str(&text) - .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; - match journal.phase { - Phase::Materializing => { - // Repo untouched; the partial tmp tree is garbage. - let _ = std::fs::remove_dir_all(&journal.tmp); - } - Phase::Carrying => { - // Some excluded paths may already sit in tmp: bring them home, - // then drop the unused new tree. - move_back(&journal.tmp, &journal.repo_root, &journal.carried); - let _ = std::fs::remove_dir_all(&journal.tmp); - } - Phase::Swapping => { - // Windows swaps through a scratch name (see `atomic_exchange`); - // whichever step it died on, the scratch holds a tree that one - // of the branches below is about to supersede. - #[cfg(windows)] - let scratch = swap_scratch(&journal.repo_root); - if !journal.repo_root.exists() && journal.tmp.exists() { - // The swap died with the repo path vacated and the new tree - // still parked at tmp: finish the move. The carried paths are - // inside tmp and come along. - std::fs::rename(&journal.tmp, &journal.repo_root)?; - } else { - // The repo path is whole, so it names exactly one tree and - // tmp holds the other: the old tree (swap done — keep it out - // of the way) or the unused new tree (swap never happened). - // Carried paths live in whichever tree is new: if that is - // still tmp, they must come back before tmp goes. - move_back(&journal.tmp, &journal.repo_root, &journal.carried); - let _ = std::fs::remove_dir_all(&journal.tmp); - } - // Whatever the scratch still holds has now been superseded by the - // branch above, exactly as `tmp` is discarded there. - #[cfg(windows)] - if let Some(scratch) = scratch { - let _ = remove_any(&scratch); - } - } - } - std::fs::remove_file(journal_path)?; - Ok(Some(journal)) -} - -fn write_journal(path: &Path, journal: &Journal) -> Result<()> { - let text = serde_json::to_string(journal) - .map_err(|error| EngineError::Restore(format!("encode journal: {error}")))?; - let tmp = path.with_extension("tmp"); - // Durability before visibility: the journal only helps if it is on the - // platter before the phase it describes begins. - // - // One writable handle carries all of it. `sync_all` is a `FlushFileBuffers` - // on Windows, which needs write access -- flushing a handle from - // `File::open` fails there with "access is denied" -- and the handle has - // to be closed before the rename, because Windows will not rename a file - // anyone still holds open. - let write = || -> std::io::Result<()> { - use std::io::Write; - let mut file = std::fs::File::create(&tmp)?; - file.write_all(text.as_bytes())?; - file.sync_all()?; - drop(file); - std::fs::rename(&tmp, path) - }; - write().map_err(|error| { - let _ = std::fs::remove_file(&tmp); - EngineError::Restore(format!("rewind: journal {}: {error}", path.display())) - })?; - Ok(()) -} - -fn prune_trash(trash_root: &Path, ttl_days: u32) { - let Ok(entries) = std::fs::read_dir(trash_root) else { - return; - }; - let ttl = std::time::Duration::from_secs(u64::from(ttl_days) * 24 * 3600); - for entry in entries.flatten() { - let Ok(metadata) = entry.metadata() else { - continue; - }; - let Ok(modified) = metadata.modified() else { - continue; - }; - if modified.elapsed().is_ok_and(|age| age > ttl) { - let _ = std::fs::remove_dir_all(entry.path()); - } - } -} - -/// The scratch name [`atomic_exchange`] swaps `a` through on Windows. -/// -/// Derived from `a` rather than randomised so that [`recover`] can name it -/// without the journal having carried it, and so a crashed swap leaves at -/// most one predictable directory behind instead of one per attempt. -#[cfg(windows)] -pub(crate) fn swap_scratch(a: &Path) -> Option { - let parent = a.parent()?; - let name = a.file_name()?.to_string_lossy().into_owned(); - Some(parent.join(format!(".{name}.{}-swap", crate::product::NAME))) -} - -/// Exchanges two directories on the same filesystem. -/// -/// **Not atomic on Windows.** There is no `RENAME_EXCHANGE` equivalent: NTFS -/// cannot swap two names in one operation, so this is three renames through -/// a scratch name in `a`'s directory. Each rename is atomic; the sequence is -/// not, and a crash can be observed between any two of them. -/// -/// What still holds is the property rewind actually needs — the repo is -/// never a mixture of the two trees. Every intermediate state has the repo -/// path either absent or naming exactly one whole tree, and [`recover`] -/// resolves each of them from the journal: the `Swapping` arm finishes the -/// move when the repo path is missing, and clears the scratch either way. -/// A crash during a journal-less [`restore_path`] leaves the same scratch -/// behind, which the next swap of that path removes before it starts. -#[cfg(windows)] -fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { - let scratch = - swap_scratch(a).ok_or_else(|| EngineError::Restore("swap path has no parent".into()))?; - // A scratch left by an interrupted swap is stale by construction: the - // recovery below never keeps it, so anything still here predates us. - let _ = remove_any(&scratch); - - std::fs::rename(a, &scratch).map_err(|error| { - EngineError::Restore(format!("swap: move aside {}: {error}", a.display())) - })?; - if let Err(error) = std::fs::rename(b, a) { - // Nothing has been published yet; put `a` back and fail clean. - let _ = std::fs::rename(&scratch, a); - return Err(EngineError::Restore(format!( - "swap: move {} into place: {error}", - b.display() - ))); - } - std::fs::rename(&scratch, b).map_err(|error| { - // `a` already holds the new tree, so the exchange has effectively - // happened; only the old tree's parking spot is wrong. Leave the - // scratch for recovery rather than unwinding a published swap. - EngineError::Restore(format!("swap: park the replaced tree: {error}")) - }) -} - -/// Atomically exchanges two directories on the same filesystem. -#[cfg(target_os = "macos")] -#[allow( - unsafe_code, - reason = "renamex_np over two live NUL-terminated paths; RENAME_SWAP is atomic on APFS" -)] -fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { - use std::os::unix::ffi::OsStrExt; - let a_c = std::ffi::CString::new(a.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - let b_c = std::ffi::CString::new(b.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - // SAFETY: both are live NUL-terminated paths; RENAME_SWAP exchanges them - // atomically on APFS. - let result = unsafe { libc::renamex_np(a_c.as_ptr(), b_c.as_ptr(), libc::RENAME_SWAP) }; - if result == 0 { - Ok(()) - } else { - Err(EngineError::Restore(format!( - "renamex_np: {}", - std::io::Error::last_os_error() - ))) - } -} - -#[cfg(target_os = "linux")] -#[allow( - unsafe_code, - reason = "renameat2 over two live NUL-terminated paths; RENAME_EXCHANGE is atomic where supported" -)] -fn atomic_exchange(a: &Path, b: &Path) -> Result<()> { - use std::os::unix::ffi::OsStrExt; - let a_c = std::ffi::CString::new(a.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - let b_c = std::ffi::CString::new(b.as_os_str().as_bytes()) - .map_err(|_| EngineError::Restore("path contains NUL".into()))?; - // SAFETY: both are live NUL-terminated paths; RENAME_EXCHANGE swaps them - // atomically on filesystems that support it. - let result = unsafe { - libc::syscall( - libc::SYS_renameat2, - libc::AT_FDCWD, - a_c.as_ptr(), - libc::AT_FDCWD, - b_c.as_ptr(), - libc::RENAME_EXCHANGE, - ) - }; - if result == 0 { - Ok(()) - } else { - Err(EngineError::Restore(format!( - "renameat2: {}", - std::io::Error::last_os_error() - ))) - } -} -#[cfg(test)] -mod tests { - use super::*; - - fn journal(repo: &Path, tmp: &Path, phase: Phase) -> Journal { - Journal { - target_generation: "00".repeat(32), - repo_root: repo.to_path_buf(), - tmp: tmp.to_path_buf(), - phase, - carried: Vec::new(), - } - } - - fn write(path: &Path, value: &Journal) { - std::fs::write(path, serde_json::to_string(value).expect("encode")).expect("write"); - } - - #[test] - fn recover_finishes_interrupted_two_step_swap() { - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - let tmp = work.path().join("repo.tmp"); - std::fs::create_dir(&tmp).expect("tmp"); - std::fs::write(tmp.join("file.txt"), b"restored").expect("seed"); - let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); - - recover(&journal_path).expect("recover"); - assert_eq!( - std::fs::read(repo.join("file.txt")).expect("read"), - b"restored" - ); - assert!(!tmp.exists()); - assert!(!journal_path.exists()); - } - - #[test] - fn recover_discards_partial_materialization() { - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - std::fs::create_dir(&repo).expect("repo"); - std::fs::write(repo.join("keep.txt"), b"live").expect("seed"); - let tmp = work.path().join("repo.tmp"); - std::fs::create_dir(&tmp).expect("tmp"); - std::fs::write(tmp.join("partial.txt"), b"half").expect("seed"); - let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Materializing)); - - recover(&journal_path).expect("recover"); - assert_eq!(std::fs::read(repo.join("keep.txt")).expect("read"), b"live"); - assert!(!tmp.exists()); - assert!(!journal_path.exists()); - } - - #[test] - fn recover_with_no_journal_is_a_noop() { - let work = tempfile::tempdir().expect("tempdir"); - assert!(recover(&work.path().join("missing.json")) - .expect("recover") - .is_none()); - } - - #[test] - fn recover_mid_carry_returns_excluded_paths_to_the_repo() { - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - let tmp = work.path().join("repo.tmp"); - std::fs::create_dir_all(repo.join("secrets")).expect("repo"); - std::fs::create_dir_all(tmp.join("secrets")).expect("tmp"); - // .env already moved into tmp; secrets/key.pem had not moved yet. - std::fs::write(tmp.join(".env"), b"LIVE").expect("moved"); - std::fs::write(repo.join("secrets/key.pem"), b"KEY").expect("unmoved"); - let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Carrying); - entry.carried = vec![PathBuf::from(".env"), PathBuf::from("secrets/key.pem")]; - write(&journal_path, &entry); - - recover(&journal_path).expect("recover"); - assert_eq!(std::fs::read(repo.join(".env")).expect("back"), b"LIVE"); - assert_eq!( - std::fs::read(repo.join("secrets/key.pem")).expect("kept"), - b"KEY" - ); - assert!(!tmp.exists(), "unused new tree removed"); - assert!(!journal_path.exists()); - } - - #[test] - fn recover_before_the_swap_keeps_carried_paths_out_of_the_discarded_tree() { - // Swapping phase, but the exchange never ran: repo is the old tree - // (minus the carried paths), tmp is the new tree holding them. - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - let tmp = work.path().join("repo.tmp"); - std::fs::create_dir_all(&repo).expect("repo"); - std::fs::create_dir_all(&tmp).expect("tmp"); - std::fs::write(repo.join("file.txt"), b"old").expect("old"); - std::fs::write(tmp.join("file.txt"), b"new").expect("new"); - std::fs::write(tmp.join(".env"), b"LIVE").expect("carried"); - let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Swapping); - entry.carried = vec![PathBuf::from(".env")]; - write(&journal_path, &entry); - - recover(&journal_path).expect("recover"); - assert_eq!(std::fs::read(repo.join("file.txt")).expect("repo"), b"old"); - assert_eq!( - std::fs::read(repo.join(".env")).expect("carried back"), - b"LIVE" - ); - assert!(!tmp.exists()); - } - - #[test] - fn recover_after_the_swap_leaves_carried_paths_in_the_new_tree() { - // Exchange completed: repo is the new tree with the carried paths, - // tmp is the old tree without them. Nothing moves; tmp goes. - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - let tmp = work.path().join("repo.tmp"); - std::fs::create_dir_all(&repo).expect("repo"); - std::fs::create_dir_all(&tmp).expect("tmp"); - std::fs::write(repo.join("file.txt"), b"new").expect("new"); - std::fs::write(repo.join(".env"), b"LIVE").expect("carried"); - std::fs::write(tmp.join("file.txt"), b"old").expect("old"); - let journal_path = work.path().join("journal.json"); - let mut entry = journal(&repo, &tmp, Phase::Swapping); - entry.carried = vec![PathBuf::from(".env")]; - write(&journal_path, &entry); - - recover(&journal_path).expect("recover"); - assert_eq!(std::fs::read(repo.join("file.txt")).expect("repo"), b"new"); - assert_eq!( - std::fs::read(repo.join(".env")).expect("still here"), - b"LIVE" - ); - assert!(!tmp.exists()); - } - - #[test] - fn journals_written_before_exclusions_still_decode() { - let text = - r#"{"target_generation":"00","repo_root":"/r","tmp":"/t","phase":"Materializing"}"#; - let journal: Journal = serde_json::from_str(text).expect("decode"); - assert!(journal.carried.is_empty()); - } - - /// The contract every platform's exchange owes the caller, asserted - /// against whichever implementation this host compiled: after it, each - /// path names the other's tree. Windows reaches that through three - /// renames rather than one syscall, so it is the arm most worth pinning. - #[test] - fn exchange_swaps_two_directories() { - let work = tempfile::tempdir().expect("tempdir"); - let left = work.path().join("left"); - let right = work.path().join("right"); - std::fs::create_dir(&left).expect("left"); - std::fs::create_dir(&right).expect("right"); - std::fs::write(left.join("who.txt"), b"left").expect("seed left"); - std::fs::write(right.join("who.txt"), b"right").expect("seed right"); - - atomic_exchange(&left, &right).expect("exchange"); - - assert_eq!(std::fs::read(left.join("who.txt")).expect("left"), b"right"); - assert_eq!( - std::fs::read(right.join("who.txt")).expect("right"), - b"left" - ); - } - - /// A failed exchange must leave the tree it was given untouched rather - /// than half-moved. On Windows this exercises the unwind between the - /// first and second rename, which is the window where the repo path is - /// vacated and nothing has replaced it yet. - #[test] - fn a_failed_exchange_leaves_the_live_tree_whole() { - let work = tempfile::tempdir().expect("tempdir"); - let live = work.path().join("live"); - std::fs::create_dir(&live).expect("live"); - std::fs::write(live.join("keep.txt"), b"precious").expect("seed"); - let missing = work.path().join("never-materialized"); - - atomic_exchange(&live, &missing).expect_err("exchange must fail"); - - assert!(live.is_dir(), "the live tree must still be a directory"); - assert_eq!( - std::fs::read(live.join("keep.txt")).expect("content survives"), - b"precious" - ); - #[cfg(windows)] - assert!( - !swap_scratch(&live).expect("scratch path").exists(), - "a failed exchange must not leave its scratch behind" - ); - } - - /// Windows swaps through a scratch directory, so a crash can leave the - /// repo path vacated with the new tree still at `tmp` and the old tree - /// parked in the scratch. Recovery has to finish the move *and* clear the - /// scratch, or the next rewind inherits a stale tree beside the repo. - #[cfg(windows)] - #[test] - fn recover_clears_the_scratch_a_windows_swap_left() { - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - let tmp = work.path().join("repo.tmp"); - std::fs::create_dir(&tmp).expect("tmp"); - std::fs::write(tmp.join("file.txt"), b"new tree").expect("seed new"); - // Died between rename one and rename two: repo vacated, old tree parked. - let scratch = swap_scratch(&repo).expect("scratch path"); - std::fs::create_dir(&scratch).expect("scratch"); - std::fs::write(scratch.join("file.txt"), b"old tree").expect("seed old"); - let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); - - recover(&journal_path).expect("recover"); - - assert_eq!( - std::fs::read(repo.join("file.txt")).expect("repo whole"), - b"new tree" - ); - assert!(!scratch.exists(), "scratch must not outlive recovery"); - assert!(!tmp.exists()); - assert!(!journal_path.exists()); - } - - /// The other Windows crash point: rename two landed, so the repo already - /// holds the new tree and only the scratch is left to clear. - #[cfg(windows)] - #[test] - fn recover_clears_the_scratch_after_the_swap_landed() { - let work = tempfile::tempdir().expect("tempdir"); - let repo = work.path().join("repo"); - std::fs::create_dir(&repo).expect("repo"); - std::fs::write(repo.join("file.txt"), b"new tree").expect("seed new"); - let tmp = work.path().join("repo.tmp"); - let scratch = swap_scratch(&repo).expect("scratch path"); - std::fs::create_dir(&scratch).expect("scratch"); - std::fs::write(scratch.join("file.txt"), b"old tree").expect("seed old"); - let journal_path = work.path().join("journal.json"); - write(&journal_path, &journal(&repo, &tmp, Phase::Swapping)); - - recover(&journal_path).expect("recover"); - - assert_eq!( - std::fs::read(repo.join("file.txt")).expect("repo whole"), - b"new tree" - ); - assert!(!scratch.exists(), "scratch must not outlive recovery"); - assert!(!journal_path.exists()); - } -} diff --git a/crates/acyclic-engine/src/store.rs b/crates/acyclic-engine/src/store.rs deleted file mode 100644 index ebd2df0..0000000 --- a/crates/acyclic-engine/src/store.rs +++ /dev/null @@ -1,355 +0,0 @@ -//! Store lifecycle: where engine state lives and how the volume opens. -//! -//! Everything lives OUTSIDE the working tree (capture snapshots the whole -//! tree and fail-closes on sockets). The repo carries only `.acyclic/config.toml`. - -use std::path::{Path, PathBuf}; - -use acyclic_fs::model::{ - AccessMode, CheckoutMode, ConsistencyMode, GenerationSelector, Lifecycle, MutationMode, - VolumeConfig, -}; -use acyclic_fs::{ - CancellationToken, Checkout, LocalAuthorityBackend, LocalFs, LocalObjectBackend, - LocalObjectsDurability, LocalOptions, LocalStreamDurability, VolumeId, WorkCounters, -}; -use serde::{Deserialize, Serialize}; - -use crate::{EngineError, Result}; - -/// Concrete checkout type for the local backend. -pub type LocalCheckout = Checkout; -/// Concrete volume type for the local backend. -pub type LocalVolume = acyclic_fs::LocalVolume; - -/// Batch limits sized for large monorepos: the fs defaults (2,048) reject any -/// baseline capture beyond ~2k paths. Immutable per volume — size generously. -const MUTATIONS_PER_BATCH: u32 = 4_194_304; - -/// Filesystem layout of one repo's store. -#[derive(Clone, Debug)] -pub struct StorePaths { - /// Store root: `//`. - pub root: PathBuf, -} - -impl StorePaths { - /// Resolves the store root for a repo. `stores_root` override comes from - /// config; the default is `~/.local/share/acyclic/stores`. - pub fn for_repo(repo_root: &Path, stores_root: Option<&Path>) -> Result { - let base = if let Some(path) = stores_root { - path.to_path_buf() - } else { - let home = std::env::var_os("HOME") - .ok_or_else(|| EngineError::Store("HOME is not set".into()))?; - Path::new(&home).join(format!(".local/share/{}/stores", crate::product::NAME)) - }; - let canonical = repo_root - .canonicalize() - .map_err(|error| EngineError::Store(format!("canonicalize repo root: {error}")))?; - let digest = blake3::hash(canonical.as_os_str().as_encoded_bytes()); - let hex = digest.to_hex(); - let short = hex.get(..16).unwrap_or(&hex); - Ok(Self { - root: base.join(short), - }) - } - - pub fn object_store(&self) -> PathBuf { - self.root.join("store") - } - pub fn index_db(&self) -> PathBuf { - self.root.join("index.db") - } - /// The daemon socket lives in a short per-user runtime directory, NOT in - /// the store: `sun_path` is capped (~104 bytes on macOS) and store roots - /// can be arbitrarily deep. - pub fn socket(&self) -> PathBuf { - let store_key = self.root.file_name().map_or_else( - || "default".into(), - |name| name.to_string_lossy().into_owned(), - ); - runtime_dir().join(format!("{store_key}.sock")) - } - pub fn pidfile(&self) -> PathBuf { - self.root.join("daemon.pid") - } - pub fn rewind_journal(&self) -> PathBuf { - self.root.join("rewind-journal.json") - } - pub fn trash(&self) -> PathBuf { - self.root.join("trash") - } - pub fn meta(&self) -> PathBuf { - self.root.join("meta.json") - } - /// Speculation cache. Deliberately NOT a table in `index_db`: the - /// pipeline thread owns that connection and writes to it synchronously, - /// so a second writer contending for `SQLite`'s write lock would block the - /// thread every hook call waits on. See `crate::spec`. - pub fn spec_db(&self) -> PathBuf { - self.root.join("spec.db") - } - /// Scratch and pid files for in-flight speculative child processes. - pub fn spec_runs(&self) -> PathBuf { - self.root.join("spec") - } -} - -/// Persisted store identity. The `VolumeId` MUST survive restarts: -/// generations only resolve on their own volume. -#[derive(Debug, Serialize, Deserialize)] -pub struct StoreMeta { - pub schema: u32, - pub repo_root: PathBuf, - pub volume_id: VolumeId, -} - -/// An opened store: the fs engine, its volume, and a writable Head checkout. -pub struct Store { - pub fs: LocalFs, - pub volume: LocalVolume, - pub checkout: LocalCheckout, - pub volume_id: VolumeId, - pub paths: StorePaths, - pub repo_root: PathBuf, -} - -/// The volume every store opens with. -/// -/// The profile is per-platform and decides how names are encoded on the way -/// in and out — see [`crate::names`], which every caller that mints a name -/// must go through. It is fixed for the life of a volume: a store created -/// under one profile cannot be reopened under another, so this must never -/// become a runtime choice. -pub(crate) fn volume_config() -> VolumeConfig { - let mut config = VolumeConfig { - profile: crate::names::profile(), - ..VolumeConfig::portable(Lifecycle::Durable) - }; - config.limits.maximum_mutations_per_batch = MUTATIONS_PER_BATCH; - config.limits.maximum_paths_per_batch = MUTATIONS_PER_BATCH; - config.limits.maximum_component_bytes = crate::names::maximum_component_bytes(); - config -} - -pub(crate) fn writable_head() -> CheckoutMode { - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - } -} - -/// Read-only pinned mode for historical generations. -pub fn read_only() -> CheckoutMode { - CheckoutMode { - access: AccessMode::ReadOnly, - consistency: ConsistencyMode::Pinned, - mutations: MutationMode::None, - } -} - -/// Barrier durability for both providers: a full device flush per journal -/// frame costs ~5ms each on Apple SSDs and a small capture issues dozens, -/// while the store only needs to survive a daemon crash — a torn tail after -/// power loss just drops the newest checkpoint. -pub fn local_options(root: impl Into) -> LocalOptions { - let mut options = LocalOptions::new(root); - options.stream.durability = LocalStreamDurability::Barrier; - options.objects.durability = LocalObjectsDurability::Barrier; - options -} - -impl Store { - /// Creates the store for a repo: directories, volume, meta record. - /// Fails if the store already exists. - pub async fn init(repo_root: &Path, paths: StorePaths) -> Result { - if paths.meta().exists() { - return Err(EngineError::Store(format!( - "store already initialized at {}", - paths.root.display() - ))); - } - std::fs::create_dir_all(paths.object_store())?; - std::fs::create_dir_all(paths.trash())?; - - let cancel = CancellationToken::new(); - let fs = LocalFs::local(local_options(paths.object_store())) - .await - .map_err(EngineError::fs("open object store"))?; - let volume_id = VolumeId::new(); - let volume = fs - .create_volume_with_id(volume_id, volume_config(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("create volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - writable_head(), - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("checkout head"))? - .value; - - let repo_root = repo_root.canonicalize()?; - let meta = StoreMeta { - schema: 1, - repo_root: repo_root.clone(), - volume_id, - }; - atomic_write_json(&paths.meta(), &meta)?; - Ok(Self { - fs, - volume, - checkout, - volume_id, - paths, - repo_root, - }) - } - - /// Opens an existing store recorded in `meta.json`. - pub async fn open(paths: StorePaths) -> Result { - let text = std::fs::read_to_string(paths.meta()).map_err(|error| { - EngineError::Store(format!( - "no store at {} ({error}); run init first", - paths.root.display() - )) - })?; - let meta: StoreMeta = serde_json::from_str(&text) - .map_err(|error| EngineError::Store(format!("meta.json: {error}")))?; - if meta.schema != 1 { - return Err(EngineError::Store(format!( - "unsupported store schema {}", - meta.schema - ))); - } - - let cancel = CancellationToken::new(); - let phase = std::time::Instant::now(); - let fs = LocalFs::local(local_options(paths.object_store())) - .await - .map_err(EngineError::fs("open object store"))?; - let objects_ms = crate::trace::ms(phase); - let phase = std::time::Instant::now(); - let volume = fs - .open_volume(meta.volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("open volume"))? - .value; - let volume_ms = crate::trace::ms(phase); - let phase = std::time::Instant::now(); - let checkout = volume - .checkout( - GenerationSelector::Head, - writable_head(), - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("checkout head"))? - .value; - crate::trace!( - "store", - "open: object store {objects_ms:.1}ms, volume {volume_ms:.1}ms, head checkout {:.1}ms", - crate::trace::ms(phase) - ); - Ok(Self { - fs, - volume, - checkout, - volume_id: meta.volume_id, - paths, - repo_root: meta.repo_root, - }) - } - - /// Opens a read-only checkout of one historical generation. - pub async fn checkout_exact( - &self, - generation: acyclic_fs::GenerationId, - ) -> Result { - let cancel = CancellationToken::new(); - Ok(self - .volume - .checkout( - GenerationSelector::Exact(generation), - read_only(), - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("checkout exact"))? - .value) - } -} - -/// Short per-user directory for daemon sockets. Created 0700 on first use. -/// -/// Deliberately NOT `std::env::temp_dir()`: that honors `TMPDIR`, which can -/// be arbitrarily deep, and `sun_path` is capped (~104 bytes on macOS). The -/// path must be short and identical across every process of this user. -#[allow(unsafe_code, reason = "getuid() has no preconditions and cannot fail")] -pub fn runtime_dir() -> PathBuf { - #[cfg(unix)] - let dir = { - // SAFETY: getuid has no preconditions and cannot fail. - let uid = unsafe { libc::getuid() }; - PathBuf::from(format!("/tmp/{}-{uid}", crate::product::NAME)) - }; - #[cfg(not(unix))] - let dir = std::env::temp_dir().join(crate::product::NAME); - let _ = std::fs::create_dir_all(&dir); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); - } - dir -} - -fn atomic_write_json(path: &Path, value: &T) -> Result<()> { - let text = serde_json::to_string_pretty(value) - .map_err(|error| EngineError::Store(format!("encode {}: {error}", path.display())))?; - let tmp = path.with_extension("json.tmp"); - std::fs::write(&tmp, text)?; - std::fs::rename(&tmp, path)?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn init_then_open_preserves_volume_identity() { - let repo = tempfile::tempdir().expect("repo dir"); - let stores = tempfile::tempdir().expect("stores dir"); - std::fs::write(repo.path().join("file.txt"), b"hi").expect("seed file"); - let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); - - let created = Store::init(repo.path(), paths.clone()).await.expect("init"); - let created_id = created.volume_id; - drop(created); - - let reopened = Store::open(paths).await.expect("open"); - assert_eq!(reopened.volume_id, created_id); - assert_eq!( - reopened.repo_root, - repo.path().canonicalize().expect("canonical repo") - ); - } - - #[tokio::test] - async fn double_init_is_refused() { - let repo = tempfile::tempdir().expect("repo dir"); - let stores = tempfile::tempdir().expect("stores dir"); - let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); - Store::init(repo.path(), paths.clone()).await.expect("init"); - assert!(Store::init(repo.path(), paths).await.is_err()); - } -} diff --git a/crates/acyclic-proto/Cargo.toml b/crates/acyclic-proto/Cargo.toml deleted file mode 100644 index 6e04816..0000000 --- a/crates/acyclic-proto/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "acyclic-proto" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false -description = "CLI <-> daemon wire types: newline-delimited JSON over the store's unix socket" - -[dependencies] -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -[lints] -workspace = true diff --git a/crates/acyclic-qual/Cargo.toml b/crates/acyclic-qual/Cargo.toml deleted file mode 100644 index 506492d..0000000 --- a/crates/acyclic-qual/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "acyclic-qual" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -publish = false -description = "Phase 0 qualification harness: proves acyclic-fs capture/restore against the Rewind acceptance criteria" - -[dependencies] -acyclic-engine = { path = "../acyclic-engine" } -acyclic-fs.workspace = true -tokio.workspace = true - -[lints] -workspace = true diff --git a/crates/acyclic-qual/src/main.rs b/crates/acyclic-qual/src/main.rs deleted file mode 100644 index 08114a8..0000000 --- a/crates/acyclic-qual/src/main.rs +++ /dev/null @@ -1,1104 +0,0 @@ -//! Phase 0 qualification harness for the Rewind milestone. -//! -//! Subcommands: -//! fixture [--with-fifo] build a small fixture repo exercising the tricky cases -//! roundtrip capture into a fresh store under , commit, -//! materialize into /restore, compare content+mode -//! -//! Exit code 0 = round-trip verified identical; 1 = mismatches or engine failure. - -#![allow( - clippy::indexing_slicing, - clippy::string_slice, - clippy::panic, - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::cast_precision_loss, - reason = "qualification harness, not shipped: a bad invocation or a broken \ - invariant should crash with its message rather than be handled" -)] -#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] - -use std::collections::BTreeMap; -use std::fs; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::time::Instant; - -use acyclic_engine::store::local_options; -use acyclic_fs::model::{ - AccessMode, CheckoutMode, ConsistencyMode, FilesystemProfile, GenerationSelector, Lifecycle, - MutationMode, VolumeConfig, -}; -use acyclic_fs::{ - capture_baseline, capture_root_identity, materialize_checkout, CaptureOptions, - MaterializeOptions, -}; -use acyclic_fs::{ - CancellationToken, CheckoutCommitOutcome, GenerationId, LocalFs, OperationId, VolumeId, - WorkCounters, -}; - -fn main() { - let args: Vec = std::env::args().skip(1).collect(); - let result = match args.first().map(String::as_str) { - Some("fixture") => fixture(&args[1..]), - Some("roundtrip") => roundtrip(&args[1..]), - Some("corpus") => corpus(&args[1..]), - Some("bench") => bench(&args[1..]), - Some("restore-gen") => restore_gen(&args[1..]), - Some("mount-smoke") => mount_smoke(&args[1..]), - Some("mount-smoke2") => mount_smoke2(&args[1..]), - Some("mount-hold") => mount_hold(&args[1..]), - Some("source-probe") => source_probe(&args[1..]), - _ => Err( - "usage: acyclic-qual fixture [--with-fifo] | roundtrip \ - | corpus | bench [rounds]" - .into(), - ), - }; - if let Err(message) = result { - eprintln!("FAIL: {message}"); - std::process::exit(1); - } -} - -type Failure = Box; - -fn engine_err(context: &str) -> impl FnOnce(E) -> Failure + '_ { - move |error| format!("{context}: {error:?}").into() -} - -// --------------------------------------------------------------------------- -// fixture -// --------------------------------------------------------------------------- - -fn fixture(args: &[String]) -> Result<(), Failure> { - let root = PathBuf::from(args.first().ok_or("fixture: missing ")?); - let with_fifo = args.iter().any(|a| a == "--with-fifo"); - fs::create_dir_all(root.join("src/nested"))?; - fs::create_dir_all(root.join("node_modules/.bin"))?; - fs::create_dir_all(root.join(".git/objects"))?; - fs::create_dir_all(root.join("empty-dir"))?; - - fs::write(root.join("README.md"), b"fixture repo\n")?; - fs::write(root.join(".gitignore"), b"generated/\n.env\n")?; - fs::write(root.join(".env"), b"SECRET=hunter2\n")?; - fs::write(root.join("src/main.rs"), b"fn main() {}\n")?; - fs::write(root.join("src/nested/mod.rs"), b"// nested\n")?; - fs::write(root.join(".git/objects/pack-data"), b"\x00\x01\x02git\n")?; - fs::write(root.join("uni-\u{00e9}\u{4e2d}.txt"), b"unicode name\n")?; - - // Deterministic 8 MiB binary file (multi-chunk content). - let mut big = Vec::with_capacity(8 * 1024 * 1024); - let mut state: u32 = 0x9e37_79b9; - while big.len() < 8 * 1024 * 1024 { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - big.extend_from_slice(&state.to_le_bytes()); - } - fs::write(root.join("assets.bin"), &big)?; - - // Executable script (mode-bit round-trip). - let script = root.join("tool.sh"); - fs::write(&script, b"#!/bin/sh\necho ok\n")?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&script, fs::Permissions::from_mode(0o755))?; - } - - // Symlinks: valid relative (the node_modules/.bin shape) and dangling. - #[cfg(unix)] - { - use std::os::unix::fs::symlink; - symlink("../../tool.sh", root.join("node_modules/.bin/tool"))?; - symlink("no-such-target", root.join("dangling-link"))?; - } - - // Hard-link pair. - fs::write(root.join("hardlink-a"), b"same inode\n")?; - fs::hard_link(root.join("hardlink-a"), root.join("hardlink-b"))?; - - if with_fifo { - let status = std::process::Command::new("mkfifo") - .arg(root.join("pipe.fifo")) - .status()?; - if !status.success() { - return Err("mkfifo failed".into()); - } - } - - println!("fixture written to {}", root.display()); - Ok(()) -} - -// --------------------------------------------------------------------------- -// restore-gen: materialize one generation from an existing store -// --------------------------------------------------------------------------- - -fn restore_gen(args: &[String]) -> Result<(), Failure> { - let store_dir = PathBuf::from(args.first().ok_or("restore-gen: missing ")?); - let volume_hex = args.get(1).ok_or("restore-gen: missing ")?; - let generation_hex = args.get(2).ok_or("restore-gen: missing ")?; - let destination = PathBuf::from(args.get(3).ok_or("restore-gen: missing ")?); - fs::create_dir_all(&destination)?; - - let volume_uuid: [u8; 16] = { - let clean: String = volume_hex.chars().filter(|c| *c != '-').collect(); - let bytes = hex_decode(&clean)?; - bytes - .as_slice() - .try_into() - .map_err(|_| "volume uuid must be 16 bytes")? - }; - let digest: [u8; 32] = hex_decode(generation_hex)? - .as_slice() - .try_into() - .map_err(|_| "generation must be 32 bytes")?; - - let runtime = tokio::runtime::Runtime::new()?; - runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume( - VolumeId::from_bytes(volume_uuid), - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("open volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Exact(GenerationId::new(acyclic_fs::Digest::from_bytes( - digest, - ))), - CheckoutMode { - access: AccessMode::ReadOnly, - consistency: ConsistencyMode::Pinned, - mutations: MutationMode::None, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout exact"))? - .value; - let receipt = materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: destination.clone(), - maximum_directory_entries: 1_024, - maximum_extent_spans: 65_536, - transfer_bytes: 8 * 1024 * 1024, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("materialize"))? - .value; - println!( - "materialized {} files / {} dirs into {}", - receipt.files, - receipt.directories, - destination.display() - ); - Ok(()) - }) -} - -fn hex_decode(text: &str) -> Result, Failure> { - if !text.len().is_multiple_of(2) { - return Err("odd hex length".into()); - } - (0..text.len()) - .step_by(2) - .map(|i| u8::from_str_radix(&text[i..i + 2], 16).map_err(|e| format!("{e}").into())) - .collect() -} - -// --------------------------------------------------------------------------- -// mount-smoke: Launch 3 gate — writable native mount of a captured checkout -// --------------------------------------------------------------------------- - -#[allow( - clippy::too_many_lines, - reason = "the gate's mount, write, unmount, and verify steps are one straight-line procedure" -)] -fn mount_smoke(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{ - mount_native, probe_native_mount, CheckoutMountSource, NativeMountRequest, SharedCheckout, - }; - use std::sync::Arc; - - let source = PathBuf::from(args.first().ok_or("mount-smoke: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("mount-smoke: missing ")?); - let store_dir = work.join("store"); - let mount_dir = work.join("mnt"); - fs::create_dir_all(&store_dir)?; - fs::create_dir_all(&mount_dir)?; - - let capabilities = probe_native_mount(); - println!( - "probe: kind={:?} available={} writable={} reason={:?}", - capabilities.kind, - capabilities.available, - capabilities.writable, - capabilities.unavailable_reason - ); - if !capabilities.available || !capabilities.writable { - return Err("native mount unavailable or read-only — fork engine gate FAILS".into()); - } - - // Capture the fixture and publish it, exactly like the daemon does. - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - - // Fresh writable Head checkout for the mount (the fork shape). - let config = volume_config(); - let (checkout, fs_engine) = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("reopen store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout head"))? - .value; - Ok::<_, Failure>((checkout, fs_engine)) - })?; - let _keep_engine_alive = fs_engine; - - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = Arc::new( - CheckoutMountSource::new(Arc::clone(&shared), config) - .map_err(engine_err("mount source"))?, - ); - let started = Instant::now(); - let mut session = mount_native( - NativeMountRequest { - mount_id: acyclic_fs::MountId::new(), - volume_id, - destination: mount_dir.clone(), - writable: true, - }, - mount_source, - ) - .map_err(engine_err("mount_native"))?; - println!( - "mounted at {} in {:?}", - mount_dir.display(), - started.elapsed() - ); - - // Everything below must not leave the mount attached on failure. - let verdict = (|| -> Result<(), Failure> { - // Read path: lazy listing + content identical to the fixture. - let started = Instant::now(); - let mounted_readme = fs::read(mount_dir.join("README.md"))?; - println!("first small read: {:?}", started.elapsed()); - if mounted_readme != fs::read(source.join("README.md"))? { - return Err("README content mismatch through mount".into()); - } - let started = Instant::now(); - let mounted_asset = fs::read(mount_dir.join("assets.bin"))?; - println!( - "8MB hydration read: {:?} ({} bytes)", - started.elapsed(), - mounted_asset.len() - ); - if mounted_asset != fs::read(source.join("assets.bin"))? { - return Err("asset content mismatch through mount".into()); - } - let mounted_link = fs::read_link(mount_dir.join("node_modules/.bin/tool"))?; - if mounted_link != Path::new("../../tool.sh") { - return Err(format!("symlink mismatch through mount: {mounted_link:?}").into()); - } - - // Write path: overlay writes visible in the mount, invisible outside. - fs::write(mount_dir.join("fork-note.txt"), b"written in the fork\n")?; - fs::write(mount_dir.join("README.md"), b"edited in the fork\n")?; - fs::create_dir(mount_dir.join("fork-dir"))?; - if fs::read(mount_dir.join("fork-note.txt"))? != b"written in the fork\n" { - return Err("write-then-read mismatch through mount".into()); - } - if fs::read(mount_dir.join("README.md"))? != b"edited in the fork\n" { - return Err("edit-then-read mismatch through mount".into()); - } - if source.join("fork-note.txt").exists() { - return Err("overlay write leaked into the source tree".into()); - } - if fs::read(source.join("README.md"))? != mounted_readme { - return Err("overlay edit leaked into the source tree".into()); - } - Ok(()) - })(); - - let stopped = session.stop(); - println!("unmounted cleanly: {stopped:?}"); - verdict?; - stopped.map_err(engine_err("unmount"))?; - - println!("MOUNT SMOKE OK: writable native mount serves, isolates, and detaches"); - Ok(()) -} - -// --------------------------------------------------------------------------- -// mount-hold: attach one mount and hold it for so failing -// operations can be probed interactively from a shell. -// --------------------------------------------------------------------------- - -fn mount_hold(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{mount_native, CheckoutMountSource, NativeMountRequest, SharedCheckout}; - use std::sync::Arc; - - let source = PathBuf::from(args.first().ok_or("mount-hold: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("mount-hold: missing ")?); - let seconds: u64 = args.get(2).map_or(Ok(60), |value| value.parse())?; - let store_dir = work.join("store"); - let mount_dir = work.join("mnt"); - fs::create_dir_all(&store_dir)?; - fs::create_dir_all(&mount_dir)?; - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - let config = volume_config(); - let checkout = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - std::mem::forget(fs_engine); - Ok::<_, Failure>(checkout) - })?; - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = - Arc::new(CheckoutMountSource::new(shared, config).map_err(engine_err("mount source"))?); - let mut session = mount_native( - NativeMountRequest { - mount_id: acyclic_fs::MountId::new(), - volume_id, - destination: mount_dir.clone(), - writable: true, - }, - mount_source, - ) - .map_err(engine_err("mount"))?; - println!("HELD: {} for {seconds}s", mount_dir.display()); - std::thread::sleep(std::time::Duration::from_secs(seconds)); - session.stop().map_err(engine_err("unmount"))?; - println!("released"); - Ok(()) -} - -// --------------------------------------------------------------------------- -// source-probe: exercise MountFilesystem directly (no kernel, no NFS) to -// pinpoint which callback fails and with what typed error. -// --------------------------------------------------------------------------- - -fn source_probe(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{CheckoutMountSource, MountFilesystem, MountPath, SharedCheckout}; - use std::sync::Arc; - - let source = - PathBuf::from(args.first().ok_or("source-probe: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("source-probe: missing ")?); - let store_dir = work.join("store"); - fs::create_dir_all(&store_dir)?; - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - let config = volume_config(); - let checkout = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - std::mem::forget(fs_engine); - Ok::<_, Failure>(checkout) - })?; - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = - CheckoutMountSource::new(shared, config).map_err(engine_err("mount source"))?; - - let readme = MountPath::root().child(b"README.md".to_vec()); - println!( - "lookup(/): {:?}", - mount_source - .lookup(&MountPath::root()) - .map(|l| l.map(|l| l.node.kind)) - ); - println!( - "lookup(README.md): {:?}", - mount_source - .lookup(&readme) - .map(|l| l.map(|l| (l.node.kind, l.node.logical_bytes))) - ); - println!( - "read_directory(/): {:?}", - mount_source - .read_directory(&MountPath::root(), None, 16) - .map(|p| p.entries.len()) - ); - match mount_source.open_file(&readme) { - Ok(file) => { - println!("open_file(README): Ok"); - println!( - " handle.lookup(): {:?}", - file.lookup().map(|l| l.node.logical_bytes) - ); - println!( - " read_range(0,13): {:?}", - file.read_range(0, 13) - .map(|b| String::from_utf8_lossy(&b).into_owned()) - ); - } - Err(error) => println!("open_file(README): ERR {error:?}"), - } - Ok(()) -} - -// --------------------------------------------------------------------------- -// mount-smoke2: TWO simultaneous native mounts from ONE process — the -// minimal repro for the fork engine's N>1 requirement. -// --------------------------------------------------------------------------- - -fn mount_smoke2(args: &[String]) -> Result<(), Failure> { - use acyclic_fs::{mount_native, CheckoutMountSource, NativeMountRequest, SharedCheckout}; - use std::sync::Arc; - - let source = - PathBuf::from(args.first().ok_or("mount-smoke2: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("mount-smoke2: missing ")?); - let store_dir = work.join("store"); - fs::create_dir_all(&store_dir)?; - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, _generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - let config = volume_config(); - - let mut sessions = Vec::new(); - for index in 0..2u32 { - let mount_dir = work.join(format!("mnt{index}")); - fs::create_dir_all(&mount_dir)?; - let checkout = runtime.block_on(async { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(&store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("open volume"))? - .value; - let checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - std::mem::forget(fs_engine); // keep engine alive for the mount - Ok::<_, Failure>(checkout) - })?; - let shared = Arc::new(SharedCheckout::new(checkout)); - let mount_source = - Arc::new(CheckoutMountSource::new(shared, config).map_err(engine_err("mount source"))?); - let started = Instant::now(); - let session = mount_native( - NativeMountRequest { - mount_id: acyclic_fs::MountId::new(), - volume_id, - destination: mount_dir.clone(), - writable: true, - }, - mount_source, - ) - .map_err(engine_err(if index == 0 { - "FIRST mount" - } else { - "SECOND mount" - }))?; - println!( - "mount {index} attached at {} in {:?}", - mount_dir.display(), - started.elapsed() - ); - let listing = fs::read_dir(&mount_dir)?.count(); - println!("mount {index} lists {listing} entries"); - sessions.push(session); - } - - for (index, mut session) in sessions.into_iter().enumerate() { - session.stop().map_err(engine_err("unmount"))?; - println!("mount {index} detached"); - } - println!("MOUNT SMOKE2 OK: two simultaneous sessions in one process"); - Ok(()) -} - -// --------------------------------------------------------------------------- -// corpus: synthetic tree of files totalling MiB, 100 per dir -// --------------------------------------------------------------------------- - -fn corpus(args: &[String]) -> Result<(), Failure> { - let root = PathBuf::from(args.first().ok_or("corpus: missing ")?); - let files: u64 = args.get(1).ok_or("corpus: missing ")?.parse()?; - let total_mb: u64 = args.get(2).ok_or("corpus: missing ")?.parse()?; - let bytes_per_file = (total_mb * 1024 * 1024) / files.max(1); - let mut payload = Vec::with_capacity(bytes_per_file as usize); - let mut state: u32 = 0x1234_5678; - while (payload.len() as u64) < bytes_per_file { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - payload.extend_from_slice(&state.to_le_bytes()); - } - let started = Instant::now(); - for index in 0..files { - let dir = root.join(format!("dir-{:05}", index / 100)); - if index % 100 == 0 { - fs::create_dir_all(&dir)?; - } - fs::write(dir.join(format!("file-{index:07}.dat")), &payload)?; - } - println!( - "corpus: {files} files x {bytes_per_file} bytes in {:?}", - started.elapsed() - ); - Ok(()) -} - -// --------------------------------------------------------------------------- -// bench: baseline capture, then watch-driven incremental capture latency -// --------------------------------------------------------------------------- - -fn bench(args: &[String]) -> Result<(), Failure> { - let source = PathBuf::from(args.first().ok_or("bench: missing ")?).canonicalize()?; - let work = PathBuf::from(args.get(1).ok_or("bench: missing ")?); - let rounds: usize = args.get(2).map_or(Ok(30), |value| value.parse())?; - let store_dir = work.join("store"); - fs::create_dir_all(&store_dir)?; - let runtime = tokio::runtime::Runtime::new()?; - runtime.block_on(bench_inner(&source, &store_dir, rounds)) -} - -#[allow( - clippy::too_many_lines, - reason = "the benchmark's setup, rounds, and report are one straight-line procedure" -)] -async fn bench_inner(source: &Path, store_dir: &Path, rounds: usize) -> Result<(), Failure> { - use acyclic_fs::model::VolumeLimits; - use acyclic_fs::{NativeWatch, NativeWatchOptions, WatchBatch}; - - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(store_dir)) - .await - .map_err(engine_err("open store"))?; - let volume = fs_engine - .create_volume(volume_config(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("create volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout"))? - .value; - - let mut watch = NativeWatch::open( - source, - NativeWatchOptions { - limits: VolumeLimits::default(), - maximum_queued_changes: 65_536, - recursive: true, - }, - ) - .map_err(engine_err("watch open"))?; - watch.begin_rescan().map_err(engine_err("begin_rescan"))?; - - let options = CaptureOptions { - source_root: source.to_path_buf(), - expected_root_identity: capture_root_identity(source).map_err(engine_err("identity"))?, - maximum_paths: 4_000_000, - maximum_extent_spans: 65_536, - }; - - let started = Instant::now(); - let receipt = capture_baseline(&mut checkout, &options, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("baseline"))? - .value; - let baseline_capture = started.elapsed(); - let started = Instant::now(); - checkout - .commit(OperationId::new(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("baseline commit"))?; - println!( - "baseline: capture {:?} + commit {:?} ({} paths, {} bytes)", - baseline_capture, - started.elapsed(), - receipt.examined_paths, - receipt.staged_file_bytes - ); - watch.finish_rescan().map_err(engine_err("finish_rescan"))?; - - let mut event_latency = Vec::with_capacity(rounds); - let mut capture_only = Vec::with_capacity(rounds); - let mut capture_checkpoint = Vec::with_capacity(rounds); - let mut capture_commit = Vec::with_capacity(rounds); - for round in 0..rounds { - let target = source.join(format!("bench-mutation-{}.txt", round % 5)); - let mutated_at = Instant::now(); - fs::write(&target, format!("round {round} at {mutated_at:?}\n"))?; - - // Poll until the watcher reports the change (event-arrival latency). - let batch = loop { - let batch = watch - .poll(4_096, WorkCounters::UNBOUNDED, &cancel) - .map_err(engine_err("poll"))? - .value; - match &batch { - WatchBatch::Changes { changes, .. } if !changes.is_empty() => break batch, - WatchBatch::Changes { .. } => { - std::thread::sleep(std::time::Duration::from_millis(2)); - } - WatchBatch::RescanRequired { reason, .. } => { - return Err(format!("rescan required mid-bench: {reason:?}").into()); - } - } - }; - event_latency.push(mutated_at.elapsed()); - - let capture_started = Instant::now(); - acyclic_fs::capture_watch_batch( - &mut checkout, - batch, - &options, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("capture_watch_batch"))?; - let captured_at = capture_started.elapsed(); - if round % 2 == 0 { - checkout - .checkpoint(WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("incremental checkpoint"))?; - capture_checkpoint.push(capture_started.elapsed()); - } else { - checkout - .commit(OperationId::new(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("incremental commit"))?; - capture_commit.push(capture_started.elapsed()); - } - capture_only.push(captured_at); - } - - report("event-arrival latency", &mut event_latency); - report("capture (no publish) latency", &mut capture_only); - report("capture+checkpoint latency", &mut capture_checkpoint); - report("capture+commit latency", &mut capture_commit); - Ok(()) -} - -fn report(label: &str, samples: &mut [std::time::Duration]) { - samples.sort(); - let p = |q: f64| samples[((samples.len() - 1) as f64 * q) as usize]; - println!( - "{label}: n={} p50={:?} p95={:?} max={:?}", - samples.len(), - p(0.50), - p(0.95), - samples[samples.len() - 1] - ); -} - -// --------------------------------------------------------------------------- -// roundtrip -// --------------------------------------------------------------------------- - -fn roundtrip(args: &[String]) -> Result<(), Failure> { - let source = PathBuf::from(args.first().ok_or("roundtrip: missing ")?) - .canonicalize() - .map_err(engine_err("canonicalize "))?; - let work = PathBuf::from(args.get(1).ok_or("roundtrip: missing ")?); - let store_dir = work.join("store"); - let restore_dir = work.join("restore"); - fs::create_dir_all(&store_dir)?; - fs::create_dir_all(&restore_dir)?; - if fs::read_dir(&restore_dir)?.next().is_some() { - return Err("roundtrip: /restore must be empty".into()); - } - - let runtime = tokio::runtime::Runtime::new()?; - let (volume_id, generation) = runtime.block_on(capture_and_commit(&source, &store_dir))?; - runtime.block_on(materialize(&store_dir, volume_id, generation, &restore_dir))?; - - let started = Instant::now(); - let mismatches = compare_trees(&source, &restore_dir)?; - println!("compare: {:?}", started.elapsed()); - - if mismatches.is_empty() { - println!("ROUNDTRIP OK: content + mode identical"); - Ok(()) - } else { - for m in &mismatches { - eprintln!("MISMATCH: {m}"); - } - Err(format!("{} mismatches", mismatches.len()).into()) - } -} - -fn volume_config() -> VolumeConfig { - // QUAL_PROFILE=portable switches the profile for differential debugging. - let profile = match std::env::var("QUAL_PROFILE").as_deref() { - Ok("portable") => FilesystemProfile::Portable, - _ => FilesystemProfile::Posix, - }; - let mut config = VolumeConfig { - profile, - ..VolumeConfig::portable(Lifecycle::Durable) - }; - // A baseline capture is one atomic authored transaction covering every - // path in the repo; the default 2,048-mutation batch cap rejects any - // real tree. Sized for large monorepos. - config.limits.maximum_mutations_per_batch = 4_194_304; - config.limits.maximum_paths_per_batch = 4_194_304; - config -} - -async fn capture_and_commit( - source: &Path, - store_dir: &Path, -) -> Result<(VolumeId, GenerationId), Failure> { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(store_dir)) - .await - .map_err(engine_err("open store"))?; - - let started = Instant::now(); - let volume = fs_engine - .create_volume(volume_config(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("create volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Head, - CheckoutMode { - access: AccessMode::ReadWrite, - consistency: ConsistencyMode::TrackingSafe, - mutations: MutationMode::PrivateOverlay, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout head"))? - .value; - println!("volume + checkout: {:?}", started.elapsed()); - - let options = CaptureOptions { - source_root: source.to_path_buf(), - expected_root_identity: capture_root_identity(source) - .map_err(engine_err("root identity"))?, - maximum_paths: 4_000_000, - maximum_extent_spans: 65_536, - }; - let started = Instant::now(); - let receipt = capture_baseline(&mut checkout, &options, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("capture_baseline"))? - .value; - println!( - "capture_baseline: {:?} (examined {} paths, {} changed, {} bytes staged)", - started.elapsed(), - receipt.examined_paths, - receipt.changed_paths, - receipt.staged_file_bytes - ); - - let started = Instant::now(); - if std::env::var_os("QUAL_CHECKPOINT_ONLY").is_some() { - let generation = checkout - .checkpoint(WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("checkpoint"))? - .value; - println!("checkpoint (no publish): {:?}", started.elapsed()); - return Ok((checkout.volume_id(), generation)); - } - let outcome = checkout - .commit(OperationId::new(), WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("commit"))? - .value; - println!("commit: {:?}", started.elapsed()); - match outcome { - CheckoutCommitOutcome::Committed { generation_id, .. } - | CheckoutCommitOutcome::AlreadyCommitted { generation_id, .. } => { - Ok((checkout.volume_id(), generation_id)) - } - other => Err(format!("commit not durable: {other:?}").into()), - } -} - -async fn materialize( - store_dir: &Path, - volume_id: VolumeId, - generation: GenerationId, - destination: &Path, -) -> Result<(), Failure> { - let cancel = CancellationToken::new(); - let fs_engine = LocalFs::local(local_options(store_dir)) - .await - .map_err(engine_err("reopen store"))?; - let volume = fs_engine - .open_volume(volume_id, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(engine_err("reopen volume"))? - .value; - let mut checkout = volume - .checkout( - GenerationSelector::Exact(generation), - CheckoutMode { - access: AccessMode::ReadOnly, - consistency: ConsistencyMode::Pinned, - mutations: MutationMode::None, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("checkout exact"))? - .value; - - let started = Instant::now(); - let receipt = materialize_checkout( - &mut checkout, - &MaterializeOptions { - destination: destination.to_path_buf(), - maximum_directory_entries: 1_024, - maximum_extent_spans: 65_536, - transfer_bytes: 8 * 1024 * 1024, - }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(engine_err("materialize_checkout"))? - .value; - println!( - "materialize: {:?} ({} files, {} dirs, {} symlinks, {} bytes written)", - started.elapsed(), - receipt.files, - receipt.directories, - receipt.symbolic_links, - receipt.written_bytes - ); - Ok(()) -} - -// --------------------------------------------------------------------------- -// tree comparison: content + kind + symlink target + mode bits (no mtime/uid) -// --------------------------------------------------------------------------- - -#[derive(Debug, PartialEq)] -enum EntryKind { - File, - Dir, - Symlink, - Other, -} - -fn walk(root: &Path) -> Result, Failure> { - let mut entries = BTreeMap::new(); - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { - for entry in fs::read_dir(&dir)? { - let path = entry?.path(); - let meta = fs::symlink_metadata(&path)?; - let relative = path.strip_prefix(root)?.to_path_buf(); - let kind = if meta.file_type().is_symlink() { - EntryKind::Symlink - } else if meta.is_dir() { - stack.push(path.clone()); - EntryKind::Dir - } else if meta.is_file() { - EntryKind::File - } else { - EntryKind::Other - }; - entries.insert(relative, kind); - } - } - Ok(entries) -} - -fn compare_trees(source: &Path, restored: &Path) -> Result, Failure> { - let left = walk(source)?; - let right = walk(restored)?; - let mut mismatches = Vec::new(); - - for (path, kind) in &left { - match right.get(path) { - None => mismatches.push(format!("{}: missing in restore", path.display())), - Some(other) if other != kind => { - mismatches.push(format!("{}: kind {kind:?} vs {other:?}", path.display())); - } - Some(_) => { - let a = source.join(path); - let b = restored.join(path); - match kind { - EntryKind::Symlink => { - let ta = fs::read_link(&a)?; - let tb = fs::read_link(&b)?; - if ta != tb { - mismatches.push(format!( - "{}: symlink target {:?} vs {:?}", - path.display(), - ta, - tb - )); - } - } - EntryKind::File => { - if !files_equal(&a, &b)? { - mismatches.push(format!("{}: content differs", path.display())); - } - if let Some(m) = mode_mismatch(&a, &b, path)? { - mismatches.push(m); - } - } - EntryKind::Dir => { - if let Some(m) = mode_mismatch(&a, &b, path)? { - mismatches.push(m); - } - } - EntryKind::Other => {} - } - } - } - } - for path in right.keys() { - if !left.contains_key(path) { - mismatches.push(format!("{}: extra in restore", path.display())); - } - } - Ok(mismatches) -} - -fn files_equal(a: &Path, b: &Path) -> Result { - let (ma, mb) = (fs::metadata(a)?, fs::metadata(b)?); - if ma.len() != mb.len() { - return Ok(false); - } - let (mut fa, mut fb) = (fs::File::open(a)?, fs::File::open(b)?); - let mut ba = vec![0u8; 1024 * 1024]; - let mut bb = vec![0u8; 1024 * 1024]; - loop { - let na = fa.read(&mut ba)?; - let nb = fb.read(&mut bb)?; - if na != nb || ba[..na] != bb[..nb] { - return Ok(false); - } - if na == 0 { - return Ok(true); - } - } -} - -#[cfg(unix)] -fn mode_mismatch(a: &Path, b: &Path, relative: &Path) -> Result, Failure> { - use std::os::unix::fs::MetadataExt; - let ma = fs::metadata(a)?.mode() & 0o7777; - let mb = fs::metadata(b)?.mode() & 0o7777; - if ma == mb { - Ok(None) - } else { - Ok(Some(format!( - "{}: mode {ma:o} vs {mb:o}", - relative.display() - ))) - } -} - -#[cfg(not(unix))] -fn mode_mismatch(_a: &Path, _b: &Path, _relative: &Path) -> Result, Failure> { - Ok(None) -} diff --git a/crates/acyclic/Cargo.toml b/crates/acyclic/Cargo.toml index 285a10b..db549fe 100644 --- a/crates/acyclic/Cargo.toml +++ b/crates/acyclic/Cargo.toml @@ -6,17 +6,22 @@ license.workspace = true repository.workspace = true homepage.workspace = true publish = false +build = "build.rs" description = "Agent-native state engine: checkpoints, rewind, and blast-radius diff for coding-agent sessions" [dependencies] -acyclic-engine = { path = "../acyclic-engine" } acyclic-fs.workspace = true -acyclic-proto = { path = "../acyclic-proto" } clap = { version = "4", features = ["derive", "env"] } rmcp = { version = "3.3.0", features = ["server", "macros", "transport-io", "schemars"] } schemars = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +rusqlite = { version = "0.40", features = ["bundled"] } +toml = "0.8" +thiserror = "2" +hex = "0.4" +blake3 = "1" +bytes = "1" tokio = { version = "1.48", features = ["rt-multi-thread", "macros", "net", "io-util", "sync", "time", "signal", "process"] } [target.'cfg(unix)'.dependencies] @@ -34,10 +39,14 @@ windows-sys = { version = "0.61", features = [ "Win32_System_JobObjects", "Win32_System_Memory", "Win32_System_Threading", + "Win32_Storage_FileSystem", ] } [dev-dependencies] tempfile = "3" +[build-dependencies] +toml = "0.8" + [lints] workspace = true diff --git a/crates/acyclic-engine/build.rs b/crates/acyclic/build.rs similarity index 100% rename from crates/acyclic-engine/build.rs rename to crates/acyclic/build.rs diff --git a/crates/acyclic/src/brief.rs b/crates/acyclic/src/brief.rs index 8d56b88..abbf324 100644 --- a/crates/acyclic/src/brief.rs +++ b/crates/acyclic/src/brief.rs @@ -5,8 +5,8 @@ //! the last session ended, what it changed, which branches it abandoned, and //! the verbs that reach the rest. -use acyclic_engine::product::NAME; -use acyclic_proto as proto; +use crate::proto; +use acyclic::product::NAME; /// Hard cap on the rendered text, including the trailing newline. pub const BUDGET_BYTES: usize = 1000; @@ -180,7 +180,7 @@ fn short(session_id: &str) -> String { } fn age(at: i64) -> String { - let delta = (acyclic_engine::unix_now() - at).max(0); + let delta = (acyclic::unix_now() - at).max(0); if delta < 60 { format!("{delta}s ago") } else if delta < 3600 { diff --git a/crates/acyclic/src/checkpoint_kind.rs b/crates/acyclic/src/checkpoint_kind.rs new file mode 100644 index 0000000..2b7ab28 --- /dev/null +++ b/crates/acyclic/src/checkpoint_kind.rs @@ -0,0 +1,133 @@ +//! Checkpoint roles stored by the plugin's timeline index. + +use serde::{Deserialize, Serialize}; + +/// Why a checkpoint observation exists. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CheckpointKind { + /// First complete observation of a source. + Baseline, + /// State immediately before a consumer operation. + Pre, + /// State immediately after a consumer operation. + Post, + /// Consumer-requested observation. + Manual, + /// Safety observation before moving the live head backward. + PreRewind, + /// Observation created while recovering a source. + Recovered, + /// Failed capture; its generation belongs to an earlier observation. + Failed, + /// A request whose state did not change. + Noop, + /// Observation created by an idle timer. + Auto, +} + +impl CheckpointKind { + /// SQL filter for user-facing rewind targets. Keep this beside the role + /// definition so timeline queries do not each repeat the list. + pub const TARGETS_SQL: &'static str = "'baseline','pre','post','manual','auto'"; + + /// Stable value used by existing timeline stores and wire consumers. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Baseline => "baseline", + Self::Pre => "pre", + Self::Post => "post", + Self::Manual => "manual", + Self::PreRewind => "pre_rewind", + Self::Recovered => "recovered", + Self::Failed => "failed", + Self::Noop => "noop", + Self::Auto => "auto", + } + } + + /// The row's generation is a real state that may be restored. + #[must_use] + pub const fn is_restorable(self) -> bool { + !matches!(self, Self::Failed) + } + + /// This observation is a user-facing rewind target and branch member. + #[must_use] + pub const fn is_target(self) -> bool { + matches!( + self, + Self::Baseline | Self::Pre | Self::Post | Self::Manual | Self::Auto + ) + } +} + +impl std::str::FromStr for CheckpointKind { + type Err = UnknownCheckpointKind; + + fn from_str(value: &str) -> Result { + match value { + "baseline" => Ok(Self::Baseline), + "pre" => Ok(Self::Pre), + "post" => Ok(Self::Post), + "manual" => Ok(Self::Manual), + "pre_rewind" => Ok(Self::PreRewind), + "recovered" => Ok(Self::Recovered), + "failed" => Ok(Self::Failed), + "noop" => Ok(Self::Noop), + "auto" => Ok(Self::Auto), + _ => Err(UnknownCheckpointKind(value.to_owned())), + } + } +} + +impl std::fmt::Display for CheckpointKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.pad(self.as_str()) + } +} + +/// A timeline store contained an unsupported checkpoint role. +#[derive(Clone, Debug, Eq, PartialEq, thiserror::Error)] +#[error("unknown checkpoint kind {0}")] +pub struct UnknownCheckpointKind(pub String); + +#[cfg(test)] +mod tests { + use super::CheckpointKind; + + #[test] + fn storage_values_and_restore_roles_are_stable() { + let roles = [ + ("baseline", true, true), + ("pre", true, true), + ("post", true, true), + ("manual", true, true), + ("pre_rewind", true, false), + ("recovered", true, false), + ("failed", false, false), + ("noop", true, false), + ("auto", true, true), + ]; + for (value, restorable, target) in roles { + let Ok(kind) = value.parse::() else { + unreachable!("known checkpoint role"); + }; + assert_eq!(kind.as_str(), value); + assert_eq!(kind.is_restorable(), restorable); + assert_eq!(kind.is_target(), target); + } + assert!("future".parse::().is_err()); + let sql_roles = CheckpointKind::TARGETS_SQL + .split(',') + .map(|role| { + role.trim_matches('\'') + .parse::() + .expect("SQL role") + }) + .collect::>(); + assert_eq!(sql_roles.len(), 5); + assert!(sql_roles.iter().all(|role| role.is_target())); + } +} diff --git a/crates/acyclic/src/client.rs b/crates/acyclic/src/client.rs index 69a2bc5..045f01a 100644 --- a/crates/acyclic/src/client.rs +++ b/crates/acyclic/src/client.rs @@ -6,8 +6,8 @@ use std::path::Path; use std::time::{Duration, Instant}; use crate::ipc::ClientStream; -use acyclic_engine::product::NAME; -use acyclic_proto as proto; +use crate::proto; +use acyclic::product::NAME; #[derive(Clone, Copy)] pub enum Spawn { @@ -25,8 +25,11 @@ pub enum Spawn { pub struct Client { stream: BufReader, next_id: u64, + call_timeout: Option, + usable: bool, } +#[derive(Debug)] pub enum ConnectError { /// No daemon and spawning was not allowed. NoDaemon, @@ -44,17 +47,17 @@ impl Client { ) -> Result { let started = std::time::Instant::now(); if let Ok(stream) = ClientStream::connect(socket) { - acyclic_engine::trace!( + acyclic::trace!( "client", "connected to running daemon at {} in {:.1}ms", crate::ipc::endpoint_display(socket), - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); return Self::from_stream(stream); } match spawn { Spawn::Never => { - acyclic_engine::trace!( + acyclic::trace!( "client", "no daemon at {} and spawning is not allowed here", crate::ipc::endpoint_display(socket) @@ -62,7 +65,7 @@ impl Client { Err(ConnectError::NoDaemon) } Spawn::Allowed | Spawn::AllowedFor(_) => { - acyclic_engine::trace!( + acyclic::trace!( "client", "no daemon at {}: spawning one", crate::ipc::endpoint_display(socket) @@ -73,10 +76,10 @@ impl Client { _ => None, }; let client = wait_for_socket(socket, child, log_path, bound); - acyclic_engine::trace!( + acyclic::trace!( "client", "daemon spawn + socket wait took {:.1}ms", - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); client } @@ -90,45 +93,135 @@ impl Client { Ok(Self { stream: BufReader::new(stream), next_id: 1, + call_timeout: None, + usable: true, }) } /// Bounds how long a single call may wait for its reply. Used by the /// pre-tool hook: an exact boundary is worth milliseconds, not seconds. pub fn set_deadline(&mut self, deadline: std::time::Duration) { - let _ = self.stream.get_ref().set_read_timeout(Some(deadline)); - let _ = self.stream.get_ref().set_write_timeout(Some(deadline)); + self.call_timeout = Some(deadline); } pub fn call(&mut self, op: proto::Op) -> Result { + if !self.usable { + return Err("daemon connection requires reconnect after a transport failure".into()); + } let id = self.next_id; self.next_id += 1; let name = format!("{op:?}"); let name = name.split([' ', '{', '(']).next().unwrap_or("?").to_owned(); let started = std::time::Instant::now(); - acyclic_engine::trace!("client", "call #{id} {name}"); - let result = self.call_inner(id, op); + acyclic::trace!("client", "call #{id} {name}"); + let result = match self.call_inner(id, op) { + Ok(proto::Payload::Ok(reply)) => Ok(*reply), + Ok(proto::Payload::Err { message }) => Err(message), + Err(error) => { + self.usable = false; + Err(error) + } + }; match &result { Ok(reply) => { let reply_name = format!("{reply:?}"); let reply_name = reply_name.split([' ', '{', '(']).next().unwrap_or("?"); - acyclic_engine::trace!( + acyclic::trace!( "client", "call #{id} {name} -> {reply_name} in {:.1}ms", - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); } - Err(message) => acyclic_engine::trace!( + Err(message) => acyclic::trace!( "client", "call #{id} {name} -> error in {:.1}ms: {}", - acyclic_engine::trace::ms(started), + acyclic::trace::ms(started), message.lines().next().unwrap_or("") ), } result } - fn call_inner(&mut self, id: u64, op: proto::Op) -> Result { + /// Take a user-requested checkpoint and wait until it has landed. + /// + /// Product consumers should use this instead of constructing the wire + /// operation themselves. Hook-specific checkpoint metadata remains on + /// [`Client::call`] until it has a real consumer-facing abstraction. + pub fn manual_checkpoint( + &mut self, + label: Option, + ) -> Result { + match self.call(proto::Op::Checkpoint { + kind: proto::CheckpointRequestKind::Manual, + session_id: None, + tool_call_id: None, + tool_name: None, + label, + wait: true, + durable: false, + })? { + proto::Reply::Checkpoint(info) => Ok(info), + other => Err(unexpected_reply("checkpoint", &other)), + } + } + + /// List checkpoints through the stable, typed client surface. + pub fn timeline( + &mut self, + session_id: Option, + turn: Option, + limit: u32, + ) -> Result, String> { + match self.call(proto::Op::Timeline { + session_id, + turn, + limit, + })? { + proto::Reply::Timeline(entries) => Ok(entries), + other => Err(unexpected_reply("timeline", &other)), + } + } + + /// Replace the working tree at a checkpoint. + pub fn rewind(&mut self, target: proto::RewindTarget) -> Result { + match self.call(proto::Op::Rewind { target, path: None })? { + proto::Reply::Rewind(info) => Ok(info), + other => Err(unexpected_reply("rewind", &other)), + } + } + + /// Restore one path while leaving the rest of the tree untouched. + pub fn restore(&mut self, checkpoint: i64, path: String) -> Result { + match self.call(proto::Op::Rewind { + target: proto::RewindTarget::Checkpoint(checkpoint), + path: Some(path), + })? { + proto::Reply::Restore(info) => Ok(info), + other => Err(unexpected_reply("restore", &other)), + } + } + + /// Compute a diff using either row ids or generation prefixes. + pub fn diff( + &mut self, + before: Option, + after: Option, + before_hex: Option, + after_hex: Option, + ) -> Result, String> { + match self.call(proto::Op::Diff { + before, + after, + before_hex, + after_hex, + })? { + proto::Reply::Diff(entries) => Ok(entries), + other => Err(unexpected_reply("diff", &other)), + } + } + + fn call_inner(&mut self, id: u64, op: proto::Op) -> Result { + let deadline = self.call_timeout.map(|timeout| Instant::now() + timeout); let request = proto::Request { v: proto::PROTOCOL_VERSION, id, @@ -136,34 +229,129 @@ impl Client { }; let mut line = serde_json::to_vec(&request).map_err(|error| error.to_string())?; line.push(b'\n'); - self.stream - .get_mut() - .write_all(&line) - .map_err(|error| format!("send: {error}"))?; - let mut response_line = String::new(); - self.stream - .read_line(&mut response_line) - .map_err(|error| format!("receive: {error}"))?; - // A daemon that exits mid-answer closes the socket, so the read - // succeeds with nothing. Left to serde that surfaced as - // "decode: EOF while parsing a value at line 1 column 0", which reads - // like corruption rather than what it is: the daemon stopped. Anyone - // running `stop` and then any other verb hit it. - parse_response(&response_line) + + let mut sent = 0; + while sent < line.len() { + self.stream + .get_ref() + .set_write_timeout(remaining(deadline)?) + .map_err(|error| format!("send timeout: {error}"))?; + let count = self + .stream + .get_mut() + .write(line.get(sent..).ok_or("send: invalid offset")?) + .map_err(|error| format!("send: {error}"))?; + if count == 0 { + return Err("send: daemon closed the connection".to_owned()); + } + sent += count; + } + let mut response_line = Vec::new(); + loop { + self.stream + .get_ref() + .set_read_timeout(remaining(deadline)?) + .map_err(|error| format!("receive timeout: {error}"))?; + let available = self + .stream + .fill_buf() + .map_err(|error| format!("receive: {error}"))?; + if available.is_empty() { + return Err("daemon stopped while answering; nothing was recorded".to_owned()); + } + let count = available + .iter() + .position(|byte| *byte == b'\n') + .map_or(available.len(), |index| index + 1); + if response_line.len() + count > 64 * 1024 * 1024 { + return Err("receive: response exceeds 64 MiB".to_owned()); + } + response_line + .extend_from_slice(available.get(..count).ok_or("receive: invalid offset")?); + self.stream.consume(count); + if response_line.last() == Some(&b'\n') { + break; + } + } + let response = parse_response(&response_line)?; + if response.id != id { + return Err(format!( + "receive: response id {} does not match request {id}", + response.id + )); + } + Ok(response.payload) } } -/// One response line to a reply. Split out from the socket so the -/// shutdown case can be tested without a daemon. -fn parse_response(line: &str) -> Result { - if line.trim().is_empty() { +fn parse_response(line: &[u8]) -> Result { + if line.iter().all(u8::is_ascii_whitespace) { return Err("daemon stopped while answering; nothing was recorded".to_owned()); } - let response: proto::Response = - serde_json::from_str(line).map_err(|error| format!("decode: {error}"))?; - match response.payload { - proto::Payload::Ok(reply) => Ok(*reply), - proto::Payload::Err { message } => Err(message), + serde_json::from_slice(line).map_err(|error| format!("decode: {error}")) +} + +fn unexpected_reply(operation: &str, reply: &proto::Reply) -> String { + format!("{operation}: unexpected daemon reply {reply:?}") +} + +fn remaining(deadline: Option) -> Result, String> { + match deadline { + Some(deadline) => { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + Err("daemon call timed out".to_owned()) + } else { + Ok(Some(left)) + } + } + None => Ok(None), + } +} + +#[cfg(all(test, windows))] +mod deadline_tests { + use super::*; + + #[test] + fn call_deadline_bounds_silent_reply() { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let socket = std::path::PathBuf::from(format!("call-{}-{nonce}", std::process::id())); + let name = crate::ipc::endpoint_display(&socket); + let (ready, connected) = std::sync::mpsc::channel(); + let server = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + runtime.block_on(async { + let pipe = tokio::net::windows::named_pipe::ServerOptions::new() + .first_pipe_instance(true) + .create(&name) + .expect("pipe"); + ready.send(()).expect("ready"); + pipe.connect().await.expect("connect"); + tokio::time::sleep(Duration::from_millis(200)).await; + }); + }); + connected.recv().expect("server ready"); + let stream = ClientStream::connect(&socket).expect("client connect"); + let mut client = Client::from_stream(stream).expect("client"); + client.set_deadline(Duration::from_millis(30)); + let started = Instant::now(); + let error = client + .call(proto::Op::Ping) + .expect_err("call should time out"); + assert!(error.contains("timed out"), "{error}"); + assert!(started.elapsed() < Duration::from_millis(180)); + let retry = client + .call(proto::Op::Ping) + .expect_err("connection is poisoned"); + assert!(retry.contains("requires reconnect"), "{retry}"); + server.join().expect("server exit"); } } @@ -307,25 +495,33 @@ fn wait_for_socket( let started = Instant::now(); let deadline = bound.unwrap_or(Duration::from_secs(30 * 60)); let mut reported = false; + let mut retry_delay = Duration::from_millis(1); loop { if let Ok(stream) = ClientStream::connect(socket) { if let Ok(mut client) = Client::from_stream(stream) { - // The daemon binds its socket before it opens the store, so - // a connect can succeed while the ping waits on the store - // open; bound the ping too so a caller with a bound never - // sits on it. - client.set_deadline(bound.unwrap_or(Duration::from_secs(60))); - if client.call(proto::Op::Ping).is_ok() { - client.set_deadline(Duration::from_secs(24 * 60 * 60)); - return Ok(client); + // The socket can accept connections before the pipeline has + // completed its baseline. Ping waits for that pipeline and + // carries a startup error back to this caller. + client.set_deadline(deadline.saturating_sub(started.elapsed())); + match client.call(proto::Op::Ping) { + Ok(_) => { + client.set_deadline(Duration::from_secs(24 * 60 * 60)); + return Ok(client); + } + Err(message) if client.usable => { + // A protocol error is the pipeline's completed startup + // result; reconnecting cannot repair that baseline. + return Err(ConnectError::Other(message)); + } + Err(_) => {} } } } if bound.is_some_and(|bound| started.elapsed() > bound) { - acyclic_engine::trace!( + acyclic::trace!( "client", "daemon still starting after {:.1}ms; not waiting", - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); return Err(ConnectError::Starting); } @@ -346,7 +542,8 @@ fn wait_for_socket( eprintln!("{NAME}: daemon starting (building the first snapshot of the tree)..."); reported = true; } - std::thread::sleep(Duration::from_millis(200)); + std::thread::sleep(retry_delay); + retry_delay = (retry_delay * 2).min(Duration::from_millis(25)); } } @@ -361,7 +558,7 @@ mod tests { // "decode: EOF while parsing a value at line 1 column 0" — which reads // as corruption. Anyone running `stop` then any other verb saw it. for line in ["", "\n", " \n"] { - let error = parse_response(line).expect_err("empty must be an error"); + let error = parse_response(line.as_bytes()).expect_err("empty must be an error"); assert!( error.contains("daemon stopped"), "unhelpful message for {line:?}: {error}" @@ -373,7 +570,7 @@ mod tests { #[test] fn malformed_json_still_reports_a_decode_error() { // Genuine corruption must stay distinguishable from a clean shutdown. - let error = parse_response("{not json").expect_err("must be an error"); + let error = parse_response(b"{not json").expect_err("must be an error"); assert!(error.starts_with("decode:"), "{error}"); } @@ -390,8 +587,11 @@ mod tests { }, }) .expect("serialize"); - let error = parse_response(&line).expect_err("must be an error"); - assert_eq!(error, "no such checkpoint"); + let response = parse_response(line.as_bytes()).expect("valid response"); + let proto::Payload::Err { message } = response.payload else { + panic!("expected error payload"); + }; + assert_eq!(message, "no such checkpoint"); } #[test] @@ -402,8 +602,8 @@ mod tests { }) .expect("serialize"); assert!(matches!( - parse_response(&line).expect("ok payload"), - proto::Reply::Pong + parse_response(line.as_bytes()).expect("ok payload").payload, + proto::Payload::Ok(reply) if matches!(*reply, proto::Reply::Pong) )); } } diff --git a/crates/acyclic-engine/src/config.rs b/crates/acyclic/src/config.rs similarity index 95% rename from crates/acyclic-engine/src/config.rs rename to crates/acyclic/src/config.rs index b49fb43..0effa18 100644 --- a/crates/acyclic-engine/src/config.rs +++ b/crates/acyclic/src/config.rs @@ -28,18 +28,15 @@ pub struct Config { /// records nothing. Zero disables it. pub auto_checkpoint_idle_ms: u64, /// The daemon exits after this long (ms) with no request, no live - /// session, no live fork and no Safe Mode session. It restarts on the + /// session and no live fork. It restarts on the /// next session start. Zero keeps it alive forever. pub daemon_idle_exit_ms: u64, /// Days a rewound-away tree is kept in the store's trash. pub trash_ttl_days: u32, /// Override for the store directory (defaults to the per-machine root). pub store_dir: Option, - /// Safe Mode: root every session in a fork by default, gated on an - /// approved diff before anything reaches the real tree. - pub dry_run: bool, - /// Safe Mode: path prefixes (relative to the repo root) no fork or - /// scratch tree may write to, enforced at the native mount layer. + /// Path prefixes (relative to the repo root) that mounted forks may not + /// write to, enforced at the native mount layer. pub guarded_paths: Vec, /// Snapshot exclusions: repo-relative paths (a file, or a directory and /// everything under it) that never enter a checkpoint. For secrets and @@ -123,7 +120,6 @@ impl Default for Config { daemon_idle_exit_ms: 3_600_000, trash_ttl_days: 7, store_dir: None, - dry_run: false, guarded_paths: Vec::new(), exclude: Vec::new(), decompose: Decompose::default(), @@ -233,16 +229,15 @@ mod tests { } #[test] - fn safe_mode_fields_parse_from_repo_config() { + fn guarded_paths_parse_from_repo_config() { let repo = tempfile::tempdir().expect("tempdir"); std::fs::create_dir(repo.path().join(crate::product::repo_config_dir())).expect("dir"); std::fs::write( repo.path().join(crate::product::repo_config_file()), - "dry_run = true\nguarded_paths = [\".env\", \"migrations/\"]\n", + "guarded_paths = [\".env\", \"migrations/\"]\n", ) .expect("write"); let config = Config::load_layered(None, repo.path()).expect("load"); - assert!(config.dry_run); assert_eq!(config.guarded_paths, vec![".env", "migrations/"]); } @@ -303,6 +298,18 @@ mod tests { assert!(Config::load_layered(None, repo.path()).is_err()); } + #[test] + fn removed_dry_run_key_is_rejected() { + let repo = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir(repo.path().join(crate::product::repo_config_dir())).expect("dir"); + std::fs::write( + repo.path().join(crate::product::repo_config_file()), + "dry_run = true\n", + ) + .expect("write"); + assert!(Config::load_layered(None, repo.path()).is_err()); + } + /// The layering the doc comment and README promise: a key set only in /// the machine config survives a repo config that does not mention it. /// Before the per-key merge, parsing the repo file discarded the whole diff --git a/crates/acyclic/src/diff.rs b/crates/acyclic/src/diff.rs new file mode 100644 index 0000000..89e8bae --- /dev/null +++ b/crates/acyclic/src/diff.rs @@ -0,0 +1,135 @@ +//! Blast-radius diff between two generations, keyed by path. +//! +//! The SDK's Merkle-aware change set resolves only changed identities to paths. + +use std::path::PathBuf; + +use acyclic_fs::kernel::FileKind; +use acyclic_fs::GenerationId; +use serde::{Deserialize, Serialize}; + +use crate::store::Store; +use crate::{EngineError, Result}; + +/// One changed path between two generations. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FileChange { + pub path: PathBuf, + pub change: ChangeKind, + pub file_kind: FileKind, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChangeKind { + Added, + Removed, + Modified, + #[serde(rename = "metadata")] + MetadataOnly, +} + +impl ChangeKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Added => "added", + Self::Removed => "removed", + Self::Modified => "modified", + Self::MetadataOnly => "metadata", + } + } + + /// The one-letter marker used by the CLI's diff output. + pub fn tag(self) -> &'static str { + match self { + Self::Added => "A", + Self::Removed => "D", + Self::Modified => "M", + Self::MetadataOnly => "m", + } + } +} + +impl std::fmt::Display for ChangeKind { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.pad(self.as_str()) + } +} + +/// Computes the path-keyed diff `before → after`. +pub async fn diff( + store: &Store, + before: GenerationId, + after: GenerationId, +) -> Result> { + if before == after { + return Ok(Vec::new()); + } + let before = store.generation(before).await?; + let after = store.generation(after).await?; + let set = before + .diff_to(&after, u32::MAX) + .await + .map_err(EngineError::fs("diff generations"))?; + let paths = set + .changed_paths(u32::MAX) + .await + .map_err(EngineError::fs("resolve changed paths"))?; + let mut changes = paths + .into_iter() + .filter_map(|path| { + let (change, file_kind) = match (&path.before, &path.after) { + (None, Some(after)) => (ChangeKind::Added, after.kind), + (Some(before), None) => (ChangeKind::Removed, before.kind), + (Some(before), Some(after)) => { + let change = if before.kind == after.kind && before.payload == after.payload { + ChangeKind::MetadataOnly + } else { + ChangeKind::Modified + }; + (change, after.kind) + } + (None, None) => return None, + }; + Some( + acyclic_fs::namespace_to_host_path(&path.path) + .map(|path| FileChange { + path, + change, + file_kind, + }) + .map_err(EngineError::fs("resolve host path")), + ) + }) + .collect::>>()?; + // Snapshots carry `.git` so rewind restores it, but a blast-radius + // report is about the working tree: object and ref churn from ordinary + // git commands would otherwise swamp the real changes. + changes.retain(|change| !is_git_internal(&change.path)); + changes.sort_by(|left, right| left.path.cmp(&right.path)); + Ok(changes) +} + +/// `.git` itself or anything beneath it, at the repo root only. +pub(crate) fn is_git_internal(path: &std::path::Path) -> bool { + path.components() + .next() + .is_some_and(|first| first.as_os_str() == ".git") +} + +#[cfg(test)] +mod tests { + use super::is_git_internal; + use std::path::Path; + + #[test] + fn only_root_git_dir_is_internal() { + assert!(is_git_internal(Path::new(".git"))); + assert!(is_git_internal(Path::new(".git/HEAD"))); + assert!(is_git_internal(Path::new(".git/objects/ab/cd"))); + assert!(!is_git_internal(Path::new(".gitignore"))); + assert!(!is_git_internal(Path::new("src/.git/config"))); + assert!(!is_git_internal(Path::new("vendor/.gitkeep"))); + assert!(!is_git_internal(Path::new("a.txt"))); + } +} diff --git a/crates/acyclic/src/exclude.rs b/crates/acyclic/src/exclude.rs new file mode 100644 index 0000000..77c1323 --- /dev/null +++ b/crates/acyclic/src/exclude.rs @@ -0,0 +1,143 @@ +//! Snapshot exclusions: paths that never enter a checkpoint. +//! +//! The store deliberately captures what git ignores, so a declared secret +//! path (`.env`, `secrets/`) would otherwise live in history for longer than +//! it lives in the working tree. `exclude` in `.acyclic/config.toml` keeps +//! such paths out, enforced in three places: +//! +//! 1. Watcher hints at or under an excluded prefix are dropped before the +//! capture runs, so the ordinary per-tool-call path never reads them. +//! 2. After any capture that may have re-walked an excluded path (the full +//! baseline, or a hint on one of its ancestors), the path is scrubbed from +//! the checkout before the generation is checkpointed. +//! 3. A full rewind carries the live excluded paths into the restored tree: +//! no checkpoint holds them, so the working copy is the only copy. +//! +//! Exclusion is not purge. A generation captured before a path was excluded +//! still holds it, and at the pinned sdk revision the fs has no way to +//! release a retained generation, so nothing can be physically removed from +//! history. That gap is documented in docs/design/implementation-rewind.md. + +use std::path::{Path, PathBuf}; + +use acyclic_fs::kernel::NamespacePath; +use acyclic_fs::CapturePolicy; + +use crate::{EngineError, Result}; + +/// Parsed `exclude` rules: repo-relative path prefixes. A rule matches the +/// path itself and everything under it; `secrets` and `secrets/` are the +/// same rule. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Exclusions { + prefixes: Vec, +} + +impl Exclusions { + /// Parses the config list. Rules must be relative and stay inside the + /// repo; an empty rule or one naming the repo root is refused, since + /// excluding everything is the same as not running the engine. + pub fn parse(patterns: &[String]) -> Result { + let config = crate::store::volume_config(); + let mut prefixes = Vec::new(); + for pattern in patterns { + let trimmed = pattern.trim().trim_end_matches('/'); + let prefix = acyclic_fs::host_path_to_namespace( + Path::new(trimmed), + config.profile, + config.limits, + ) + .map_err(|_| { + EngineError::Config(format!( + "exclude rule {pattern:?} must be a non-empty relative path inside the repo" + )) + })?; + prefixes.push(prefix); + } + prefixes.sort(); + prefixes.dedup(); + let mut canonical = Vec::::new(); + for prefix in prefixes { + if canonical + .last() + .is_none_or(|ancestor| !prefix.is_within(ancestor)) + { + canonical.push(prefix); + } + } + Ok(Self { + prefixes: canonical, + }) + } + + pub fn is_empty(&self) -> bool { + self.prefixes.is_empty() + } + + /// Every rule as a repo-relative host path. + pub fn host_paths(&self) -> Vec { + self.prefixes + .iter() + .filter_map(|prefix| acyclic_fs::namespace_to_host_path(prefix).ok()) + .collect() + } + + /// True when `relative` is an excluded path or lies under one. + pub fn covers_host(&self, relative: &Path) -> bool { + let config = crate::store::volume_config(); + acyclic_fs::host_path_to_namespace(relative, config.profile, config.limits) + .is_ok_and(|path| self.covers(&path)) + } + + /// True when `path` is an excluded path or lies under one. + pub fn covers(&self, path: &NamespacePath) -> bool { + let candidate = self + .prefixes + .partition_point(|prefix| prefix <= path) + .checked_sub(1) + .and_then(|index| self.prefixes.get(index)); + candidate.is_some_and(|prefix| path.is_within(prefix)) + } + + /// Canonical SDK capture policy shared by baseline, watcher, and direct + /// subtree reconciliation. + pub fn capture_policy(&self) -> Result { + CapturePolicy::excluding(self.prefixes.clone()) + .map_err(|error| EngineError::Fs(format!("capture policy: {error}"))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rules(list: &[&str]) -> Exclusions { + Exclusions::parse(&list.iter().map(|s| s.to_string()).collect::>()).expect("parse") + } + + #[test] + fn rules_normalize_and_reject_escapes() { + let parsed = rules(&[".env", "secrets/", "./build/out/"]); + assert!(parsed.covers_host(Path::new(".env"))); + assert!(parsed.covers_host(Path::new("secrets/key.pem"))); + assert!(parsed.covers_host(Path::new("build/out"))); + assert!(!parsed.covers_host(Path::new("build"))); + assert!(!parsed.covers_host(Path::new(".env.example"))); + assert!(Exclusions::parse(&["../x".into()]).is_err()); + assert!(Exclusions::parse(&["/etc".into()]).is_err()); + assert!(Exclusions::parse(&["".into()]).is_err()); + assert!(Exclusions::parse(&["./".into()]).is_err()); + assert_eq!( + parsed.host_paths(), + vec![ + PathBuf::from(".env"), + PathBuf::from("build/out"), + PathBuf::from("secrets") + ] + ); + assert_eq!( + rules(&["secrets/deep", "secrets", "secrets/deeper"]).host_paths(), + vec![PathBuf::from("secrets")] + ); + } +} diff --git a/crates/acyclic/src/fork.rs b/crates/acyclic/src/fork.rs new file mode 100644 index 0000000..58bd178 --- /dev/null +++ b/crates/acyclic/src/fork.rs @@ -0,0 +1,200 @@ +//! Fork engine primitives (Launch 3). +//! +//! A fork is a writable overlay mount of a Head checkout: reads hydrate +//! lazily from the store (O(1) creation, `node_modules` included), writes +//! accumulate in that checkout's private overlay — invisible to the real +//! tree and to every other fork. The pipeline mints fork checkouts (it owns +//! the volume); the daemon owns the mount sessions (they must live in the +//! long-lived process). +//! +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use acyclic_fs::model::VolumeConfig; +use acyclic_fs::SharedCheckout; +use acyclic_fs::{ + probe_native_mount, GenerationId, LocalAuthorityBackend, LocalObjectBackend, NativeMountKind, + VolumeId, +}; + +/// What the host can do for fork mounts, probed once at daemon start and +/// reported by `status`/`init`. +#[derive(Clone, Debug)] +pub struct MountCapability { + /// Human name of the provider this build would use ("fuse", "nfs loopback"). + pub provider: &'static str, + pub available: bool, + /// Why not, when unavailable. + pub reason: Option, +} + +/// Live probe of the native mount provider. +pub fn mount_capability() -> MountCapability { + let probe = probe_native_mount(); + let provider = match probe.kind { + Some(NativeMountKind::LinuxFuse) => "fuse", + Some(NativeMountKind::MacOsNfs) => "nfs loopback", + Some(NativeMountKind::WindowsProjFs) => "projfs", + None => "none", + }; + MountCapability { + provider, + available: probe.available, + reason: probe.unavailable_reason, + } +} + +/// Platform-specific instructions for making mounts available. Printed by +/// `init` and `install` when the probe fails, so a user learns on day one +/// rather than the day they first try a fork. +pub fn mount_setup_hint() -> &'static str { + if cfg!(target_os = "linux") { + concat!( + "forks require usable /dev/fuse. To enable mounts:\n", + " sudo modprobe fuse # load the kernel module\n", + " sudo usermod -aG fuse \"$USER\" # if /dev/fuse is group-restricted; log in again\n", + " docker run --device /dev/fuse --cap-add SYS_ADMIN ... # inside a container", + ) + } else if cfg!(target_os = "macos") { + concat!( + "forks require the built-in NFS mount tools (/sbin/mount_nfs, /sbin/umount), which\n", + "are missing or blocked by policy. No extra software is needed on macOS;\n", + "ask your administrator to allow loopback NFS mounts.", + ) + } else if cfg!(windows) { + concat!( + "forks require the optional Windows Projected File System feature. Enable\n", + "Client-ProjFS in Windows Features. ProjFS safely rejects cross-root\n", + "directory moves that it cannot capture atomically.", + ) + } else { + "native mounts are required for forks and are not supported on this platform." + } +} + +/// The mount-safe checkout wrapper for the local backend. +pub type SharedLocalCheckout = SharedCheckout; + +/// What the pipeline hands the daemon for one new fork. +pub struct ForkSeed { + pub shared: Arc, + pub config: VolumeConfig, + pub volume_id: VolumeId, + /// Published head the fork was cut from; promote conflicts are judged + /// against movement past this point. + pub base: GenerationId, +} + +/// Result of a promote request. +#[derive(Clone, Debug)] +pub enum PromoteOutcome { + Promoted { + generation: GenerationId, + /// None when the fork had no writes (nothing to land). + old_tree: Option, + }, + /// The mainline moved past the fork's base — v1 surfaces the conflict + /// legibly instead of merging. + Conflict { message: String }, +} + +/// Where a repo's fork workspaces live: a sibling of the repo, outside the +/// working tree so capture never sees them. +pub fn forks_root(repo_root: &Path) -> Option { + let parent = repo_root.parent()?; + let name = repo_root.file_name()?.to_string_lossy(); + Some(parent.join(format!(".{name}.forks"))) +} + +/// The single mountpoint projecting every fork as a routed subdirectory. +pub fn forks_mount_root(repo_root: &Path) -> Option { + Some(forks_root(repo_root)?.join("mnt")) +} + +/// Removes the single routed mount and workspace directory a dead daemon +/// left behind. An absent root is the only successful no-op. +pub fn sweep_stale_forks(repo_root: &Path) -> Result<(), String> { + let Some(root) = forks_root(repo_root) else { + return Err("repo root has no parent for fork workspaces".to_owned()); + }; + if !root.try_exists().map_err(|error| error.to_string())? { + return Ok(()); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + let mount = root.join("mnt"); + // FUSE-T's go-nfsv4 helpers outlive a killed daemon and wedge the + // vendor's tiny shared NFS port pool for every future mount on the + // host — reap any helper serving one of OUR workspaces first. + #[cfg(target_os = "macos")] + if mount.try_exists().map_err(|error| error.to_string())? { + let status = std::process::Command::new("pkill") + .arg("-f") + .arg(format!("go-nfsv4.*{}", mount.display())) + .status() + .map_err(|error| format!("stop stale NFS helper: {error}"))?; + if !status.success() && status.code() != Some(1) { + return Err(format!("stop stale NFS helper: {status}")); + } + let status = std::process::Command::new("umount") + .arg("-f") + .arg(&mount) + .status() + .map_err(|error| format!("unmount stale fork workspace: {error}"))?; + if !status.success() { + return Err(format!("unmount stale fork workspace: {status}")); + } + } + #[cfg(target_os = "linux")] + if mount.try_exists().map_err(|error| error.to_string())? { + let status = std::process::Command::new("fusermount") + .arg("-u") + .arg(&mount) + .status() + .map_err(|error| format!("unmount stale fork workspace: {error}"))?; + if !status.success() { + return Err(format!("unmount stale fork workspace: {status}")); + } + } + std::fs::remove_dir_all(&root) + .map_err(|error| format!("remove stale fork workspace {}: {error}", root.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn probe_names_a_provider() { + let capability = mount_capability(); + assert_ne!(capability.provider, ""); + assert_eq!(capability.available, capability.reason.is_none()); + assert!(!mount_setup_hint().is_empty()); + } + + #[test] + fn forks_root_is_a_hidden_sibling() { + let root = forks_root(Path::new("/work/my-repo")).expect("root"); + assert_eq!(root, Path::new("/work/.my-repo.forks")); + } + + #[test] + fn sweep_removes_stale_directories() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + std::fs::create_dir(&repo).expect("repo"); + let root = forks_root(&repo).expect("root"); + std::fs::create_dir_all(root.join("dead-fork")).expect("stale"); + std::fs::write(root.join("dead-fork/leftover"), b"x").expect("file"); + + sweep_stale_forks(&repo).expect("sweep"); + assert!(!root.exists()); + } + + #[test] + fn sweeping_an_absent_root_is_a_no_op() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + std::fs::create_dir(&repo).expect("repo"); + sweep_stale_forks(&repo).expect("absent root"); + } +} diff --git a/crates/acyclic-engine/src/guard.rs b/crates/acyclic/src/guard.rs similarity index 95% rename from crates/acyclic-engine/src/guard.rs rename to crates/acyclic/src/guard.rs index be4919c..41c4152 100644 --- a/crates/acyclic-engine/src/guard.rs +++ b/crates/acyclic/src/guard.rs @@ -5,7 +5,7 @@ //! interposition, and its routing is single-level and non-overlaying — it //! cannot layer a read-only view over part of a writable fork. Guarding //! therefore wraps the *inner* source (a fork's `CheckoutMountSource`) -//! directly, before it is ever registered as a route or shadow mount, so +//! directly, before it is registered as a fork route, so //! guarded prefixes work at any depth. use acyclic_fs::kernel::FileMetadata; @@ -44,18 +44,25 @@ impl GuardedPrefix { /// Splits a configured guarded path into the component bytes a /// [`MountPath`] carries. /// -/// The encoding has to be the host's (see [`crate::names`]), not UTF-8: these +/// The encoding has to be the host's, not UTF-8: these /// components are compared byte-for-byte against the names the mount layer /// hands us. Comparing UTF-8 against a host that speaks UTF-16LE never /// matches, and a guard that never matches fails *open* — every guarded path /// would silently accept writes. fn parse_guarded_path(path: &str) -> Vec> { - path.trim() - .trim_matches('/') - .split('/') - .filter(|component| !component.is_empty()) - .map(crate::names::str_to_bytes) - .collect() + let config = crate::store::volume_config(); + acyclic_fs::host_path_to_namespace( + Path::new(path.trim().trim_matches('/')), + config.profile, + config.limits, + ) + .map(|path| { + path.components() + .iter() + .map(|name| name.as_bytes().to_vec()) + .collect() + }) + .unwrap_or_default() } /// Wraps one [`MountFilesystem`] and rejects mutating calls under any @@ -99,7 +106,7 @@ impl GuardedMountFilesystem { /// Whether a mutating call to `path` must be refused: either it falls /// under a configured guarded prefix, or its leaf is a macOS `AppleDouble` /// sidecar (`._X`). Sidecars are written by the macOS client over the - /// mount to carry a file's xattrs / resource fork; in a Safe Mode or fork + /// mount to carry a file's xattrs / resource fork; in a fork /// projection they are pure transport noise that would otherwise pollute /// the session diff and litter the real tree on apply, and a guarded /// file's metadata must not leak into one either. Dropping every sidecar @@ -124,7 +131,7 @@ impl GuardedMountFilesystem { /// bytes the mount layer carries, and on a UTF-16LE host an ASCII literal /// matches nothing. fn is_appledouble(path: &MountPath) -> bool { - let prefix = crate::names::str_to_bytes("._"); + let prefix = parse_guarded_path("._").pop().unwrap_or_default(); path.components() .last() .is_some_and(|leaf| leaf.starts_with(&prefix)) @@ -466,6 +473,15 @@ impl MountFilesystem for GuardedMountFilesystem { self.guard(path)?; self.inner.capture_host_path(source_root, path) } + + fn capture_host_subtree( + &self, + source_root: &Path, + path: &MountPath, + ) -> Result<(), MountSourceError> { + self.guard(path)?; + self.inner.capture_host_subtree(source_root, path) + } } #[cfg(test)] @@ -484,7 +500,8 @@ mod tests { fn test_path(components: &[&str]) -> MountPath { let mut path = MountPath::root(); for component in components { - path = path.child(crate::names::str_to_bytes(component)); + let bytes = parse_guarded_path(component).pop().expect("component"); + path = path.child(bytes); } path } diff --git a/crates/acyclic/src/hook.rs b/crates/acyclic/src/hook.rs index cd4c787..8c506ac 100644 --- a/crates/acyclic/src/hook.rs +++ b/crates/acyclic/src/hook.rs @@ -10,8 +10,8 @@ use std::io::Read; use std::path::Path; use std::time::Duration; -use acyclic_engine::product::NAME; -use acyclic_proto as proto; +use crate::proto; +use acyclic::product::NAME; use crate::client::{Client, ConnectError, Spawn}; @@ -136,7 +136,7 @@ impl HookEvent { pub fn run(repo: &Path, event: &str) -> i32 { let Some(event) = HookEvent::parse(event) else { - acyclic_engine::trace!("hook", "unknown event {event:?}: ignored"); + acyclic::trace!("hook", "unknown event {event:?}: ignored"); return 0; }; // Reading stdin can't hang the agent: hosts close it after writing. @@ -164,7 +164,7 @@ pub fn run(repo: &Path, event: &str) -> i32 { } else { Spawn::Never }; - acyclic_engine::trace!( + acyclic::trace!( "hook", "event {}: daemon spawn {}; pre-tool waits (bounded), post-tool enqueues (ack before capture)", event.as_arg(), @@ -395,12 +395,11 @@ mod tests { fn scratch() -> (tempfile::TempDir, std::path::PathBuf) { let dir = tempfile::tempdir().expect("tempdir"); let repo = dir.path().join("repo"); - std::fs::create_dir_all(repo.join(acyclic_engine::product::repo_config_dir())) - .expect("repo"); + std::fs::create_dir_all(repo.join(acyclic::product::repo_config_dir())).expect("repo"); let stores = dir.path().join("stores"); std::fs::create_dir_all(&stores).expect("stores"); std::fs::write( - repo.join(acyclic_engine::product::repo_config_file()), + repo.join(acyclic::product::repo_config_file()), format!("store_dir = {:?}\n", stores.to_string_lossy()), ) .expect("config"); diff --git a/crates/acyclic-engine/src/index.rs b/crates/acyclic/src/index.rs similarity index 78% rename from crates/acyclic-engine/src/index.rs rename to crates/acyclic/src/index.rs index 36589b9..c3f8994 100644 --- a/crates/acyclic-engine/src/index.rs +++ b/crates/acyclic/src/index.rs @@ -7,10 +7,11 @@ use std::path::Path; +pub use crate::checkpoint_kind::CheckpointKind; use acyclic_fs::{Digest, GenerationId}; use rusqlite::{params, Connection, OptionalExtension}; -use crate::{EngineError, Result}; +use crate::Result; /// One checkpoint row. #[derive(Clone, Debug, PartialEq)] @@ -69,58 +70,7 @@ impl CheckpointRow { /// `failed` rows carry the generation from BEFORE the failed capture — /// restoring one would claim a state the row does not represent. pub fn is_restorable(&self) -> bool { - self.kind != CheckpointKind::Failed - } -} - -/// Why a checkpoint exists. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum CheckpointKind { - Baseline, - Pre, - Post, - Manual, - PreRewind, - Recovered, - Failed, - Noop, - /// Taken by the idle timer, not any request: the safety net for hosts - /// with no lifecycle-hook API (see `Pipeline::auto_checkpoint`). - Auto, -} - -impl CheckpointKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Baseline => "baseline", - Self::Pre => "pre", - Self::Post => "post", - Self::Manual => "manual", - Self::PreRewind => "pre_rewind", - Self::Recovered => "recovered", - Self::Failed => "failed", - Self::Noop => "noop", - Self::Auto => "auto", - } - } - - fn parse(text: &str) -> Result { - Ok(match text { - "baseline" => Self::Baseline, - "pre" => Self::Pre, - "post" => Self::Post, - "manual" => Self::Manual, - "pre_rewind" => Self::PreRewind, - "recovered" => Self::Recovered, - "failed" => Self::Failed, - "noop" => Self::Noop, - "auto" => Self::Auto, - other => { - return Err(EngineError::Store(format!( - "unknown checkpoint kind {other}" - ))) - } - }) + self.kind.is_restorable() } } @@ -198,11 +148,14 @@ impl Index { kind: CheckpointKind, attribution: &Attribution, ) -> Result { - let turn = self.effective_turn(attribution)?; + let transaction = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; if let Some(session_id) = attribution.session_id.as_deref() { - self.ensure_session(session_id)?; + ensure_session_on(&transaction, session_id)?; } - self.connection.execute( + let turn = effective_turn_on(&transaction, attribution)?; + transaction.execute( "INSERT INTO checkpoints (generation, created_at, kind, session_id, tool_call_id, tool_name, label, turn, rewind_target) @@ -219,25 +172,9 @@ impl Index { attribution.rewind_target, ], )?; - Ok(self.connection.last_insert_rowid()) - } - - /// The turn a new checkpoint belongs to: the explicit one, else the - /// session's latest recorded turn (hooks don't know turn numbers; the - /// `UserPromptSubmit` hook records them and tool hooks inherit). - fn effective_turn(&self, attribution: &Attribution) -> Result> { - if attribution.turn.is_some() { - return Ok(attribution.turn); - } - let Some(session_id) = attribution.session_id.as_deref() else { - return Ok(None); - }; - let turn: Option = self.connection.query_row( - "SELECT MAX(turn) FROM turns WHERE session_id = ?1", - params![session_id], - |row| row.get(0), - )?; - Ok(turn) + let id = transaction.last_insert_rowid(); + transaction.commit()?; + Ok(id) } /// Records a failed capture attempt (no generation advanced: the previous @@ -248,8 +185,14 @@ impl Index { error: &str, attribution: &Attribution, ) -> Result { - let turn = self.effective_turn(attribution)?; - self.connection.execute( + let transaction = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + if let Some(session_id) = attribution.session_id.as_deref() { + ensure_session_on(&transaction, session_id)?; + } + let turn = effective_turn_on(&transaction, attribution)?; + transaction.execute( "INSERT INTO checkpoints (generation, created_at, kind, session_id, tool_call_id, tool_name, error, turn) VALUES (?1, ?2, 'failed', ?3, ?4, ?5, ?6, ?7)", @@ -263,7 +206,9 @@ impl Index { turn, ], )?; - Ok(self.connection.last_insert_rowid()) + let id = transaction.last_insert_rowid(); + transaction.commit()?; + Ok(id) } /// Marks every checkpoint up to `through_id` as covered by an authority @@ -307,8 +252,9 @@ impl Index { .query_row( &format!( "SELECT {CHECKPOINT_COLUMNS} - FROM checkpoints WHERE kind IN ('baseline','pre','post','manual','auto') - ORDER BY id DESC LIMIT 1" + FROM checkpoints WHERE kind IN ({}) + ORDER BY id DESC LIMIT 1", + CheckpointKind::TARGETS_SQL ), [], row_to_checkpoint, @@ -324,8 +270,9 @@ impl Index { .query_row( &format!( "SELECT {CHECKPOINT_COLUMNS} FROM checkpoints - WHERE id < ?1 AND kind IN ('baseline','pre','post','manual','auto') - ORDER BY id DESC LIMIT 1" + WHERE id < ?1 AND kind IN ({}) + ORDER BY id DESC LIMIT 1", + CheckpointKind::TARGETS_SQL ), params![id], row_to_checkpoint, @@ -427,8 +374,9 @@ impl Index { .query_row( &format!( "SELECT {CHECKPOINT_COLUMNS} FROM checkpoints - WHERE session_id = ?1 AND kind IN ('baseline','pre','post','manual','auto') - ORDER BY id DESC LIMIT 1" + WHERE session_id = ?1 AND kind IN ({}) + ORDER BY id DESC LIMIT 1", + CheckpointKind::TARGETS_SQL ), params![session_id], row_to_checkpoint, @@ -456,8 +404,9 @@ impl Index { pub fn between(&self, after_id: i64, before_id: i64) -> Result> { let mut statement = self.connection.prepare(&format!( "SELECT {CHECKPOINT_COLUMNS} FROM checkpoints - WHERE id > ?1 AND id < ?2 AND kind IN ('baseline','pre','post','manual','auto') - ORDER BY id ASC" + WHERE id > ?1 AND id < ?2 AND kind IN ({}) + ORDER BY id ASC", + CheckpointKind::TARGETS_SQL ))?; let rows = statement.query_map(params![after_id, before_id], row_to_checkpoint)?; rows.map(|row| row.map_err(Into::into)).collect() @@ -466,16 +415,20 @@ impl Index { /// Records the start of a conversation turn; returns its 1-based number. /// The prompt is truncated to an excerpt on a char boundary. pub fn turn_started(&mut self, session_id: &str, prompt: &str) -> Result { - self.ensure_session(session_id)?; - let next: i64 = self.connection.query_row( + let transaction = self + .connection + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + ensure_session_on(&transaction, session_id)?; + let next: i64 = transaction.query_row( "SELECT COALESCE(MAX(turn), 0) + 1 FROM turns WHERE session_id = ?1", params![session_id], |row| row.get(0), )?; - self.connection.execute( + transaction.execute( "INSERT INTO turns(session_id, turn, started_at, prompt) VALUES (?1, ?2, ?3, ?4)", params![session_id, next, now(), excerpt(prompt)], )?; + transaction.commit()?; Ok(next) } @@ -566,19 +519,6 @@ impl Index { Ok(()) } - /// A turn or checkpoint for a session the daemon never saw start (its - /// `SessionStart` hook fired while the daemon was down) still gets a - /// session row, so `sessions`, `diff --turn`, and `brief` can find it. - /// The host is unknown at this point; a later `session_started` is - /// ignored by the primary key, so the row keeps its earliest start. - fn ensure_session(&mut self, session_id: &str) -> Result<()> { - self.connection.execute( - "INSERT OR IGNORE INTO sessions(session_id, host, started_at) VALUES (?1, NULL, ?2)", - params![session_id, now()], - )?; - Ok(()) - } - pub fn session_ended(&mut self, session_id: &str) -> Result<()> { self.connection.execute( "UPDATE sessions SET ended_at = ?2 WHERE session_id = ?1", @@ -588,6 +528,35 @@ impl Index { } } +fn ensure_session_on(connection: &Connection, session_id: &str) -> Result<()> { + // A turn or checkpoint for a session whose SessionStart hook was missed + // still needs a discoverable session row. A later session_started call + // fills in its host without replacing this earlier start time. + connection.execute( + "INSERT OR IGNORE INTO sessions(session_id, host, started_at) VALUES (?1, NULL, ?2)", + params![session_id, now()], + )?; + Ok(()) +} + +/// Use an explicit turn when supplied; otherwise inherit the session's most +/// recent turn. Checkpoint insertion and this lookup share one write transaction. +fn effective_turn_on(connection: &Connection, attribution: &Attribution) -> Result> { + if attribution.turn.is_some() { + return Ok(attribution.turn); + } + let Some(session_id) = attribution.session_id.as_deref() else { + return Ok(None); + }; + connection + .query_row( + "SELECT MAX(turn) FROM turns WHERE session_id = ?1", + params![session_id], + |row| row.get(0), + ) + .map_err(Into::into) +} + fn row_to_checkpoint(row: &rusqlite::Row<'_>) -> rusqlite::Result { let corrupt = |message: &str| { rusqlite::Error::FromSqlConversionFailure( @@ -606,7 +575,9 @@ fn row_to_checkpoint(row: &rusqlite::Row<'_>) -> rusqlite::Result id: row.get(0)?, generation: GenerationId::new(Digest::from_bytes(bytes)), created_at: row.get(2)?, - kind: CheckpointKind::parse(&kind_text).map_err(|error| corrupt(&error.to_string()))?, + kind: kind_text + .parse::() + .map_err(|error| corrupt(&error.to_string()))?, published: row.get::<_, i64>(4)? != 0, session_id: row.get(5)?, tool_call_id: row.get(6)?, @@ -695,9 +666,19 @@ fn ensure_auto_kind_allowed(connection: &Connection) -> Result<()> { if sql.contains("'auto'") { return Ok(()); } - connection.execute_batch( - "BEGIN IMMEDIATE; - ALTER TABLE checkpoints RENAME TO checkpoints_pre_auto; + let saved_schema = connection + .prepare( + "SELECT type, sql FROM sqlite_master + WHERE tbl_name = 'checkpoints' AND type IN ('index', 'trigger') AND sql IS NOT NULL + ORDER BY CASE type WHEN 'index' THEN 0 ELSE 1 END, name", + )? + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::>>()?; + let transaction = connection.unchecked_transaction()?; + transaction.execute_batch( + "ALTER TABLE checkpoints RENAME TO checkpoints_pre_auto; CREATE TABLE checkpoints( id INTEGER PRIMARY KEY, generation BLOB NOT NULL, @@ -719,25 +700,60 @@ fn ensure_auto_kind_allowed(connection: &Connection) -> Result<()> { tool_name, label, error, turn, rewind_target FROM checkpoints_pre_auto; DROP TABLE checkpoints_pre_auto; - CREATE INDEX IF NOT EXISTS checkpoints_by_generation ON checkpoints(generation); - CREATE INDEX IF NOT EXISTS checkpoints_by_session ON checkpoints(session_id, id); - COMMIT;", + ", + )?; + for (_, definition) in saved_schema { + transaction.execute_batch(&definition)?; + } + transaction.execute_batch( + "CREATE INDEX IF NOT EXISTS checkpoints_by_generation ON checkpoints(generation); + CREATE INDEX IF NOT EXISTS checkpoints_by_session ON checkpoints(session_id, id);", )?; + transaction.commit()?; Ok(()) } /// Bounded, single-line prompt excerpt: whitespace runs collapsed, cut on a /// char boundary at [`PROMPT_EXCERPT_BYTES`] with an ellipsis. +/// +/// The original prompt is scanned only until the excerpt fills. pub fn excerpt(prompt: &str) -> String { - let collapsed = prompt.split_whitespace().collect::>().join(" "); - if collapsed.len() <= PROMPT_EXCERPT_BYTES { - return collapsed; - } - let mut cut = PROMPT_EXCERPT_BYTES - 1; - while !collapsed.is_char_boundary(cut) { - cut -= 1; + let mut collapsed = String::with_capacity(PROMPT_EXCERPT_BYTES); + let mut words = prompt.split_whitespace().peekable(); + while let Some(word) = words.next() { + let separator = usize::from(!collapsed.is_empty()); + let remaining = PROMPT_EXCERPT_BYTES.saturating_sub(collapsed.len() + separator); + if word.len() > remaining { + let target = PROMPT_EXCERPT_BYTES - 1; + if separator != 0 && collapsed.len() < target { + collapsed.push(' '); + } + if word.len() >= target.saturating_sub(collapsed.len()) { + for character in word.chars() { + if collapsed.len() + character.len_utf8() > target { + break; + } + collapsed.push(character); + } + } else { + collapsed.push_str(word); + } + collapsed.push('…'); + return collapsed; + } + if separator != 0 { + collapsed.push(' '); + } + collapsed.push_str(word); + if collapsed.len() == PROMPT_EXCERPT_BYTES && words.peek().is_some() { + if let Some((index, _)) = collapsed.char_indices().next_back() { + collapsed.truncate(index); + } + collapsed.push('…'); + return collapsed; + } } - format!("{}…", collapsed.get(..cut).unwrap_or(&collapsed)) + collapsed } fn now() -> i64 { @@ -883,6 +899,48 @@ mod tests { assert!(late.started_at <= started); } + #[test] + fn concurrent_turn_writers_get_distinct_numbers() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("index.db"); + let mut first = Index::open(&path).expect("first index"); + let mut second = Index::open(&path).expect("second index"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let other_barrier = barrier.clone(); + let other = std::thread::spawn(move || { + other_barrier.wait(); + second.turn_started("shared", "second") + }); + barrier.wait(); + let first_turn = first.turn_started("shared", "first").expect("first turn"); + let second_turn = other.join().expect("writer thread").expect("second turn"); + assert_ne!(first_turn, second_turn); + assert_eq!(first.turns(Some("shared")).expect("turns").len(), 2); + } + + #[test] + fn failed_capture_backfills_its_session_and_inherits_the_latest_turn() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut index = Index::open(&dir.path().join("index.db")).expect("open"); + let attribution = Attribution { + session_id: Some("missed-start".into()), + ..Attribution::default() + }; + let first = index + .record_failure(generation(1), "capture failed", &attribution) + .expect("first failed capture"); + assert_eq!(index.by_id(first).expect("query").expect("row").turn, None); + assert_eq!(index.sessions(10).expect("sessions").len(), 1); + index.turn_started("missed-start", "retry").expect("turn"); + let second = index + .record_failure(generation(1), "capture failed again", &attribution) + .expect("second failed capture"); + assert_eq!( + index.by_id(second).expect("query").expect("row").turn, + Some(1) + ); + } + #[test] fn turns_link_checkpoints_and_resolve_both_ways() { let dir = tempfile::tempdir().expect("tempdir"); @@ -1015,6 +1073,45 @@ mod tests { assert!(cut.len() <= PROMPT_EXCERPT_BYTES + "…".len()); assert!(cut.ends_with('…')); assert_eq!(excerpt("a b\n\tc"), "a b c"); + let exact = "x".repeat(PROMPT_EXCERPT_BYTES); + assert_eq!(excerpt(&exact), exact); + assert_eq!( + excerpt(&format!("{exact} tail")), + format!("{}…", "x".repeat(PROMPT_EXCERPT_BYTES - 1)) + ); + let boundary = format!("{} é", "x".repeat(PROMPT_EXCERPT_BYTES - 2)); + assert_eq!( + excerpt(&boundary), + format!("{} …", "x".repeat(PROMPT_EXCERPT_BYTES - 2)) + ); + } + + #[test] + fn excerpt_matches_the_full_normalization_model() { + fn full(prompt: &str) -> String { + let collapsed = prompt.split_whitespace().collect::>().join(" "); + if collapsed.len() <= PROMPT_EXCERPT_BYTES { + return collapsed; + } + let mut cut = PROMPT_EXCERPT_BYTES - 1; + while !collapsed.is_char_boundary(cut) { + cut -= 1; + } + format!("{}…", collapsed.get(..cut).expect("character boundary")) + } + + let atoms = ["a", " ", "\n", "é", "☃", "𐍈", " ", "xyz"]; + let mut state = 0x9e37_79b9_7f4a_7c15_u64; + for _ in 0..1_000 { + let mut prompt = String::new(); + for _ in 0..400 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + prompt.push_str(atoms[(state as usize) % atoms.len()]); + } + assert_eq!(excerpt(&prompt), full(&prompt)); + } } #[test] @@ -1031,7 +1128,9 @@ mod tests { published INTEGER NOT NULL DEFAULT 0, session_id TEXT, tool_call_id TEXT, tool_name TEXT, label TEXT, error TEXT); INSERT INTO checkpoints(generation, created_at, kind) - VALUES (zeroblob(32), 1, 'post');", + VALUES (zeroblob(32), 1, 'post'); + CREATE INDEX checkpoints_by_generation ON checkpoints(generation); + CREATE INDEX checkpoints_by_session ON checkpoints(session_id, id);", ) .expect("seed"); } @@ -1062,7 +1161,11 @@ mod tests { tool_call_id TEXT, tool_name TEXT, label TEXT, error TEXT, turn INTEGER, rewind_target INTEGER); INSERT INTO checkpoints(generation, created_at, kind) - VALUES (zeroblob(32), 1, 'post');", + VALUES (zeroblob(32), 1, 'post'); + CREATE TABLE checkpoint_audit(inserted INTEGER NOT NULL); + CREATE INDEX custom_post_idx ON checkpoints(created_at) WHERE kind = 'post'; + CREATE TRIGGER audit_checkpoint_insert AFTER INSERT ON checkpoints + BEGIN INSERT INTO checkpoint_audit(inserted) VALUES (NEW.id); END;", ) .expect("seed"); } @@ -1078,5 +1181,25 @@ mod tests { .expect("auto checkpoint should now be a valid kind"); let row = index.by_id(row_id).expect("q").expect("row"); assert_eq!(row.kind, CheckpointKind::Auto); + let indexes: Vec = index + .connection + .prepare("PRAGMA index_list(checkpoints)") + .expect("prepare index list") + .query_map([], |row| row.get(1)) + .expect("index list") + .collect::>() + .expect("index names"); + assert!(indexes + .iter() + .any(|name| name == "checkpoints_by_generation")); + assert!(indexes.iter().any(|name| name == "checkpoints_by_session")); + assert!(indexes.iter().any(|name| name == "custom_post_idx")); + let audit_id: i64 = index + .connection + .query_row("SELECT inserted FROM checkpoint_audit", [], |row| { + row.get(0) + }) + .expect("custom trigger fired after migration"); + assert_eq!(audit_id, row_id); } } diff --git a/crates/acyclic/src/install.rs b/crates/acyclic/src/install.rs index 3ead803..31391df 100644 --- a/crates/acyclic/src/install.rs +++ b/crates/acyclic/src/install.rs @@ -8,14 +8,14 @@ //! API, so they get `acyclic mcp` (see `crate::mcp`) registered as an MCP //! server in whatever config file that host reads — project-scoped and //! checked in where the host supports it, the user's global config where -//! it doesn't. agents-md is the fallback for anything shell-capable: a -//! cheatsheet block in AGENTS.md and no hooks at all. +//! it doesn't. The agents-md adapter gives shell-capable hosts a cheatsheet +//! block in AGENTS.md and no hooks. //! //! Each `HostAdapter` below documents exactly what its host gets. The //! README's per-host table is the user-facing version of the same list, //! with how far each adapter has been verified. -use acyclic_engine::product::{self, NAME, NPM_PACKAGE, PYPI_PACKAGE}; +use acyclic::product::{self, NAME, NPM_PACKAGE, PYPI_PACKAGE}; use std::path::{Path, PathBuf}; use serde_json::{json, Value}; @@ -238,9 +238,6 @@ fn merge_hooks(path: &Path, host: &str) -> Result<(), String> { let root_map = root .as_object_mut() .ok_or_else(|| format!("{} is not an object", path.display()))?; - if host == "codex" { - remove_flat_codex_hooks(root_map); - } let hooks = root_map.entry("hooks").or_insert(json!({})); let hooks = hooks.as_object_mut().ok_or("hooks is not an object")?; merge_event_hooks(hooks, "hooks", host)?; @@ -250,22 +247,6 @@ fn merge_hooks(path: &Path, host: &str) -> Result<(), String> { Ok(()) } -/// Earlier releases wrote Codex's events at the top level of `hooks.json` -/// (`{"PreToolUse": [...]}`), a shape Codex 0.154 silently ignores. Drop -/// our entries from that layout so a re-install moves them under `hooks`; -/// anything a user put there is left alone. -fn remove_flat_codex_hooks(root: &mut serde_json::Map) { - for (event, _, _) in hook_events("codex") { - let Some(entries) = root.get_mut(event).and_then(Value::as_array_mut) else { - continue; - }; - entries.retain(|entry| !is_ours(entry)); - if entries.is_empty() { - root.remove(event); - } - } -} - /// Merges our entries into a map keyed by event name (the value under /// `"hooks"`). Same idempotency contract as `merge_hooks`. fn merge_event_hooks( @@ -326,7 +307,7 @@ fn is_our_command(command: &str) -> bool { /// `~/.codex/config.toml` / `.codex/config.toml`. If that also means they /// share whatever fires `.codex/hooks.json`'s lifecycle events, this /// adapter may already cover the desktop app and IDE extension too, with no -/// new code — verify that first. The MCP fallback is already known to +/// new code — verify that first. MCP is already known to /// work (docs/manual-testing.md): `[mcp_servers.]` with `command`, /// `args` and `default_tools_approval_mode = "approve"` (without it a /// non-interactive session rejects every call). It is TOML, so a writer @@ -349,7 +330,7 @@ fn codex(repo: &Path) -> Result<(), String> { /// and `type` directly rather than Claude/Codex's nested `hooks` array), /// plus an always-applied project rule so the model has the CLI verbs in /// context. Cursor's hook events and payload shape differ from Claude -/// Code/Codex (see `hook::Payload`'s `conversation_id`/`command` fallbacks +/// Code/Codex (see `hook::Payload`'s host-specific fields /// and `hook::run`'s Cursor-only `{"permission":"allow"}` reply), so this /// writes Cursor's own event names rather than reusing `hook_events()`. fn cursor(repo: &Path) -> Result<(), String> { @@ -714,7 +695,7 @@ fn write_atomic(path: &Path, text: &str) -> Result<(), String> { file.write_all(text.as_bytes()).map_err(stringify)?; file.sync_all().map_err(stringify)?; drop(file); - std::fs::rename(&tmp, path).map_err(stringify) + acyclic_fs::durable_rename(&tmp, path, acyclic_fs::RenameMode::Replace).map_err(stringify) } /// Where a host reads its MCP config. This decides both the path and the @@ -1462,9 +1443,7 @@ effective values in ONE line before the first fork, e.g. Depth starts at 1 and increases by one per round. 1. `{{name}} fork -n `. It records the fork base as a checkpoint - itself, so do not checkpoint first. Note each id and path and whether - it says `(mount)` or `(copy)`. Copy forks cost time proportional to - the tree: keep them few and short-lived. + itself, so do not checkpoint first. Note each id and mounted path. 2. Dispatch ALL subagents in one turn, one per fork, using the CHILD PROMPT below. Do not keep one approach for yourself. 3. **Freeze.** Make NO edits to the real tree while forks are live. A @@ -1551,7 +1530,7 @@ After the final round: `{{name}} diff ` and summarise the blast radius, ignoring `m` (metadata-only) lines. Mention anything a script or generator wrote. -## 6. Failure and fallback +## 6. Failure handling - A promote that reports `N file(s) conflict` has written conflict markers into THAT FORK (the mainline is untouched) and moved the fork @@ -1827,44 +1806,6 @@ mod tests { ); } - #[test] - fn codex_reinstall_migrates_the_legacy_flat_layout() { - let dir = tempfile::tempdir().expect("tempdir"); - let hooks_path = dir.path().join(".codex/hooks.json"); - std::fs::create_dir_all(dir.path().join(".codex")).expect("mkdir"); - let legacy = json!({ - "PreToolUse": [ - { "hooks": [{ "type": "command", "command": "echo user-hook" }] }, - { "hooks": [{ "type": "command", - "command": format!("ACYCLIC_HOST=codex {NAME} hook pre-tool") }] } - ], - "SessionEnd": [ - { "hooks": [{ "type": "command", - "command": format!("ACYCLIC_HOST=codex {NAME} hook session-end") }] } - ] - }); - std::fs::write(&hooks_path, legacy.to_string()).expect("seed"); - - codex(dir.path()).expect("install"); - let value: Value = - serde_json::from_str(&std::fs::read_to_string(&hooks_path).expect("read")) - .expect("json"); - assert_eq!( - value["PreToolUse"].as_array().map(Vec::len), - Some(1), - "user hook kept" - ); - assert!( - value.get("SessionEnd").is_none(), - "emptied legacy key removed" - ); - assert_eq!( - value["hooks"]["PreToolUse"].as_array().map(Vec::len), - Some(1), - "ours lives under hooks now" - ); - } - #[test] fn cursor_install_is_idempotent_and_writes_rule() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/acyclic/src/ipc.rs b/crates/acyclic/src/ipc.rs index 92a535b..e6f5679 100644 --- a/crates/acyclic/src/ipc.rs +++ b/crates/acyclic/src/ipc.rs @@ -6,7 +6,7 @@ //! the endpoint naming, the "already running" check, and teardown differ. //! //! Callers name the endpoint with the socket path from -//! [`acyclic_engine::store::Paths::socket`] on every platform. On Windows +//! [`acyclic::store::Paths::socket`] on every platform. On Windows //! that path is never created on disk — it only supplies the stable, //! per-store name the pipe is built from. @@ -58,7 +58,7 @@ fn pipe_name(socket: &Path) -> String { } }) .collect(); - format!(r"\\.\pipe\{}-{key}", acyclic_engine::product::NAME) + format!(r"\\.\pipe\{}-{key}", acyclic::product::NAME) } // ---------------------------------------------------------------- client @@ -68,7 +68,13 @@ pub struct ClientStream { #[cfg(unix)] inner: std::os::unix::net::UnixStream, #[cfg(windows)] - inner: std::fs::File, + inner: tokio::net::windows::named_pipe::NamedPipeClient, + #[cfg(windows)] + runtime: tokio::runtime::Runtime, + #[cfg(windows)] + read_timeout: std::cell::Cell>, + #[cfg(windows)] + write_timeout: std::cell::Cell>, } impl ClientStream { @@ -83,19 +89,31 @@ impl ClientStream { } #[cfg(windows)] { + use tokio::net::windows::named_pipe::ClientOptions; // A named pipe is opened like a file. Every server instance being // momentarily busy is normal under concurrent hooks, so a short // bounded retry stands in for WaitNamedPipe. const BUSY: i32 = 231; // ERROR_PIPE_BUSY let name = pipe_name(socket); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build()?; let mut last = None; for _ in 0..20 { - match std::fs::OpenOptions::new() - .read(true) - .write(true) - .open(&name) - { - Ok(file) => return Ok(Self { inner: file }), + let opened = { + let _entered = runtime.enter(); + ClientOptions::new().open(&name) + }; + match opened { + Ok(inner) => { + return Ok(Self { + inner, + runtime, + read_timeout: std::cell::Cell::new(None), + write_timeout: std::cell::Cell::new(None), + }); + } Err(error) => { if error.raw_os_error() != Some(BUSY) { return Err(error); @@ -109,9 +127,7 @@ impl ClientStream { } } - /// Bounds a single read. Windows named pipes opened as files carry no - /// per-handle timeout, so this is a no-op there — see the deadline note - /// in `docs/windows-verification.md`. + /// Bounds a single read, including Windows overlapped named-pipe reads. pub fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { #[cfg(unix)] { @@ -119,12 +135,12 @@ impl ClientStream { } #[cfg(windows)] { - let _ = timeout; + self.read_timeout.set(timeout); Ok(()) } } - /// Bounds a single write. No-op on Windows, as for reads. + /// Bounds a single write, including Windows overlapped named-pipe writes. pub fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { #[cfg(unix)] { @@ -132,7 +148,7 @@ impl ClientStream { } #[cfg(windows)] { - let _ = timeout; + self.write_timeout.set(timeout); Ok(()) } } @@ -140,16 +156,97 @@ impl ClientStream { impl io::Read for ClientStream { fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.inner.read(buf) + #[cfg(unix)] + { + self.inner.read(buf) + } + #[cfg(windows)] + { + use tokio::io::AsyncReadExt as _; + match self.read_timeout.get() { + Some(timeout) => self + .runtime + .block_on(async { tokio::time::timeout(timeout, self.inner.read(buf)).await }) + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "named-pipe read timed out") + })?, + None => self.runtime.block_on(async { self.inner.read(buf).await }), + } + } } } impl io::Write for ClientStream { fn write(&mut self, buf: &[u8]) -> io::Result { - self.inner.write(buf) + #[cfg(unix)] + { + self.inner.write(buf) + } + #[cfg(windows)] + { + use tokio::io::AsyncWriteExt as _; + match self.write_timeout.get() { + Some(timeout) => self + .runtime + .block_on(async { tokio::time::timeout(timeout, self.inner.write(buf)).await }) + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "named-pipe write timed out") + })?, + None => self.runtime.block_on(async { self.inner.write(buf).await }), + } + } } fn flush(&mut self) -> io::Result<()> { - self.inner.flush() + #[cfg(unix)] + { + self.inner.flush() + } + #[cfg(windows)] + { + Ok(()) + } + } +} + +#[cfg(all(test, windows))] +mod deadline_tests { + use super::*; + use std::io::Read as _; + use std::time::{Duration, Instant}; + + #[test] + fn named_pipe_read_deadline_is_enforced() { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let socket = std::path::PathBuf::from(format!("deadline-{}-{nonce}", std::process::id())); + let name = pipe_name(&socket); + let (ready, connected) = std::sync::mpsc::channel(); + let server = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime"); + runtime.block_on(async { + let pipe = create_pipe_instance(&name, true).expect("pipe"); + ready.send(()).expect("ready"); + pipe.connect().await.expect("connect"); + tokio::time::sleep(Duration::from_millis(200)).await; + }); + }); + connected.recv().expect("server ready"); + let mut client = ClientStream::connect(&socket).expect("client connect"); + client + .set_read_timeout(Some(Duration::from_millis(30))) + .expect("deadline"); + let started = Instant::now(); + let error = client + .read(&mut [0_u8; 1]) + .expect_err("read should time out"); + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert!(started.elapsed() < Duration::from_millis(180)); + server.join().expect("server exit"); } } diff --git a/crates/acyclic-engine/src/lib.rs b/crates/acyclic/src/lib.rs similarity index 99% rename from crates/acyclic-engine/src/lib.rs rename to crates/acyclic/src/lib.rs index 6c03408..f5d45ba 100644 --- a/crates/acyclic-engine/src/lib.rs +++ b/crates/acyclic/src/lib.rs @@ -40,6 +40,7 @@ pub fn short_hex(hex: &str) -> &str { hex.get(..12).unwrap_or(hex) } +pub mod checkpoint_kind; pub mod config; pub mod diff; pub mod exclude; @@ -47,7 +48,6 @@ pub mod fork; pub mod guard; pub mod index; pub mod merge; -pub mod names; pub mod pipeline; pub mod product; pub mod rewind; diff --git a/crates/acyclic/src/main.rs b/crates/acyclic/src/main.rs index e2d0166..d9f5fe9 100644 --- a/crates/acyclic/src/main.rs +++ b/crates/acyclic/src/main.rs @@ -19,15 +19,15 @@ mod hook; mod install; mod ipc; mod mcp; +mod proto; mod server; mod spec_runner; mod speculate; use std::path::{Path, PathBuf}; -use acyclic_engine::product::{self, NAME}; -use acyclic_engine::short_hex; -use acyclic_proto as proto; +use acyclic::product::{self, NAME}; +use acyclic::short_hex; use clap::{Parser, Subcommand}; use client::{Client, ConnectError, Spawn}; @@ -61,9 +61,6 @@ enum Command { /// a queued snapshot can include edits made before it runs. #[arg(long, conflicts_with = "durable")] no_wait: bool, - /// Accepted for compatibility: waiting is now the default. - #[arg(long, hide = true)] - wait: bool, /// Also publish to the durable authority (coarse boundary). #[arg(long)] durable: bool, @@ -225,17 +222,6 @@ enum Command { /// Record a host session ending (hook use). #[command(hide = true)] SessionEnd { session_id: String }, - /// Safe Mode: commit a session's shadow fork and show what it would - /// change, without touching the real tree yet. - #[command(hide = true)] - SessionResolve { session_id: String }, - /// Safe Mode: apply a `session-resolve`d session's changes to the real - /// tree. - #[command(hide = true)] - SessionApply { session_id: String }, - /// Safe Mode: discard a `session-resolve`d session without applying it. - #[command(hide = true)] - SessionDiscard { session_id: String }, /// Internal: the per-repo daemon process. #[command(name = "__daemon", hide = true)] Daemon { repo_root: PathBuf }, @@ -259,17 +245,41 @@ fn main() { libc::signal(libc::SIGPIPE, libc::SIG_DFL); } } + if matches!(cli.command, Command::Daemon { .. }) { + std::process::exit(run(cli, Path::new("."))); + } let repo_arg = cli.repo.clone().unwrap_or_else(|| PathBuf::from(".")); - // Before touching the repo path (canonicalize, config load, socket): a - // crashed Safe Mode daemon can leave a dead shadow mount over the repo - // root that wedges every stat under it. Force-unmount it first (a no-op - // unless a bounded probe shows the mount is genuinely wedged), so the - // real tree is back before we read anything. - acyclic_engine::fork::reap_dead_shadow(&repo_arg); + let (repo_arg, recovered) = acyclic::rewind::recover_before_repo_open(&repo_arg) + .unwrap_or_else(|error| { + eprintln!("{}: rewind recovery: {error}", product::NAME); + std::process::exit(1); + }); let repo = repo_arg.canonicalize().unwrap_or_else(|error| { eprintln!("{}: bad repo path: {error}", product::NAME); std::process::exit(1); }); + let recovery = match recovered { + Some(recovered) if recovered.reconcile_head => (|| -> Result<(), String> { + let config = acyclic::config::Config::load(&repo).map_err(|error| error.to_string())?; + let stores_root = config.store_dir.as_ref().map(PathBuf::from); + let paths = acyclic::store::StorePaths::for_repo(&repo, stores_root.as_deref()) + .map_err(|error| error.to_string())?; + let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; + let mut store = runtime + .block_on(acyclic::store::Store::open(&repo, paths)) + .map_err(|error| error.to_string())?; + runtime + .block_on(acyclic::rewind::recover_workspace(&mut store, recovered)) + .map_err(|error| error.to_string())?; + acyclic::rewind::finish_recovery(&repo).map_err(|error| error.to_string()) + })(), + Some(_) => acyclic::rewind::finish_recovery(&repo).map_err(|error| error.to_string()), + None => Ok(()), + }; + if let Err(error) = recovery { + eprintln!("{}: finish rewind recovery: {error}", product::NAME); + std::process::exit(1); + } if cli.repo.is_none() && stranded_in_trash(&repo) { // A rewind or promote swaps the repo directory's inode; a shell that // was inside it now resolves its cwd to the replaced tree in trash. @@ -287,7 +297,7 @@ fn main() { /// True when `repo` (canonical) lies inside a store's trash (an ancestor /// named `trash` whose parent is a store root, marked by `meta.json`), or -/// inside a rewind's sibling fallback trash directory (`..acyclic-trash-*`). +/// inside a rewind's sibling trash directory (`..acyclic-trash-*`). fn stranded_in_trash(repo: &Path) -> bool { repo.ancestors().any(|ancestor| { let Some(name) = ancestor.file_name().map(|name| name.to_string_lossy()) else { @@ -360,7 +370,7 @@ fn run(cli: Cli, repo: &Path) -> i32 { return 1; } }; - match execute(&mut client, command) { + match execute(&mut client, command, repo) { Ok(()) => 0, Err(message) => { eprintln!("{}: {message}", product::NAME); @@ -389,10 +399,10 @@ fn step_aside() { #[cfg(not(windows))] fn step_aside() {} -fn store_paths(repo: &Path) -> Result { - let config = acyclic_engine::config::Config::load(repo).map_err(|error| error.to_string())?; +fn store_paths(repo: &Path) -> Result { + let config = acyclic::config::Config::load(repo).map_err(|error| error.to_string())?; let stores_root = config.store_dir.as_ref().map(PathBuf::from); - acyclic_engine::store::StorePaths::for_repo(repo, stores_root.as_deref()) + acyclic::store::StorePaths::for_repo(repo, stores_root.as_deref()) .map_err(|error| error.to_string()) } @@ -402,21 +412,18 @@ fn connect(repo: &Path, spawn: Spawn) -> Result { Client::connect(&paths.socket(), repo, &log, spawn) } -/// One line on what forks and Safe Mode can do here, plus setup steps when -/// the host lacks a mount provider. Shown by `init` and `install`. +/// One line on native fork support, plus setup steps when the host lacks a +/// mount provider. Shown by `init` and `install`. fn print_mount_capability() { - let capability = acyclic_engine::fork::mount_capability(); + let capability = acyclic::fork::mount_capability(); if capability.available { - println!( - "mounts: {} (forks and Safe Mode available)", - capability.provider - ); + println!("mounts: {} (forks mount)", capability.provider); } else { println!( "mounts: unavailable ({})", capability.reason.as_deref().unwrap_or("unknown reason") ); - println!("{}", acyclic_engine::fork::mount_setup_hint()); + println!("{}", acyclic::fork::mount_setup_hint()); } } @@ -533,7 +540,7 @@ fn print_speculation(spec: &proto::SpecStatus) { } fn policy(repo: &Path) -> i32 { - match acyclic_engine::config::Config::load(repo) { + match acyclic::config::Config::load(repo) { Ok(config) => { let d = config.decompose; println!("fan_out = {}", d.fan_out); @@ -560,28 +567,28 @@ fn policy(repo: &Path) -> i32 { fn init(repo: &Path) -> i32 { let result = (|| -> Result<(), String> { - let config = - acyclic_engine::config::Config::load(repo).map_err(|error| error.to_string())?; + let config = acyclic::config::Config::load(repo).map_err(|error| error.to_string())?; let stores_root = config.store_dir.as_ref().map(PathBuf::from); - let paths = acyclic_engine::store::StorePaths::for_repo(repo, stores_root.as_deref()) + let paths = acyclic::store::StorePaths::for_repo(repo, stores_root.as_deref()) .map_err(|error| error.to_string())?; if paths.meta().exists() { println!("store already exists at {}", paths.root.display()); } else { let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; runtime - .block_on(acyclic_engine::store::Store::init(repo, paths.clone())) + .block_on(acyclic::store::Store::init(repo, paths.clone())) .map_err(|error| error.to_string())?; println!("store created at {}", paths.root.display()); } - // Spawning the daemon builds (or refreshes) the baseline. + // Spawning opens metadata and the watcher without scanning descendants. + // The first content-dependent operation establishes the baseline. let mut client = connect(repo, Spawn::Allowed).map_err(|error| match error { ConnectError::NoDaemon => "daemon failed to start".to_owned(), ConnectError::Starting => "daemon is still starting".to_owned(), ConnectError::Other(message) => message, })?; client.call(proto::Op::Ping)?; - println!("daemon ready — checkpointing is on"); + println!("daemon ready — checkpointing activates on first use"); print_mount_capability(); Ok(()) })(); @@ -598,12 +605,11 @@ fn init(repo: &Path) -> i32 { clippy::too_many_lines, reason = "one arm per Command; each arm is a single call plus its printing" )] -fn execute(client: &mut Client, command: Command) -> Result<(), String> { +fn execute(client: &mut Client, command: Command, repo: &Path) -> Result<(), String> { match command { Command::Checkpoint { message, no_wait, - wait: _, durable, kind, session_id, @@ -636,14 +642,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { turn, limit, } => { - let reply = client.call(proto::Op::Timeline { - session_id: session, - turn, - limit, - })?; - let proto::Reply::Timeline(entries) = reply else { - return Err("unexpected reply".into()); - }; + let entries = client.timeline(session, turn, limit)?; if entries.is_empty() { println!("no checkpoints yet"); return Ok(()); @@ -836,13 +835,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { } Command::Restore { checkpoint, paths } => { for path in paths { - let reply = client.call(proto::Op::Rewind { - target: proto::RewindTarget::Checkpoint(checkpoint), - path: Some(path), - })?; - let proto::Reply::Restore(info) = reply else { - return Err("unexpected reply".into()); - }; + let info = client.restore(checkpoint, path)?; match info.action { proto::RestoreAction::Removed => { println!("{}: absent at #{}, removed", info.path, info.checkpoint); @@ -881,10 +874,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { } } step_aside(); - let reply = client.call(proto::Op::Rewind { target, path: None })?; - let proto::Reply::Rewind(info) = reply else { - return Err("unexpected reply".into()); - }; + let info = client.rewind(target)?; println!("restored checkpoint #{}", info.restored_checkpoint); println!("old tree kept at {}", info.old_tree); println!("note: {}", info.warning); @@ -905,15 +895,7 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { let (after, after_hex) = checkpoint_ref(after.as_deref())?; (before, after, before_hex, after_hex) }; - let reply = client.call(proto::Op::Diff { - before, - after, - before_hex, - after_hex, - })?; - let proto::Reply::Diff(entries) = reply else { - return Err("unexpected reply".into()); - }; + let entries = client.diff(before, after, before_hex, after_hex)?; print_diff(&entries); Ok(()) } @@ -926,18 +908,12 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { return Err("unexpected reply".into()); }; for entry in &entries { - println!("fork {} ({}) {}", entry.id, entry.mode, entry.path); + println!("fork {} {}", entry.id, entry.path); } println!( "{} fork(s) ready — work in them freely; `{NAME} promote ` keeps a winner", entries.len() ); - if entries.iter().any(|entry| entry.mode == "copy") { - println!( - "note: no mount provider on this host, so these are full copies \ - (`{NAME} status` explains; promote works the same)" - ); - } Ok(()) } Command::Forks => { @@ -959,9 +935,8 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { ) }; println!( - "{} {} {} {} base {}{conflict}", + "{} {} {} base {}{conflict}", entry.id, - entry.mode, age(entry.created_at), entry.path, short_hex(&entry.base) @@ -1028,15 +1003,11 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { .map_or_else(|| "none".into(), |id| format!("#{id}")) ); println!("unpublished: {}", info.unpublished); - println!("store size: {}", human_bytes(info.store_bytes)); if info.mount_available { - println!( - "mounts: {} (forks mount, Safe Mode on)", - info.mount_provider - ); + println!("mounts: {} (forks mount)", info.mount_provider); } else { println!( - "mounts: unavailable ({}) — forks copy, Safe Mode off", + "mounts: unavailable ({}) — forks disabled", info.mount_reason.as_deref().unwrap_or("unknown reason") ); } @@ -1071,7 +1042,15 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { } Command::Stop => { client.call(proto::Op::Stop)?; - println!("daemon stopping"); + let pidfile = store_paths(repo)?.pidfile(); + let started = std::time::Instant::now(); + while pidfile.exists() { + if started.elapsed() >= std::time::Duration::from_secs(30) { + return Err("daemon did not finish stopping within 30 seconds".into()); + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + println!("daemon stopped"); Ok(()) } Command::SessionStart { session_id, host } => { @@ -1082,50 +1061,6 @@ fn execute(client: &mut Client, command: Command) -> Result<(), String> { client.call(proto::Op::SessionEnd { session_id })?; Ok(()) } - Command::SessionResolve { session_id } => { - let reply = client.call(proto::Op::SessionResolve { session_id })?; - let proto::Reply::SessionPending(info) = reply else { - return Err("unexpected reply".into()); - }; - if info.diff.is_empty() { - println!("session {}: no changes", info.session_id); - return Ok(()); - } - for entry in &info.diff { - println!("{} {}", entry.change.tag(), entry.path); - } - println!( - "{} paths changed; run `{NAME} session-apply {}` to land them or \ - `{NAME} session-discard {}` to throw them away", - info.diff.len(), - info.session_id, - info.session_id - ); - Ok(()) - } - Command::SessionApply { session_id } => { - step_aside(); - let reply = client.call(proto::Op::SessionApply { session_id })?; - let proto::Reply::Promote(info) = reply else { - return Err("unexpected reply".into()); - }; - match info.old_tree { - Some(old_tree) => { - println!( - "applied: working tree now at {}", - short_hex(&info.generation) - ); - println!("old tree kept at {old_tree}"); - println!("note: {}", info.warning); - } - None => println!("session had no changes; nothing to land"), - } - Ok(()) - } - Command::SessionDiscard { session_id } => { - client.call(proto::Op::SessionDiscard { session_id })?; - Ok(()) - } Command::Init | Command::Policy | Command::Daemon { .. } @@ -1225,7 +1160,7 @@ pub(crate) fn short_session(session_id: &str) -> String { } fn age(created_at: i64) -> String { - let delta = (acyclic_engine::unix_now() - created_at).max(0); + let delta = (acyclic::unix_now() - created_at).max(0); if delta < 60 { format!("{delta}s ago") } else if delta < 3600 { diff --git a/crates/acyclic/src/mcp.rs b/crates/acyclic/src/mcp.rs index 837b849..c03cac2 100644 --- a/crates/acyclic/src/mcp.rs +++ b/crates/acyclic/src/mcp.rs @@ -3,7 +3,7 @@ //! that support it alongside hooks (Cursor). Exposes the same verbs already //! surfaced to every other host via `AGENTS_MD_BLOCK`/`SELF_ROLLBACK_SKILL` //! (see `install.rs`) as MCP tools, translating each call directly into the -//! `acyclic-proto::Op` the daemon already understands. The MCP adapters in +//! `proto::Op` the daemon already understands. The MCP adapters in //! `install.rs` register it with each host; `tests/acceptance/mcp-e2e.sh` //! drives it end-to-end. //! @@ -19,8 +19,8 @@ use std::path::{Path, PathBuf}; -use acyclic_engine::product::{self, NAME}; -use acyclic_proto as proto; +use crate::proto; +use acyclic::product::{self, NAME}; use rmcp::{ handler::server::wrapper::Parameters, model::{ErrorCode, Implementation, ServerCapabilities, ServerInfo}, @@ -104,16 +104,7 @@ where /// recorded afterwards (the same thing the CLI prints), so the caller can /// refer to the post-restore state without another `timeline` call. fn restore_one(client: &mut Client, checkpoint: i64, path: String) -> Result { - let reply = call( - client, - proto::Op::Rewind { - target: proto::RewindTarget::Checkpoint(checkpoint), - path: Some(path), - }, - )?; - let proto::Reply::Restore(info) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let info = client.restore(checkpoint, path).map_err(internal_error)?; let what = match info.action { proto::RestoreAction::Removed => format!("absent at #{}, removed", info.checkpoint), _ => format!("restored from #{}", info.checkpoint), @@ -211,21 +202,9 @@ impl McpServer { Parameters(params): Parameters, ) -> Result { with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Checkpoint { - kind: proto::CheckpointRequestKind::Manual, - session_id: None, - tool_call_id: None, - tool_name: None, - label: params.message, - wait: true, - durable: false, - }, - )?; - let proto::Reply::Checkpoint(info) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let info = client + .manual_checkpoint(params.message) + .map_err(internal_error)?; Ok(format!("checkpoint #{} ({})", info.row_id, info.generation)) }) .await @@ -240,17 +219,9 @@ impl McpServer { Parameters(params): Parameters, ) -> Result { with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Timeline { - session_id: None, - turn: None, - limit: params.limit.unwrap_or(50), - }, - )?; - let proto::Reply::Timeline(entries) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let entries = client + .timeline(None, None, params.limit.unwrap_or(50)) + .map_err(internal_error)?; if entries.is_empty() { return Ok("no checkpoints yet".into()); } @@ -341,16 +312,9 @@ impl McpServer { )); } with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Rewind { - target: proto::RewindTarget::Checkpoint(params.checkpoint), - path: None, - }, - )?; - let proto::Reply::Rewind(info) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let info = client + .rewind(proto::RewindTarget::Checkpoint(params.checkpoint)) + .map_err(internal_error)?; Ok(format!( "restored checkpoint #{}\nold tree kept at {}\nnote: {}", info.restored_checkpoint, info.old_tree, info.warning @@ -366,18 +330,9 @@ impl McpServer { )] async fn diff(&self, Parameters(params): Parameters) -> Result { with_daemon(self.repo.clone(), move |client| { - let reply = call( - client, - proto::Op::Diff { - before: params.before, - after: params.after, - before_hex: None, - after_hex: None, - }, - )?; - let proto::Reply::Diff(entries) = reply else { - return Err(internal_error("unexpected reply".into())); - }; + let entries = client + .diff(params.before, params.after, None, None) + .map_err(internal_error)?; if entries.is_empty() { return Ok("no changes".into()); } diff --git a/crates/acyclic-engine/src/merge.rs b/crates/acyclic/src/merge.rs similarity index 58% rename from crates/acyclic-engine/src/merge.rs rename to crates/acyclic/src/merge.rs index 029b51b..e1a4a50 100644 --- a/crates/acyclic-engine/src/merge.rs +++ b/crates/acyclic/src/merge.rs @@ -4,149 +4,39 @@ //! **H** ("theirs"), and the fork snapshot **F** ("ours"; the fork is //! `ours` because it merges *into* the mainline, graphcoder's convention). //! [`plan`] walks the union of changed paths, applies the entry-level -//! decision table, runs [`merge3`] on regular text files both sides +//! decision table, runs the SDK text driver on regular text files both sides //! changed, and returns everything the daemon needs to either land the //! merge or rebase the fork with conflict markers. Nothing here writes to //! the working tree. //! -//! `merge3`, its trailing-newline rule, and [`has_conflict_markers`] are -//! verbatim ports of graphcoder's `lib/compute/src/merge.rs` and -//! `local/src/lib/worktree/conflict/markers.ts`. +//! The SDK owns marker and newline semantics so every consumer sees the same result. -use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; -use acyclic_fs::kernel::{FileKind, FileMetadata, MetadataField, NamespacePath}; -use acyclic_fs::{ByteRange, CancellationToken, GenerationId, WorkCounters}; +use acyclic_fs::kernel::{FileKind, FileMetadata, FileRecord, MetadataField, NamespacePath}; +pub use acyclic_fs::text_merge::{ + conflict_hunks, has_conflict_markers, ByteConflictKind as ConflictKind, +}; +use acyclic_fs::text_merge::{ + merge_bytes, ByteMerge as ContentMerge, ByteMergeError, ByteMergeLimits, +}; +use acyclic_fs::{ + AuthoredMutation, ByteRange, CancellationToken, GenerationId, ResolvedFileRangeReadRequest, + WorkCounters, +}; use bytes::Bytes; -use crate::diff::{self, RecordSummary}; -use crate::rewind::{namespace_path, validate_relative}; +use crate::diff; +use crate::rewind::validate_relative; use crate::store::{LocalCheckout, Store}; use crate::{EngineError, Result}; /// Default `[merge] max_file_bytes`. pub const DEFAULT_MAX_FILE_BYTES: u64 = 4 * 1024 * 1024; const TRANSFER_BYTES: u64 = 8 * 1024 * 1024; +const FILE_READ_CONCURRENCY: usize = 32; const PAGE_ENTRIES: u32 = 1_024; -const BINARY_PROBE_BYTES: usize = 8 * 1024; - -// --------------------------------------------------------------------------- -// merge3: graphcoder's three-way text merge, ported verbatim -// --------------------------------------------------------------------------- - -/// Result of a 3-way merge operation. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Merge3Result { - /// True if merge completed without conflicts. - pub clean: bool, - /// Merged content. If clean=false, contains conflict markers. - pub content: String, -} - -/// Ensures a string ends with a newline so conflict markers always occupy -/// complete lines. Borrows when nothing needs adding. -fn ensure_trailing_newline(s: &str) -> Cow<'_, str> { - if s.is_empty() || s.ends_with('\n') { - Cow::Borrowed(s) - } else { - Cow::Owned(format!("{s}\n")) - } -} - -/// Whether the clean result keeps a trailing newline: if ours and theirs -/// agree, that; else if ours agrees with base, theirs decides; else ours. -fn merged_has_trailing_newline(base: &str, ours: &str, theirs: &str) -> bool { - let base_has_newline = base.ends_with('\n'); - let ours_has_newline = ours.ends_with('\n'); - let theirs_has_newline = theirs.ends_with('\n'); - if ours_has_newline == theirs_has_newline { - ours_has_newline - } else if ours_has_newline == base_has_newline { - theirs_has_newline - } else { - ours_has_newline - } -} - -/// Performs a 3-way merge (diffy, diff3 conflict style, marker length 7). -/// -/// Conflict marker format: -/// ```text -/// <<<<<<< {ours_name} -/// ... ours content ... -/// ||||||| original -/// ... base content ... -/// ======= -/// ... theirs content ... -/// >>>>>>> {theirs_name} -/// ``` -pub fn merge3( - base: &str, - ours: &str, - theirs: &str, - ours_name: &str, - theirs_name: &str, -) -> Merge3Result { - use diffy::{ConflictStyle, MergeOptions}; - - let mut options = MergeOptions::new(); - options - .set_conflict_style(ConflictStyle::Diff3) - .set_conflict_marker_length(7); - - let base_norm = ensure_trailing_newline(base); - let ours_norm = ensure_trailing_newline(ours); - let theirs_norm = ensure_trailing_newline(theirs); - - match options.merge(&base_norm, &ours_norm, &theirs_norm) { - Ok(mut merged) => { - if !merged_has_trailing_newline(base, ours, theirs) { - merged.pop(); - } - Merge3Result { - clean: true, - content: merged, - } - } - Err(merged_with_conflicts) => { - let content = merged_with_conflicts - .replace("<<<<<<< ours", &format!("<<<<<<< {ours_name}")) - .replace(">>>>>>> theirs", &format!(">>>>>>> {theirs_name}")); - Merge3Result { - clean: false, - content, - } - } - } -} - -/// Whether `content` still holds a generated conflict block: a `<<<<<<< ` -/// marker at the start of a line AND a `=======` line. Recognizes the -/// `(modified)` / `(deleted)` label variants by construction. -pub fn has_conflict_markers(content: &str) -> bool { - let opens = content - .split_inclusive('\n') - .any(|line| line.starts_with("<<<<<<< ") || line.starts_with("<<<<<<<\t")); - if !opens { - return false; - } - content - .lines() - .any(|line| line == "=======" || line == "=======\r") -} - -/// Number of conflict blocks in marker-bearing content. -pub fn conflict_hunks(content: &str) -> u32 { - content - .lines() - .filter(|line| line.starts_with("<<<<<<< ")) - .count() - .try_into() - .unwrap_or(u32::MAX) -} - // --------------------------------------------------------------------------- // Per-path decision table // --------------------------------------------------------------------------- @@ -214,39 +104,6 @@ impl std::fmt::Display for Reason { } } -/// What kind of conflict a marker-bearing file carries. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] -pub enum ConflictKind { - /// Overlapping hunks; `hunks` counts the blocks. - Hunks, - /// The fork modified a file the mainline deleted. - TheirsDeleted, - /// The mainline modified a file the fork deleted. - OursDeleted, -} - -/// Outcome of merging one regular file that both sides changed. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ContentMerge { - Merged(Vec), - Conflicted { - bytes: Vec, - hunks: u32, - kind: ConflictKind, - }, -} - -/// The text gate: UTF-8, no NUL in the probe window, under the size cap. -fn text_gate<'a>(bytes: &'a [u8], limits: &MergeLimits) -> std::result::Result<&'a str, Reason> { - if bytes.len() as u64 > limits.max_file_bytes { - return Err(Reason::TooLarge); - } - if bytes.iter().take(BINARY_PROBE_BYTES).any(|byte| *byte == 0) { - return Err(Reason::Binary); - } - std::str::from_utf8(bytes).map_err(|_| Reason::Binary) -} - /// Merges one regular file both sides changed. `None` on a side means that /// side deleted the file; `base` is `None` when both sides added it. pub fn merge_file( @@ -257,82 +114,20 @@ pub fn merge_file( theirs_name: &str, limits: &MergeLimits, ) -> std::result::Result { - fn gate<'a>( - side: Option<&'a [u8]>, - limits: &MergeLimits, - ) -> std::result::Result, Reason> { - match side { - None => Ok(None), - Some(bytes) => text_gate(bytes, limits).map(Some), - } - } - let base_text = gate(base, limits)?; - let ours_text = gate(ours, limits)?; - let theirs_text = gate(theirs, limits)?; - match (ours_text, theirs_text) { - (Some(ours), Some(theirs)) => { - if ours == theirs { - return Ok(ContentMerge::Merged(ours.as_bytes().to_vec())); - } - let result = merge3( - base_text.unwrap_or(""), - ours, - theirs, - ours_name, - theirs_name, - ); - if result.clean { - Ok(ContentMerge::Merged(result.content.into_bytes())) - } else { - let hunks = conflict_hunks(&result.content); - Ok(ContentMerge::Conflicted { - bytes: result.content.into_bytes(), - hunks, - kind: ConflictKind::Hunks, - }) - } - } - (Some(ours), None) => Ok(modify_delete( - base_text.unwrap_or(""), - ours, - &format!("{ours_name} (modified)"), - &format!("{theirs_name} (deleted)"), - ConflictKind::TheirsDeleted, - )), - (None, Some(theirs)) => Ok(modify_delete( - base_text.unwrap_or(""), - theirs, - &format!("{ours_name} (deleted)"), - &format!("{theirs_name} (modified)"), - ConflictKind::OursDeleted, - )), - (None, None) => Ok(ContentMerge::Merged(Vec::new())), - } -} - -/// A modify/delete conflict is always a conflict: one block holding the -/// surviving content against nothing, with the base in the middle. -fn modify_delete( - base: &str, - kept: &str, - ours_label: &str, - theirs_label: &str, - kind: ConflictKind, -) -> ContentMerge { - let base = ensure_trailing_newline(base); - let kept = ensure_trailing_newline(kept); - let (ours_block, theirs_block) = match kind { - ConflictKind::TheirsDeleted => (kept.as_ref(), ""), - _ => ("", kept.as_ref()), - }; - let content = format!( - "<<<<<<< {ours_label}\n{ours_block}||||||| original\n{base}=======\n{theirs_block}>>>>>>> {theirs_label}\n" - ); - ContentMerge::Conflicted { - bytes: content.into_bytes(), - hunks: 1, - kind, - } + merge_bytes( + base, + ours, + theirs, + ours_name, + theirs_name, + ByteMergeLimits { + max_bytes: limits.max_file_bytes, + }, + ) + .map_err(|error| match error { + ByteMergeError::Binary => Reason::Binary, + ByteMergeError::TooLarge => Reason::TooLarge, + }) } // --------------------------------------------------------------------------- @@ -444,28 +239,166 @@ fn descendants<'a>( .filter(move |next| next.as_path() != path) } -/// Paths whose content differs between two summary maps (added, removed, -/// or kind/payload changed). Metadata-only differences do not count. -fn changed_paths( - before: &BTreeMap, - after: &BTreeMap, -) -> BTreeSet { - let mut set = BTreeSet::new(); - for (path, summary) in before { - match after.get(path) { - Some(other) if other.same_content(summary) => {} - _ => { - set.insert(path.clone()); +async fn changed_paths( + before: &crate::store::LocalGeneration, + after: &crate::store::LocalGeneration, +) -> Result> { + let changes = before + .diff_to(after, u32::MAX) + .await + .map_err(EngineError::fs("diff merge generations"))? + .changed_paths(u32::MAX) + .await + .map_err(EngineError::fs("resolve merge paths"))?; + changes + .into_iter() + .filter(|change| match (change.before, change.after) { + (Some(before), Some(after)) => { + before.kind != after.kind + || (before.kind != FileKind::Directory && before.payload != after.payload) } - } + _ => true, + }) + .map(|change| { + acyclic_fs::namespace_to_host_path(&change.path) + .map_err(EngineError::fs("resolve merge host path")) + }) + .filter(|path| match path { + Ok(path) => !diff::is_git_internal(path), + Err(_) => true, + }) + .collect() +} + +async fn records_at( + generation: &crate::store::LocalGeneration, + paths: &[PathBuf], +) -> Result> { + if paths.is_empty() { + return Ok(BTreeMap::new()); } - for path in after.keys() { - if !before.contains_key(path) { - set.insert(path.clone()); - } + let namespaces = paths + .iter() + .map(|path| namespace_of(path)) + .collect::>>()?; + let records = generation + .lookup_paths( + &namespaces, + WorkCounters::UNBOUNDED, + &CancellationToken::new(), + ) + .await + .map_err(EngineError::fs("lookup merge paths"))? + .value; + Ok(paths + .iter() + .cloned() + .zip(records) + .filter_map(|(path, record)| record.map(|record| (path, record))) + .collect()) +} + +struct MergeInputs { + base: BTreeMap, + theirs: BTreeMap, + ours: BTreeMap, + ours_changed: BTreeSet, + theirs_changed: BTreeSet, +} + +fn regular_paths(records: &BTreeMap) -> Vec { + records + .iter() + .filter(|(_, record)| record.kind == FileKind::Regular) + .map(|(path, _)| path.clone()) + .collect() +} + +async fn regular_contents( + store: &Store, + generation: GenerationId, + records: &BTreeMap, +) -> Result>> { + let paths = regular_paths(records); + let contents = read_files(store, generation, &paths).await?; + paths + .into_iter() + .zip(contents) + .map(|(path, content)| { + content + .map(|content| (path.clone(), content)) + .ok_or_else(|| { + EngineError::Fs(format!("{}: regular file is absent", path.display())) + }) + }) + .collect() +} + +async fn record_modes( + store: &Store, + generation: GenerationId, + records: &BTreeMap, +) -> Result>> { + if records.is_empty() { + return Ok(BTreeMap::new()); } - set.retain(|path| !diff::is_git_internal(path)); - set + let checkout = store.checkout_exact(generation).await?; + let reader = checkout + .pinned_reader() + .map_err(EngineError::fs("open pinned reader"))?; + let paths = records + .keys() + .map(|path| namespace_of(path)) + .collect::>>()?; + let metadata = reader + .describe_files(&paths, WorkCounters::UNBOUNDED, &CancellationToken::new()) + .await + .map_err(EngineError::fs("batch read metadata"))? + .value; + Ok(records + .keys() + .cloned() + .zip(metadata) + .map(|(path, description)| { + let mode = description.and_then(|description| match description.metadata.posix_mode { + MetadataField::Value(mode) => Some(mode & 0o7777), + MetadataField::Unavailable => None, + }); + (path, mode) + }) + .collect()) +} + +fn regular_content<'a>(contents: &'a BTreeMap>, path: &Path) -> Result<&'a [u8]> { + contents + .get(path) + .map(Vec::as_slice) + .ok_or_else(|| EngineError::Fs(format!("{}: regular content is absent", path.display()))) +} + +fn mode_at(modes: &BTreeMap>, path: &Path) -> Option { + modes.get(path).copied().flatten() +} + +async fn merge_inputs( + store: &Store, + base: GenerationId, + theirs: GenerationId, + ours: GenerationId, +) -> Result { + let base_generation = store.generation(base).await?; + let theirs_generation = store.generation(theirs).await?; + let ours_generation = store.generation(ours).await?; + let ours_changed = changed_paths(&base_generation, &ours_generation).await?; + let theirs_changed = changed_paths(&base_generation, &theirs_generation).await?; + let paths: Vec<_> = ours_changed.union(&theirs_changed).cloned().collect(); + Ok(MergeInputs { + base: records_at(&base_generation, &paths).await?, + theirs: records_at(&theirs_generation, &paths).await?, + ours: records_at(&ours_generation, &paths).await?, + ours_changed, + theirs_changed, + }) } /// Computes the merge of fork `ours` onto mainline `theirs` from `base`. @@ -481,15 +414,23 @@ pub async fn plan( ours_name: &str, limits: &MergeLimits, ) -> Result { - let base_map = diff::summaries(store, base).await?; - let theirs_map = diff::summaries(store, theirs).await?; - let ours_map = diff::summaries(store, ours).await?; - let ours_changed = changed_paths(&base_map, &ours_map); - let theirs_changed = changed_paths(&base_map, &theirs_map); - - let mut base_checkout = store.checkout_exact(base).await?; - let mut theirs_checkout = store.checkout_exact(theirs).await?; - let mut ours_checkout = store.checkout_exact(ours).await?; + let inputs = merge_inputs(store, base, theirs, ours).await?; + let MergeInputs { + base: base_map, + theirs: theirs_map, + ours: ours_map, + ours_changed, + theirs_changed, + } = inputs; + + let (base_contents, theirs_contents, ours_contents, base_modes, theirs_modes, ours_modes) = tokio::try_join!( + regular_contents(store, base, &base_map), + regular_contents(store, theirs, &theirs_map), + regular_contents(store, ours, &ours_map), + record_modes(store, base, &base_map), + record_modes(store, theirs, &theirs_map), + record_modes(store, ours, &ours_map), + )?; let mut plan = MergePlan::default(); let mut decided_root: Option = None; @@ -516,7 +457,7 @@ pub async fn plan( let b = base_map.get(path); let h = theirs_map.get(path); let f = ours_map.get(path); - let kind = |summary: Option<&RecordSummary>| summary.map(|s| s.kind); + let kind = |record: Option<&FileRecord>| record.map(|record| record.kind); match (kind(b), kind(f), kind(h)) { // Nothing to decide here: both deleted it (or both changed it // inside a now-absent dir), or independent edits below one @@ -529,11 +470,11 @@ pub async fn plan( ) => {} // Regular files on both sides. (_, Some(FileKind::Regular), Some(FileKind::Regular)) => { - if f.and_then(|s| s.payload) == h.and_then(|s| s.payload) { + if f.map(|record| record.payload) == h.map(|record| record.payload) { continue; } let base_bytes = match kind(b) { - Some(FileKind::Regular) => Some(read_regular(&mut base_checkout, path).await?), + Some(FileKind::Regular) => Some(regular_content(&base_contents, path)?), None => None, Some(_) => { plan.refusals.push(Refusal { @@ -543,21 +484,21 @@ pub async fn plan( continue; } }; - let ours_bytes = read_regular(&mut ours_checkout, path).await?; - let theirs_bytes = read_regular(&mut theirs_checkout, path).await?; + let ours_bytes = regular_content(&ours_contents, path)?; + let theirs_bytes = regular_content(&theirs_contents, path)?; let mode = merged_mode( - read_mode(&mut base_checkout, path).await?, - read_mode(&mut ours_checkout, path).await?, - read_mode(&mut theirs_checkout, path).await?, + mode_at(&base_modes, path), + mode_at(&ours_modes, path), + mode_at(&theirs_modes, path), ); push_content( &mut plan, path, mode, merge_file( - base_bytes.as_deref(), - Some(&ours_bytes), - Some(&theirs_bytes), + base_bytes, + Some(ours_bytes), + Some(theirs_bytes), ours_name, "mainline", limits, @@ -567,18 +508,18 @@ pub async fn plan( // Modify/delete in either direction. (Some(FileKind::Regular), Some(FileKind::Regular), None) | (Some(FileKind::Regular), None, Some(FileKind::Regular)) => { - let base_bytes = read_regular(&mut base_checkout, path).await?; + let base_bytes = regular_content(&base_contents, path)?; let (ours_bytes, theirs_bytes, mode) = if f.is_some() { ( - Some(read_regular(&mut ours_checkout, path).await?), + Some(regular_content(&ours_contents, path)?), None, - read_mode(&mut ours_checkout, path).await?, + mode_at(&ours_modes, path), ) } else { ( None, - Some(read_regular(&mut theirs_checkout, path).await?), - read_mode(&mut theirs_checkout, path).await?, + Some(regular_content(&theirs_contents, path)?), + mode_at(&theirs_modes, path), ) }; push_content( @@ -586,9 +527,9 @@ pub async fn plan( path, mode, merge_file( - Some(&base_bytes), - ours_bytes.as_deref(), - theirs_bytes.as_deref(), + Some(base_bytes), + ours_bytes, + theirs_bytes, ours_name, "mainline", limits, @@ -614,7 +555,7 @@ pub async fn plan( } // Symlinks retargeted identically are fine. (_, Some(FileKind::SymbolicLink), Some(FileKind::SymbolicLink)) - if f.and_then(|s| s.payload) == h.and_then(|s| s.payload) => {} + if f.map(|record| record.payload) == h.map(|record| record.payload) => {} // Everything else is a kind clash: a file on one side and a // directory or symlink on the other, symlinks retargeted // differently, or a file that became a directory on both sides. @@ -670,99 +611,97 @@ fn merged_mode(base: Option, ours: Option, theirs: Option) -> Opt } pub(crate) fn namespace_of(path: &Path) -> Result { - let components = validate_relative(path)?; - namespace_path(&components, acyclic_fs::model::VolumeLimits::default()) + validate_relative(path)?; + let config = crate::store::volume_config(); + acyclic_fs::host_path_to_namespace(path, config.profile, config.limits) + .map_err(EngineError::fs("host path to namespace")) } -/// Whole content of a regular file at `path` in `checkout`. -pub(crate) async fn read_regular(checkout: &mut LocalCheckout, path: &Path) -> Result> { +/// Contents of regular files in one generation, preserving request order. +/// Opens and authenticates the generation once and resolves every namespace +/// path in one SDK batch; absent and non-regular entries remain `None`. +pub async fn read_files( + store: &Store, + generation: GenerationId, + paths: &[PathBuf], +) -> Result>>> { + let checkout = store.checkout_exact(generation).await?; + let namespaces = paths + .iter() + .map(|path| namespace_of(path)) + .collect::>>()?; + if namespaces.is_empty() { + return Ok(Vec::new()); + } let cancel = CancellationToken::new(); - let namespace = namespace_of(path)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) + let reader = checkout + .pinned_reader() + .map_err(EngineError::fs("open pinned reader"))?; + let files = reader + .resolve_files(&namespaces, WorkCounters::UNBOUNDED, &cancel) .await - .map_err(EngineError::fs("lookup"))? + .map_err(EngineError::fs("resolve files"))? .value; - let record = lookup - .record - .ok_or_else(|| EngineError::Fs(format!("{}: absent", path.display())))?; - let length = match record.payload { - acyclic_fs::kernel::FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, - acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => logical_bytes, - _ => { - return Err(EngineError::Fs(format!( - "{}: not a regular file", - path.display() - ))) - } - }; let limits = checkout.volume_config().limits; let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); - let mut out = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); - let mut offset = 0; - while offset < length { - let take = chunk.min(length - offset); - let read = checkout - .read_file_range( - &namespace, - ByteRange { + let mut contents = vec![None; paths.len()]; + let mut pending = Vec::new(); + let mut destinations = Vec::new(); + for (index, file) in files.iter().enumerate() { + let Some(file) = file else { + continue; + }; + let description = file.description(); + if description.kind != FileKind::Regular { + continue; + } + let length = description.logical_bytes; + let display_path = paths + .get(index) + .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))?; + let capacity = usize::try_from(length).map_err(|_| { + EngineError::Fs(format!( + "{}: file is too large for this host", + display_path.display() + )) + })?; + *contents + .get_mut(index) + .ok_or_else(|| EngineError::Fs("batch lookup returned too many entries".into()))? = + Some(Vec::with_capacity(capacity)); + let mut offset = 0_u64; + while offset < length { + let take = chunk.min(length - offset); + pending.push(ResolvedFileRangeReadRequest { + file, + range: ByteRange { offset, length: take, }, - WorkCounters::UNBOUNDED, - &cancel, - ) - .await - .map_err(EngineError::fs("read file range"))? - .value; - out.extend_from_slice(&read.bytes); - offset += take; - } - Ok(out) -} - -/// Content of `path` in `generation` if it is a regular file there. -pub async fn read_file( - store: &Store, - generation: GenerationId, - path: &Path, -) -> Result>> { - let mut checkout = store.checkout_exact(generation).await?; - let cancel = CancellationToken::new(); - let namespace = namespace_of(path)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - match lookup.record { - Some(record) if record.kind == FileKind::Regular => { - Ok(Some(read_regular(&mut checkout, path).await?)) + }); + destinations.push(index); + offset += take; } - _ => Ok(None), } -} - -async fn read_mode(checkout: &mut LocalCheckout, path: &Path) -> Result> { - let cancel = CancellationToken::new(); - let namespace = namespace_of(path)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) + let reads = reader + .read_resolved_ranges( + &pending, + FILE_READ_CONCURRENCY, + WorkCounters::UNBOUNDED, + &cancel, + ) .await - .map_err(EngineError::fs("lookup"))? + .map_err(EngineError::fs("read resolved file ranges"))? .value; - if lookup.record.is_none() { - return Ok(None); + for (destination, read) in destinations.into_iter().zip(reads) { + contents + .get_mut(destination) + .ok_or_else(|| EngineError::Fs("resolved read has an invalid destination".into()))? + .as_mut() + .ok_or_else(|| EngineError::Fs("resolved read lost its destination".into()))? + .extend_from_slice(&read.bytes); } - let metadata = checkout - .read_metadata(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("read metadata"))? - .value; - Ok(match metadata.posix_mode { - MetadataField::Value(mode) => Some(mode & 0o7777), - MetadataField::Unavailable => None, - }) + Ok(contents) } // --------------------------------------------------------------------------- @@ -785,57 +724,49 @@ pub async fn apply_entries( entries: &[(PathBuf, Entry)], ) -> Result<()> { let cancel = CancellationToken::new(); - for (path, entry) in entries { - let namespace = namespace_of(path)?; - // Boxed per entry: keeps the facade's large futures off the caller's - // stack frame. - Box::pin(apply_entry(store, dst, &namespace, entry, &cancel)).await?; - } - Ok(()) -} - -async fn apply_entry( - store: &Store, - dst: &mut LocalCheckout, - namespace: &NamespacePath, - entry: &Entry, - cancel: &CancellationToken, -) -> Result<()> { - { - { - match entry { - Entry::Regular { bytes, mode } => { - remove_subtree(dst, namespace, cancel).await?; - ensure_parents(dst, namespace, cancel).await?; - dst.create_file( - namespace.clone(), - Bytes::copy_from_slice(bytes), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create file"))?; - if let Some(mode) = mode { - let metadata = FileMetadata { - posix_mode: MetadataField::Value(*mode), - ..FileMetadata::default() - }; - dst.set_metadata( - namespace.clone(), - metadata, - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("set metadata"))?; - } - } - Entry::FromGeneration { generation } => { - let mut src = store.checkout_exact(*generation).await?; - remove_subtree(dst, namespace, cancel).await?; - ensure_parents(dst, namespace, cancel).await?; - copy_node(&mut src, dst, namespace, cancel).await?; + let namespaces = entries + .iter() + .map(|(path, _)| namespace_of(path)) + .collect::>>()?; + ensure_entry_parents(dst, &namespaces, &cancel).await?; + let mut source = None; + for ((_, entry), namespace) in entries.iter().zip(&namespaces) { + match entry { + Entry::Regular { bytes, mode } => { + source = None; + remove_subtree(dst, namespace, &cancel).await?; + let metadata = FileMetadata { + posix_mode: mode.map_or(MetadataField::Unavailable, MetadataField::Value), + ..FileMetadata::default() + }; + dst.apply_authored_transaction( + vec![AuthoredMutation::CreateFile { + path: namespace.clone(), + bytes: Bytes::copy_from_slice(bytes), + metadata, + }], + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(EngineError::fs("create merge file"))?; + } + Entry::FromGeneration { generation } => { + let needs_checkout = + source + .as_ref() + .is_none_or(|(current, _): &(GenerationId, LocalCheckout)| { + current != generation + }); + if needs_checkout { + source = Some((*generation, store.checkout_exact(*generation).await?)); } + let src = &mut source + .as_mut() + .ok_or_else(|| EngineError::Fs("source checkout missing".into()))? + .1; + remove_subtree(dst, namespace, &cancel).await?; + copy_node(src, dst, namespace, &cancel).await?; } } } @@ -843,25 +774,47 @@ async fn apply_entry( } /// Creates missing ancestor directories of `namespace` in `dst`. -async fn ensure_parents( +async fn ensure_entry_parents( dst: &mut LocalCheckout, - namespace: &NamespacePath, + namespaces: &[NamespacePath], cancel: &CancellationToken, ) -> Result<()> { let limits = dst.volume_config().limits; - let components = namespace.components(); - for depth in 1..components.len() { - let parent = NamespacePath::new(components.iter().take(depth).cloned().collect(), limits) - .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?; - let lookup = dst - .lookup_no_follow(&parent, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - if lookup.record.is_none() { - dst.create_directory(parent, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("create directory"))?; + let mut parents = BTreeSet::new(); + for namespace in namespaces { + let components = namespace.components(); + for depth in 1..components.len() { + parents.insert( + NamespacePath::new(components.iter().take(depth).cloned().collect(), limits) + .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}")))?, + ); + } + } + let parents = parents.into_iter().collect::>(); + if parents.is_empty() { + return Ok(()); + } + let lookups = dst + .lookup_batch_no_follow(&parents, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("batch parent lookup"))? + .value; + for (parent, lookup) in parents.into_iter().zip(lookups.entries) { + match lookup.record { + Some(record) if record.kind == FileKind::Directory => {} + Some(_) => { + return Err(EngineError::Fs(format!( + "{}: parent is not a directory", + acyclic_fs::namespace_to_host_path(&parent) + .map_err(EngineError::fs("resolve parent path"))? + .display() + ))) + } + None => { + dst.create_directory(parent, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("create directory"))?; + } } } Ok(()) @@ -878,34 +831,82 @@ fn child_path( .map_err(|error| EngineError::Fs(format!("namespace path: {error:?}"))) } -async fn list_children( - checkout: &mut LocalCheckout, - namespace: &NamespacePath, +async fn resolved_child_paths( + reader: &acyclic_fs::PinnedReader, + directory: &NamespacePath, + limits: acyclic_fs::model::VolumeLimits, cancel: &CancellationToken, -) -> Result> { - let mut names = Vec::new(); +) -> Result> +where + A: acyclic_fs::AsyncAuthorityStore, + O: acyclic_fs::AsyncObjectStore, +{ + let mut children = Vec::new(); let mut after = None; loop { - let page = checkout - .list_directory_records( - namespace, + let page = reader + .resolve_directory_page( + directory, after.as_ref(), PAGE_ENTRIES, WorkCounters::UNBOUNDED, cancel, ) .await - .map_err(EngineError::fs("list directory"))? + .map_err(EngineError::fs("resolve subtree page"))? .value; for entry in &page.entries { - names.push(entry.name.clone()); + children.push(child_path(directory, &entry.name, limits)?); + } + if !page.has_more { + break; + } + after = Some( + page.entries + .last() + .ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))? + .name + .clone(), + ); + } + Ok(children) +} + +async fn resolved_children( + reader: &acyclic_fs::PinnedReader, + directory: &NamespacePath, + limits: acyclic_fs::model::VolumeLimits, + cancel: &CancellationToken, +) -> Result)>> +where + A: acyclic_fs::AsyncAuthorityStore, + O: acyclic_fs::AsyncObjectStore, +{ + let mut children = Vec::new(); + let mut after = None; + loop { + let page = reader + .resolve_directory_page( + directory, + after.as_ref(), + PAGE_ENTRIES, + WorkCounters::UNBOUNDED, + cancel, + ) + .await + .map_err(EngineError::fs("resolve subtree page"))? + .value; + let next = page.entries.last().map(|entry| entry.name.clone()); + for entry in page.entries { + children.push((child_path(directory, &entry.name, limits)?, entry.file)); } - match page.entries.last() { - Some(last) if page.has_more => after = Some(last.name.clone()), - _ => break, + if !page.has_more { + break; } + after = + Some(next.ok_or_else(|| EngineError::Fs("paged directory returned no cursor".into()))?); } - Ok(names) + Ok(children) } /// Removes `namespace` from `dst`, recursively for directories; absent is @@ -917,32 +918,52 @@ async fn remove_subtree( namespace: &NamespacePath, cancel: &CancellationToken, ) -> Result<()> { - let limits = dst.volume_config().limits; - // (path, children_expanded) - let mut stack: Vec<(NamespacePath, bool)> = vec![(namespace.clone(), false)]; - while let Some((path, expanded)) = stack.pop() { - if expanded { - dst.remove(path, None, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("remove"))?; - continue; + let mut frontier = vec![namespace.clone()]; + let reader = dst.snapshot_reader(); + let roots = reader + .resolve_files(&frontier, WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("resolve subtree root"))? + .value; + let mut nodes = frontier + .drain(..) + .zip(roots) + .filter_map(|(path, file)| file.map(|file| (path, file))) + .collect::>(); + let mut levels = Vec::new(); + while !nodes.is_empty() { + let limits = dst.volume_config().limits; + let mut next = Vec::new(); + for directory in nodes + .iter() + .filter(|(_, file)| file.description().kind == FileKind::Directory) + .map(|(path, _)| path) + { + next.extend(resolved_children(&reader, directory, limits, cancel).await?); } - let lookup = dst - .lookup_no_follow(&path, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - let Some(record) = lookup.record else { - continue; - }; - if record.kind == FileKind::Directory { - let children = list_children(dst, &path, cancel).await?; - stack.push((path.clone(), true)); - for name in children { - stack.push((child_path(&path, &name, limits)?, false)); - } - } else { - stack.push((path, true)); + levels.push( + nodes + .into_iter() + .map(|(path, file)| (path, file.file_id())) + .collect::>(), + ); + nodes = next; + } + let maximum = usize::try_from(dst.volume_config().limits.maximum_mutations_per_batch) + .unwrap_or(usize::MAX) + .max(1); + for level in levels.into_iter().rev() { + let removals = level + .into_iter() + .map(|(path, file_id)| AuthoredMutation::Remove { + path, + expected_file_id: Some(file_id), + }) + .collect::>(); + for chunk in removals.chunks(maximum) { + dst.apply_authored_transaction(chunk.to_vec(), WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("remove subtree frontier"))?; } } Ok(()) @@ -957,110 +978,139 @@ async fn copy_node( namespace: &NamespacePath, cancel: &CancellationToken, ) -> Result<()> { - let limits = src.volume_config().limits; - let mut queue: Vec = vec![namespace.clone()]; - while let Some(path) = queue.pop() { - let lookup = src - .lookup_no_follow(&path, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - let Some(record) = lookup.record else { - continue; - }; - let metadata = src - .read_metadata(&path, WorkCounters::UNBOUNDED, cancel) + let mut frontier = vec![namespace.clone()]; + while !frontier.is_empty() { + let reader = src + .pinned_reader() + .map_err(EngineError::fs("open pinned reader"))?; + let files = reader + .resolve_files(&frontier, WorkCounters::UNBOUNDED, cancel) .await - .map_err(EngineError::fs("read metadata"))? - .value; - match record.kind { - FileKind::Regular => { - let bytes = read_regular_ns(src, &path, &record.payload, cancel).await?; - dst.create_file( - path.clone(), - Bytes::from(bytes), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create file"))?; - } - FileKind::SymbolicLink => { - let target = src - .read_symbolic_link(&path, WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("read symlink"))? - .value; - dst.create_symbolic_link( - path.clone(), - Bytes::copy_from_slice(&target), - WorkCounters::UNBOUNDED, - cancel, - ) - .await - .map_err(EngineError::fs("create symlink"))?; - } - FileKind::Directory => { - dst.create_directory(path.clone(), WorkCounters::UNBOUNDED, cancel) - .await - .map_err(EngineError::fs("create directory"))?; - for name in list_children(src, &path, cancel).await? { - queue.push(child_path(&path, &name, limits)?); + .map_err(EngineError::fs("batch subtree metadata"))? + .value + .into_iter(); + let nodes = frontier + .into_iter() + .zip(files) + .filter_map(|(path, file)| file.map(|file| (path, file))) + .collect::>(); + if nodes.is_empty() { + break; + } + let files = nodes.iter().map(|(_, file)| file).collect::>(); + let regular = read_regular_frontier(&reader, &files, cancel).await?; + let limits = src.volume_config().limits; + frontier = Vec::new(); + for (directory, _) in nodes + .iter() + .filter(|(_, file)| file.description().kind == FileKind::Directory) + { + frontier.extend(resolved_child_paths(&reader, directory, limits, cancel).await?); + } + let mut mutations = Vec::with_capacity(nodes.len()); + for (index, (path, file)) in nodes.into_iter().enumerate() { + let metadata = file.description().metadata; + match file.description().kind { + FileKind::Regular => { + let bytes = regular.get(index).ok_or_else(|| { + EngineError::Fs("subtree read omitted a regular file".into()) + })?; + mutations.push(AuthoredMutation::CreateFile { + path, + bytes: Bytes::from(bytes.clone()), + metadata, + }); + } + FileKind::SymbolicLink => { + let target = file + .read_symbolic_link(WorkCounters::UNBOUNDED, cancel) + .await + .map_err(EngineError::fs("read resolved symlink"))? + .value; + mutations.push(AuthoredMutation::CreateSymbolicLink { + path, + target, + metadata, + }); + } + FileKind::Directory => { + mutations.push(AuthoredMutation::CreateDirectory { path, metadata }); + } + other => { + return Err(EngineError::Fs(format!( + "cannot copy a {other:?} node (only files, symlinks, and directories)" + ))) } - } - other => { - return Err(EngineError::Fs(format!( - "cannot copy a {other:?} node (only files, symlinks, and directories)" - ))) } } - if let MetadataField::Value(mode) = metadata.posix_mode { - let set = FileMetadata { - posix_mode: MetadataField::Value(mode), - ..FileMetadata::default() - }; - dst.set_metadata(path, set, WorkCounters::UNBOUNDED, cancel) + let maximum = usize::try_from(dst.volume_config().limits.maximum_mutations_per_batch) + .unwrap_or(usize::MAX) + .saturating_div(2) + .max(1); + for chunk in mutations.chunks(maximum) { + dst.apply_authored_transaction(chunk.to_vec(), WorkCounters::UNBOUNDED, cancel) .await - .map_err(EngineError::fs("set metadata"))?; + .map_err(EngineError::fs("create subtree frontier"))?; } } Ok(()) } -async fn read_regular_ns( - checkout: &mut LocalCheckout, - namespace: &NamespacePath, - payload: &acyclic_fs::kernel::FilePayload, +async fn read_regular_frontier( + reader: &acyclic_fs::PinnedReader, + files: &[&acyclic_fs::ResolvedFile], cancel: &CancellationToken, -) -> Result> { - let length = match payload { - acyclic_fs::kernel::FilePayload::InlineRegular(inline) => inline.as_bytes().len() as u64, - acyclic_fs::kernel::FilePayload::Regular { logical_bytes, .. } => *logical_bytes, - _ => return Err(EngineError::Fs("regular file with foreign payload".into())), - }; - let limits = checkout.volume_config().limits; - let chunk = TRANSFER_BYTES.min(limits.maximum_read_bytes.max(1)); - let mut out = Vec::with_capacity(usize::try_from(length).unwrap_or(0)); - let mut offset = 0; - while offset < length { - let take = chunk.min(length - offset); - let read = checkout - .read_file_range( - namespace, - ByteRange { - offset, +) -> Result>> +where + A: acyclic_fs::AsyncAuthorityStore, + O: acyclic_fs::AsyncObjectStore, +{ + let mut contents = vec![Vec::new(); files.len()]; + let mut offsets = vec![0_u64; files.len()]; + loop { + let mut destinations = Vec::new(); + let mut pending = Vec::new(); + for (index, (file, offset)) in files.iter().zip(&mut offsets).enumerate() { + let description = file.description(); + if description.kind != FileKind::Regular { + continue; + } + let length = description.logical_bytes; + if *offset >= length { + continue; + } + let take = TRANSFER_BYTES.min(length - *offset); + pending.push(ResolvedFileRangeReadRequest { + file, + range: ByteRange { + offset: *offset, length: take, }, + }); + destinations.push(index); + *offset += take; + } + if pending.is_empty() { + break; + } + let chunks = reader + .read_resolved_ranges( + &pending, + FILE_READ_CONCURRENCY, WorkCounters::UNBOUNDED, cancel, ) .await - .map_err(EngineError::fs("read file range"))? + .map_err(EngineError::fs("batch read resolved subtree ranges"))? .value; - out.extend_from_slice(&read.bytes); - offset += take; + for (index, chunk) in destinations.into_iter().zip(chunks) { + contents + .get_mut(index) + .ok_or_else(|| EngineError::Fs("batch read returned an invalid index".into()))? + .extend_from_slice(&chunk.bytes); + } } - Ok(out) + Ok(contents) } /// Which of `paths` the repo's `.gitignore` rules ignore, per @@ -1133,35 +1183,16 @@ pub async fn materialize_paths( dir: &Path, paths: &[PathBuf], ) -> Result<()> { - let mut checkout = store.checkout_exact(generation).await?; - let limits = checkout.volume_config().limits; - let cancel = CancellationToken::new(); - for path in paths { - let namespace = namespace_of(path)?; - let destination = dir.join(path); - crate::rewind::remove_any(&destination)?; - let lookup = checkout - .lookup_no_follow(&namespace, WorkCounters::UNBOUNDED, &cancel) - .await - .map_err(EngineError::fs("lookup"))? - .value; - let Some(record) = lookup.record else { - continue; - }; - if let Some(parent) = destination.parent() { - std::fs::create_dir_all(parent)?; - } - crate::rewind::write_node( - &mut checkout, - &namespace, - record.kind, - &record.payload, - &destination, - limits, - &cancel, + let generation = store.generation(generation).await?; + generation + .restore_host_paths( + paths, + acyclic_fs::HostPathReplacement::LiveMount, + &acyclic_fs::MaterializeOptions::native(dir), + &acyclic_fs::CancellationToken::new(), ) - .await?; - } + .await + .map_err(EngineError::fs("materialize paths"))?; Ok(()) } @@ -1171,7 +1202,7 @@ pub fn subtree_roots(paths: &[PathBuf]) -> Vec { sorted.sort(); let mut roots: Vec = Vec::new(); for path in sorted { - if !roots.iter().any(|root| path.starts_with(root)) { + if roots.last().is_none_or(|root| !path.starts_with(root)) { roots.push(path); } } @@ -1193,7 +1224,7 @@ mod tests { let base = "line 1\nline 2\nline 3\n"; let ours = "OURS line 1\nline 2\nline 3\n"; let theirs = "line 1\nline 2\nTHEIRS line 3\n"; - let result = merge3(base, ours, theirs, "child", "parent"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "child", "parent"); assert!(result.clean); assert_eq!(result.content, "OURS line 1\nline 2\nTHEIRS line 3\n"); } @@ -1203,7 +1234,8 @@ mod tests { let base = "line 1\nline 2\nline 3\n"; let ours = "OURS line 1\nline 2\nline 3\n"; let theirs = "THEIRS line 1\nline 2\nline 3\n"; - let result = merge3(base, ours, theirs, "child-agent", "parent-agent"); + let result = + acyclic_fs::text_merge::merge_text(base, ours, theirs, "child-agent", "parent-agent"); assert!(!result.clean); assert!(result.content.contains("<<<<<<< child-agent\n")); assert!(result.content.contains("||||||| original\n")); @@ -1216,14 +1248,15 @@ mod tests { let base = "line 1\nline 2\n"; let ours = "SAME line 1\nline 2\n"; let theirs = "SAME line 1\nline 2\n"; - let result = merge3(base, ours, theirs, "child", "parent"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "child", "parent"); assert!(result.clean); assert_eq!(result.content, "SAME line 1\nline 2\n"); } #[test] fn conflict_markers_have_proper_newlines() { - let result = merge3("content", "ours", "theirs", "child", "parent"); + let result = + acyclic_fs::text_merge::merge_text("content", "ours", "theirs", "child", "parent"); assert!(!result.clean); assert!(result.content.contains("\n|||||||")); assert!(result.content.contains("\n=======\n")); @@ -1232,40 +1265,26 @@ mod tests { #[test] fn empty_base_ours_added_theirs_empty() { - let result = merge3("", "added", "", "child", "parent"); + let result = acyclic_fs::text_merge::merge_text("", "added", "", "child", "parent"); assert!(result.clean); assert_eq!(result.content, "added"); } #[test] fn empty_base_both_add_different() { - let result = merge3("", "ours\n", "theirs\n", "child", "parent"); + let result = + acyclic_fs::text_merge::merge_text("", "ours\n", "theirs\n", "child", "parent"); assert!(!result.clean); assert!(result.content.contains("<<<<<<< child")); assert!(result.content.contains(">>>>>>> parent")); } - // --- trailing newline rule -------------------------------------------- - - #[test] - fn trailing_newline_rule() { - // ours and theirs agree: keep theirs' (== ours') state. - assert!(merged_has_trailing_newline("a", "a\n", "b\n")); - assert!(!merged_has_trailing_newline("a\n", "a", "b")); - // ours == base, theirs decides. - assert!(!merged_has_trailing_newline("a\n", "a\n", "b")); - assert!(merged_has_trailing_newline("a", "a", "b\n")); - // ours differs from base and theirs: ours decides. - assert!(!merged_has_trailing_newline("a\n", "x", "b\n")); - assert!(merged_has_trailing_newline("a", "x\n", "b")); - } - #[test] fn fork_appends_without_trailing_newline_and_head_edits_top() { let base = "one\ntwo\nthree\n"; let ours = "one\ntwo\nthree\nfour"; let theirs = "ONE\ntwo\nthree\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); assert!(result.clean); assert_eq!(result.content, "ONE\ntwo\nthree\nfour"); } @@ -1275,7 +1294,7 @@ mod tests { let base = "a\r\nb\r\nc\r\n"; let ours = "A\r\nb\r\nc\r\n"; let theirs = "a\r\nb\r\nC\r\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); assert!(result.clean); assert_eq!(result.content, "A\r\nb\r\nC\r\n"); } @@ -1285,7 +1304,7 @@ mod tests { let base = "1\n2\n3\n4\n"; let ours = "1x\n2\n3\n4\n"; let theirs = "1\n2y\n3\n4\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); // Adjacent edits are a conflict for diff3 (git behaves the same); // whichever way diffy decides, the result must be consistent with clean. if result.clean { @@ -1300,7 +1319,7 @@ mod tests { let base = "a\nb\nc\nd\n"; let ours = "a\nd\n"; let theirs = "a\nB\nc\nd\n"; - let result = merge3(base, ours, theirs, "fork", "mainline"); + let result = acyclic_fs::text_merge::merge_text(base, ours, theirs, "fork", "mainline"); assert!(!result.clean); assert_eq!(conflict_hunks(&result.content), 1); } diff --git a/crates/acyclic-engine/src/pipeline.rs b/crates/acyclic/src/pipeline.rs similarity index 74% rename from crates/acyclic-engine/src/pipeline.rs rename to crates/acyclic/src/pipeline.rs index 8a24942..7ebffa8 100644 --- a/crates/acyclic-engine/src/pipeline.rs +++ b/crates/acyclic/src/pipeline.rs @@ -9,11 +9,14 @@ use std::path::PathBuf; use std::time::{Duration, Instant}; use acyclic_fs::model::VolumeLimits; -use acyclic_fs::{capture_baseline, capture_root_identity, capture_watch_batch, CaptureOptions}; +use acyclic_fs::{ + capture_baseline_with_policy, capture_root_identity, capture_subtrees_with_policy, + capture_watch_batch_with_policy, CaptureOptions, +}; use acyclic_fs::{ CancellationToken, CheckoutCommitOutcome, GenerationId, MountPublication, NativeWatch, NativeWatchOptions, OperationId, WatchBatch, WatchChange, WatchEpoch, WatchSequence, - WorkCounters, + WorkCounters, WorkspaceRestore, }; use tokio::sync::{mpsc, oneshot}; @@ -22,7 +25,7 @@ use std::sync::Arc; use crate::config::Config; use crate::diff::{self, FileChange}; use crate::exclude::Exclusions; -use crate::fork::{ForkSeed, PromoteOutcome, SessionResolveOutcome, SharedLocalCheckout}; +use crate::fork::{ForkSeed, PromoteOutcome, SharedLocalCheckout}; use crate::index::{Attribution, CheckpointKind, CheckpointRow, Index}; use crate::rewind::{self, RestoreOutcome, RewindOutcome}; use crate::store::Store; @@ -33,9 +36,64 @@ const MAXIMUM_EXTENT_SPANS: u32 = 65_536; const WATCH_QUEUE: u32 = 65_536; const POLL_CHANGES: u32 = 16_384; +#[cfg(target_os = "windows")] +const CONTINUITY_MAGIC: &[u8; 8] = b"ACCONT\0\x01"; +#[cfg(target_os = "windows")] +const CONTINUITY_BYTES: usize = 88; + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy)] +struct ContinuityRecord { + generation: GenerationId, + row: i64, + usn: acyclic_fs::WindowsUsnCheckpoint, +} + +#[cfg(target_os = "windows")] +impl ContinuityRecord { + fn encode(self) -> [u8; CONTINUITY_BYTES] { + let mut bytes = [0_u8; CONTINUITY_BYTES]; + bytes[..8].copy_from_slice(CONTINUITY_MAGIC); + bytes[8..40].copy_from_slice(self.generation.digest().as_bytes()); + bytes[40..48].copy_from_slice(&self.row.to_le_bytes()); + bytes[48..].copy_from_slice(&self.usn.to_bytes()); + bytes + } + + fn decode(bytes: &[u8]) -> Option { + if bytes.len() != CONTINUITY_BYTES || !bytes.starts_with(CONTINUITY_MAGIC) { + return None; + } + let mut generation = [0_u8; 32]; + generation.copy_from_slice(bytes.get(8..40)?); + let mut row = [0_u8; 8]; + row.copy_from_slice(bytes.get(40..48)?); + let row = i64::from_le_bytes(row); + if row <= 0 { + return None; + } + Some(Self { + generation: GenerationId::new(acyclic_fs::Digest::from_bytes(generation)), + row, + usn: acyclic_fs::WindowsUsnCheckpoint::from_bytes(bytes.get(48..)?).ok()?, + }) + } +} + +fn fork_moved(base: GenerationId) -> PromoteOutcome { + PromoteOutcome::Conflict { + message: format!( + "the working tree moved past the fork's base ({}); promote in v1 requires an unmoved mainline — rewind to the base or re-fork and re-apply", + crate::generation_hex(base) + ), + } +} + /// Pipeline state reported by `status`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum State { + /// Metadata and watcher are open; no repository descendants were scanned. + NeedsBaseline, Baselining, Ready, Rewinding, @@ -191,43 +249,14 @@ enum Request { paths: Vec, reply: oneshot::Sender>, }, - /// Restores one path from `target` into `root` (a copy fork's directory). - RestorePathInto { - target: GenerationId, - root: PathBuf, - path: PathBuf, - reply: oneshot::Sender>, - }, - /// Copy-mode fork: write `generation` out to `destination`. - Materialize { - generation: GenerationId, - destination: PathBuf, - reply: oneshot::Sender>, - }, Promote { shared: Arc, base: GenerationId, label: String, reply: oneshot::Sender>, }, - ResolveSession { - shared: Arc, - base: GenerationId, - label: String, - reply: oneshot::Sender>, - }, - ApplySession { - generation: GenerationId, - base: GenerationId, - label: String, - reply: oneshot::Sender>, - }, Status { - reply: oneshot::Sender, - }, - SetShadowed { - active: bool, - reply: oneshot::Sender>, + reply: oneshot::Sender>, }, SessionStarted { session_id: String, @@ -243,6 +272,20 @@ enum Request { }, } +impl Request { + const fn requires_ready(&self) -> bool { + !matches!( + self, + Self::Status { .. } + | Self::TurnStarted { .. } + | Self::RecordGeneration { .. } + | Self::SessionStarted { .. } + | Self::SessionEnded { .. } + | Self::Shutdown { .. } + ) + } +} + /// Cloneable handle used by the daemon to talk to the pipeline. #[derive(Clone)] pub struct PipelineHandle { @@ -534,34 +577,6 @@ impl PipelineHandle { )? } - /// Restores one path from `target` into `root` rather than the working tree. - pub async fn restore_path_into( - &self, - target: GenerationId, - root: PathBuf, - path: PathBuf, - ) -> Result { - request!( - self, - RestorePathInto { - target: target, - root: root, - path: path - } - )? - } - - /// Copy-mode fork: materializes `generation` into `destination`. - pub async fn materialize(&self, generation: GenerationId, destination: PathBuf) -> Result<()> { - request!( - self, - Materialize { - generation: generation, - destination: destination - } - )? - } - pub async fn promote( &self, shared: Arc, @@ -578,54 +593,8 @@ impl PipelineHandle { )? } - /// Safe Mode's commit half of promote: commits the overlay (or reports - /// a conflict) without touching the real tree, so the caller can show - /// an approval-gated diff before deciding whether to `apply_session`. - pub async fn resolve_session( - &self, - shared: Arc, - base: GenerationId, - label: String, - ) -> Result { - request!( - self, - ResolveSession { - shared: shared, - base: base, - label: label - } - )? - } - - /// Safe Mode's swap half of promote: lands an already-`resolve_session`d - /// generation onto the real tree. - pub async fn apply_session( - &self, - generation: GenerationId, - base: GenerationId, - label: String, - ) -> Result { - request!( - self, - ApplySession { - generation: generation, - base: base, - label: label - } - )? - } - pub async fn status(&self) -> Result { - request!(self, Status {}) - } - - /// Safe Mode: while a session's fork is shadow-mounted over the real repo - /// root, mainline capture must pause — the pipeline's watcher would - /// otherwise observe the fork's content and the mount/unmount lifecycle - /// (which can map to the volume root) instead of real-tree mutations. - /// `resolve_session`/`apply_session` clear it and rebuild the watcher. - pub async fn set_shadowed(&self, active: bool) -> Result<()> { - request!(self, SetShadowed { active: active })? + request!(self, Status {})? } pub async fn session_started(&self, session_id: String, host: String) -> Result<()> { @@ -670,7 +639,7 @@ enum RootHint { /// Removes hints that target the volume root from a batch. The engine /// refuses any mutation of the root ("mutation cannot target the volume /// root"), and such hints only ever come from mount lifecycle events on the -/// repo directory (Safe Mode shadowing it, or a fork session tearing down), +/// repo directory (for example, a fork session tearing down), /// never from the user's edits. fn strip_root_hints(batch: WatchBatch) -> (WatchBatch, RootHint) { let WatchBatch::Changes { @@ -739,11 +708,7 @@ pub fn spawn( let (sender, receiver) = mpsc::channel(1024); let thread = std::thread::Builder::new() .name(format!("{}-pipeline", crate::product::NAME)) - // The fs facade's futures are large and a few of them nest per - // request (a subtree copy, a restore); the 2 MiB default is tight - // in debug builds. Virtual reservation only: untouched pages cost - // nothing. - .stack_size(32 * 1024 * 1024) + .stack_size(8 * 1024 * 1024) .spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_time() @@ -759,19 +724,17 @@ struct Pipeline { store: Store, index: Index, config: Config, - watch: NativeWatch, + watch: Option, options: CaptureOptions, /// Paths that never enter a checkpoint (`exclude` in the config). exclusions: Exclusions, + capture_policy: acyclic_fs::CapturePolicy, cancel: CancellationToken, state: State, last_generation: GenerationId, last_checkpoint_row: Option, checkpoints_since_commit: u32, last_activity: Instant, - /// A Safe Mode session's fork is shadow-mounted over the repo root: - /// mainline capture is suspended until `resolve_session`/`apply_session`. - shadowed: bool, /// Watcher invalidations and recovery rescans since start; see /// [`WatcherHealth`]. watcher_health: WatcherHealth, @@ -839,8 +802,9 @@ async fn run(store: Store, index: Index, config: Config, mut receiver: mpsc::Rec clippy::match_same_arms, reason = "the arms look identical but each `reply` is a differently typed sender" )] -fn fail_request(request: Request, message: &str) { - let error = || EngineError::Store(message.to_owned()); +fn fail_request(request: Request, error: impl std::fmt::Display) { + let message = error.to_string(); + let error = || EngineError::Store(message.clone()); match request { Request::Checkpoint { reply, .. } => drop(reply.send(Err(error()))), Request::Commit { reply } => drop(reply.send(Err(error()))), @@ -848,30 +812,19 @@ fn fail_request(request: Request, message: &str) { Request::Diff { reply, .. } => drop(reply.send(Err(error()))), Request::Fork { reply } => drop(reply.send(Err(error()))), Request::Promote { reply, .. } => drop(reply.send(Err(error()))), - Request::Materialize { reply, .. } => drop(reply.send(Err(error()))), Request::ScratchCheckout { reply, .. } => drop(reply.send(Err(error()))), Request::PublishHead { reply } => drop(reply.send(Err(error()))), Request::MergePlan { reply, .. } => drop(reply.send(Err(error()))), Request::BuildGeneration { reply, .. } => drop(reply.send(Err(error()))), Request::ApplyToOverlay { reply, .. } => drop(reply.send(Err(error()))), Request::ReadFiles { reply, .. } => drop(reply.send(Err(error()))), - Request::RestorePathInto { reply, .. } => drop(reply.send(Err(error()))), Request::MaterializePaths { reply, .. } => drop(reply.send(Err(error()))), Request::RecordGeneration { reply, .. } => drop(reply.send(Err(error()))), Request::SnapshotOverlay { reply, .. } => drop(reply.send(Err(error()))), - Request::ResolveSession { reply, .. } => drop(reply.send(Err(error()))), - Request::ApplySession { reply, .. } => drop(reply.send(Err(error()))), Request::RestorePath { reply, .. } => drop(reply.send(Err(error()))), Request::RestorePaths { reply, .. } => drop(reply.send(Err(error()))), Request::TurnStarted { reply, .. } => drop(reply.send(Err(error()))), - Request::SetShadowed { reply, .. } => drop(reply.send(Err(error()))), - Request::Status { reply } => drop(reply.send(StatusReport { - state: State::Baselining, - last_checkpoint: None, - unpublished: 0, - checkpoints_since_commit: 0, - watcher: WatcherHealth::default(), - })), + Request::Status { reply } => drop(reply.send(Err(error()))), Request::SessionStarted { reply, .. } | Request::SessionEnded { reply, .. } => { drop(reply.send(Err(error()))); } @@ -884,51 +837,108 @@ impl Pipeline { let cancel = CancellationToken::new(); let repo_root: PathBuf = store.repo_root.clone(); - let mut watch = NativeWatch::open( - &repo_root, - NativeWatchOptions { - limits: VolumeLimits::default(), - maximum_queued_changes: WATCH_QUEUE, - recursive: true, - }, - ) - .map_err(EngineError::fs("open watcher"))?; - watch - .begin_rescan() - .map_err(EngineError::fs("begin rescan"))?; - let options = CaptureOptions { - source_root: repo_root.clone(), - expected_root_identity: capture_root_identity(&repo_root) - .map_err(EngineError::fs("root identity"))?, + source_root: repo_root, + // First filesystem demand replaces this before any capture. Keeping + // startup free of native root access lets metadata-only consumers + // run even when the repository is temporarily unavailable. + expected_root_identity: acyclic_fs::NativeRootIdentity::from_bytes([0; 16]), maximum_paths: MAXIMUM_CAPTURE_PATHS, maximum_extent_spans: MAXIMUM_EXTENT_SPANS, }; let exclusions = Exclusions::parse(&config.exclude)?; - let mut pipeline = Self { + let capture_policy = exclusions.capture_policy()?; + let pipeline = Self { store, index, config, - watch, + watch: None, options, exclusions, + capture_policy, cancel, - state: State::Baselining, + state: State::NeedsBaseline, last_generation: GenerationId::new(acyclic_fs::Digest::ZERO), last_checkpoint_row: None, checkpoints_since_commit: 0, last_activity: Instant::now(), - shadowed: false, watcher_health: WatcherHealth::default(), auto_pending: None, }; - pipeline.baseline(CheckpointKind::Baseline).await?; Ok(pipeline) } + #[cfg(target_os = "windows")] + fn admit_continuity(store: &Store, index: &Index, watch: &mut NativeWatch) -> Result { + let bytes = match std::fs::read(store.paths.continuity()) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(_) => return Ok(false), + }; + let Some(record) = ContinuityRecord::decode(&bytes) else { + return Ok(false); + }; + let Some(row) = index.latest()? else { + return Ok(false); + }; + if !row.published + || row.id != record.row + || row.generation != record.generation + || store.checkout.generation_id() != record.generation + { + return Ok(false); + } + Ok(matches!( + watch + .accept_windows_usn_baseline(record.usn) + .map_err(EngineError::fs("admit Windows continuity"))?, + Ok(acyclic_fs::WindowsUsnContinuity::Unchanged) + )) + } + + async fn ensure_ready(&mut self) -> Result<()> { + if self.state == State::Ready { + return Ok(()); + } + if self.watch.is_none() { + let repo_root = self.store.repo_root.clone(); + self.options.expected_root_identity = + capture_root_identity(&repo_root).map_err(EngineError::fs("root identity"))?; + let mut watch = NativeWatch::open( + &repo_root, + NativeWatchOptions { + limits: VolumeLimits::default(), + maximum_queued_changes: WATCH_QUEUE, + recursive: true, + }, + ) + .map_err(EngineError::fs("open watcher"))?; + #[cfg(target_os = "windows")] + if Self::admit_continuity(&self.store, &self.index, &mut watch)? { + let latest = self.index.latest()?.ok_or_else(|| { + EngineError::Store("continuity admitted without an index row".into()) + })?; + self.last_generation = latest.generation; + self.last_checkpoint_row = Some(latest.id); + self.watch = Some(watch); + self.state = State::Ready; + return Ok(()); + } + watch + .begin_rescan() + .map_err(EngineError::fs("begin rescan"))?; + self.watch = Some(watch); + } + self.baseline(CheckpointKind::Baseline).await + } + /// Full baseline: capture the whole tree, checkpoint, finish the watcher /// rescan, publish. Used at startup and after RescanRequired/rewind. + #[allow( + clippy::too_many_lines, + reason = "rescan retries share one publication boundary" + )] async fn baseline(&mut self, kind: CheckpointKind) -> Result<()> { crate::trace!( "pipeline", @@ -936,81 +946,130 @@ impl Pipeline { ); let baseline_started = Instant::now(); self.state = State::Baselining; - // A baseline requires a clean checkout. Mid-session (watcher - // invalidation, rewind) the overlay holds uncommitted captures: - // publish them first. At startup this is a no-op. - let phase = Instant::now(); - self.commit_engine().await?; - let precommit_ms = crate::trace::ms(phase); - let phase = Instant::now(); - capture_baseline( - &mut self.store.checkout, - &self.options, - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("capture baseline"))?; - let capture_ms = crate::trace::ms(phase); - let phase = Instant::now(); - self.scrub_exclusions().await?; - let generation = self.checkpoint_engine().await?; - let snapshot_ms = crate::trace::ms(phase); - let row = self - .index - .record(generation, kind, &Attribution::default())?; - self.last_generation = generation; - self.last_checkpoint_row = Some(row); - - // Changes that raced the baseline arrive as the rescan-completion - // batch; fold them in before declaring Ready. - let batch = self - .watch - .finish_rescan() - .map_err(EngineError::fs("finish rescan"))?; - // A root hint here is already covered by the rescan that just ran. - let (batch, root) = strip_root_hints(batch); - if root != RootHint::None { + // A newly created hard link can race the rescan tail. Restart with a + // fresh watcher so the next full capture sees every alias together. + for attempt in 0..3 { + #[cfg(target_os = "windows")] + let continuity = acyclic_fs::capture_windows_usn_checkpoint(&self.store.repo_root).ok(); + // A baseline requires a clean checkout. Mid-session (watcher + // invalidation, rewind) the overlay holds uncommitted captures: + // publish them first. At startup this is a no-op. + let phase = Instant::now(); + self.commit_engine().await?; + let precommit_ms = crate::trace::ms(phase); + let phase = Instant::now(); + capture_baseline_with_policy( + &mut self.store.checkout, + &self.options, + &self.capture_policy, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await + .map_err(EngineError::fs("capture baseline"))?; + let capture_ms = crate::trace::ms(phase); + // Changes that raced the baseline arrive as the rescan-completion + // batch; fold them in before declaring Ready. + let batch = self + .watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? + .finish_rescan() + .map_err(EngineError::fs("finish rescan"))?; + // A root hint here is already covered by the rescan that just ran. + let (batch, root) = strip_root_hints(batch); + if matches!(batch, WatchBatch::RescanRequired { .. }) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some("rescan tail invalidated".into()); + self.reset_watch().await?; + crate::trace!( + "pipeline", + "baseline rescan tail invalidated; retry {}", + attempt + 1 + ); + continue; + } + if root != RootHint::None { + crate::trace!( + "pipeline", + "rescan tail: root hint ({root:?}) dropped, covered by the rescan" + ); + } + if let WatchBatch::Changes { ref changes, .. } = batch { + if !changes.is_empty() { + let captured = capture_watch_batch_with_policy( + &mut self.store.checkout, + batch, + &self.options, + &self.capture_policy, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await; + if let Err(failure) = captured { + if matches!( + failure.error, + acyclic_fs::CaptureError::RescanRequired { .. } + ) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some(failure.error.to_string()); + self.reset_watch().await?; + crate::trace!( + "pipeline", + "baseline rescan tail needs full capture; retry {}", + attempt + 1 + ); + continue; + } + return Err(EngineError::fs("capture rescan tail")(failure)); + } + } + } + let phase = Instant::now(); + let generation = self.checkpoint_engine().await?; + let snapshot_ms = crate::trace::ms(phase); + let row = self + .index + .record(generation, kind, &Attribution::default())?; + self.last_generation = generation; + self.last_checkpoint_row = Some(row); + let phase = Instant::now(); + self.commit_engine().await?; + #[cfg(target_os = "windows")] + if let Some(usn) = continuity { + crate::store::durable_replace( + &self.store.paths.continuity(), + &ContinuityRecord { + generation, + row, + usn, + } + .encode(), + )?; + } + let postcommit_ms = crate::trace::ms(phase); + self.state = State::Ready; + if kind == CheckpointKind::Recovered { + let ms = crate::trace::ms(baseline_started); + self.watcher_health.recovery_rescans += 1; + self.watcher_health.recovery_ms_total += ms; + self.watcher_health.last_recovery_ms = ms; + } crate::trace!( "pipeline", - "rescan tail: root hint ({root:?}) dropped, covered by the rescan" + "baseline phases: pre-commit {precommit_ms:.1}ms, full capture {capture_ms:.1}ms, \ + snapshot {snapshot_ms:.1}ms, post-commit {postcommit_ms:.1}ms" ); + crate::trace!( + "pipeline", + "baseline done in {:.1}ms; state Ready", + crate::trace::ms(baseline_started) + ); + return Ok(()); } - if let WatchBatch::Changes { ref changes, .. } = batch { - if !changes.is_empty() { - capture_watch_batch( - &mut self.store.checkout, - batch, - &self.options, - WorkCounters::UNBOUNDED, - &self.cancel, - ) - .await - .map_err(EngineError::fs("capture rescan tail"))?; - self.scrub_exclusions().await?; - } - } - let phase = Instant::now(); - self.commit_engine().await?; - let postcommit_ms = crate::trace::ms(phase); - self.state = State::Ready; - if kind == CheckpointKind::Recovered { - let ms = crate::trace::ms(baseline_started); - self.watcher_health.recovery_rescans += 1; - self.watcher_health.recovery_ms_total += ms; - self.watcher_health.last_recovery_ms = ms; - } - crate::trace!( - "pipeline", - "baseline phases: pre-commit {precommit_ms:.1}ms, full capture {capture_ms:.1}ms, \ - scrub+snapshot {snapshot_ms:.1}ms, rescan tail+post-commit {postcommit_ms:.1}ms" - ); - crate::trace!( - "pipeline", - "baseline done in {:.1}ms; state Ready", - crate::trace::ms(baseline_started) - ); - Ok(()) + Err(EngineError::Store( + "baseline rescan repeatedly invalidated".into(), + )) } /// Handles one request; returns true when the pipeline should exit. @@ -1020,6 +1079,12 @@ impl Pipeline { )] async fn handle(&mut self, request: Request) -> bool { self.last_activity = Instant::now(); + if request.requires_ready() { + if let Err(error) = self.ensure_ready().await { + fail_request(request, error); + return false; + } + } match request { Request::Checkpoint { kind, @@ -1186,12 +1251,9 @@ impl Pipeline { reply, } => { let result = Box::pin(async { - let mut out = Vec::with_capacity(paths.len()); - for path in paths { - let bytes = crate::merge::read_file(&self.store, generation, &path).await?; - out.push((path, bytes)); - } - Ok(out) + let contents = + crate::merge::read_files(&self.store, generation, &paths).await?; + Ok(paths.into_iter().zip(contents).collect()) }) .await; let _ = reply.send(result); @@ -1213,26 +1275,6 @@ impl Pipeline { let _ = reply.send(result); false } - Request::RestorePathInto { - target, - root, - path, - reply, - } => { - let result = - Box::pin(rewind::restore_path_into(&self.store, target, &root, &path)).await; - let _ = reply.send(result); - false - } - Request::Materialize { - generation, - destination, - reply, - } => { - let _ = reply - .send(rewind::materialize_into(&self.store, generation, &destination).await); - false - } Request::Promote { shared, base, @@ -1242,37 +1284,14 @@ impl Pipeline { let _ = reply.send(self.promote(shared, base, &label).await); false } - Request::ResolveSession { - shared, - base, - label, - reply, - } => { - let _ = reply.send(self.resolve_session(shared, base, &label).await); - false - } - Request::ApplySession { - generation, - base, - label, - reply, - } => { - let _ = reply.send(self.apply_session(generation, base, &label).await); - false - } Request::Status { reply } => { - let _ = reply.send(StatusReport { + let _ = reply.send(Ok(StatusReport { state: self.state, last_checkpoint: self.last_checkpoint_row, unpublished: self.index.unpublished_count().unwrap_or(0), checkpoints_since_commit: self.checkpoints_since_commit, watcher: self.watcher_health.clone(), - }); - false - } - Request::SetShadowed { active, reply } => { - self.shadowed = active; - let _ = reply.send(Ok(())); + })); false } Request::SessionStarted { @@ -1283,7 +1302,10 @@ impl Pipeline { let result = async { self.index.session_started(&session_id, &host)?; // Session start is a coarse boundary. - self.commit_engine().await + if self.state == State::Ready { + self.commit_engine().await?; + } + Ok(()) } .await; let _ = reply.send(result); @@ -1292,7 +1314,10 @@ impl Pipeline { Request::SessionEnded { session_id, reply } => { let result = async { self.index.session_ended(&session_id)?; - self.commit_engine().await + if self.state == State::Ready { + self.commit_engine().await?; + } + Ok(()) } .await; let _ = reply.send(result); @@ -1313,26 +1338,6 @@ impl Pipeline { attribution: &Attribution, ) -> Result { let started = Instant::now(); - if self.shadowed { - // A Safe Mode session's fork is mounted over the repo root; the - // real tree is frozen and the watcher sees only fork/mount noise. - // Record a noop so the hook gets a clean reply, but never drain - // the watcher or snapshot the mainline mid-session. - crate::trace!( - "pipeline", - "checkpoint kind={:?}: shadowed -> noop row, watcher untouched", - kind - ); - let row = self - .index - .record(self.last_generation, CheckpointKind::Noop, attribution)?; - self.last_checkpoint_row = Some(row); - return Ok(CheckpointOutcome { - row_id: row, - generation: self.last_generation, - kind: CheckpointKind::Noop, - }); - } if self.state != State::Ready { // A failed recovery leaves state at Baselining; a request is the // natural moment to retry rather than staying down forever. @@ -1396,14 +1401,18 @@ impl Pipeline { }) } - /// Polls the watcher until it stays quiet for `quiesce_ms` (capped at - /// `quiesce_cap_ms`), capturing every non-empty batch. Returns whether - /// anything was captured. + /// Captures every available watcher batch, then waits for `quiesce_ms` + /// of quiet after the first change (capped at `quiesce_cap_ms`). An empty + /// first poll returns immediately. Returns whether anything was captured. + #[allow( + clippy::too_many_lines, + reason = "watch invalidation and capture share one poll loop" + )] async fn drain_watcher(&mut self) -> Result { let quiesce = Duration::from_millis(self.config.quiesce_ms); let cap = Duration::from_millis(self.config.quiesce_cap_ms); let started = Instant::now(); - let mut last_change = Instant::now(); + let mut last_change = None; let mut changed = false; let mut polls = 0u32; let mut batches = 0u32; @@ -1412,6 +1421,8 @@ impl Pipeline { polls += 1; let batch = self .watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? .poll(POLL_CHANGES, WorkCounters::UNBOUNDED, &self.cancel) .map_err(EngineError::fs("watch poll"))? .value; @@ -1422,7 +1433,6 @@ impl Pipeline { "drain: watcher hinted at the volume root ({root:?}); dropped" ); } - let (batch, scrub) = self.exclusions.filter_batch(batch); if root == RootHint::Structural { // The repo directory itself changed identity (a mount came // or went): re-baseline on a fresh watcher rather than apply @@ -1439,29 +1449,44 @@ impl Pipeline { WatchBatch::Changes { ref changes, .. } if !changes.is_empty() => { batches += 1; hints += changes.len(); - capture_watch_batch( + let captured = capture_watch_batch_with_policy( &mut self.store.checkout, batch, &self.options, + &self.capture_policy, WorkCounters::UNBOUNDED, &self.cancel, ) - .await - .map_err(EngineError::fs("capture watch batch"))?; - if scrub { - self.scrub_exclusions().await?; + .await; + if let Err(failure) = captured { + if matches!( + failure.error, + acyclic_fs::CaptureError::RescanRequired { .. } + ) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some(failure.error.to_string()); + self.reset_watch().await?; + self.baseline(CheckpointKind::Recovered).await?; + return Ok(true); + } + return Err(EngineError::fs("capture watch batch")(failure)); } changed = true; - last_change = Instant::now(); + last_change = Some(Instant::now()); } WatchBatch::Changes { .. } => { - if last_change.elapsed() >= quiesce || started.elapsed() >= cap { + if last_change.is_none() + || last_change.is_some_and(|last| last.elapsed() >= quiesce) + || started.elapsed() >= cap + { crate::trace!( "pipeline", "drain: done after {:.1}ms, {polls} polls, {batches} batch(es), \ {hints} hint(s); stopped by {}", crate::trace::ms(started), - if started.elapsed() >= cap { + if last_change.is_none() { + "empty poll" + } else if started.elapsed() >= cap { "cap" } else { "quiesce window" @@ -1493,19 +1518,6 @@ impl Pipeline { } } - /// Drops excluded paths a capture may have pulled into the checkout, - /// before the generation they would otherwise land in is checkpointed. - async fn scrub_exclusions(&mut self) -> Result<()> { - let removed = self.exclusions.scrub(&mut self.store.checkout).await?; - if removed > 0 { - crate::trace!( - "pipeline", - "exclusions: scrubbed {removed} excluded path(s) from the checkout" - ); - } - Ok(()) - } - /// `checkpoint()`: snapshot without authority publish. The fast path. async fn checkpoint_engine(&mut self) -> Result { Ok(self @@ -1565,14 +1577,23 @@ impl Pipeline { /// The safety net for hosts with no lifecycle-hook API (Claude Desktop /// over MCP): a checkpoint no request asked for, taken once the watcher /// has been quiet for `auto_checkpoint_idle_ms`. Runs on every idle - /// tick: drains whatever the watcher has (cheap, bounded by - /// `quiesce_ms`; usually nothing, since hook-driven hosts drain on their - /// own pre/post-tool checkpoints), then records a row only once a full - /// tick has passed with no further changes and nothing else has - /// checkpointed them in the meantime. Never records a row when nothing - /// changed, so it cannot spam the timeline. + /// tick: drains whatever the watcher has (an empty watcher returns after + /// one poll; hook-driven hosts usually drained on their own pre/post-tool + /// checkpoints), then records a row only once a full tick has passed with + /// no further changes and nothing else has checkpointed them in the + /// meantime. Never records a row when nothing changed, so it cannot spam + /// the timeline. async fn auto_checkpoint(&mut self) { - if self.config.auto_checkpoint_idle_ms == 0 || self.shadowed || self.state != State::Ready { + if self.config.auto_checkpoint_idle_ms == 0 { + return; + } + // An idle daemon must remain O(1) in repository size. The first + // consumer operation that needs authenticated contents performs the + // baseline; a timer alone is not such an operation. + if self.state == State::NeedsBaseline { + return; + } + if self.state != State::Ready { return; } // Like `idle_commit`, failures here are advisory: the pending state @@ -1682,20 +1703,7 @@ impl Pipeline { self.last_checkpoint_row = Some(safety_row); self.commit_engine().await?; - let outcome = rewind::execute( - &self.store, - target.generation, - self.config.trash_ttl_days, - &self.exclusions, - ) - .await; - - // The swap replaced the repo directory's inode: the pinned root - // identity and the watcher both point at the old tree. Rebuild both, - // then re-baseline. - self.reset_watch().await?; - self.baseline(CheckpointKind::Recovered).await?; - outcome + self.swap_root_and_rebaseline(target.generation).await } /// Single-path restore, bracketed by checkpoints: a `manual` safety row @@ -1745,7 +1753,7 @@ impl Pipeline { &parent, )?)); } - capture_watch_batch( + let captured = capture_watch_batch_with_policy( &mut self.store.checkout, WatchBatch::Changes { epoch: WatchEpoch::from_u64(0), @@ -1754,12 +1762,47 @@ impl Pipeline { changes: hints, }, &self.options, + &self.capture_policy, + WorkCounters::UNBOUNDED, + &self.cancel, + ) + .await; + if let Err(failure) = captured { + if matches!( + failure.error, + acyclic_fs::CaptureError::RescanRequired { .. } + ) { + self.watcher_health.invalidations += 1; + self.watcher_health.last_reason = Some(failure.error.to_string()); + self.reset_watch().await?; + self.baseline(CheckpointKind::Recovered).await?; + return Ok(()); + } + return Err(EngineError::fs("capture restored paths")(failure)); + } + Ok(()) + } + + /// Reconciles exact restored roots immediately, including all descendants + /// of a directory that was replaced or removed. The SDK unions the host + /// and checkout subtrees, so stale descendants are removed without waiting + /// for the native watcher to enumerate the write. + async fn capture_restored_subtrees(&mut self, paths: &[PathBuf]) -> Result<()> { + let roots = paths + .iter() + .map(|path| crate::merge::namespace_of(path)) + .collect::>>()?; + capture_subtrees_with_policy( + &mut self.store.checkout, + &roots, + &self.options, + &self.capture_policy, WorkCounters::UNBOUNDED, &self.cancel, ) .await - .map_err(EngineError::fs("capture restored paths"))?; - self.scrub_exclusions().await + .map_err(EngineError::fs("capture restored subtrees"))?; + Ok(()) } /// Restores `paths` from `target` as one event: at most one safety row @@ -1771,12 +1814,8 @@ impl Pipeline { safety: bool, label: Option, ) -> Result { - if self.shadowed { - return Err(EngineError::Restore( - "a Safe Mode session is shadowing the repo root; resolve it first".into(), - )); - } - for path in paths { + let paths = crate::merge::subtree_roots(paths); + for path in &paths { if self.exclusions.covers_host(path) { return Err(EngineError::Restore(format!( "{} is excluded from snapshots (`exclude` in {}); no checkpoint holds it", @@ -1789,7 +1828,7 @@ impl Pipeline { self.reset_watch().await?; self.baseline(CheckpointKind::Recovered).await?; } - let what = match paths { + let what = match paths.as_slice() { [path] => format!("{} from #{}", path.display(), target.id), _ => format!("{} path(s) from #{}", paths.len(), target.id), }; @@ -1817,7 +1856,7 @@ impl Pipeline { let write_started = Instant::now(); let mut outcomes = Vec::with_capacity(paths.len()); - for path in paths { + for path in &paths { outcomes.push(rewind::restore_path(&self.store, target.generation, path).await?); } let write_ms = crate::trace::ms(write_started); @@ -1827,21 +1866,20 @@ impl Pipeline { // later. The echo is harmless when it comes: a modified hint on a // path whose content already matches captures nothing, and the // staged sibling's create+rename resolves to an absent path. - // Direct capture describes each path with one hint, which is exact - // for a regular file or symlink and wrong for a subtree (a removed - // or replaced directory needs a hint per descendant). Anything - // else waits for the native watcher, which delivers those. + // Leaf batches avoid directory traversal. Directory and absent roots + // use the SDK subtree reconciler, which unions live and stored + // descendants and therefore captures replacements and removals exactly. let post_started = Instant::now(); let all_leaves = paths.iter().all(|path| { std::fs::symlink_metadata(self.store.repo_root.join(path)) .is_ok_and(|metadata| metadata.is_file() || metadata.is_symlink()) }); let capture_mode = if all_leaves { - self.capture_paths_directly(paths).await?; + self.capture_paths_directly(&paths).await?; "direct capture" } else { - self.drain_watcher().await?; - "watcher drain (a path is a directory or absent)" + self.capture_restored_subtrees(&paths).await?; + "direct subtree capture" }; let post_ms = crate::trace::ms(post_started); let capture_started = Instant::now(); @@ -1908,16 +1946,13 @@ impl Pipeline { async fn scratch_checkout(&mut self, base: GenerationId) -> Result> { let checkout = self .store - .volume + .workspace .checkout( acyclic_fs::model::GenerationSelector::Exact(base), crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, ) .await - .map_err(EngineError::fs("scratch checkout"))? - .value; + .map_err(EngineError::fs("scratch checkout"))?; Ok(Arc::new(SharedLocalCheckout::with_publication( checkout, MountPublication::Manual, @@ -1970,22 +2005,19 @@ impl Pipeline { // front of a fork. let checkout = self .store - .volume + .workspace .checkout( acyclic_fs::model::GenerationSelector::Exact(base), crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, ) .await - .map_err(EngineError::fs("fork checkout"))? - .value; + .map_err(EngineError::fs("fork checkout"))?; let config = checkout.volume_config(); let volume_id = checkout.volume_id(); Ok(ForkSeed { // Native close/flush must never publish: sibling forks share one // volume head, so a seal on close makes the next fork's mutations - // Stale. Promote/resolve are the only commits. + // stale. Promote is the only commit. shared: Arc::new(SharedLocalCheckout::with_publication( checkout, MountPublication::Manual, @@ -2021,6 +2053,13 @@ impl Pipeline { self.last_checkpoint_row = Some(safety_row); self.commit_engine().await?; + // The fork may have been cut from an unpublished checkpoint. Publishing + // that same generation advances the authority sequence, so its checkout + // must rebase before committing even though the mainline did not move. + if self.store.checkout.generation_id() != base { + return Ok(fork_moved(base)); + } + let outcome = { let mut guard = shared.lock().await; if !guard.has_pending_mutations() { @@ -2031,6 +2070,17 @@ impl Pipeline { old_tree: None, }); } + let rebased = guard + .rebase_head(0, WorkCounters::UNBOUNDED, &self.cancel) + .await + .map_err(EngineError::fs("fork rebase"))? + .value; + if matches!( + rebased, + acyclic_fs::kernel::RebaseDecision::Conflicted { .. } + ) { + return Ok(fork_moved(base)); + } guard .commit(OperationId::new(), WorkCounters::UNBOUNDED, &self.cancel) .await @@ -2041,14 +2091,7 @@ impl Pipeline { CheckoutCommitOutcome::Committed { generation_id, .. } | CheckoutCommitOutcome::AlreadyCommitted { generation_id, .. } => generation_id, CheckoutCommitOutcome::Conflict { .. } | CheckoutCommitOutcome::Fenced { .. } => { - return Ok(PromoteOutcome::Conflict { - message: format!( - "the working tree moved past the fork's base \ - ({}); promote in v1 requires an unmoved mainline — \ - rewind to the base or re-fork and re-apply", - crate::generation_hex(base) - ), - }); + return Ok(fork_moved(base)); } other => { return Err(EngineError::Fs(format!( @@ -2062,16 +2105,13 @@ impl Pipeline { self.state = State::Rewinding; self.store.checkout = self .store - .volume + .workspace .checkout( acyclic_fs::model::GenerationSelector::Head, crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, ) .await - .map_err(EngineError::fs("refresh checkout"))? - .value; + .map_err(EngineError::fs("refresh checkout"))?; let row = self.index.record( generation, CheckpointKind::Manual, @@ -2083,150 +2123,75 @@ impl Pipeline { self.last_generation = generation; self.last_checkpoint_row = Some(row); - let swap = rewind::execute( - &self.store, - generation, - self.config.trash_ttl_days, - &self.exclusions, - ) - .await; - self.reset_watch().await?; - self.baseline(CheckpointKind::Recovered).await?; - let swap = swap?; + let swap = self.swap_root_and_rebaseline(generation).await?; Ok(PromoteOutcome::Promoted { generation, old_tree: Some(swap.old_tree), }) } - /// The commit half of promote, split out for Safe Mode: publishes the - /// mainline safety net and commits the fork's overlay, but never - /// touches the real tree — the caller diffs `base` against the - /// returned generation and decides whether to `apply_session` it. - async fn resolve_session( - &mut self, - shared: Arc, - base: GenerationId, - label: &str, - ) -> Result { - // The server unmounts the shadow before calling us, so the pipeline - // watcher's queue is full of mount-teardown hints — some of which map - // to the volume root and would fail capture outright. Leave shadow - // mode and swap in a fresh watcher on the now-real tree to DISCARD - // that queue. The pipeline's own checkout never moved during the - // session (mainline checkpoints no-op while shadowed), so it still - // sits at `base`; we must not re-baseline/publish it here or the - // mainline HEAD would advance past the fork and the overlay commit - // below would spuriously conflict. - if self.shadowed { - self.shadowed = false; - self.reset_watch().await?; - let _ = self - .watch - .finish_rescan() - .map_err(EngineError::fs("finish rescan"))?; - self.state = State::Ready; - } else { - // No shadow (direct callers, e.g. tests): the watcher is trusted, - // so fold any pending real-tree change into the safety net. - self.drain_watcher().await?; - } - let safety = self.checkpoint_engine().await?; - let safety_row = self.index.record( - safety, - CheckpointKind::PreRewind, - &Attribution { - label: Some(format!("before {label}")), - ..Attribution::default() - }, - )?; - self.last_generation = safety; - self.last_checkpoint_row = Some(safety_row); - // The safety checkpoint stays UNPUBLISHED (checkpoint, not commit): - // publishing it would move the authority head off `base`, and Safe - // Mode must leave the head at base until `apply_session` so that - // `session-discard` truly changes nothing and the next session forks - // cleanly. Unpublished checkpoint generations are fully restorable. - let _ = base; // conflict against a moved mainline is detected at apply - - // Snapshot the fork's overlay as an unpublished generation: diffable - // and rewind-applyable by id, without advancing the head. - let generation = { - let guard = shared.lock().await; - if !guard.has_pending_mutations() { - return Ok(SessionResolveOutcome::NoChanges); - } - guard - .checkpoint(WorkCounters::UNBOUNDED, &self.cancel) - .await - .map_err(EngineError::fs("session checkpoint"))? - .value - }; - Ok(SessionResolveOutcome::Resolved { generation }) - } - - /// The swap half of promote, split out for Safe Mode: lands an - /// already-committed generation (from `resolve_session`) onto the real - /// tree, exactly like `promote`'s own tail. - async fn apply_session( + /// Stops the old event source before the intentional root exchange. An + /// old callback must never publish a hint into the new watcher's epoch. + async fn swap_root_and_rebaseline( &mut self, generation: GenerationId, - base: GenerationId, - label: &str, - ) -> Result { - self.state = State::Rewinding; + ) -> Result { + drop(self.watch.take()); + let prepared = rewind::prepare( + &self.store, + generation, + self.config.trash_ttl_days, + &self.exclusions, + ) + .await?; + let current = self + .store + .workspace + .head() + .await + .map_err(EngineError::fs("workspace head before rewind"))?; + let target = self + .store + .workspace + .generation(generation) + .await + .map_err(EngineError::fs("rewind generation"))?; + prepared.mark_restoring_head()?; + match self + .store + .workspace + .restore_generation( + &target, + current.id(), + crate::rewind::publication_key(generation)?, + ) + .await + .map_err(EngineError::fs("restore workspace generation"))? + { + WorkspaceRestore::Restored(_) + | WorkspaceRestore::AlreadyRestored(_) + | WorkspaceRestore::Current(_) => {} + WorkspaceRestore::Stale(_) + | WorkspaceRestore::Fenced + | WorkspaceRestore::IdempotencyConflict => { + return Err(EngineError::Store( + "workspace head changed during rewind".into(), + )); + } + } + let outcome = prepared.publish(); self.store.checkout = self .store - .volume + .workspace .checkout( acyclic_fs::model::GenerationSelector::Head, crate::store::writable_head(), - WorkCounters::UNBOUNDED, - &self.cancel, ) .await - .map_err(EngineError::fs("refresh checkout"))? - .value; - // Same v1 stance as promote: if the mainline published anything since - // the fork's base, the head has moved off `base` and applying would - // clobber it. Detected here rather than at resolve, because resolve - // deliberately leaves the head at base (unpublished overlay). - if self.store.checkout.generation_id() != base { - self.state = State::Ready; - return Ok(PromoteOutcome::Conflict { - message: format!( - "the working tree moved past the session's base ({}); \ - Safe Mode in v1 requires an unmoved mainline — rewind to \ - the base or start a new session", - crate::generation_hex(base) - ), - }); - } - let row = self.index.record( - generation, - CheckpointKind::Manual, - &Attribution { - label: Some(label.to_owned()), - ..Attribution::default() - }, - )?; - self.last_generation = generation; - self.last_checkpoint_row = Some(row); - - let swap = rewind::execute( - &self.store, - generation, - self.config.trash_ttl_days, - &self.exclusions, - ) - .await; + .map_err(EngineError::fs("refresh rewind checkout"))?; self.reset_watch().await?; self.baseline(CheckpointKind::Recovered).await?; - let swap = swap?; - Ok(PromoteOutcome::Promoted { - generation, - old_tree: Some(swap.old_tree), - }) + outcome } /// Reopens the watcher and recomputes the capture root identity — needed @@ -2239,16 +2204,20 @@ impl Pipeline { let repo_root = self.store.repo_root.clone(); self.options.expected_root_identity = capture_root_identity(&repo_root).map_err(EngineError::fs("root identity"))?; - self.watch = NativeWatch::open( - &repo_root, - NativeWatchOptions { - limits: VolumeLimits::default(), - maximum_queued_changes: WATCH_QUEUE, - recursive: true, - }, - ) - .map_err(EngineError::fs("reopen watcher"))?; + self.watch = Some( + NativeWatch::open( + &repo_root, + NativeWatchOptions { + limits: VolumeLimits::default(), + maximum_queued_changes: WATCH_QUEUE, + recursive: true, + }, + ) + .map_err(EngineError::fs("reopen watcher"))?, + ); self.watch + .as_mut() + .ok_or_else(|| EngineError::Store("watcher is inactive".into()))? .begin_rescan() .map_err(EngineError::fs("begin rescan"))?; Ok(()) @@ -2258,8 +2227,9 @@ impl Pipeline { #[cfg(test)] mod root_hint_tests { use super::*; - use acyclic_fs::kernel::{LogicalName, NamespacePath}; + use acyclic_fs::kernel::NamespacePath; use acyclic_fs::{WatchEpoch, WatchSequence}; + use std::path::Path; fn root() -> NamespacePath { NamespacePath::new(Vec::new(), VolumeLimits::default()).unwrap() @@ -2267,13 +2237,8 @@ mod root_hint_tests { fn file(name: &str) -> NamespacePath { let limits = VolumeLimits::default(); - let name = LogicalName::new( - crate::names::encoding(), - crate::names::str_to_bytes(name), - limits.maximum_component_bytes, - ) - .unwrap(); - NamespacePath::new(vec![name], limits).unwrap() + acyclic_fs::host_path_to_namespace(Path::new(name), crate::store::host_profile(), limits) + .unwrap() } fn batch(changes: Vec) -> WatchBatch { @@ -2345,3 +2310,100 @@ mod root_hint_tests { assert!(matches!(out, WatchBatch::RescanRequired { .. })); } } + +#[cfg(all(test, target_os = "windows"))] +mod continuity_tests { + use super::*; + + fn record() -> ContinuityRecord { + let mut bytes = [0_u8; 40]; + bytes[..8].copy_from_slice(b"ACYUSN\0\x01"); + bytes[8..24].copy_from_slice(&[7; 16]); + bytes[24..32].copy_from_slice(&19_u64.to_le_bytes()); + bytes[32..40].copy_from_slice(&23_u64.to_le_bytes()); + let usn = acyclic_fs::WindowsUsnCheckpoint::from_bytes(&bytes).expect("canonical fixture"); + ContinuityRecord { + generation: GenerationId::new(acyclic_fs::Digest::from_bytes([3; 32])), + row: 7, + usn, + } + } + + #[test] + fn continuity_record_is_canonical_and_rejects_torn_state() { + let record = record(); + let bytes = record.encode(); + let decoded = ContinuityRecord::decode(&bytes).expect("canonical record"); + assert_eq!(decoded.generation, record.generation); + assert_eq!(decoded.row, record.row); + assert_eq!(decoded.usn, record.usn); + assert!(ContinuityRecord::decode(&bytes[..CONTINUITY_BYTES - 1]).is_none()); + let mut wrong_version = bytes; + wrong_version[0] ^= 0xff; + assert!(ContinuityRecord::decode(&wrong_version).is_none()); + let mut invalid_row = bytes; + invalid_row[40..48].copy_from_slice(&0_i64.to_le_bytes()); + assert!(ContinuityRecord::decode(&invalid_row).is_none()); + } + + #[test] + fn durable_replacement_keeps_only_the_complete_new_record() { + let directory = tempfile::tempdir().expect("store"); + let path = directory.path().join("continuity.bin"); + let first = record().encode(); + crate::store::durable_replace(&path, &first).expect("first replace"); + let mut second = record(); + second.row += 1; + crate::store::durable_replace(&path, &second.encode()).expect("second replace"); + let stored = std::fs::read(path).expect("read marker"); + assert_eq!( + ContinuityRecord::decode(&stored).expect("complete").row, + second.row + ); + } +} + +#[cfg(test)] +mod readiness_tests { + use super::*; + + #[test] + fn metadata_requests_do_not_force_a_repository_scan() { + let (status_reply, _) = oneshot::channel(); + assert!(!Request::Status { + reply: status_reply + } + .requires_ready()); + + let (turn_reply, _) = oneshot::channel(); + assert!(!Request::TurnStarted { + session_id: "session".to_owned(), + prompt: "prompt".to_owned(), + reply: turn_reply, + } + .requires_ready()); + + let (started_reply, _) = oneshot::channel(); + assert!(!Request::SessionStarted { + session_id: "session".to_owned(), + host: "host".to_owned(), + reply: started_reply, + } + .requires_ready()); + + let (ended_reply, _) = oneshot::channel(); + assert!(!Request::SessionEnded { + session_id: "session".to_owned(), + reply: ended_reply, + } + .requires_ready()); + + let (checkpoint_reply, _) = oneshot::channel(); + assert!(Request::Checkpoint { + kind: CheckpointKind::Manual, + attribution: Attribution::default(), + reply: checkpoint_reply, + } + .requires_ready()); + } +} diff --git a/crates/acyclic-engine/src/product.rs b/crates/acyclic/src/product.rs similarity index 100% rename from crates/acyclic-engine/src/product.rs rename to crates/acyclic/src/product.rs diff --git a/crates/acyclic-proto/src/lib.rs b/crates/acyclic/src/proto.rs similarity index 81% rename from crates/acyclic-proto/src/lib.rs rename to crates/acyclic/src/proto.rs index 0e68046..5917f8d 100644 --- a/crates/acyclic-proto/src/lib.rs +++ b/crates/acyclic/src/proto.rs @@ -8,7 +8,61 @@ use std::str::FromStr; use serde::{Deserialize, Serialize}; -pub const PROTOCOL_VERSION: u32 = 1; +pub use acyclic::diff::ChangeKind; +pub use acyclic::index::CheckpointKind; +pub use acyclic::rewind::RestoreAction; + +#[cfg(test)] +mod domain_wire_tests { + use super::{ChangeKind, CheckpointKind, RestoreAction}; + + #[test] + fn domain_enums_preserve_version_one_wire_names() { + let checkpoint_kinds = [ + (CheckpointKind::Baseline, "baseline"), + (CheckpointKind::Pre, "pre"), + (CheckpointKind::Post, "post"), + (CheckpointKind::Manual, "manual"), + (CheckpointKind::PreRewind, "pre_rewind"), + (CheckpointKind::Recovered, "recovered"), + (CheckpointKind::Failed, "failed"), + (CheckpointKind::Noop, "noop"), + (CheckpointKind::Auto, "auto"), + ]; + for (kind, name) in checkpoint_kinds { + let encoded = serde_json::to_string(&kind).unwrap(); + assert_eq!(encoded, format!("\"{name}\"")); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + kind + ); + } + let changes = [ + (ChangeKind::Added, "added"), + (ChangeKind::Removed, "removed"), + (ChangeKind::Modified, "modified"), + (ChangeKind::MetadataOnly, "metadata"), + ]; + for (kind, name) in changes { + let encoded = serde_json::to_string(&kind).unwrap(); + assert_eq!(encoded, format!("\"{name}\"")); + assert_eq!(serde_json::from_str::(&encoded).unwrap(), kind); + } + for (action, name) in [ + (RestoreAction::Restored, "restored"), + (RestoreAction::Removed, "removed"), + ] { + let encoded = serde_json::to_string(&action).unwrap(); + assert_eq!(encoded, format!("\"{name}\"")); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + action + ); + } + } +} + +pub const PROTOCOL_VERSION: u32 = 2; /// The kinds a client may ask for. Bookkeeping kinds (`baseline`, /// `noop`, `auto`, ...) exist only on replies: the daemon decides those. @@ -51,93 +105,6 @@ impl fmt::Display for CheckpointRequestKind { } } -/// Why a checkpoint row exists, as the timeline reports it. Mirrors the -/// engine's `index::CheckpointKind`; the daemon converts between them. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum CheckpointKind { - Baseline, - Pre, - Post, - Manual, - PreRewind, - Recovered, - Failed, - Noop, - Auto, -} - -impl CheckpointKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Baseline => "baseline", - Self::Pre => "pre", - Self::Post => "post", - Self::Manual => "manual", - Self::PreRewind => "pre_rewind", - Self::Recovered => "recovered", - Self::Failed => "failed", - Self::Noop => "noop", - Self::Auto => "auto", - } - } -} - -impl fmt::Display for CheckpointKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad(self.as_str()) - } -} - -/// What a single-path restore did to the path. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RestoreAction { - /// The path now has the checkpoint's contents. - Restored, - /// The path was absent at the checkpoint, so it was removed. - Removed, -} - -/// How a path differs between two checkpoints. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ChangeKind { - Added, - Removed, - Modified, - /// Mode or other metadata only; contents are identical. - Metadata, -} - -impl ChangeKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Added => "added", - Self::Removed => "removed", - Self::Modified => "modified", - Self::Metadata => "metadata", - } - } - - /// The one-letter tag `diff` listings use: A / D / M, and `m` for - /// metadata-only, so real blast radius stands out from noise. - pub fn tag(self) -> &'static str { - match self { - Self::Added => "A", - Self::Removed => "D", - Self::Modified => "M", - Self::Metadata => "m", - } - } -} - -impl fmt::Display for ChangeKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.pad(self.as_str()) - } -} - #[derive(Debug, Serialize, Deserialize)] pub struct Request { /// Protocol version; mismatches are rejected. @@ -272,26 +239,6 @@ pub enum Op { #[serde(rename = "fork")] id: String, }, - /// Safe Mode: forks one checkout and mounts it directly at the repo - /// root for the session's duration (see `dry_run` in `.acyclic/config.toml`). - SessionFork { - session_id: String, - }, - /// Safe Mode: unmounts the session's shadow mount, commits its overlay, - /// and returns the resulting diff without touching the real tree yet. - /// The fork is held pending `SessionApply`/`SessionDiscard`. - SessionResolve { - session_id: String, - }, - /// Safe Mode: applies a `SessionResolve`d session's changes to the real - /// tree (the deferred half of promote). - SessionApply { - session_id: String, - }, - /// Safe Mode: discards a `SessionResolve`d session without applying it. - SessionDiscard { - session_id: String, - }, } fn default_fork_count() -> u32 { @@ -346,8 +293,6 @@ pub enum Reply { Summary(SummaryInfo), Forks(Vec), Promote(PromoteInfo), - /// A `SessionResolve`d session awaiting `SessionApply`/`SessionDiscard`. - SessionPending(SessionPendingInfo), } /// One turn's summary, and where it came from. @@ -368,19 +313,10 @@ pub struct SummaryInfo { pub lead_ms: Option, } -#[derive(Debug, Serialize, Deserialize)] -pub struct SessionPendingInfo { - pub session_id: String, - pub diff: Vec, -} - #[derive(Debug, Serialize, Deserialize)] pub struct ForkEntry { pub id: String, pub path: String, - /// "mount" (routed native mount) or "copy" (materialized directory). - #[serde(default = "default_fork_mode")] - pub mode: String, /// Hex of the published generation the fork was cut from. pub base: String, pub created_at: i64, @@ -450,9 +386,8 @@ pub struct StatusInfo { pub state: String, pub last_checkpoint: Option, pub unpublished: u64, - pub store_bytes: u64, pub repo_root: String, - /// Mount provider this daemon would use for forks and Safe Mode. + /// Mount provider this daemon would use for forks. #[serde(default)] pub mount_provider: String, #[serde(default)] @@ -649,7 +584,3 @@ pub struct DiffEntry { #[serde(default)] pub ignored: bool, } - -fn default_fork_mode() -> String { - "mount".to_owned() -} diff --git a/crates/acyclic/src/rewind.rs b/crates/acyclic/src/rewind.rs new file mode 100644 index 0000000..c123578 --- /dev/null +++ b/crates/acyclic/src/rewind.rs @@ -0,0 +1,1049 @@ +//! Full-tree rewind: materialize the target generation into a sibling temp +//! directory, atomically exchange it with the working tree, keep the old tree +//! beside the repository, and journal every phase so kill -9 leaves the repo fully-old or +//! fully-new — never mixed. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use acyclic_fs::{ + durable_rename, prepare_native_exchange_with_recovery, publish_native_exchange, + recover_native_exchange, HostPathReplacement, HostPathRestore, IdempotencyKey, + MaterializeOptions, NativeExchangeJournal, RenameMode, +}; +use acyclic_fs::{CancellationToken, GenerationId, WorkCounters}; +use serde::{Deserialize, Serialize}; + +use crate::exclude::Exclusions; +use crate::store::Store; +use crate::{EngineError, Result}; + +static PARK_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +pub(crate) fn publication_key(generation: GenerationId) -> Result { + let bytes = generation.digest().as_bytes()[..16] + .try_into() + .map_err(|_| EngineError::Store("invalid generation digest".into()))?; + Ok(IdempotencyKey::from_bytes(bytes)) +} + +/// What a completed rewind reports back. +#[derive(Clone, Debug)] +pub struct RewindOutcome { + pub restored: GenerationId, + /// Where the replaced tree was retained beside the repository. + pub old_tree: PathBuf, + /// User-facing caveat: open editors keep inodes from the old tree. + pub warning: &'static str, +} + +pub(crate) struct PreparedRewind<'a> { + store: &'a Store, + target: GenerationId, + tmp: PathBuf, + parent: PathBuf, + name: String, + journal_path: PathBuf, + staging_journal_path: PathBuf, + carried: Vec, + trash_ttl_days: u32, +} + +/// What a single-path restore did to the working tree. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RestoreAction { + /// The path now matches the checkpoint's content. + Restored, + /// The path was absent at the checkpoint and has been removed. + Removed, +} + +/// Result of [`restore_path`]. +#[derive(Clone, Debug)] +pub struct RestoreOutcome { + pub path: PathBuf, + pub action: RestoreAction, +} + +/// Restores ONE path (file, symlink, or directory subtree) from `target` +/// into the working tree, leaving every other path untouched. The content +/// is staged in a hidden sibling and swapped in atomically (directory +/// subtrees use the same exchange as a full rewind), so a crash leaves the +/// path either old or new. Called from the pipeline, which brackets it with +/// checkpoints so the timeline records the restore. +pub async fn restore_path( + store: &Store, + target: GenerationId, + relative: &Path, +) -> Result { + let root = store.repo_root.clone(); + let generation = store.generation(target).await?; + let cancel = CancellationToken::new(); + let restored = generation + .restore_host_path( + relative, + HostPathReplacement::Atomic, + &MaterializeOptions::native(&root), + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(|error| EngineError::Restore(error.to_string()))?; + Ok(RestoreOutcome { + path: relative.to_path_buf(), + action: match restored.value { + HostPathRestore::Restored => RestoreAction::Restored, + HostPathRestore::Removed => RestoreAction::Removed, + }, + }) +} + +pub(crate) fn validate_relative(relative: &Path) -> Result>> { + let config = crate::store::volume_config(); + let namespace = acyclic_fs::host_path_to_namespace(relative, config.profile, config.limits) + .map_err(|_| { + EngineError::Restore(format!( + "{}: path must be relative to the repo root and stay inside it", + relative.display() + )) + })?; + Ok(namespace + .components() + .iter() + .map(|name| name.as_bytes().to_vec()) + .collect()) +} + +pub(crate) fn remove_any(path: &Path) -> std::io::Result<()> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path), + Ok(_) => std::fs::remove_file(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +/// Crash-recovery journal. Present on disk only while a swap is in flight. +#[derive(Debug, Serialize, Deserialize)] +struct LegacyJournal { + pub target_generation: String, + pub repo_root: PathBuf, + pub tmp: PathBuf, + pub phase: LegacyPhase, + /// Excluded paths (repo-relative) moved from the live tree into `tmp` + /// before the swap. Absent in journals written before exclusions. + #[serde(default)] + pub carried: Vec, +} + +#[derive(Serialize, Deserialize)] +struct StagingMarker { + temporary: PathBuf, +} + +/// Returns every listed path present in `from` to `into` during legacy +/// journal recovery. +/// A conflict or I/O failure keeps the journal and both trees for retry. +fn move_back(from: &Path, into: &Path, relative: &[PathBuf]) -> Result<()> { + for path in relative { + let source = from.join(path); + let destination = into.join(path); + if !path_exists(&source)? { + continue; + } + if path_exists(&destination)? { + return Err(EngineError::Restore(format!( + "rewind recovery found both copies of carried path {}", + path.display() + ))); + } + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::rename(&source, &destination)?; + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +enum LegacyPhase { + /// Materializing into tmp; repo untouched. Recovery: delete tmp. + Materializing, + /// The prepared tree is complete and the SDK head is being restored. + RestoringHead, + /// Excluded paths moving from repo into tmp. Recovery: move back any + /// that already moved, then delete tmp. + Carrying, + /// Root exchange in flight. Recovery makes the repo whole and parks every + /// displaced tree; Windows may also have an intermediate scratch tree. + Swapping, +} + +/// Result of reconciling an interrupted root replacement. +#[derive(Debug)] +pub struct RecoveredSwap { + /// The target was already published when a Windows parking rename failed. + pub published: bool, + /// The displaced tree retained beside the repository. + pub old_tree: Option, + /// Generation named by the durable journal. + pub target: GenerationId, + /// The durable SDK head was part of this operation and must be reconciled. + pub reconcile_head: bool, +} + +pub async fn recover_workspace(store: &mut Store, recovered: RecoveredSwap) -> Result<()> { + store.recover_workspace_head(&recovered).await?; + if recovered.published { + return Ok(()); + } + let text = match std::fs::read_to_string(store.paths.rewind_journal()) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + let journal: LegacyJournal = serde_json::from_str(&text) + .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; + if journal.phase != LegacyPhase::RestoringHead { + return Ok(()); + } + let locator = publish_locator(&journal.repo_root, &store.paths.rewind_journal())?; + let exchange = publish_native_exchange( + &store.paths.rewind_journal(), + &journal.repo_root, + &journal.tmp, + publication_key(recovered.target)?, + journal.carried, + ) + .map_err(|error| EngineError::Restore(format!("recover publish tree: {error}")))?; + if !exchange.published { + return Err(EngineError::Restore( + "rewind recovery did not publish prepared tree".into(), + )); + } + if let Some(displaced) = exchange.displaced { + let parent = journal + .repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let name = journal + .repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))? + .to_string_lossy(); + let _ = park_replaced_tree(&displaced, parent, &name)?; + } + let _ = std::fs::remove_file(locator); + Ok(()) +} + +/// Stored beside the repository so startup can find the journal even while +/// the Windows exchange has temporarily removed the repository name. The +/// store location may come from a config file inside that missing tree. +#[derive(Serialize, Deserialize)] +struct RecoveryLocator { + repo_root: PathBuf, + journal: PathBuf, +} + +fn locator_path(repo_root: &Path) -> Result { + let parent = repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let name = repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))? + .to_string_lossy(); + Ok(parent.join(format!(".{name}.{}-rewind.json", crate::product::NAME))) +} + +/// Recovers before loading the repository's config or canonicalizing its root. +/// Both operations can fail after the first Windows rename has removed it. +pub fn recover_before_repo_open(repo_root: &Path) -> Result<(PathBuf, Option)> { + let canonical = match repo_root.canonicalize() { + Ok(path) => path, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let parent = if parent.as_os_str().is_empty() { + Path::new(".") + } else { + parent + }; + parent.canonicalize()?.join( + repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))?, + ) + } + Err(error) => return Err(error.into()), + }; + #[cfg(windows)] + let canonical = { + let mut canonical = canonical; + if let (Some(parent), Some(name)) = (canonical.parent(), canonical.file_name()) { + let name = name.to_string_lossy(); + let suffix = format!(".{}-swap", crate::product::NAME); + if let Some(original) = name + .strip_prefix('.') + .and_then(|name| name.strip_suffix(&suffix)) + { + let candidate = parent.join(original); + if locator_path(&candidate)?.exists() { + // Windows holds the current directory open. Leave the old + // tree before recovery renames it back to the repository. + std::env::set_current_dir(parent)?; + canonical = candidate; + } + } + } + canonical + }; + let locator_path = locator_path(&canonical)?; + let text = match std::fs::read_to_string(&locator_path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok((canonical, None)), + Err(error) => return Err(error.into()), + }; + let locator: RecoveryLocator = serde_json::from_str(&text) + .map_err(|error| EngineError::Restore(format!("rewind locator: {error}")))?; + if locator.repo_root != canonical { + return Err(EngineError::Restore("rewind locator repo mismatch".into())); + } + let recovered = recover(&locator.journal)?; + Ok((canonical, recovered)) +} + +/// Acknowledges that the store head was reconciled after pre-open recovery. +pub fn finish_recovery(repo_root: &Path) -> Result<()> { + let path = locator_path(repo_root)?; + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +/// Executes a rewind against the store's repo. Called from the pipeline with +/// captures paused; the caller re-baselines afterwards. Excluded paths are +/// carried from the live tree into the restored one: no checkpoint holds +/// them, so the working copy is the only copy. +pub(crate) async fn prepare<'a>( + store: &'a Store, + target: GenerationId, + trash_ttl_days: u32, + exclusions: &Exclusions, +) -> Result> { + let repo = &store.repo_root; + let parent = repo + .parent() + .ok_or_else(|| EngineError::Restore("repo root has no parent".into()))?; + let name = repo + .file_name() + .ok_or_else(|| EngineError::Restore("repo root has no name".into()))? + .to_string_lossy() + .into_owned(); + let nonce = std::process::id(); + let tmp = parent.join(format!(".{name}.{}-tmp-{nonce}", crate::product::NAME)); + let journal_path = store.paths.rewind_journal(); + let staging_journal_path = journal_path.with_extension("staging.json"); + + // A failed previous exchange may have left a complete tree in scratch. + // Resolve its journal before reusing either the temporary name or journal. + recover(&journal_path)?; + let _ = std::fs::remove_file(&staging_journal_path); + + // 1. Materialize the target into an empty sibling directory. A tmp left + // by an earlier attempt that failed before the swap is stale by + // construction -- the name carries this daemon's pid, and a rewind that + // got as far as the swap removes it -- so clear it rather than refusing + // every later rewind with "already exists". + let _ = remove_any(&tmp); + std::fs::create_dir(&tmp).map_err(|error| { + EngineError::Restore(format!("rewind: stage {}: {error}", tmp.display())) + })?; + write_staging_marker( + &staging_journal_path, + &StagingMarker { + temporary: tmp.clone(), + }, + )?; + let generation = store.generation(target).await?; + let cancel = CancellationToken::new(); + generation + .materialize( + &MaterializeOptions::native(&tmp), + WorkCounters::UNBOUNDED, + &cancel, + ) + .await + .map_err(|error| { + let _ = std::fs::remove_dir_all(&tmp); + let _ = std::fs::remove_file(&staging_journal_path); + EngineError::Restore(format!("materialize: {error:?}")) + })?; + + let carried: Vec = exclusions + .host_paths() + .into_iter() + .filter(|relative| std::fs::symlink_metadata(repo.join(relative)).is_ok()) + .collect(); + Ok(PreparedRewind { + store, + target, + tmp, + parent: parent.to_path_buf(), + name, + journal_path, + staging_journal_path, + carried, + trash_ttl_days, + }) +} + +impl PreparedRewind<'_> { + pub(crate) fn mark_restoring_head(&self) -> Result<()> { + std::fs::remove_file(&self.staging_journal_path)?; + prepare_native_exchange_with_recovery( + &self.journal_path, + &self.store.repo_root, + &self.tmp, + publication_key(self.target)?, + self.carried.clone(), + self.target.digest().as_bytes().to_vec(), + ) + .map_err(|error| EngineError::Restore(format!("prepare tree publication: {error}"))) + } + + pub(crate) fn publish(self) -> Result { + let repo = &self.store.repo_root; + let locator = publish_locator(repo, &self.journal_path)?; + let exchange = publish_native_exchange( + &self.journal_path, + repo, + &self.tmp, + publication_key(self.target)?, + self.carried, + ) + .map_err(|error| EngineError::Restore(format!("publish tree: {error}")))?; + if !exchange.published { + return Err(EngineError::Restore( + "native exchange recovered without publishing the target tree".into(), + )); + } + + let displaced = exchange.displaced.unwrap_or(self.tmp); + let old_tree = park_replaced_tree(&displaced, &self.parent, &self.name)?; + let _ = std::fs::remove_file(locator); + prune_sibling_trash(repo, self.trash_ttl_days); + + Ok(RewindOutcome { + restored: self.target, + old_tree, + warning: "reload your editor: open files still point at the replaced tree", + }) + } +} + +/// Moves the replaced tree out of the way and returns where it landed. +/// +/// The destination is a sibling so the rename stays on the repository volume. +fn park_replaced_tree(tmp: &Path, parent: &Path, name: &str) -> Result { + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |duration| duration.as_nanos()); + let unique = format!( + "{stamp}-{}-{}", + std::process::id(), + PARK_SEQUENCE.fetch_add(1, Ordering::Relaxed) + ); + let sibling = parent.join(format!(".{name}.{}-trash-{unique}", crate::product::NAME)); + durable_rename(tmp, &sibling, RenameMode::NoReplace).map_err(|error| { + EngineError::Restore(format!( + "rewind: park the replaced tree at {}: {error}", + sibling.display() + )) + })?; + Ok(sibling) +} + +/// Startup crash recovery. Reads the journal (if any) and finishes or unwinds +/// the interrupted rewind so the repo is whole before the pipeline baselines. +pub fn recover(journal_path: &Path) -> Result> { + recover_staging(&journal_path.with_extension("staging.json"))?; + let text = match std::fs::read_to_string(journal_path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if let Ok(journal) = serde_json::from_str::(&text) { + let target = decode_generation(&hex::encode(&journal.recovery))?; + let outcome = recover_native_exchange(journal_path) + .map_err(|error| EngineError::Restore(format!("recover native exchange: {error}")))?; + return Ok(Some(RecoveredSwap { + published: outcome.published, + old_tree: outcome.displaced, + target, + reconcile_head: true, + })); + } + recover_legacy(journal_path, &text) +} + +fn recover_staging(path: &Path) -> Result<()> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + let marker: StagingMarker = serde_json::from_str(&text) + .map_err(|error| EngineError::Restore(format!("rewind staging journal: {error}")))?; + remove_any(&marker.temporary)?; + std::fs::remove_file(path)?; + Ok(()) +} + +fn recover_legacy(journal_path: &Path, text: &str) -> Result> { + let journal: LegacyJournal = serde_json::from_str(text) + .map_err(|error| EngineError::Restore(format!("rewind journal: {error}")))?; + let target = decode_generation(&journal.target_generation)?; + #[cfg(windows)] + let mut published = false; + #[cfg(not(windows))] + let published = false; + let mut old_tree = None; + match journal.phase { + LegacyPhase::Materializing => { + // Repo untouched; the partial tmp tree is garbage. + let _ = std::fs::remove_dir_all(&journal.tmp); + } + LegacyPhase::RestoringHead => { + return Ok(Some(RecoveredSwap { + published, + old_tree, + target, + reconcile_head: true, + })) + } + LegacyPhase::Carrying => { + // Some excluded paths may already sit in tmp: bring them home, + // then drop the unused new tree. + move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; + let _ = std::fs::remove_dir_all(&journal.tmp); + } + LegacyPhase::Swapping => { + #[cfg(windows)] + let scratch = swap_scratch(&journal.repo_root); + #[cfg(windows)] + let scratch_present = scratch + .as_deref() + .map(path_exists) + .transpose()? + .unwrap_or(false); + let repo_present = path_exists(&journal.repo_root)?; + let tmp_present = path_exists(&journal.tmp)?; + #[cfg(windows)] + { + published = repo_present && scratch_present && !tmp_present; + } + #[cfg(windows)] + if scratch_present && tmp_present && repo_present { + return Err(EngineError::Restore( + "rewind recovery found three live trees; refusing to discard any".into(), + )); + } + #[cfg(windows)] + if !repo_present && scratch_present { + // Rename #1 landed but the publication may not have. Restore + // the old tree, including excluded paths carried into tmp. + let old = scratch + .as_deref() + .ok_or_else(|| EngineError::Restore("rewind scratch path is missing".into()))?; + move_back(&journal.tmp, old, &journal.carried)?; + durable_rename(old, &journal.repo_root, RenameMode::NoReplace)?; + } else if !repo_present && tmp_present { + // A journal from an older implementation can have no scratch. + // Make the repo whole before attempting store startup. + durable_rename(&journal.tmp, &journal.repo_root, RenameMode::NoReplace)?; + } else { + // If the exchange never started, carried paths are still in + // tmp and must return to the live tree. After a completed + // exchange they are already in the live tree. + move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; + } + #[cfg(not(windows))] + if !repo_present && tmp_present { + durable_rename(&journal.tmp, &journal.repo_root, RenameMode::NoReplace)?; + } else { + move_back(&journal.tmp, &journal.repo_root, &journal.carried)?; + } + if !path_exists(&journal.repo_root)? { + return Err(EngineError::Restore( + "rewind recovery could not find a complete repository tree".into(), + )); + } + let parent = journal + .repo_root + .parent() + .ok_or_else(|| EngineError::Restore("rewind repo root has no parent".into()))?; + let name = journal + .repo_root + .file_name() + .ok_or_else(|| EngineError::Restore("rewind repo root has no name".into()))? + .to_string_lossy(); + if path_exists(&journal.tmp)? { + old_tree = Some(park_replaced_tree(&journal.tmp, parent, &name)?); + } + #[cfg(windows)] + if let Some(scratch) = scratch { + if path_exists(&scratch)? { + old_tree = Some(park_replaced_tree(&scratch, parent, &name)?); + } + } + } + } + std::fs::remove_file(journal_path)?; + #[cfg(unix)] + sync_parent(journal_path)?; + Ok(Some(RecoveredSwap { + published, + old_tree, + target, + reconcile_head: false, + })) +} + +fn decode_generation(encoded: &str) -> Result { + let bytes = hex::decode(encoded) + .map_err(|error| EngineError::Restore(format!("rewind generation: {error}")))?; + let digest: [u8; 32] = bytes + .try_into() + .map_err(|_| EngineError::Restore("rewind generation must be 32 bytes".into()))?; + Ok(GenerationId::new(acyclic_fs::Digest::from_bytes(digest))) +} + +fn path_exists(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +fn write_staging_marker(path: &Path, marker: &StagingMarker) -> Result<()> { + let text = serde_json::to_string(marker) + .map_err(|error| EngineError::Restore(format!("encode staging marker: {error}")))?; + let tmp = path.with_extension("tmp"); + let mut file = std::fs::File::create(&tmp)?; + use std::io::Write; + file.write_all(text.as_bytes())?; + file.sync_all()?; + drop(file); + durable_rename(&tmp, path, RenameMode::Replace)?; + Ok(()) +} + +fn write_locator(path: &Path, locator: &RecoveryLocator) -> Result<()> { + use std::io::Write; + let text = serde_json::to_vec(locator) + .map_err(|error| EngineError::Restore(format!("encode rewind locator: {error}")))?; + let tmp = path.with_extension("tmp"); + let mut file = std::fs::File::create(&tmp)?; + file.write_all(&text)?; + file.sync_all()?; + drop(file); + durable_rename(&tmp, path, RenameMode::Replace)?; + Ok(()) +} + +fn publish_locator(repo: &Path, journal: &Path) -> Result { + let path = locator_path(repo)?; + write_locator( + &path, + &RecoveryLocator { + repo_root: repo.to_path_buf(), + journal: journal.to_path_buf(), + }, + )?; + Ok(path) +} + +#[cfg(unix)] +fn sync_parent(path: &Path) -> Result<()> { + std::fs::File::open( + path.parent() + .ok_or_else(|| EngineError::Restore("rewind path has no parent".into()))?, + )? + .sync_all()?; + Ok(()) +} + +fn prune_sibling_trash(repo: &Path, ttl_days: u32) { + let Some(parent) = repo.parent() else { + return; + }; + let Some(name) = repo.file_name() else { + return; + }; + let prefix = format!( + ".{}.{}-trash-", + name.to_string_lossy(), + crate::product::NAME + ); + let Ok(entries) = std::fs::read_dir(parent) else { + return; + }; + let ttl = std::time::Duration::from_secs(u64::from(ttl_days) * 24 * 3600); + for entry in entries.flatten() { + if !entry.file_name().to_string_lossy().starts_with(&prefix) { + continue; + } + let Ok(metadata) = entry.metadata() else { + continue; + }; + let Ok(modified) = metadata.modified() else { + continue; + }; + if modified.elapsed().is_ok_and(|age| age > ttl) { + let _ = std::fs::remove_dir_all(entry.path()); + } + } +} + +/// The scratch name [`atomic_exchange`] swaps `a` through on Windows. +/// +/// Derived from `a` rather than randomised so that [`recover`] can name it +/// without the journal having carried it, and so a crashed swap leaves at +/// most one predictable directory behind instead of one per attempt. +#[cfg(windows)] +pub(crate) fn swap_scratch(a: &Path) -> Option { + let parent = a.parent()?; + let name = a.file_name()?.to_string_lossy().into_owned(); + Some(parent.join(format!(".{name}.{}-swap", crate::product::NAME))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parking_replaced_trees_never_reuses_a_name() -> Result<()> { + let work = tempfile::tempdir()?; + let mut parked = Vec::new(); + for index in 0..3 { + let tmp = work.path().join(format!("tmp-{index}")); + std::fs::create_dir(&tmp)?; + std::fs::write(tmp.join("old.txt"), index.to_string())?; + let destination = park_replaced_tree(&tmp, work.path(), "repo")?; + assert_eq!( + std::fs::read_to_string(destination.join("old.txt"))?, + index.to_string() + ); + parked.push(destination); + } + assert!(parked[0] != parked[1] && parked[1] != parked[2] && parked[0] != parked[2]); + Ok(()) + } + + fn recover(journal_path: &Path) -> Result> { + super::recover(journal_path) + } + + fn parked_tree_contains(repo: &Path, file: &str, expected: &[u8]) -> bool { + repo.parent() + .and_then(|parent| std::fs::read_dir(parent).ok()) + .into_iter() + .flatten() + .filter_map(std::result::Result::ok) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .contains(".acyclic-trash-") + }) + .any(|entry| std::fs::read(entry.path().join(file)).ok().as_deref() == Some(expected)) + } + + fn journal(repo: &Path, tmp: &Path, phase: LegacyPhase) -> LegacyJournal { + LegacyJournal { + target_generation: "00".repeat(32), + repo_root: repo.to_path_buf(), + tmp: tmp.to_path_buf(), + phase, + carried: Vec::new(), + } + } + + fn write(path: &Path, value: &LegacyJournal) { + std::fs::write(path, serde_json::to_string(value).expect("encode")).expect("write"); + } + + #[test] + fn recover_finishes_interrupted_two_step_swap() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir(&tmp).expect("tmp"); + std::fs::write(tmp.join("file.txt"), b"restored").expect("seed"); + let journal_path = work.path().join("journal.json"); + write(&journal_path, &journal(&repo, &tmp, LegacyPhase::Swapping)); + + let recovered = recover(&journal_path) + .expect("recover") + .expect("recovered journal"); + assert!(!recovered.published); + assert_eq!( + std::fs::read(repo.join("file.txt")).expect("read"), + b"restored" + ); + assert!(!tmp.exists()); + assert!(!journal_path.exists()); + } + + #[test] + fn recover_discards_partial_materialization() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + std::fs::create_dir(&repo).expect("repo"); + std::fs::write(repo.join("keep.txt"), b"live").expect("seed"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir(&tmp).expect("tmp"); + std::fs::write(tmp.join("partial.txt"), b"half").expect("seed"); + let journal_path = work.path().join("journal.json"); + write( + &journal_path, + &journal(&repo, &tmp, LegacyPhase::Materializing), + ); + + recover(&journal_path).expect("recover"); + assert_eq!(std::fs::read(repo.join("keep.txt")).expect("read"), b"live"); + assert!(!tmp.exists()); + assert!(!journal_path.exists()); + } + + #[test] + fn recover_discards_separate_partial_staging() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + std::fs::create_dir(&repo).expect("repo"); + std::fs::write(repo.join("keep.txt"), b"live").expect("seed"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir(&tmp).expect("tmp"); + std::fs::write(tmp.join("partial.txt"), b"half").expect("seed"); + let journal_path = work.path().join("journal.json"); + let staging_path = journal_path.with_extension("staging.json"); + write_staging_marker( + &staging_path, + &StagingMarker { + temporary: tmp.clone(), + }, + ) + .expect("write staging marker"); + + recover(&journal_path).expect("recover"); + assert_eq!(std::fs::read(repo.join("keep.txt")).expect("read"), b"live"); + assert!(!tmp.exists()); + assert!(!staging_path.exists()); + } + + #[test] + fn recover_with_no_journal_is_a_noop() { + let work = tempfile::tempdir().expect("tempdir"); + assert!(recover(&work.path().join("missing.json")) + .expect("recover") + .is_none()); + } + + #[test] + fn recover_mid_carry_returns_excluded_paths_to_the_repo() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir_all(repo.join("secrets")).expect("repo"); + std::fs::create_dir_all(tmp.join("secrets")).expect("tmp"); + // .env already moved into tmp; secrets/key.pem had not moved yet. + std::fs::write(tmp.join(".env"), b"LIVE").expect("moved"); + std::fs::write(repo.join("secrets/key.pem"), b"KEY").expect("unmoved"); + let journal_path = work.path().join("journal.json"); + let mut entry = journal(&repo, &tmp, LegacyPhase::Carrying); + entry.carried = vec![PathBuf::from(".env"), PathBuf::from("secrets/key.pem")]; + write(&journal_path, &entry); + + recover(&journal_path).expect("recover"); + assert_eq!(std::fs::read(repo.join(".env")).expect("back"), b"LIVE"); + assert_eq!( + std::fs::read(repo.join("secrets/key.pem")).expect("kept"), + b"KEY" + ); + assert!(!tmp.exists(), "unused new tree removed"); + assert!(!journal_path.exists()); + } + + #[test] + fn recover_before_the_swap_keeps_carried_paths_out_of_the_discarded_tree() { + // Swapping phase, but the exchange never ran: repo is the old tree + // (minus the carried paths), tmp is the new tree holding them. + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir_all(&repo).expect("repo"); + std::fs::create_dir_all(&tmp).expect("tmp"); + std::fs::write(repo.join("file.txt"), b"old").expect("old"); + std::fs::write(tmp.join("file.txt"), b"new").expect("new"); + std::fs::write(tmp.join(".env"), b"LIVE").expect("carried"); + let journal_path = work.path().join("journal.json"); + let mut entry = journal(&repo, &tmp, LegacyPhase::Swapping); + entry.carried = vec![PathBuf::from(".env")]; + write(&journal_path, &entry); + + recover(&journal_path).expect("recover"); + assert_eq!(std::fs::read(repo.join("file.txt")).expect("repo"), b"old"); + assert_eq!( + std::fs::read(repo.join(".env")).expect("carried back"), + b"LIVE" + ); + assert!(!tmp.exists()); + } + + #[test] + fn recover_keeps_both_trees_and_journal_on_carried_path_conflict() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir(&repo).expect("repo"); + std::fs::create_dir(&tmp).expect("tmp"); + std::fs::write(repo.join(".env"), b"repo copy").expect("repo data"); + std::fs::write(tmp.join(".env"), b"staged copy").expect("staged data"); + let journal_path = work.path().join("journal.json"); + let mut entry = journal(&repo, &tmp, LegacyPhase::Carrying); + entry.carried = vec![PathBuf::from(".env")]; + write(&journal_path, &entry); + + recover(&journal_path).expect_err("conflicting copies require inspection"); + assert_eq!( + std::fs::read(repo.join(".env")).expect("repo"), + b"repo copy" + ); + assert_eq!( + std::fs::read(tmp.join(".env")).expect("staged"), + b"staged copy" + ); + assert!(journal_path.exists()); + } + + #[test] + fn recover_after_the_swap_leaves_carried_paths_in_the_new_tree() { + // Exchange completed: repo is the new tree with the carried paths, + // tmp is the old tree without them. Nothing moves; tmp goes. + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir_all(&repo).expect("repo"); + std::fs::create_dir_all(&tmp).expect("tmp"); + std::fs::write(repo.join("file.txt"), b"new").expect("new"); + std::fs::write(repo.join(".env"), b"LIVE").expect("carried"); + std::fs::write(tmp.join("file.txt"), b"old").expect("old"); + let journal_path = work.path().join("journal.json"); + let mut entry = journal(&repo, &tmp, LegacyPhase::Swapping); + entry.carried = vec![PathBuf::from(".env")]; + write(&journal_path, &entry); + + recover(&journal_path).expect("recover"); + assert_eq!(std::fs::read(repo.join("file.txt")).expect("repo"), b"new"); + assert_eq!( + std::fs::read(repo.join(".env")).expect("still here"), + b"LIVE" + ); + assert!(!tmp.exists()); + assert!(parked_tree_contains(&repo, "file.txt", b"old")); + } + + #[test] + fn journals_written_before_exclusions_still_decode() { + let text = + r#"{"target_generation":"00","repo_root":"/r","tmp":"/t","phase":"Materializing"}"#; + let journal: LegacyJournal = serde_json::from_str(text).expect("decode"); + assert!(journal.carried.is_empty()); + } + + /// A crash after Windows rename #1 must roll back to the old tree and + /// retain the staged new tree in trash for recovery or inspection. + #[cfg(windows)] + #[test] + fn startup_recovers_before_the_repository_path_exists() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work + .path() + .canonicalize() + .expect("canonical parent") + .join("repo"); + let scratch = swap_scratch(&repo).expect("scratch path"); + let staged = work.path().join("staged"); + std::fs::create_dir(&scratch).expect("old tree"); + std::fs::write(scratch.join("old.txt"), b"original").expect("old file"); + std::fs::create_dir(&staged).expect("new tree"); + std::fs::write(staged.join("new.txt"), b"replacement").expect("new file"); + let store = work.path().join("custom-store"); + std::fs::create_dir(&store).expect("custom store"); + let journal_path = store.join("rewind-journal.json"); + write( + &journal_path, + &journal(&repo, &staged, LegacyPhase::Swapping), + ); + let locator = locator_path(&repo).expect("locator path"); + write_locator( + &locator, + &RecoveryLocator { + repo_root: repo.clone(), + journal: journal_path.clone(), + }, + ) + .expect("locator"); + + recover_before_repo_open(&repo).expect("startup recovery"); + assert_eq!( + std::fs::read(repo.join("old.txt")).expect("old tree"), + b"original" + ); + assert!(!journal_path.exists()); + assert!(locator.exists()); + finish_recovery(&repo).expect("finish recovery"); + assert!(!locator.exists()); + } + + #[cfg(windows)] + #[test] + fn recover_preserves_both_trees_after_first_windows_rename() { + let work = tempfile::tempdir().expect("tempdir"); + let repo = work.path().join("repo"); + let tmp = work.path().join("repo.tmp"); + std::fs::create_dir(&tmp).expect("tmp"); + std::fs::write(tmp.join("file.txt"), b"new tree").expect("seed new"); + std::fs::write(tmp.join(".env"), b"carried live data").expect("carry"); + // Died between rename one and rename two: repo vacated, old tree parked. + let scratch = swap_scratch(&repo).expect("scratch path"); + std::fs::create_dir(&scratch).expect("scratch"); + std::fs::write(scratch.join("file.txt"), b"old tree").expect("seed old"); + let journal_path = work.path().join("journal.json"); + let mut entry = journal(&repo, &tmp, LegacyPhase::Swapping); + entry.carried = vec![PathBuf::from(".env")]; + write(&journal_path, &entry); + + recover(&journal_path).expect("recover"); + + assert_eq!( + std::fs::read(repo.join("file.txt")).expect("repo whole"), + b"old tree" + ); + assert!(!scratch.exists(), "scratch must not outlive recovery"); + assert!(!tmp.exists()); + assert!(!journal_path.exists()); + assert_eq!( + std::fs::read(repo.join(".env")).expect("carried data survived"), + b"carried live data" + ); + assert!(parked_tree_contains(&repo, "file.txt", b"new tree")); + } +} diff --git a/crates/acyclic/src/server.rs b/crates/acyclic/src/server.rs index f898ce0..4dbf0e6 100644 --- a/crates/acyclic/src/server.rs +++ b/crates/acyclic/src/server.rs @@ -6,24 +6,22 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use crate::ipc; -use acyclic_engine::config::Config; -use acyclic_engine::fork::{ - self, ForkMode, MountCapability, PromoteOutcome, SessionResolveOutcome, SharedLocalCheckout, -}; -use acyclic_engine::guard::GuardedMountFilesystem; -use acyclic_engine::index::{Attribution, CheckpointKind, CheckpointRow, Index}; -use acyclic_engine::merge::{self, Entry}; -use acyclic_engine::pipeline::{self, PipelineHandle}; -use acyclic_engine::product::NAME; -use acyclic_engine::spec::SpeculateConfig; -use acyclic_engine::store::{Store, StorePaths}; -use acyclic_engine::{rewind, EngineError}; +use crate::proto; +use acyclic::config::Config; +use acyclic::fork::{self, MountCapability, SharedLocalCheckout}; +use acyclic::guard::GuardedMountFilesystem; +use acyclic::index::{Attribution, CheckpointKind, CheckpointRow, Index}; +use acyclic::merge::{self, Entry}; +use acyclic::pipeline::{self, PipelineHandle}; +use acyclic::product::NAME; +use acyclic::spec::SpeculateConfig; +use acyclic::store::{Store, StorePaths}; +use acyclic::{rewind, EngineError}; use acyclic_fs::model::VolumeConfig; use acyclic_fs::{ - mount_native, mount_native_over_existing, CheckoutMountSource, MountFilesystem, - NativeMountRequest, NativeMountSession, RoutedMountSource, + mount_native, CheckoutMountSource, MountFilesystem, NativeMountRequest, NativeMountSession, + RoutedMountSource, }; -use acyclic_proto as proto; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::sync::{Mutex, Notify}; @@ -36,11 +34,8 @@ struct ForkState { /// The generation promote is judged against. Starts at the fork's cut /// point; a conflicting promote rebases the fork and moves it to the /// head it was rebased onto. - base: acyclic_engine::GenerationId, + base: acyclic::GenerationId, entry: proto::ForkEntry, - /// Copy-mode forks only: the materialized directory the user works in. - /// Captured back into `shared` at promote, removed at drop/promote. - copy_dir: Option, /// Set by a conflicting promote until the markers are gone. conflict: Option, } @@ -52,27 +47,6 @@ struct OpenConflict { paths: Vec, } -/// One Safe Mode session: its fork and the shadow mount that projects it -/// directly at the real repo root for the session's duration. Only one can -/// be active at a time -- shadowing is a whole-path substitution, so two -/// sessions can't both shadow the same repo root concurrently. -struct DrySession { - fork_id: String, - session_id: String, - shared: Arc, - base: acyclic_engine::GenerationId, - mount: NativeMountSession, -} - -/// A `SessionResolve`d session awaiting `SessionApply`/`SessionDiscard`. Its -/// overlay is already committed to the store under `generation`; nothing -/// has touched the real tree yet. -struct PendingSession { - generation: acyclic_engine::GenerationId, - base: acyclic_engine::GenerationId, - label: String, -} - /// The one native session projecting every fork through the router. /// Mounted lazily on the first fork, unmounted when the last route goes. struct ForkMount { @@ -169,42 +143,33 @@ fn spawn_speculation( } pub fn run(repo_root: &Path) -> Result<(), String> { - // FIRST, before anything reads through `repo_root`: a Safe Mode shadow - // mount from a crashed daemon leaves the repo root a dead NFS mountpoint - // that wedges every stat/open under it (Config::load, canonicalize, ...). - // The force-unmount acts on the mountpoint path itself without touching - // the dead server, so the real tree reappears before we read the config. let startup = std::time::Instant::now(); let mut phase = std::time::Instant::now(); let mut lap = |name: &str| { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "startup: {name} {:.1}ms (t+{:.1}ms)", - acyclic_engine::trace::ms(phase), - acyclic_engine::trace::ms(startup) + acyclic::trace::ms(phase), + acyclic::trace::ms(startup) ); phase = std::time::Instant::now(); }; - fork::sweep_stale_dry_session(repo_root); - lap("sweep stale dry-run session"); - - let config = Config::load(repo_root).map_err(|error| error.to_string())?; + let (repo_root, early_recovered) = + rewind::recover_before_repo_open(repo_root).map_err(|error| error.to_string())?; + lap("rewind recovery before repo open"); + let config = Config::load(&repo_root).map_err(|error| error.to_string())?; lap("config load"); let stores_root = config.store_dir.as_ref().map(PathBuf::from); - let paths = StorePaths::for_repo(repo_root, stores_root.as_deref()) + let paths = StorePaths::for_repo(&repo_root, stores_root.as_deref()) .map_err(|error| error.to_string())?; - // Finish or unwind any rewind that a crash interrupted BEFORE the store - // opens and the pipeline baselines; sweep fork dirs a dead daemon left - // mounted (fork sessions do not survive the daemon). - rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?; + // Finish or unwind any rewind that a crash interrupted before serving + // stateful operations. Fork cleanup is delayed until forks are requested. + let recovered = match early_recovered { + Some(recovered) => Some(recovered), + None => rewind::recover(&paths.rewind_journal()).map_err(|error| error.to_string())?, + }; lap("rewind journal recovery"); - fork::sweep_stale_forks(repo_root); - lap("sweep stale forks"); - // Same reason as the fork sweep, and the same moment: a model run a - // crashed daemon left behind is still running, and still billing. - crate::spec_runner::sweep_stale_runs(&paths.spec_runs()); - lap("sweep stale spec runs"); let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; // Socket + pidfile FIRST, before the store opens: a client can then @@ -214,27 +179,28 @@ pub fn run(repo_root: &Path) -> Result<(), String> { // is removed; a live one refuses the second daemon via bind failure. let listener = bind_socket(&runtime, &paths)?; lap("socket bind + pidfile"); - let store = runtime - .block_on(Store::open(paths.clone())) + let mut store = runtime + .block_on(Store::open(&repo_root, paths.clone())) .map_err(|error| error.to_string())?; + if let Some(recovered) = recovered { + runtime + .block_on(rewind::recover_workspace(&mut store, recovered)) + .map_err(|error| error.to_string())?; + rewind::finish_recovery(&repo_root).map_err(|error| error.to_string())?; + } let repo_root = store.repo_root.clone(); lap("store open"); let index = Index::open(&paths.index_db()).map_err(|error| error.to_string())?; lap("index open"); let (handle, pipeline_thread) = pipeline::spawn(store, index, config.clone()); - lap("pipeline thread spawn (baseline runs on it)"); + lap("pipeline metadata thread spawn"); let shutdown = Arc::new(Notify::new()); - let mut mounts = fork::mount_capability(); + let mounts = fork::mount_capability(); lap("native mount probe"); - // Test hook: exercise the copy-fork paths on a host that has mounts. - if std::env::var_os("ACYCLIC_FORCE_COPY_FORKS").is_some() { - mounts.available = false; - mounts.reason = Some("ACYCLIC_FORCE_COPY_FORKS is set".into()); - } if !mounts.available { eprintln!( - "{NAME} daemon: mounts unavailable ({}): forks fall back to copies, Safe Mode is off", + "{NAME} daemon: mounts unavailable ({}): forks are disabled", mounts.reason.as_deref().unwrap_or("unknown reason") ); } @@ -247,7 +213,6 @@ pub fn run(repo_root: &Path) -> Result<(), String> { mounts, handle: handle.clone(), index_db: paths.index_db(), - store_root: paths.root.clone(), repo_root, config, shutdown: shutdown.clone(), @@ -256,12 +221,10 @@ pub fn run(repo_root: &Path) -> Result<(), String> { router: Arc::new(RoutedMountSource::new()), session: None, })), - dry_session: Arc::new(Mutex::new(None)), - pending: Arc::new(Mutex::new(HashMap::new())), + fork_cleanup_pending: Arc::new(Mutex::new(true)), spec, live_sessions: Arc::new(Mutex::new(HashSet::new())), last_activity: Arc::new(Mutex::new(Instant::now())), - store_bytes: Arc::new(Mutex::new(None)), }; runtime.block_on(serve_until_done(server, listener, shutdown, handle)); @@ -277,20 +240,16 @@ pub fn run(repo_root: &Path) -> Result<(), String> { #[derive(Clone)] struct Server { - /// Probed once at start: decides fork mode and gates Safe Mode. + /// Probed once at start to decide whether forks mount or materialize. mounts: MountCapability, handle: PipelineHandle, index_db: PathBuf, - store_root: PathBuf, repo_root: PathBuf, config: Config, shutdown: Arc, forks: Arc>>, fork_mount: Arc>, - /// The one active Safe Mode session shadow-mounted at `repo_root`, if any. - dry_session: Arc>>, - /// Sessions that resolved (committed) but haven't been applied/discarded. - pending: Arc>>, + fork_cleanup_pending: Arc>, /// The speculation scheduler, when it is enabled. `None` makes every /// call site a no-op, so the default path costs nothing. spec: Option>, @@ -299,12 +258,20 @@ struct Server { live_sessions: Arc>>, /// When the last request arrived; the idle-exit clock. last_activity: Arc>, - /// `store size` for `status`, refreshed in the background: walking the - /// object directory costs ~1s on an aged store. - store_bytes: Arc>>, } impl Server { + async fn ensure_fork_workspace_clean(&self) -> Result<(), String> { + let mut pending = self.fork_cleanup_pending.lock().await; + if !*pending { + return Ok(()); + } + let repo_root = self.repo_root.clone(); + tokio::task::block_in_place(|| fork::sweep_stale_forks(&repo_root))?; + *pending = false; + Ok(()) + } + async fn serve(&self, stream: ipc::ServerStream) { let (read, mut write) = tokio::io::split(stream); let mut lines = BufReader::new(read).lines(); @@ -336,22 +303,22 @@ impl Server { async fn dispatch(&self, op: proto::Op) -> proto::Payload { let name = op_name(&op); let started = std::time::Instant::now(); - acyclic_engine::trace!("daemon", "op {name} received"); + acyclic::trace!("daemon", "op {name} received"); let payload = match self.dispatch_inner(op).await { Ok(reply) => { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "op {name} -> {} in {:.1}ms", reply_name(&reply), - acyclic_engine::trace::ms(started) + acyclic::trace::ms(started) ); proto::Payload::Ok(Box::new(reply)) } Err(message) => { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "op {name} -> error in {:.1}ms: {}", - acyclic_engine::trace::ms(started), + acyclic::trace::ms(started), message.lines().next().unwrap_or("") ); err(message) @@ -361,39 +328,11 @@ impl Server { } /// True when nothing has needed this daemon for `idle`: no request, no - /// open session, no live fork, no Safe Mode session, nothing pending. + /// open session, and no live fork. async fn idle_for(&self, idle: Duration) -> bool { self.last_activity.lock().await.elapsed() >= idle && self.live_sessions.lock().await.is_empty() && self.forks.lock().await.is_empty() - && self.dry_session.lock().await.is_none() - && self.pending.lock().await.is_empty() - } - - /// The store's size on disk, from a cache that a background walk - /// refreshes once it is a minute old. Only the very first call walks. - async fn store_size(&self) -> u64 { - const FRESH: Duration = Duration::from_secs(60); - let root = self.store_root.join("store"); - let cached = *self.store_bytes.lock().await; - match cached { - Some((at, bytes)) if at.elapsed() < FRESH => bytes, - Some((_, bytes)) => { - let cache = Arc::clone(&self.store_bytes); - tokio::task::spawn_blocking(move || { - let fresh = directory_bytes(&root); - if let Ok(mut slot) = cache.try_lock() { - *slot = Some((Instant::now(), fresh)); - } - }); - bytes - } - None => { - let bytes = tokio::task::block_in_place(|| directory_bytes(&root)); - *self.store_bytes.lock().await = Some((Instant::now(), bytes)); - bytes - } - } } #[allow( @@ -404,14 +343,16 @@ impl Server { async fn dispatch_inner(&self, op: proto::Op) -> Result { *self.last_activity.lock().await = Instant::now(); match op { - proto::Op::Ping => Ok(proto::Reply::Pong), + proto::Op::Ping => { + self.handle.status().await.map_err(stringify)?; + Ok(proto::Reply::Pong) + } proto::Op::Status => { let status = self.handle.status().await.map_err(stringify)?; Ok(proto::Reply::Status(proto::StatusInfo { state: format!("{:?}", status.state).to_lowercase(), last_checkpoint: status.last_checkpoint, unpublished: status.unpublished, - store_bytes: self.store_size().await, repo_root: self.repo_root.display().to_string(), mount_provider: self.mounts.provider.to_owned(), mount_available: self.mounts.available, @@ -445,7 +386,7 @@ impl Server { rewind_target: None, }; if wait { - acyclic_engine::trace!( + acyclic::trace!( "daemon", "checkpoint: WAIT path (reply after the capture lands; durable={durable})" ); @@ -460,14 +401,14 @@ impl Server { Ok(proto::Reply::Checkpoint(proto::CheckpointInfo { row_id: outcome.row_id, generation: hex_generation(outcome.generation), - kind: wire_kind(outcome.kind), + kind: outcome.kind, })) } else { // Enqueue-ack: the hook path. Admission into the FIFO // happens BEFORE the ack, so a stop arriving after the // ack queues behind the capture instead of dropping it. // Failures land in the index as `failed`. - acyclic_engine::trace!( + acyclic::trace!( "daemon", "checkpoint: ENQUEUE path (ack on admission, capture runs behind)" ); @@ -566,7 +507,7 @@ impl Server { id: row.id, generation: hex_generation(row.generation), created_at: row.created_at, - kind: wire_kind(row.kind), + kind: row.kind, published: row.published, session_id: row.session_id, host, @@ -633,10 +574,7 @@ impl Server { Ok(proto::Reply::Restore(proto::RestoreInfo { checkpoint: row_id, path: outcome.path.display().to_string(), - action: match outcome.action { - rewind::RestoreAction::Restored => proto::RestoreAction::Restored, - rewind::RestoreAction::Removed => proto::RestoreAction::Removed, - }, + action: outcome.action, recorded_checkpoint, })) } @@ -710,9 +648,6 @@ impl Server { .session_started(session_id.clone(), host) .await .map_err(stringify)?; - if self.config.dry_run { - self.session_fork(session_id).await?; - } Ok(proto::Reply::Unit) } proto::Op::SessionEnd { session_id } => { @@ -735,12 +670,7 @@ impl Server { .map(|(id, _)| id.clone()) .collect(); for id in scratch_ids { - let fork = self.forks.lock().await.remove(&id); - let copy_dir = fork.and_then(|fork| fork.copy_dir); - if let Err(error) = self.discard_fork_workspace(&id, copy_dir.as_deref()).await - { - eprintln!("{NAME} daemon: drop scratch fork {id}: {error}"); - } + self.remove_fork(&id).await?; } // The session that just ended is the one the NEXT session's // brief will describe, and nothing is asking for it yet: @@ -765,13 +695,6 @@ impl Server { if let Some(spec) = self.spec.as_ref() { spec.shutdown().await; } - // Unmount an active Safe Mode shadow first: it sits directly - // on the real repo root, so this must never be left mounted - // once the daemon that owns it is gone. - if let Some(mut session) = self.dry_session.lock().await.take() { - let _ = tokio::task::block_in_place(|| session.mount.stop()); - } - self.pending.lock().await.clear(); // Detach the fork session before the pipeline goes away: its // callback runtimes reach into the shared checkouts. self.forks.lock().await.clear(); @@ -790,9 +713,15 @@ impl Server { if count == 0 || count > 16 { return Err("fork count must be 1..=16".into()); } - if self.mounts.fork_mode() == ForkMode::Copy { - return self.fork_copies(count, session_id).await; + if !self.mounts.available { + return Err(format!( + "native {} mounts are unavailable: {}\n{}", + self.mounts.provider, + self.mounts.reason.as_deref().unwrap_or("unknown reason"), + fork::mount_setup_hint() + )); } + self.ensure_fork_workspace_clean().await?; let root = fork::forks_mount_root(&self.repo_root) .ok_or("repo root has no parent for fork workspaces")?; let mut created = Vec::new(); @@ -804,8 +733,7 @@ impl Server { let entry = proto::ForkEntry { id: id.clone(), path: root.join(&id).display().to_string(), - mode: ForkMode::Mount.as_str().to_owned(), - base: acyclic_engine::generation_hex(seed.base), + base: acyclic::generation_hex(seed.base), created_at: unix_now(), session_id: session_id.clone(), conflict_paths: Vec::new(), @@ -817,7 +745,6 @@ impl Server { shared: seed.shared, base: seed.base, entry: clone_entry(&entry), - copy_dir: None, conflict: None, }, ); @@ -835,11 +762,7 @@ impl Server { Ok(proto::Reply::Forks(entries)) } proto::Op::ForkDrop { id } => { - let mut forks = self.forks.lock().await; - let fork = forks.remove(&id).ok_or(format!("no fork {id}"))?; - drop(forks); - self.discard_fork_workspace(&id, fork.copy_dir.as_deref()) - .await?; + self.remove_fork(&id).await?; Ok(proto::Reply::Unit) } proto::Op::Promote { id } => { @@ -863,18 +786,18 @@ impl Server { paths: files.iter().map(|file| PathBuf::from(&file.path)).collect(), }); fork.base = theirs; - fork.entry.base = acyclic_engine::generation_hex(theirs); + fork.entry.base = acyclic::generation_hex(theirs); fork.entry.conflict_paths = files.iter().map(|file| file.path.clone()).collect(); fork.entry.conflict = Some(proto::ConflictInfo { - base: acyclic_engine::generation_hex(conflict_base), - ours: acyclic_engine::generation_hex(ours), - theirs: acyclic_engine::generation_hex(theirs), + base: acyclic::generation_hex(conflict_base), + ours: acyclic::generation_hex(ours), + theirs: acyclic::generation_hex(theirs), }); let fork_path = fork.entry.path.clone(); self.keep_fork(id, fork).await?; return Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(theirs), + generation: acyclic::generation_hex(theirs), old_tree: None, warning: String::new(), replayed_paths: 0, @@ -901,14 +824,11 @@ impl Server { } Ok(landed) => landed, }; - // Landed: the fork is consumed. Drop its route (mount fork) - // or its directory (copy fork). - if let Err(error) = self - .discard_fork_workspace(&id, fork.copy_dir.as_deref()) - .await - { - eprintln!("{NAME} daemon: discard fork {id} after promote: {error}"); - } + // Landed: the fork is consumed. A teardown failure must be + // visible because the route is still live and the fork must + // remain tracked until a later drop can finish it. + self.forks.lock().await.insert(id.clone(), fork); + self.remove_fork(&id).await?; match landed { Landed::Replayed { generation, @@ -917,7 +837,7 @@ impl Server { kept, moved, } => Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(generation), + generation: acyclic::generation_hex(generation), old_tree: None, warning: if moved { "the mainline had moved; the fork's paths were merged onto it in place" @@ -934,7 +854,7 @@ impl Server { })), Landed::Nothing { generation, kept } => { Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(generation), + generation: acyclic::generation_hex(generation), old_tree: None, warning: String::new(), replayed_paths: 0, @@ -949,29 +869,10 @@ impl Server { } } proto::Op::ForkDiff { id } => { - let (base, shared, copy_dir) = { + let (base, overlay) = { let forks = self.forks.lock().await; let fork = forks.get(&id).ok_or(format!("no fork {id}"))?; - (fork.base, Arc::clone(&fork.shared), fork.copy_dir.clone()) - }; - // Mounted fork: its writes already sit in its own overlay, so - // snapshot that. Copy fork: read the directory into a - // scratch overlay pinned at the base (native capture cannot - // read through the mount itself: NFS lacks the extent ioctl). - // Either way the fork stays promotable afterwards. - let overlay = match copy_dir { - None => shared, - Some(dir) => { - let scratch = self - .handle - .scratch_checkout(base) - .await - .map_err(stringify)?; - fork::capture_copy(&scratch, &dir) - .await - .map_err(stringify)?; - scratch - } + (fork.base, Arc::clone(&fork.shared)) }; let changes = if overlay.lock().await.has_pending_mutations() { let generation = self @@ -986,216 +887,19 @@ impl Server { } else { Vec::new() }; - // Timestamps differ on a copy fork by construction; only - // content is a blast radius. Ok(proto::Reply::Diff( self.annotate_ignored( changes .into_iter() .filter(|change| { - change.change != acyclic_engine::diff::ChangeKind::MetadataOnly + change.change != acyclic::diff::ChangeKind::MetadataOnly }) .map(diff_entry) .collect(), ), )) } - proto::Op::SessionFork { session_id } => { - self.session_fork(session_id).await?; - Ok(proto::Reply::Unit) - } - proto::Op::SessionResolve { session_id } => { - self.invalidate(&crate::speculate::Cause::Session(session_id.clone())); - let mut slot = self.dry_session.lock().await; - let session = slot - .take() - .filter(|session| session.session_id == session_id) - .ok_or_else(|| format!("no active Safe Mode session {session_id}"))?; - drop(slot); - // Unmount first: the real tree must reappear before we ask - // the engine to touch it, and no new writes can race the - // commit below. - let DrySession { - fork_id, - session_id, - shared, - base, - mut mount, - } = session; - tokio::task::block_in_place(|| mount.stop()) - .map_err(|error| format!("unmount: {error:?}"))?; - let label = format!("safe mode session {fork_id}"); - let outcome = self - .handle - .resolve_session(Arc::clone(&shared), base, label.clone()) - .await - .map_err(stringify)?; - match outcome { - SessionResolveOutcome::NoChanges => { - Ok(proto::Reply::SessionPending(proto::SessionPendingInfo { - session_id, - diff: Vec::new(), - })) - } - SessionResolveOutcome::Resolved { generation } => { - let changes = self - .handle - .diff(base, generation) - .await - .map_err(stringify)?; - self.pending.lock().await.insert( - session_id.clone(), - PendingSession { - generation, - base, - label, - }, - ); - Ok(proto::Reply::SessionPending(proto::SessionPendingInfo { - session_id, - diff: changes.into_iter().map(diff_entry).collect(), - })) - } - SessionResolveOutcome::Conflict { message } => Err(message), - } - } - proto::Op::SessionApply { session_id } => { - let mut pending = self.pending.lock().await; - let session = pending - .remove(&session_id) - .ok_or_else(|| format!("no resolved Safe Mode session {session_id}"))?; - drop(pending); - let outcome = self - .handle - .apply_session(session.generation, session.base, session.label) - .await - .map_err(stringify)?; - match outcome { - PromoteOutcome::Promoted { - generation, - old_tree, - } => Ok(proto::Reply::Promote(proto::PromoteInfo { - generation: acyclic_engine::generation_hex(generation), - old_tree: old_tree.map(|path| path.display().to_string()), - warning: "reload your editor: open files still point at the replaced tree" - .into(), - replayed_paths: 0, - merged_files: 0, - conflicts: Vec::new(), - fork_path: None, - kept_mainline: Vec::new(), - mainline_moved: false, - })), - PromoteOutcome::Conflict { message } => Err(message), - } - } - proto::Op::SessionDiscard { session_id } => { - self.pending.lock().await.remove(&session_id); - Ok(proto::Reply::Unit) - } - } - } - - /// Forks one checkout and shadow-mounts it directly at `repo_root` for - /// `session_id`'s duration (Safe Mode's session redirection). Only one - /// Safe Mode session can be active per repo at a time. - async fn session_fork(&self, session_id: String) -> Result<(), String> { - if !self.mounts.available { - return Err(format!( - "Safe Mode needs a mount provider and this host has none ({}).\n{}", - self.mounts.reason.as_deref().unwrap_or("unknown reason"), - fork::mount_setup_hint() - )); - } - if self.dry_session.lock().await.is_some() { - return Err("a Safe Mode session is already active for this repo".to_owned()); - } - let seed = self.handle.fork().await.map_err(stringify)?; - let shared = Arc::clone(&seed.shared); - let config = seed.config; - let guarded_paths = self.config.guarded_paths.clone(); - let volume_id = seed.volume_id; - let destination = self.repo_root.clone(); - let mount = tokio::task::block_in_place(move || { - let source = CheckoutMountSource::new(shared, config) - .map_err(|error| format!("mount source: {error:?}"))?; - let source: Arc = - if GuardedMountFilesystem::is_active(&guarded_paths) { - Arc::new(GuardedMountFilesystem::new( - Arc::new(source), - &guarded_paths, - )) - } else { - Arc::new(source) - }; - mount_native_over_existing( - NativeMountRequest { - mount_id: acyclic_engine::MountId::new(), - volume_id, - destination, - writable: true, - }, - source, - ) - .map_err(|error| format!("shadow mount: {error:?}")) - })?; - *self.dry_session.lock().await = Some(DrySession { - fork_id: short_id(), - session_id, - shared: seed.shared, - base: seed.base, - mount, - }); - // The fork now shadows the real repo root: suspend mainline capture - // until resolve/apply, or the pipeline watcher captures the shadow's - // content and the mount lifecycle instead of real-tree mutations. - self.handle.set_shadowed(true).await.map_err(stringify)?; - Ok(()) - } - - /// Copy-mode forks: materialize the base generation into a real - /// directory per fork. Same ids, lifecycle, and promote semantics as - /// mounted forks; creation is O(tree) instead of O(1). - async fn fork_copies( - &self, - count: u32, - session_id: Option, - ) -> Result { - let root = fork::forks_copy_root(&self.repo_root) - .ok_or("repo root has no parent for fork workspaces")?; - std::fs::create_dir_all(&root).map_err(|error| error.to_string())?; - let mut created = Vec::new(); - for _ in 0..count { - let seed = self.handle.fork().await.map_err(stringify)?; - let id = short_id(); - let dir = root.join(&id); - self.handle - .materialize(seed.base, dir.clone()) - .await - .map_err(stringify)?; - let entry = proto::ForkEntry { - id: id.clone(), - path: dir.display().to_string(), - mode: ForkMode::Copy.as_str().to_owned(), - base: acyclic_engine::generation_hex(seed.base), - created_at: unix_now(), - session_id: session_id.clone(), - conflict_paths: Vec::new(), - conflict: None, - }; - self.forks.lock().await.insert( - id, - ForkState { - shared: seed.shared, - base: seed.base, - entry: clone_entry(&entry), - copy_dir: Some(dir), - conflict: None, - }, - ); - created.push(entry); } - Ok(proto::Reply::Forks(created)) } /// Lands a fork in place. The fork's paths are merged onto the current @@ -1210,39 +914,17 @@ impl Server { fork: &ForkState, label: &str, ) -> Result { - // Get at the fork's overlay. A mounted fork keeps serving while we - // work: its snapshot is taken under the checkout lock, and the - // route is detached only once the fork has landed. (Detaching - // first and re-attaching on a conflict left the kernel's negative - // name cache hiding the fork on Linux FUSE, which cannot - // invalidate a route name.) - let overlay = match fork.copy_dir.as_deref() { - Some(dir) => { - let scratch = self - .handle - .scratch_checkout(fork.base) - .await - .map_err(stringify)?; - fork::capture_copy(&scratch, dir).await.map_err(stringify)?; - scratch - } - None => Arc::clone(&fork.shared), - }; + let overlay = Arc::clone(&fork.shared); // A rebased fork must have resolved its markers before it can land. if let Some(conflict) = fork.conflict.as_ref() { self.refuse_unresolved_markers(&overlay, conflict).await?; } let publish_started = std::time::Instant::now(); let head = self.handle.publish_head().await.map_err(stringify)?; - acyclic_engine::trace!( + acyclic::trace!( "daemon", - "promote {id}: publish_head {:.1}ms; {} fork, mainline {} since the fork's base", - acyclic_engine::trace::ms(publish_started), - if fork.copy_dir.is_some() { - "copy" - } else { - "mount" - }, + "promote {id}: publish_head {:.1}ms; mainline {} since the fork's base", + acyclic::trace::ms(publish_started), if head == fork.base { "UNMOVED (plain in-place write)" } else { @@ -1250,16 +932,9 @@ impl Server { } ); let landed = self - .merge_onto_head( - id, - overlay, - fork.base, - head, - fork.copy_dir.as_deref(), - label, - ) + .merge_onto_head(id, overlay, fork.base, head, label) .await; - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: outcome {}", match &landed { @@ -1302,6 +977,17 @@ impl Server { Ok(()) } + /// Detaches a fork's live route before forgetting its state. If teardown + /// fails, the entry remains visible and a caller can retry the drop. + async fn remove_fork(&self, id: &str) -> Result<(), String> { + if !self.forks.lock().await.contains_key(id) { + return Err(format!("no fork {id}")); + } + self.detach_route(id).await?; + self.forks.lock().await.remove(id); + Ok(()) + } + /// Builds the mount source for a fork's shared checkout and routes it /// under the one native session (mounting it on the first route). async fn attach_route( @@ -1331,9 +1017,10 @@ impl Server { ) })?; let mut mount = self.fork_mount.lock().await; + let route = route_name(id)?; mount .router - .add_route(route_name(id), source) + .add_route(route.clone(), source) .map_err(|error| format!("route: {error:?}"))?; // The ONE session, mounted lazily on the first fork. A route // insert is all later forks pay. @@ -1345,7 +1032,7 @@ impl Server { let session = tokio::task::block_in_place(move || { mount_native( NativeMountRequest { - mount_id: acyclic_engine::MountId::new(), + mount_id: acyclic::MountId::new(), volume_id, destination: dest, writable: true, @@ -1357,7 +1044,7 @@ impl Server { match session { Ok(session) => mount.session = Some(session), Err(error) => { - tokio::task::block_in_place(|| mount.router.remove_route(&route_name(id))); + tokio::task::block_in_place(|| mount.router.remove_route(&route)); return Err(error); } } @@ -1419,9 +1106,8 @@ impl Server { &self, id: &str, overlay: Arc, - base: acyclic_engine::GenerationId, - head: acyclic_engine::GenerationId, - copy_dir: Option<&Path>, + base: acyclic::GenerationId, + head: acyclic::GenerationId, label: &str, ) -> Result { let moved = head != base; @@ -1437,17 +1123,17 @@ impl Server { .snapshot_overlay(Arc::clone(&overlay)) .await .map_err(stringify)?; - let snapshot_ms = acyclic_engine::trace::ms(snapshot_started); + let snapshot_ms = acyclic::trace::ms(snapshot_started); let plan_started = std::time::Instant::now(); let mut plan = self .handle .merge_plan(base, head, snapshot, format!("fork {id}")) .await .map_err(stringify)?; - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: snapshot_overlay {snapshot_ms:.1}ms, merge_plan {:.1}ms", - acyclic_engine::trace::ms(plan_started) + acyclic::trace::ms(plan_started) ); // Gitignored paths (bytecode caches, build output, .env) are not // merge payload: a fork's copy never blocks a promote, the @@ -1461,13 +1147,13 @@ impl Server { let ignore_started = std::time::Instant::now(); let ignored = tokio::task::block_in_place(|| merge::ignored_paths(&self.repo_root, &contested)); - let ignore_ms = acyclic_engine::trace::ms(ignore_started); + let ignore_ms = acyclic::trace::ms(ignore_started); let kept: Vec = plan .keep_mainline_for(&ignored) .iter() .map(|path| path.display().to_string()) .collect(); - acyclic_engine::trace!( + acyclic::trace!( "daemon", "merge plan: take_ours={} take_theirs={} merged={} conflicted={} refused={} kept_mainline={}", plan.take_ours.len(), @@ -1488,7 +1174,7 @@ impl Server { "the working tree moved past the fork's base ({}) and {} path(s) cannot be merged:\n{}\n\ One fork must own those paths: re-fork from the current tree and redo that part, \ or rewind to the base. The fork is untouched", - acyclic_engine::generation_hex(base), + acyclic::generation_hex(base), plan.refusals.len(), lines.join("\n") )); @@ -1546,7 +1232,7 @@ impl Server { .await .map_err(stringify)? }; - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: gitignore check {ignore_ms:.1}ms ({} contested), landing source {} in \ {:.1}ms ({} replayed subtree(s), {} merged file(s))", @@ -1556,7 +1242,7 @@ impl Server { } else { "= built merged generation" }, - acyclic_engine::trace::ms(build_started), + acyclic::trace::ms(build_started), plan.take_ours.len(), plan.merged.len() ); @@ -1586,13 +1272,13 @@ impl Server { rebased, format!( "fork {id} rebased onto {} ({} conflict(s))", - acyclic_engine::short_hex(&acyclic_engine::generation_hex(head)), + acyclic::short_hex(&acyclic::generation_hex(head)), plan.conflicted.len() ), ) .await .map_err(stringify)?; - self.rebase_fork(id, snapshot, rebased, copy_dir).await?; + self.rebase_fork(id, snapshot, rebased).await?; let mut files: Vec = plan .conflicted .iter() @@ -1642,7 +1328,7 @@ impl Server { ) .await .map_err(stringify)?; - let pre_ms = acyclic_engine::trace::ms(land_started); + let pre_ms = acyclic::trace::ms(land_started); // One restore for every landing path: one drain, one timeline row // carrying the promote label, however many paths the fork touched. // publish_head captured the tree just now, so no safety row either. @@ -1669,16 +1355,16 @@ impl Server { landing.len() ) })?; - let restore_ms = acyclic_engine::trace::ms(restore_started); + let restore_ms = acyclic::trace::ms(restore_started); let written = u32::try_from(restored.outcomes.len()).unwrap_or(u32::MAX); let landed_generation = restored.generation; // No inline publish: the landed row is a checkpoint like any other // and the idle timer publishes it. Authority publish is O(tree). - acyclic_engine::trace!( + acyclic::trace!( "daemon", "promote {id}: landing {written} path(s): record rows {pre_ms:.1}ms, \ restore {restore_ms:.1}ms, land total {:.1}ms", - acyclic_engine::trace::ms(land_started) + acyclic::trace::ms(land_started) ); Ok(Landed::Replayed { generation: landed_generation, @@ -1689,15 +1375,13 @@ impl Server { }) } - /// Makes the fork workspace equal to `rebased`: writes R − F into the - /// fork's directory, through the mount for a mounted fork. The real - /// tree is never touched. + /// Makes the mounted fork equal to `rebased` by writing R − F through + /// the mount. The real tree is never touched. async fn rebase_fork( &self, id: &str, - snapshot: acyclic_engine::GenerationId, - rebased: acyclic_engine::GenerationId, - copy_dir: Option<&Path>, + snapshot: acyclic::GenerationId, + rebased: acyclic::GenerationId, ) -> Result<(), String> { let changed: Vec = content_changes( self.handle @@ -1709,77 +1393,47 @@ impl Server { .map(|change| change.path) .collect(); let roots = merge::subtree_roots(&changed); - if let Some(dir) = copy_dir { - for root in &roots { - self.handle - .restore_path_into(rebased, dir.to_path_buf(), root.clone()) - .await - .map_err(stringify)?; - } - } else { - // A mounted fork: write THROUGH the mount, never behind it. - // The driver and the kernel keep name and attribute caches - // that only their own operations update; a write via the - // checkout leaves a file the fork had deleted invisible for - // good, and the FUSE transport has no invalidation at all. - let dir = fork::forks_mount_root(&self.repo_root) - .ok_or("repo root has no parent for fork workspaces")? - .join(id); - self.handle - .materialize_paths(rebased, dir, roots) - .await - .map_err(stringify)?; - } - Ok(()) - } - - /// Drops whatever backs a fork: its route for mounted forks, its - /// directory for copy forks. - async fn discard_fork_workspace( - &self, - id: &str, - copy_dir: Option<&Path>, - ) -> Result<(), String> { - match copy_dir { - Some(dir) => { - std::fs::remove_dir_all(dir) - .map_err(|error| format!("remove fork copy: {error}"))?; - Self::remove_if_empty(dir.parent()); - Ok(()) - } - None => self.detach_route(id).await, - } - } - - fn remove_if_empty(dir: Option<&Path>) { - if let Some(dir) = dir { - let _ = std::fs::remove_dir(dir); - let _ = dir.parent().map(std::fs::remove_dir); - } + // Write THROUGH the mount, never behind it. The driver and kernel + // keep name and attribute caches that only their own operations + // update. + let dir = fork::forks_mount_root(&self.repo_root) + .ok_or("repo root has no parent for fork workspaces")? + .join(id); + self.handle + .materialize_paths(rebased, dir, roots) + .await + .map_err(stringify) } /// Removes one fork's route; the session unmounts (and the mount root /// disappears) when the last route goes, freeing the FUSE-T pool slot. async fn detach_route(&self, id: &str) -> Result<(), String> { let mut mount = self.fork_mount.lock().await; - // Dropping a route drops its CheckoutMountSource, which owns a tokio - // runtime — runtimes must never be dropped on an async worker. - tokio::task::block_in_place(|| mount.router.remove_route(&route_name(id))); - // The kernel may hold a positive entry cache for the removed name - // (FSKit caches until told otherwise): invalidate it eagerly. - if let Some(session) = mount.session.as_ref() { - if let Err(error) = tokio::task::block_in_place(|| session.invalidate(&route_name(id))) - { - eprintln!("{NAME} daemon: invalidate {id}: {error:?}"); + let route = route_name(id)?; + let last_route = mount.router.route_count() == 1; + if !last_route { + if let Some(session) = mount.session.as_ref() { + tokio::task::block_in_place(|| session.invalidate(&route)) + .map_err(|error| format!("invalidate {id}: {error:?}"))?; } } - if mount.router.is_empty() { - if let Some(mut session) = mount.session.take() { + if last_route { + if let Some(session) = mount.session.as_mut() { tokio::task::block_in_place(|| session.stop()) .map_err(|error| format!("unmount: {error:?}"))?; } + mount.session.take(); + } + // Dropping a route drops its CheckoutMountSource, which owns a tokio + // runtime — runtimes must never be dropped on an async worker. For + // the last route, stop the fallible kernel session first so failure + // leaves both the route and fork state intact for an exact retry. + if !tokio::task::block_in_place(|| mount.router.remove_route(&route)) { + return Err(format!("fork {id} has no mount route")); + } + if last_route { if let Some(root) = fork::forks_mount_root(&self.repo_root) { - let _ = std::fs::remove_dir_all(root); + let _ = std::fs::remove_dir(root); } } Ok(()) @@ -2121,7 +1775,7 @@ struct PendingBranch { prompt: Option, checkpoints: i64, /// Generations to diff for the branch's size, when the two differ. - diff: Option<(acyclic_engine::GenerationId, acyclic_engine::GenerationId)>, + diff: Option<(acyclic::GenerationId, acyclic::GenerationId)>, } /// Every branch the session rewound away from, with the work it would take @@ -2174,12 +1828,10 @@ fn abandoned_branches( /// Content changes only. A rewind or restore rewrites mtimes on every path /// it materializes, so metadata-only rows are noise for "what changed". -fn content_changes( - changes: Vec, -) -> Vec { +fn content_changes(changes: Vec) -> Vec { changes .into_iter() - .filter(|change| change.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|change| change.change != acyclic::diff::ChangeKind::MetadataOnly) .collect() } @@ -2194,21 +1846,27 @@ fn err(message: String) -> proto::Payload { /// the `ProjFS` provider decodes every entry name it is handed as UTF-16LE, /// and hands an id passed as raw ASCII back to the user as mojibake. Ids are /// hex, so this is a widening on Windows and a copy everywhere else. -fn route_name(id: &str) -> Vec { - acyclic_engine::names::str_to_bytes(id) +fn route_name(id: &str) -> Result, String> { + let config = acyclic::store::volume_config(); + acyclic_fs::host_path_to_namespace(Path::new(id), config.profile, config.limits) + .map_err(|error| format!("route name: {error}"))? + .components() + .first() + .map(|name| name.as_bytes().to_vec()) + .ok_or_else(|| "route name is empty".to_owned()) } fn short_id() -> String { // UUIDv7 leads with timestamp bits (identical across nearby calls); // the tail is the random section. - acyclic_engine::MountId::new().into_bytes()[10..] + acyclic::MountId::new().into_bytes()[10..] .iter() .map(|byte| format!("{byte:02x}")) .collect() } fn unix_now() -> i64 { - acyclic_engine::unix_now() + acyclic::unix_now() } /// How a fork ended up in the real tree. @@ -2217,7 +1875,7 @@ enum Landed { /// `merged` of the paths were produced by a three-way content merge; /// `moved` says whether the mainline had moved past the fork's base. Replayed { - generation: acyclic_engine::GenerationId, + generation: acyclic::GenerationId, paths: u32, merged: u32, kept: Vec, @@ -2225,14 +1883,14 @@ enum Landed { }, /// No content changes to land. Nothing { - generation: acyclic_engine::GenerationId, + generation: acyclic::GenerationId, kept: Vec, }, /// Nothing landed: the fork was rebased onto `theirs` and `files` /// carry conflict markers in the fork workspace. Conflicted { - theirs: acyclic_engine::GenerationId, - ours: acyclic_engine::GenerationId, + theirs: acyclic::GenerationId, + ours: acyclic::GenerationId, files: Vec, kept: Vec, }, @@ -2273,7 +1931,6 @@ fn clone_entry(entry: &proto::ForkEntry) -> proto::ForkEntry { proto::ForkEntry { id: entry.id.clone(), path: entry.path.clone(), - mode: entry.mode.clone(), base: entry.base.clone(), created_at: entry.created_at, session_id: entry.session_id.clone(), @@ -2300,33 +1957,14 @@ fn engine_kind(kind: proto::CheckpointRequestKind) -> CheckpointKind { } } -fn wire_kind(kind: CheckpointKind) -> proto::CheckpointKind { - match kind { - CheckpointKind::Baseline => proto::CheckpointKind::Baseline, - CheckpointKind::Pre => proto::CheckpointKind::Pre, - CheckpointKind::Post => proto::CheckpointKind::Post, - CheckpointKind::Manual => proto::CheckpointKind::Manual, - CheckpointKind::PreRewind => proto::CheckpointKind::PreRewind, - CheckpointKind::Recovered => proto::CheckpointKind::Recovered, - CheckpointKind::Failed => proto::CheckpointKind::Failed, - CheckpointKind::Noop => proto::CheckpointKind::Noop, - CheckpointKind::Auto => proto::CheckpointKind::Auto, - } -} - #[allow( clippy::needless_pass_by_value, reason = "used as `.map(diff_entry)` over an owning iterator" )] -fn diff_entry(change: acyclic_engine::diff::FileChange) -> proto::DiffEntry { +fn diff_entry(change: acyclic::diff::FileChange) -> proto::DiffEntry { proto::DiffEntry { path: change.path.display().to_string(), - change: match change.change { - acyclic_engine::diff::ChangeKind::Added => proto::ChangeKind::Added, - acyclic_engine::diff::ChangeKind::Removed => proto::ChangeKind::Removed, - acyclic_engine::diff::ChangeKind::Modified => proto::ChangeKind::Modified, - acyclic_engine::diff::ChangeKind::MetadataOnly => proto::ChangeKind::Metadata, - }, + change: change.change, file_kind: format!("{:?}", change.file_kind).to_lowercase(), ignored: false, } @@ -2336,7 +1974,7 @@ fn timeline_entry(row: CheckpointRow) -> proto::TimelineEntry { proto::TimelineEntry { id: row.id, created_at: row.created_at, - kind: wire_kind(row.kind), + kind: row.kind, published: row.published, session_id: row.session_id, tool_name: row.tool_name, @@ -2346,27 +1984,6 @@ fn timeline_entry(row: CheckpointRow) -> proto::TimelineEntry { } } -fn hex_generation(generation: acyclic_engine::GenerationId) -> String { - acyclic_engine::generation_hex(generation) -} - -fn directory_bytes(root: &Path) -> u64 { - let mut total = 0u64; - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { - let Ok(entries) = std::fs::read_dir(&dir) else { - continue; - }; - for entry in entries.flatten() { - let Ok(metadata) = entry.metadata() else { - continue; - }; - if metadata.is_dir() { - stack.push(entry.path()); - } else { - total += metadata.len(); - } - } - } - total +fn hex_generation(generation: acyclic::GenerationId) -> String { + acyclic::generation_hex(generation) } diff --git a/crates/acyclic-engine/src/spec.rs b/crates/acyclic/src/spec.rs similarity index 100% rename from crates/acyclic-engine/src/spec.rs rename to crates/acyclic/src/spec.rs diff --git a/crates/acyclic/src/spec_runner.rs b/crates/acyclic/src/spec_runner.rs index 68c3dea..a55676a 100644 --- a/crates/acyclic/src/spec_runner.rs +++ b/crates/acyclic/src/spec_runner.rs @@ -24,9 +24,9 @@ use std::path::{Path, PathBuf}; use std::time::Duration; // Only the Unix sweep names the product; on Windows there is no sweep. -#[cfg(unix)] -use acyclic_engine::product::NAME; -use acyclic_engine::spec::RunOutcome; +#[cfg(all(unix, test))] +use acyclic::product::NAME; +use acyclic::spec::RunOutcome; use tokio::io::AsyncWriteExt; use tokio::process::Command; @@ -67,7 +67,7 @@ impl RunSpace { } fn record(&self, pgid: i32, argv0: &str) { - let line = format!("{pgid} {} {argv0}\n", acyclic_engine::unix_now()); + let line = format!("{pgid} {} {argv0}\n", acyclic::unix_now()); let _ = std::fs::write(&self.pid_file, line); } @@ -351,18 +351,18 @@ impl Drop for OwnedJob { } } -/// Kills anything a dead daemon left running, before the store opens. +/// Kills anything a dead daemon left running when speculation is enabled. /// /// Matched on the recorded command name as well as the pid, so a reused pid /// belonging to something else is never signalled — the check costs one /// `ps` and removes the whole class of mistake. -#[cfg(unix)] +#[cfg(all(unix, test))] #[allow( unsafe_code, reason = "kill(pid, 0) only tests for the process's existence; the \ killpg that follows is guarded by a command-name match" )] -pub fn sweep_stale_runs(spec_runs: &Path) { +fn sweep_stale_runs(spec_runs: &Path) { let pids = spec_runs.join("pids"); let Ok(entries) = std::fs::read_dir(&pids) else { return; @@ -401,12 +401,9 @@ pub fn sweep_stale_runs(spec_runs: &Path) { /// Windows needs no sweep: each run's job object carries /// `KILL_ON_JOB_CLOSE` and the daemon holds its only handle, so a daemon /// that dies — however it dies — takes the run's whole tree with it. -#[cfg(not(unix))] -pub fn sweep_stale_runs(_spec_runs: &Path) {} - /// Whether the live process really is the run we recorded, rather than /// whatever inherited its pid. -#[cfg(unix)] +#[cfg(all(unix, test))] fn command_name_matches(pid: i32, expected: &str) -> bool { let Ok(output) = std::process::Command::new("ps") .args(["-o", "comm=", "-p", &pid.to_string()]) @@ -593,6 +590,7 @@ mod tests { }); } + #[cfg(unix)] #[test] fn sweeping_an_absent_directory_is_a_no_op() { let dir = tempfile::tempdir().expect("tempdir"); diff --git a/crates/acyclic/src/speculate.rs b/crates/acyclic/src/speculate.rs index 59ac60b..e64b640 100644 --- a/crates/acyclic/src/speculate.rs +++ b/crates/acyclic/src/speculate.rs @@ -22,13 +22,13 @@ use std::path::PathBuf; use std::time::Duration; -use acyclic_engine::index::Index; -use acyclic_engine::pipeline::PipelineHandle; -use acyclic_engine::spec::{ +use crate::proto; +use acyclic::index::Index; +use acyclic::pipeline::PipelineHandle; +use acyclic::spec::{ RunId, RunOutcome, SpecEvent as LogEvent, SpecEventRow, SpecHit, SpecKey, SpecKind, SpecMetrics, SpecState, SpecStore, SpeculateConfig, }; -use acyclic_proto as proto; use tokio::sync::{mpsc, oneshot}; /// Bound on the claim path's wait for `spec.db`. Short on purpose: the one @@ -107,7 +107,7 @@ impl SpecHandle { /// request handlers, several of them on the hook path. pub fn notify(&self, event: SpecEvent) { if let Err(error) = self.events.try_send(event) { - acyclic_engine::trace!("spec", "event dropped: {error}"); + acyclic::trace!("spec", "event dropped: {error}"); } } @@ -155,7 +155,7 @@ impl SpecHandle { }; let info = serde_json::from_str(&hit.body).ok(); let _ = store.log(&event(LogEvent::ClaimHit, Some(hit.lead_ms))); - acyclic_engine::trace!("spec", "brief claimed, {}ms ahead", hit.lead_ms); + acyclic::trace!("spec", "brief claimed, {}ms ahead", hit.lead_ms); info } @@ -299,10 +299,7 @@ pub fn spawn( let spec_db = deps.spec_db.clone(); let thread_config = config.clone(); let thread = std::thread::Builder::new() - .name(format!("{}-speculate", acyclic_engine::product::NAME)) - // The fs futures a diff pulls in are large; match the pipeline's - // headroom rather than the 2 MiB default. - .stack_size(32 * 1024 * 1024) + .name(format!("{}-speculate", acyclic::product::NAME)) .spawn(move || { // `enable_all`, not just timers: a model run is a child // process, which needs the IO and signal drivers. @@ -314,7 +311,7 @@ pub fn spawn( Err(error) => { eprintln!( "{}: speculation runtime: {error}; speculation off", - acyclic_engine::product::NAME + acyclic::product::NAME ); return; } @@ -371,7 +368,7 @@ async fn run(config: SpeculateConfig, deps: SpecDeps, mut receiver: mpsc::Receiv let Ok(mut store) = SpecStore::open(&deps.spec_db, SCHEDULER_BUSY_TIMEOUT) else { eprintln!( "{}: speculation cache unavailable; speculation off", - acyclic_engine::product::NAME + acyclic::product::NAME ); return; }; @@ -379,7 +376,7 @@ async fn run(config: SpeculateConfig, deps: SpecDeps, mut receiver: mpsc::Receiv // `running` is the one state that is never retryable. if let Ok(swept) = store.sweep_orphans() { if swept > 0 { - acyclic_engine::trace!("spec", "swept {swept} orphaned run(s) from a dead daemon"); + acyclic::trace!("spec", "swept {swept} orphaned run(s) from a dead daemon"); } } let mut scheduler = Scheduler { @@ -444,7 +441,7 @@ impl Scheduler { self.in_flight = Some(run); return; } - acyclic_engine::trace!("spec", "cancelling the summary run: {cause:?}"); + acyclic::trace!("spec", "cancelling the summary run: {cause:?}"); let _ = run.cancel.send(()); let _ = self.store.finish(run.run, &RunOutcome::Cancelled); self.log(LogEvent::Cancel, &run.session_id, run.turn, None, None); @@ -463,7 +460,7 @@ impl Scheduler { self.in_flight = None; let (event, bytes) = match &report.outcome { RunOutcome::Ready { body } => { - acyclic_engine::trace!( + acyclic::trace!( "spec", "summary for turn {} ready in {}ms", report.turn, @@ -607,7 +604,7 @@ impl Scheduler { let (cancel, cancelled) = oneshot::channel(); let done = done.clone(); let owner = session_id.to_owned(); - acyclic_engine::trace!("spec", "pre-firing the summarizer for turn {turn}"); + acyclic::trace!("spec", "pre-firing the summarizer for turn {turn}"); tokio::spawn(async move { let started = std::time::Instant::now(); let outcome = crate::spec_runner::run(spec, &space, cancelled).await; @@ -642,11 +639,7 @@ fn turn_range( index: &Index, session_id: &str, turn: i64, -) -> Option<( - acyclic_engine::GenerationId, - acyclic_engine::GenerationId, - String, -)> { +) -> Option<(acyclic::GenerationId, acyclic::GenerationId, String)> { let row = index.turn(session_id, turn).ok()??; let (first, last) = (row.first_checkpoint?, row.last_checkpoint?); let base = index.latest_target_before(first).ok()??; @@ -667,9 +660,9 @@ fn turn_range( fn render_prompt( config: &SpeculateConfig, prompt: &str, - changes: &[acyclic_engine::diff::FileChange], + changes: &[acyclic::diff::FileChange], ) -> String { - use acyclic_engine::diff::ChangeKind; + use acyclic::diff::ChangeKind; let instruction = if config.prompt_template.is_empty() { SUMMARY_PROMPT.to_owned() } else { @@ -741,7 +734,7 @@ async fn speculate_brief(config: &SpeculateConfig, deps: &SpecDeps, store: &mut let wall_ms = i64::try_from(started.elapsed().as_millis()).unwrap_or(i64::MAX); match &outcome { RunOutcome::Ready { body } => { - acyclic_engine::trace!("spec", "brief precomputed in {wall_ms}ms"); + acyclic::trace!("spec", "brief precomputed in {wall_ms}ms"); log( store, LogEvent::Ready, diff --git a/crates/acyclic/src/store.rs b/crates/acyclic/src/store.rs new file mode 100644 index 0000000..01dab72 --- /dev/null +++ b/crates/acyclic/src/store.rs @@ -0,0 +1,595 @@ +//! Store lifecycle: where engine state lives and how the volume opens. +//! +//! Everything lives OUTSIDE the working tree (capture snapshots the whole +//! tree and fail-closes on sockets). The repo carries only `.acyclic/config.toml`. + +#[cfg(target_os = "windows")] +use std::io::Write; +use std::path::{Path, PathBuf}; + +use acyclic_fs::model::{ + AccessMode, CheckoutMode, ConsistencyMode, GenerationSelector, Lifecycle, MutationMode, + VolumeConfig, +}; +use acyclic_fs::{ + CancellationToken, Checkout, LocalAuthorityBackend, LocalFs, LocalObjectBackend, + LocalObjectsDurability, LocalOptions, LocalStreamDurability, VolumeId, WorkCounters, + WorkspaceRestore, +}; +use serde::{Deserialize, Serialize}; + +use crate::{EngineError, Result}; + +/// Concrete checkout type for the local backend. +pub type LocalCheckout = Checkout; +/// Concrete workspace type for the local backend. +pub type LocalWorkspace = acyclic_fs::Workspace; +/// Concrete immutable generation type for the local backend. +pub type LocalGeneration = acyclic_fs::Generation; + +/// Batch limits sized for large monorepos: the fs defaults (2,048) reject any +/// baseline capture beyond ~2k paths. Immutable per volume — size generously. +const MUTATIONS_PER_BATCH: u32 = 4_194_304; + +/// Filesystem layout of one repo's store. +#[derive(Clone, Debug)] +pub struct StorePaths { + /// Store root: `//`. + pub root: PathBuf, +} + +impl StorePaths { + /// Resolves the store root for a repo. `stores_root` override comes from + /// config; the default is `~/.local/share/acyclic/stores`. + pub fn for_repo(repo_root: &Path, stores_root: Option<&Path>) -> Result { + let canonical = repo_root + .canonicalize() + .map_err(|error| EngineError::Store(format!("canonicalize repo root: {error}")))?; + let base = if let Some(path) = stores_root { + if path.is_absolute() { + path.to_path_buf() + } else { + canonical.join(path) + } + } else { + let home = std::env::var_os("HOME") + .ok_or_else(|| EngineError::Store("HOME is not set".into()))?; + Path::new(&home).join(format!(".local/share/{}/stores", crate::product::NAME)) + }; + let base = canonicalize_planned(&base)?; + let digest = blake3::hash(canonical.as_os_str().as_encoded_bytes()); + let hex = digest.to_hex(); + let short = hex.get(..16).unwrap_or(&hex); + let paths = Self { + root: base.join(short), + }; + paths.ensure_outside_repo(&canonical)?; + Ok(paths) + } + + fn ensure_outside_repo(&self, repo_root: &Path) -> Result<()> { + let repo = repo_root.canonicalize()?; + for path in [ + self.root.clone(), + self.object_store(), + self.index_db(), + self.spec_db(), + self.spec_runs(), + self.pidfile(), + self.rewind_journal(), + self.trash(), + self.meta(), + #[cfg(target_os = "windows")] + self.continuity(), + self.root.join("daemon.log"), + ] { + let resolved = canonicalize_planned(&path)?; + if resolved.starts_with(&repo) { + return Err(EngineError::Store(format!( + "store must be outside the repository: {}", + resolved.display() + ))); + } + } + Ok(()) + } + + pub fn object_store(&self) -> PathBuf { + self.root.join("store") + } + pub fn index_db(&self) -> PathBuf { + self.root.join("index.db") + } + /// The daemon socket lives in a short per-user runtime directory, NOT in + /// the store: `sun_path` is capped (~104 bytes on macOS) and store roots + /// can be arbitrarily deep. + pub fn socket(&self) -> PathBuf { + let store_key = self.root.file_name().map_or_else( + || "default".into(), + |name| name.to_string_lossy().into_owned(), + ); + runtime_dir().join(format!("{store_key}.sock")) + } + pub fn pidfile(&self) -> PathBuf { + self.root.join("daemon.pid") + } + pub fn rewind_journal(&self) -> PathBuf { + self.root.join("rewind-journal.json") + } + pub fn trash(&self) -> PathBuf { + self.root.join("trash") + } + pub fn meta(&self) -> PathBuf { + self.root.join("meta.json") + } + #[cfg(target_os = "windows")] + pub fn continuity(&self) -> PathBuf { + self.root.join("continuity.bin") + } + /// Speculation cache. Deliberately NOT a table in `index_db`: the + /// pipeline thread owns that connection and writes to it synchronously, + /// so a second writer contending for `SQLite`'s write lock would block the + /// thread every hook call waits on. See `crate::spec`. + pub fn spec_db(&self) -> PathBuf { + self.root.join("spec.db") + } + /// Scratch and pid files for in-flight speculative child processes. + pub fn spec_runs(&self) -> PathBuf { + self.root.join("spec") + } +} + +fn canonicalize_planned(path: &Path) -> std::io::Result { + let mut cursor = path; + let mut missing = Vec::new(); + loop { + match cursor.canonicalize() { + Ok(mut resolved) => { + for name in missing.into_iter().rev() { + resolved.push(name); + } + return Ok(resolved); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if cursor + .symlink_metadata() + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + return Err(std::io::Error::other("dangling store path symlink")); + } + let name = cursor.file_name().ok_or(error)?; + missing.push(name.to_os_string()); + cursor = cursor + .parent() + .ok_or_else(|| std::io::Error::other("store path has no existing ancestor"))?; + } + Err(error) => return Err(error), + } + } +} + +/// Persisted store identity. The `VolumeId` MUST survive restarts: +/// generations only resolve on their own volume. +#[derive(Debug, Serialize, Deserialize)] +pub struct StoreMeta { + pub schema: u32, + pub repo_root: PathBuf, + pub volume_id: VolumeId, +} + +/// An opened store: the fs engine, its volume, and a writable Head checkout. +pub struct Store { + pub fs: LocalFs, + pub workspace: LocalWorkspace, + pub checkout: LocalCheckout, + pub volume_id: VolumeId, + pub paths: StorePaths, + pub repo_root: PathBuf, +} + +/// The volume every store opens with. +/// +/// The profile is per-platform and decides how names are encoded on the way +/// in and out. It is fixed for the life of a volume: a store created +/// under one profile cannot be reopened under another, so this must never +/// become a runtime choice. +pub fn volume_config() -> VolumeConfig { + let mut config = VolumeConfig { + profile: host_profile(), + ..VolumeConfig::portable(Lifecycle::Durable) + }; + config.limits.maximum_mutations_per_batch = MUTATIONS_PER_BATCH; + config.limits.maximum_paths_per_batch = MUTATIONS_PER_BATCH; + config.limits.maximum_component_bytes = if cfg!(windows) { 510 } else { 255 }; + config +} + +pub(crate) const fn host_profile() -> acyclic_fs::model::FilesystemProfile { + #[cfg(windows)] + return acyclic_fs::model::FilesystemProfile::Windows; + #[cfg(unix)] + return acyclic_fs::model::FilesystemProfile::Posix; + #[cfg(not(any(unix, windows)))] + acyclic_fs::model::FilesystemProfile::Portable +} + +pub(crate) fn writable_head() -> CheckoutMode { + CheckoutMode { + access: AccessMode::ReadWrite, + consistency: ConsistencyMode::TrackingSafe, + mutations: MutationMode::PrivateOverlay, + } +} + +/// Read-only pinned mode for historical generations. +pub fn read_only() -> CheckoutMode { + CheckoutMode { + access: AccessMode::ReadOnly, + consistency: ConsistencyMode::Pinned, + mutations: MutationMode::None, + } +} + +/// Exact local durability supported by this platform. +pub fn local_options(root: impl Into) -> LocalOptions { + let mut options = LocalOptions::new(root); + #[cfg(target_vendor = "apple")] + { + // Apple exposes an ordered barrier that does not drain the device cache. + options.stream.durability = LocalStreamDurability::Barrier; + options.objects.durability = LocalObjectsDurability::Barrier; + } + #[cfg(not(target_vendor = "apple"))] + { + options.stream.durability = LocalStreamDurability::FullFlush; + options.objects.durability = LocalObjectsDurability::FullFlush; + } + options +} + +impl Store { + /// Reconciles an SDK head after crash recovery completed a host tree swap. + pub async fn recover_workspace_head( + &mut self, + recovered: &crate::rewind::RecoveredSwap, + ) -> Result<()> { + if !recovered.reconcile_head || !recovered.published { + return Ok(()); + } + let current = self + .workspace + .head() + .await + .map_err(EngineError::fs("workspace head"))?; + if current.id() != recovered.target { + let target = self + .workspace + .generation(recovered.target) + .await + .map_err(EngineError::fs("recover rewind generation"))?; + let key = acyclic_fs::IdempotencyKey::from_bytes( + recovered.target.digest().as_bytes()[..16] + .try_into() + .map_err(|_| EngineError::Store("invalid generation digest".into()))?, + ); + match self + .workspace + .restore_generation(&target, current.id(), key) + .await + .map_err(EngineError::fs("recover workspace rewind"))? + { + WorkspaceRestore::Restored(_) + | WorkspaceRestore::AlreadyRestored(_) + | WorkspaceRestore::Current(_) => {} + WorkspaceRestore::Stale(_) + | WorkspaceRestore::Fenced + | WorkspaceRestore::IdempotencyConflict => { + return Err(EngineError::Store( + "workspace head changed during rewind recovery".into(), + )); + } + } + } + self.checkout = self + .workspace + .checkout(GenerationSelector::Head, writable_head()) + .await + .map_err(EngineError::fs("refresh recovered workspace"))?; + Ok(()) + } + + /// Creates the store for a repo: directories, volume, meta record. + /// Fails if the store already exists. + pub async fn init(repo_root: &Path, paths: StorePaths) -> Result { + paths.ensure_outside_repo(repo_root)?; + if paths.meta().exists() { + return Err(EngineError::Store(format!( + "store already initialized at {}", + paths.root.display() + ))); + } + std::fs::create_dir_all(&paths.root)?; + paths.ensure_outside_repo(repo_root)?; + std::fs::create_dir_all(paths.object_store())?; + std::fs::create_dir_all(paths.trash())?; + paths.ensure_outside_repo(repo_root)?; + + let cancel = CancellationToken::new(); + let fs = LocalFs::local(local_options(paths.object_store())) + .await + .map_err(EngineError::fs("open object store"))?; + let volume_id = VolumeId::new(); + let volume = fs + .create_volume_with_id(volume_id, volume_config(), WorkCounters::UNBOUNDED, &cancel) + .await + .map_err(EngineError::fs("create volume"))? + .value; + let workspace = fs + .adopt_volume_workspace("main", volume) + .map_err(EngineError::fs("adopt workspace"))?; + let checkout = workspace + .checkout(GenerationSelector::Head, writable_head()) + .await + .map_err(EngineError::fs("checkout head"))?; + + let repo_root = repo_root.canonicalize()?; + let meta = StoreMeta { + schema: 1, + repo_root: repo_root.clone(), + volume_id, + }; + atomic_write_json(&paths.meta(), &meta)?; + Ok(Self { + fs, + workspace, + checkout, + volume_id, + paths, + repo_root, + }) + } + + /// Opens an existing store recorded in `meta.json`. + pub async fn open(repo_root: &Path, paths: StorePaths) -> Result { + paths.ensure_outside_repo(repo_root)?; + let text = std::fs::read_to_string(paths.meta()).map_err(|error| { + EngineError::Store(format!( + "no store at {} ({error}); run init first", + paths.root.display() + )) + })?; + let meta: StoreMeta = serde_json::from_str(&text) + .map_err(|error| EngineError::Store(format!("meta.json: {error}")))?; + if meta.schema != 1 { + return Err(EngineError::Store(format!( + "unsupported store schema {}", + meta.schema + ))); + } + let expected_repo = repo_root.canonicalize()?; + if meta.repo_root.canonicalize()? != expected_repo { + return Err(EngineError::Store( + "store belongs to a different repository".into(), + )); + } + paths.ensure_outside_repo(&expected_repo)?; + + let cancel = CancellationToken::new(); + let phase = std::time::Instant::now(); + let fs = LocalFs::local(local_options(paths.object_store())) + .await + .map_err(EngineError::fs("open object store"))?; + let objects_ms = crate::trace::ms(phase); + let phase = std::time::Instant::now(); + let volume = fs + .open_volume(meta.volume_id, WorkCounters::UNBOUNDED, &cancel) + .await + .map_err(EngineError::fs("open volume"))? + .value; + let volume_ms = crate::trace::ms(phase); + let phase = std::time::Instant::now(); + let workspace = fs + .adopt_volume_workspace("main", volume) + .map_err(EngineError::fs("adopt workspace"))?; + let checkout = workspace + .checkout(GenerationSelector::Head, writable_head()) + .await + .map_err(EngineError::fs("checkout head"))?; + crate::trace!( + "store", + "open: object store {objects_ms:.1}ms, volume {volume_ms:.1}ms, head checkout {:.1}ms", + crate::trace::ms(phase) + ); + Ok(Self { + fs, + workspace, + checkout, + volume_id: meta.volume_id, + paths, + repo_root: meta.repo_root, + }) + } + + /// Opens a read-only checkout of one historical generation. + pub async fn checkout_exact( + &self, + generation: acyclic_fs::GenerationId, + ) -> Result { + self.workspace + .checkout(GenerationSelector::Exact(generation), read_only()) + .await + .map_err(EngineError::fs("checkout exact")) + } + + /// Opens one authenticated immutable generation handle. + pub async fn generation( + &self, + generation: acyclic_fs::GenerationId, + ) -> Result { + self.workspace + .generation(generation) + .await + .map_err(EngineError::fs("open exact generation")) + } +} + +/// Short per-user directory for daemon sockets. Created 0700 on first use. +/// +/// Deliberately NOT `std::env::temp_dir()`: that honors `TMPDIR`, which can +/// be arbitrarily deep, and `sun_path` is capped (~104 bytes on macOS). The +/// path must be short and identical across every process of this user. +#[allow(unsafe_code, reason = "getuid() has no preconditions and cannot fail")] +pub fn runtime_dir() -> PathBuf { + #[cfg(unix)] + let dir = { + // SAFETY: getuid has no preconditions and cannot fail. + let uid = unsafe { libc::getuid() }; + PathBuf::from(format!("/tmp/{}-{uid}", crate::product::NAME)) + }; + #[cfg(not(unix))] + let dir = std::env::temp_dir().join(crate::product::NAME); + let _ = std::fs::create_dir_all(&dir); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); + } + dir +} + +fn atomic_write_json(path: &Path, value: &T) -> Result<()> { + let text = serde_json::to_string_pretty(value) + .map_err(|error| EngineError::Store(format!("encode {}: {error}", path.display())))?; + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, text)?; + acyclic_fs::durable_rename(&tmp, path, acyclic_fs::RenameMode::Replace)?; + Ok(()) +} + +#[cfg(target_os = "windows")] +pub(crate) fn durable_replace(path: &Path, bytes: &[u8]) -> Result<()> { + let tmp = path.with_extension("bin.tmp"); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&tmp)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + acyclic_fs::durable_rename(&tmp, path, acyclic_fs::RenameMode::Replace)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn init_then_open_preserves_volume_identity() { + let repo = tempfile::tempdir().expect("repo dir"); + let stores = tempfile::tempdir().expect("stores dir"); + std::fs::write(repo.path().join("file.txt"), b"hi").expect("seed file"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); + + let created = Store::init(repo.path(), paths.clone()).await.expect("init"); + let created_id = created.volume_id; + drop(created); + + let reopened = Store::open(repo.path(), paths).await.expect("open"); + assert_eq!(reopened.volume_id, created_id); + assert_eq!( + reopened.repo_root, + repo.path().canonicalize().expect("canonical repo") + ); + } + + #[tokio::test] + async fn double_init_is_refused() { + let repo = tempfile::tempdir().expect("repo dir"); + let stores = tempfile::tempdir().expect("stores dir"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); + Store::init(repo.path(), paths.clone()).await.expect("init"); + assert!(Store::init(repo.path(), paths).await.is_err()); + } + + #[tokio::test] + async fn store_inside_repo_is_rejected_on_init_and_open() { + let repo = tempfile::tempdir().expect("repo dir"); + let paths = StorePaths { + root: repo.path().join(".stores").join("fixture"), + }; + assert!(StorePaths::for_repo(repo.path(), Some(&repo.path().join(".stores"))).is_err()); + assert!(matches!( + Store::init(repo.path(), paths.clone()).await, + Err(EngineError::Store(message)) if message.contains("outside the repository") + )); + assert!(!paths.root.exists()); + assert!(!paths.meta().exists()); + + std::fs::create_dir_all(&paths.root).expect("create invalid root"); + atomic_write_json( + &paths.meta(), + &StoreMeta { + schema: 1, + repo_root: repo.path().canonicalize().expect("canonical repo"), + volume_id: VolumeId::new(), + }, + ) + .expect("seed meta"); + assert!(matches!( + Store::open(repo.path(), paths).await, + Err(EngineError::Store(message)) if message.contains("outside the repository") + )); + } + + #[test] + fn relative_store_dir_resolves_from_repo_before_daemon_changes_cwd() { + let repo = tempfile::tempdir().expect("repo dir"); + let paths = StorePaths::for_repo(repo.path(), Some(Path::new("../stores"))) + .expect("resolve relative store"); + assert!(paths.root.is_absolute()); + assert!(paths.root.starts_with( + repo.path() + .canonicalize() + .expect("canonical repo") + .parent() + .expect("repo parent") + .join("stores") + )); + } + + #[tokio::test] + async fn store_open_rejects_a_different_requested_repository() { + let repo = tempfile::tempdir().expect("repo dir"); + let other = tempfile::tempdir().expect("other repo dir"); + let stores = tempfile::tempdir().expect("stores dir"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); + drop(Store::init(repo.path(), paths.clone()).await.expect("init")); + assert!(matches!( + Store::open(other.path(), paths).await, + Err(EngineError::Store(message)) if message.contains("different repository") + )); + } + + #[cfg(unix)] + #[tokio::test] + async fn store_child_symlink_into_repo_is_rejected() { + let repo = tempfile::tempdir().expect("repo dir"); + let stores = tempfile::tempdir().expect("stores dir"); + let target = repo.path().join("misplaced-store"); + std::fs::create_dir(&target).expect("target dir"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("resolve paths"); + std::fs::create_dir_all(&paths.root).expect("store root"); + std::os::unix::fs::symlink(&target, paths.object_store()).expect("store link"); + assert!(StorePaths::for_repo(repo.path(), Some(stores.path())).is_err()); + assert!(matches!( + Store::init(repo.path(), paths).await, + Err(EngineError::Store(message)) if message.contains("outside the repository") + )); + assert_eq!( + std::fs::read_dir(target).expect("target entries").count(), + 0 + ); + } +} diff --git a/crates/acyclic-engine/src/trace.rs b/crates/acyclic/src/trace.rs similarity index 100% rename from crates/acyclic-engine/src/trace.rs rename to crates/acyclic/src/trace.rs diff --git a/crates/acyclic-engine/tests/fork.rs b/crates/acyclic/tests/fork.rs similarity index 69% rename from crates/acyclic-engine/tests/fork.rs rename to crates/acyclic/tests/fork.rs index 730cfa6..e060219 100644 --- a/crates/acyclic-engine/tests/fork.rs +++ b/crates/acyclic/tests/fork.rs @@ -14,13 +14,12 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; -use acyclic_engine::config::Config; -use acyclic_engine::fork::{PromoteOutcome, SessionResolveOutcome}; -use acyclic_engine::index::{Attribution, CheckpointKind, Index}; -use acyclic_engine::pipeline::{self, PipelineHandle}; -use acyclic_engine::store::{Store, StorePaths}; -use acyclic_fs::kernel::{LogicalName, NamespacePath}; -use acyclic_fs::model::VolumeLimits; +use acyclic::config::Config; +use acyclic::fork::PromoteOutcome; +use acyclic::index::{Attribution, CheckpointKind, Index}; +use acyclic::pipeline::{self, PipelineHandle, State}; +use acyclic::store::{Store, StorePaths}; +use acyclic_fs::kernel::NamespacePath; use acyclic_fs::{CancellationToken, WorkCounters}; fn fast_config() -> Config { @@ -85,31 +84,24 @@ impl Rig { /// Built in the host's encoding rather than as a portable path: these names /// stand in for what the mount layer writes, and a diff decodes them with /// the same encoding on the way back out. -fn namespace(path: &str) -> NamespacePath { - let limits = VolumeLimits::default(); - let names = path - .split('/') - .filter(|part| !part.is_empty()) - .map(|part| { - LogicalName::new( - acyclic_engine::names::encoding(), - acyclic_engine::names::str_to_bytes(part), - limits.maximum_component_bytes, - ) - .expect("name") - }) - .collect(); - NamespacePath::new(names, limits).expect("namespace path") +fn namespace(path: &str, config: acyclic_fs::model::VolumeConfig) -> NamespacePath { + acyclic_fs::host_path_to_namespace( + Path::new(path.trim_start_matches('/')), + config.profile, + config.limits, + ) + .expect("namespace path") } /// Writes into a fork's overlay exactly as a mount callback would: through /// the shared checkout. -async fn write_in_fork(seed: &acyclic_engine::fork::ForkSeed, path: &str, bytes: &[u8]) { +async fn write_in_fork(seed: &acyclic::fork::ForkSeed, path: &str, bytes: &[u8]) { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard .create_file( - namespace(path), + namespace(path, config), bytes::Bytes::copy_from_slice(bytes), WorkCounters::UNBOUNDED, &cancel, @@ -118,6 +110,23 @@ async fn write_in_fork(seed: &acyclic_engine::fork::ForkSeed, path: &str, bytes: .expect("create file in fork overlay"); } +fn assert_watcher_recovers_after_swap(rig: &Rig) { + rig.runtime.block_on(async { + rig.handle + .checkpoint(CheckpointKind::Post, Attribution::default()) + .await + .expect("checkpoint after root swap"); + let status = rig.handle.status().await.expect("status after root swap"); + assert_eq!(status.state, State::Ready, "{status:?}"); + #[cfg(not(target_os = "macos"))] + assert_eq!(status.watcher.invalidations, 0, "{status:?}"); + // FSEvents can replay a root move after the new watcher opens. A + // bounded recovery scan is safe; discarding that hint is not. + #[cfg(target_os = "macos")] + assert!(status.watcher.invalidations <= 1, "{status:?}"); + }); +} + /// P1 + P5: promote lands the fork's exact content and records attribution. #[test] fn promote_lands_fork_changes() { @@ -162,74 +171,7 @@ fn promote_lands_fork_changes() { assert!(rows.iter().any(|row| { row.kind == CheckpointKind::Manual && row.label.as_deref() == Some("promote test-fork") })); - rig.finish(); -} - -/// Safe Mode: `resolve_session` + `apply_session` together must land a fork's -/// changes identically to promote's single-shot version. -#[test] -fn resolve_then_apply_session_lands_fork_changes() { - let rig = Rig::start(); - let (diffable_base, outcome) = rig.runtime.block_on(async { - let seed = rig.handle.fork().await.expect("fork"); - write_in_fork(&seed, "/fork-note.txt", b"written in fork\n").await; - let resolved = rig - .handle - .resolve_session( - Arc::clone(&seed.shared), - seed.base, - "safe-mode session".into(), - ) - .await - .expect("resolve_session"); - let SessionResolveOutcome::Resolved { generation } = resolved else { - panic!("expected Resolved, got {resolved:?}"); - }; - let diff = rig.handle.diff(seed.base, generation).await.expect("diff"); - let outcome = rig - .handle - .apply_session(generation, seed.base, "safe-mode session".into()) - .await - .expect("apply_session"); - (diff, outcome) - }); - - // The pre-apply diff already shows the new file, before anything landed. - assert!(diffable_base - .iter() - .any(|change| change.path == Path::new("fork-note.txt"))); - - let PromoteOutcome::Promoted { old_tree, .. } = outcome else { - panic!("expected Promoted, got {outcome:?}"); - }; - assert!(old_tree.is_some(), "a real change must swap the tree"); - assert_eq!( - std::fs::read(rig.repo_path().join("fork-note.txt")).expect("landed file"), - b"written in fork\n" - ); - rig.finish(); -} - -/// Safe Mode: discarding a resolved session (never calling `apply_session`) -/// must leave the real tree completely untouched. -#[test] -fn resolved_session_left_unapplied_leaves_zero_trace() { - let rig = Rig::start(); - rig.runtime.block_on(async { - let seed = rig.handle.fork().await.expect("fork"); - write_in_fork(&seed, "/fork-note.txt", b"written in fork\n").await; - let resolved = rig - .handle - .resolve_session( - Arc::clone(&seed.shared), - seed.base, - "safe-mode session".into(), - ) - .await - .expect("resolve_session"); - assert!(matches!(resolved, SessionResolveOutcome::Resolved { .. })); - }); - assert!(!rig.repo_path().join("fork-note.txt").exists()); + assert_watcher_recovers_after_swap(&rig); rig.finish(); } diff --git a/crates/acyclic-engine/tests/merge.rs b/crates/acyclic/tests/merge.rs similarity index 89% rename from crates/acyclic-engine/tests/merge.rs rename to crates/acyclic/tests/merge.rs index 85791b0..415175a 100644 --- a/crates/acyclic-engine/tests/merge.rs +++ b/crates/acyclic/tests/merge.rs @@ -15,15 +15,14 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; -use acyclic_engine::config::Config; -use acyclic_engine::fork::ForkSeed; -use acyclic_engine::index::{Attribution, CheckpointKind, Index}; -use acyclic_engine::merge::{self, ConflictKind, Entry, Reason}; -use acyclic_engine::pipeline::{self, PipelineHandle}; -use acyclic_engine::store::{Store, StorePaths}; -use acyclic_engine::GenerationId; -use acyclic_fs::kernel::{LogicalName, NamespacePath}; -use acyclic_fs::model::VolumeLimits; +use acyclic::config::Config; +use acyclic::fork::ForkSeed; +use acyclic::index::{Attribution, CheckpointKind, Index}; +use acyclic::merge::{self, ConflictKind, Entry, Reason}; +use acyclic::pipeline::{self, PipelineHandle}; +use acyclic::store::{Store, StorePaths}; +use acyclic::GenerationId; +use acyclic_fs::kernel::NamespacePath; use acyclic_fs::{CancellationToken, WorkCounters}; fn fast_config() -> Config { @@ -117,27 +116,19 @@ impl Rig { /// Native capture stores names as POSIX bytes; lookups must use the same /// encoding or they miss (a portable-encoded name is a different key). -fn namespace(path: &str) -> NamespacePath { - let limits = VolumeLimits::default(); - let names = path - .trim_start_matches('/') - .split('/') - .map(|component| { - LogicalName::new( - acyclic_engine::names::encoding(), - acyclic_engine::names::str_to_bytes(component), - limits.maximum_component_bytes, - ) - .expect("logical name") - }) - .collect(); - NamespacePath::new(names, limits).expect("namespace path") +fn namespace(path: &str, config: acyclic_fs::model::VolumeConfig) -> NamespacePath { + acyclic_fs::host_path_to_namespace( + Path::new(path.trim_start_matches('/')), + config.profile, + config.limits, + ) + .expect("namespace path") } async fn fork_write(seed: &ForkSeed, path: &str, bytes: &[u8]) { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; - let ns = namespace(path); + let ns = namespace(path, guard.volume_config()); let existing = guard .lookup_no_follow(&ns, WorkCounters::UNBOUNDED, &cancel) .await @@ -163,8 +154,14 @@ async fn fork_write(seed: &ForkSeed, path: &str, bytes: &[u8]) { async fn fork_remove(seed: &ForkSeed, path: &str) { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard - .remove(namespace(path), None, WorkCounters::UNBOUNDED, &cancel) + .remove( + namespace(path, config), + None, + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("remove in fork overlay"); } @@ -189,6 +186,32 @@ fn p(s: &str) -> PathBuf { PathBuf::from(s) } +#[test] +fn batch_reads_preserve_order_and_non_regular_absence() { + let rig = Rig::start(); + rig.runtime.block_on(async { + let generation = rig.handle.publish_head().await.expect("head"); + let contents = rig + .handle + .read_files( + generation, + vec![p("src/shared.txt"), p("missing"), p("src"), p("bin.dat")], + ) + .await + .expect("batch read"); + assert_eq!( + contents, + vec![ + (p("src/shared.txt"), Some(b"1\n2\n3\n".to_vec())), + (p("missing"), None), + (p("src"), None), + (p("bin.dat"), Some(b"\x00\x01\x02".to_vec())), + ] + ); + }); + rig.finish(); +} + /// The plan over a realistic overlap: disjoint paths on both sides, a file /// merged by content, an identical change, and independent additions under /// one directory. Then M = H + entries holds exactly the expected tree. @@ -295,7 +318,7 @@ fn plan_and_merge_generation_over_real_generations() { .await .expect("diff") .into_iter() - .filter(|c| c.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|c| c.change != acyclic::diff::ChangeKind::MetadataOnly) .map(|c| c.path) .collect(); assert_eq!(changed, plan.landing_paths()); @@ -411,7 +434,7 @@ fn conflicts_are_collected_and_r_carries_markers() { .await .expect("diff") .into_iter() - .filter(|c| c.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|c| c.change != acyclic::diff::ChangeKind::MetadataOnly) .map(|c| c.path) .collect(); let roots = merge::subtree_roots(&changes); @@ -437,7 +460,7 @@ fn conflicts_are_collected_and_r_carries_markers() { .await .expect("diff") .into_iter() - .filter(|c| c.change != acyclic_engine::diff::ChangeKind::MetadataOnly) + .filter(|c| c.change != acyclic::diff::ChangeKind::MetadataOnly) .map(|c| c.path) .collect(); assert!(remaining.is_empty(), "overlay must equal R: {remaining:?}"); @@ -460,9 +483,10 @@ fn refusals_name_paths_and_reasons() { { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard .remove( - namespace("/src/same.txt"), + namespace("/src/same.txt", config), None, WorkCounters::UNBOUNDED, &cancel, @@ -471,7 +495,7 @@ fn refusals_name_paths_and_reasons() { .expect("rm"); guard .create_symbolic_link( - namespace("/src/same.txt"), + namespace("/src/same.txt", config), bytes::Bytes::from_static(b"shared.txt"), WorkCounters::UNBOUNDED, &cancel, @@ -578,16 +602,25 @@ fn nested_directory_subtree_copies_into_merge_generation() { { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard - .create_directory(namespace("/docs"), WorkCounters::UNBOUNDED, &cancel) + .create_directory(namespace("/docs", config), WorkCounters::UNBOUNDED, &cancel) .await .expect("mkdir"); guard - .create_directory(namespace("/docs/deep"), WorkCounters::UNBOUNDED, &cancel) + .create_directory( + namespace("/docs/deep", config), + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("mkdir"); guard - .create_directory(namespace("/docs/deep/er"), WorkCounters::UNBOUNDED, &cancel) + .create_directory( + namespace("/docs/deep/er", config), + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("mkdir"); } @@ -634,12 +667,17 @@ fn materialize_paths_writes_a_generation_into_a_directory() { { let cancel = CancellationToken::new(); let mut guard = seed.shared.lock().await; + let config = guard.volume_config(); guard - .create_directory(namespace("/docs"), WorkCounters::UNBOUNDED, &cancel) + .create_directory(namespace("/docs", config), WorkCounters::UNBOUNDED, &cancel) .await .expect("mkdir"); guard - .create_directory(namespace("/docs/deep"), WorkCounters::UNBOUNDED, &cancel) + .create_directory( + namespace("/docs/deep", config), + WorkCounters::UNBOUNDED, + &cancel, + ) .await .expect("mkdir"); } diff --git a/crates/acyclic-engine/tests/pipeline.rs b/crates/acyclic/tests/pipeline.rs similarity index 75% rename from crates/acyclic-engine/tests/pipeline.rs rename to crates/acyclic/tests/pipeline.rs index f52982e..100bf60 100644 --- a/crates/acyclic-engine/tests/pipeline.rs +++ b/crates/acyclic/tests/pipeline.rs @@ -11,11 +11,11 @@ use std::path::{Path, PathBuf}; use std::time::Duration; -use acyclic_engine::config::Config; -use acyclic_engine::diff::{self, ChangeKind}; -use acyclic_engine::index::{Attribution, CheckpointKind, Index}; -use acyclic_engine::pipeline; -use acyclic_engine::store::{Store, StorePaths}; +use acyclic::config::Config; +use acyclic::diff::{self, ChangeKind}; +use acyclic::index::{Attribution, CheckpointKind, Index}; +use acyclic::pipeline; +use acyclic::store::{Store, StorePaths}; fn read_only_index(path: &Path) -> Index { Index::open(path).expect("open index") @@ -33,6 +33,69 @@ fn fast_config() -> Config { } } +#[test] +fn native_startup_failure_is_deferred_until_filesystem_demand() { + let repo = tempfile::tempdir().expect("repo"); + let stores = tempfile::tempdir().expect("stores"); + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("paths"); + let runtime = tokio::runtime::Runtime::new().expect("runtime"); + let store = runtime + .block_on(Store::init(repo.path(), paths.clone())) + .expect("init store"); + let index = Index::open(&paths.index_db()).expect("index"); + std::fs::remove_dir(repo.path()).expect("remove empty repo before watcher opens"); + let (handle, thread) = pipeline::spawn(store, index, fast_config()); + + let status = runtime + .block_on(handle.status()) + .expect("metadata-only startup remains available"); + assert_eq!(status.state, pipeline::State::NeedsBaseline); + let error = runtime + .block_on(handle.checkpoint(CheckpointKind::Manual, Attribution::default())) + .expect_err("filesystem demand must fail"); + assert!(error.to_string().contains("root identity")); + runtime.block_on(handle.shutdown()).expect("shutdown"); + thread.join().expect("pipeline thread"); +} + +/// Starting the daemon and polling metadata must stay constant in repository +/// size. Even an enabled idle timer cannot authenticate content before a real +/// content operation asks for it. +#[test] +fn idle_metadata_traffic_never_establishes_the_baseline() { + let repo = tempfile::tempdir().expect("repo"); + let stores = tempfile::tempdir().expect("stores"); + std::fs::write(repo.path().join("a.txt"), b"one\n").expect("seed"); + + let paths = StorePaths::for_repo(repo.path(), Some(stores.path())).expect("paths"); + let runtime = tokio::runtime::Runtime::new().expect("runtime"); + let store = runtime + .block_on(Store::init(repo.path(), paths.clone())) + .expect("init store"); + let index = Index::open(&paths.index_db()).expect("index"); + let config = Config { + auto_checkpoint_idle_ms: 20, + ..fast_config() + }; + let (handle, thread) = pipeline::spawn(store, index, config); + + runtime.block_on(async { + let deadline = tokio::time::Instant::now() + Duration::from_millis(150); + while tokio::time::Instant::now() < deadline { + let status = handle.status().await.expect("status"); + assert_eq!(status.state, pipeline::State::NeedsBaseline); + assert_eq!(status.last_checkpoint, None); + tokio::time::sleep(Duration::from_millis(10)).await; + } + handle.shutdown().await.expect("shutdown"); + }); + thread.join().expect("pipeline thread"); + assert!(read_only_index(&paths.index_db()) + .latest() + .expect("query") + .is_none()); +} + #[test] fn checkpoint_rewind_journey() { let repo = tempfile::tempdir().expect("repo"); @@ -106,15 +169,25 @@ fn checkpoint_rewind_journey() { ); // Diff before → after names exactly the changed paths. + handle + .checkpoint(CheckpointKind::Post, Attribution::default()) + .await + .expect("checkpoint after rewind"); let status = handle.status().await.expect("status"); assert_eq!(status.state, pipeline::State::Ready); + #[cfg(not(target_os = "macos"))] + assert_eq!(status.watcher.invalidations, 0, "{status:?}"); + #[cfg(target_os = "macos")] + assert!(status.watcher.invalidations <= 1, "{status:?}"); handle.shutdown().await.expect("shutdown"); (before.generation, after.generation) }); thread.join().expect("pipeline thread"); // Diff runs against the reopened store (no daemon needed). - let store = runtime.block_on(Store::open(paths)).expect("reopen"); + let store = runtime + .block_on(Store::open(repo.path(), paths)) + .expect("reopen"); let changes = runtime .block_on(diff::diff(&store, pre_generation, post_generation)) .expect("diff"); @@ -129,9 +202,8 @@ fn checkpoint_rewind_journey() { assert!(by_name.contains(&(PathBuf::from(".env"), ChangeKind::Removed))); } -/// The safety net for hosts with no lifecycle-hook API (Claude Desktop over -/// MCP): an edit with no `handle.checkpoint()` call at all still gets -/// checkpointed once the idle timer fires. +/// Once a consumer establishes the authenticated baseline, the safety net for +/// hosts with no lifecycle-hook API still checkpoints later unannounced edits. #[test] fn idle_timer_auto_checkpoints_changes_no_host_asked_for() { let repo = tempfile::tempdir().expect("repo"); @@ -157,10 +229,13 @@ fn idle_timer_auto_checkpoints_changes_no_host_asked_for() { let (handle, thread) = pipeline::spawn(store, index, config); runtime.block_on(async { - // A round trip first, so baseline capture is guaranteed done before - // the edit — otherwise the edit can race into the baseline itself - // and leave nothing pending for the idle timer to find. - let baseline_row = handle.status().await.expect("status").last_checkpoint; + let baseline_row = Some( + handle + .checkpoint(CheckpointKind::Baseline, Attribution::default()) + .await + .expect("baseline") + .row_id, + ); // No hook, no explicit checkpoint call — just an edit, like a host // with no lifecycle-hook API would produce. @@ -227,10 +302,12 @@ fn zero_auto_checkpoint_idle_ms_disables_the_idle_timer() { let (handle, thread) = pipeline::spawn(store, index, config); runtime.block_on(async { - // Baseline done first, so the edit is guaranteed to be pending - // rather than absorbed into the baseline (which would pass this - // test for the wrong reason). - handle.status().await.expect("status"); + // An explicit checkpoint is the readiness boundary when background + // auto capture is disabled; status deliberately remains scan free. + handle + .checkpoint(CheckpointKind::Manual, Attribution::default()) + .await + .expect("baseline checkpoint"); std::fs::write(repo.path().join("a.txt"), b"two\n").expect("edit"); tokio::time::sleep(Duration::from_millis(500)).await; handle.shutdown().await.expect("shutdown"); @@ -238,9 +315,10 @@ fn zero_auto_checkpoint_idle_ms_disables_the_idle_timer() { thread.join().expect("pipeline thread"); let index = read_only_index(&paths.index_db()); - // Only the baseline row from init: the idle timer never ran. + // The explicit readiness request may be a baseline or a noop immediately + // after it; the disabled idle timer must add no later row. let latest = index.latest().expect("query").expect("a row exists"); - assert_eq!(latest.kind, CheckpointKind::Baseline); + assert_eq!(latest.kind, CheckpointKind::Noop); } /// An idle tick that has drained an edit into the checkout but not yet @@ -274,7 +352,10 @@ fn requested_checkpoint_records_changes_an_idle_tick_already_drained() { let (handle, thread) = pipeline::spawn(store, index, config); let outcome = runtime.block_on(async { - handle.status().await.expect("status"); + handle + .checkpoint(CheckpointKind::Manual, Attribution::default()) + .await + .expect("establish baseline"); std::fs::write(repo.path().join("a.txt"), b"two\n").expect("edit"); tokio::time::sleep(Duration::from_millis(600)).await; let outcome = handle @@ -321,7 +402,10 @@ fn periodic_requests_do_not_postpone_the_auto_checkpoint() { let (handle, thread) = pipeline::spawn(store, index, config); runtime.block_on(async { - handle.status().await.expect("status"); + handle + .checkpoint(CheckpointKind::Baseline, Attribution::default()) + .await + .expect("baseline"); std::fs::write(repo.path().join("a.txt"), b"two\n").expect("edit"); // Poll far more often than the idle interval, for far longer. for _ in 0..40 { @@ -354,8 +438,12 @@ fn enqueued_checkpoint_survives_immediate_shutdown() { let (handle, thread) = pipeline::spawn(store, index, fast_config()); runtime.block_on(async { - // Sync on Ready first: a write issued during the startup baseline is - // captured by it, and the enqueued checkpoint would be a noop. + // An explicit checkpoint is the readiness boundary; status remains + // metadata only and does not scan the repository. + handle + .checkpoint(CheckpointKind::Manual, Attribution::default()) + .await + .expect("baseline checkpoint"); let status = handle.status().await.expect("status"); assert_eq!(status.state, pipeline::State::Ready); std::fs::write(repo.path().join("file.txt"), b"after\n").expect("edit"); @@ -400,7 +488,19 @@ fn single_path_restore_leaves_the_rest_alone() { std::fs::write(root.join("src/main.rs"), b"v1\n").expect("seed"); std::fs::write(root.join("src/deep/a.txt"), b"a1\n").expect("seed"); std::fs::write(root.join("src/deep/b.txt"), b"b1\n").expect("seed"); + std::fs::write(root.join("src/deep/hard-a.txt"), b"linked\n").expect("seed hard link"); + std::fs::hard_link( + root.join("src/deep/hard-a.txt"), + root.join("src/deep/hard-b.txt"), + ) + .expect("hard link"); + std::fs::hard_link( + root.join("src/deep/hard-a.txt"), + root.join("outside-hard.txt"), + ) + .expect("hard link outside restored subtree"); std::fs::write(root.join("other.txt"), b"keep\n").expect("seed"); + std::fs::write(root.join("source.txt"), b"link source\n").expect("seed"); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -425,9 +525,15 @@ fn single_path_restore_leaves_the_rest_alone() { .expect("v1"); // Mutate everything. + // A new hard link can arrive as a lone watcher hint. The SDK must + // request a baseline instead of recording it as an independent file. + std::fs::hard_link(root.join("source.txt"), root.join("late-link.txt")) + .expect("late hard link"); std::fs::write(root.join("src/main.rs"), b"v2\n").expect("edit"); std::fs::write(root.join("src/deep/a.txt"), b"a2\n").expect("edit"); std::fs::remove_file(root.join("src/deep/b.txt")).expect("rm"); + std::fs::remove_file(root.join("src/deep/hard-b.txt")).expect("rm hard link"); + std::fs::write(root.join("src/deep/hard-a.txt"), b"changed\n").expect("edit hard link"); std::fs::write(root.join("src/deep/c.txt"), b"c2\n").expect("add"); std::fs::write(root.join("other.txt"), b"changed\n").expect("edit"); std::fs::write(root.join("new.txt"), b"new\n").expect("add"); @@ -454,10 +560,7 @@ fn single_path_restore_leaves_the_rest_alone() { .restore_path(target(v1.row_id), "src/main.rs".into()) .await .expect("restore file"); - assert_eq!( - outcome.action, - acyclic_engine::rewind::RestoreAction::Restored - ); + assert_eq!(outcome.action, acyclic::rewind::RestoreAction::Restored); assert_eq!( std::fs::read(root.join("src/main.rs")).expect("read"), b"v1\n" @@ -486,20 +589,50 @@ fn single_path_restore_leaves_the_rest_alone() { b"b1\n" ); assert!(!root.join("src/deep/c.txt").exists()); + std::fs::write(root.join("src/deep/hard-a.txt"), b"relinked\n") + .expect("write restored hard link"); + assert_eq!( + std::fs::read(root.join("src/deep/hard-b.txt")).expect("read restored hard link"), + b"relinked\n" + ); + assert_eq!( + std::fs::read(root.join("outside-hard.txt")).expect("read untouched outside link"), + b"changed\n" + ); assert_eq!( std::fs::read(root.join("other.txt")).expect("read"), b"changed\n" ); + // Batched callers may contain duplicates and descendants of a root. + // The pipeline must materialize and reconcile the minimal root once. + std::fs::write(root.join("src/deep/a.txt"), b"a3\n").expect("edit again"); + let restored = handle + .restore_paths( + target(v1.row_id), + vec![ + "src/deep/a.txt".into(), + "src/deep".into(), + "src/deep".into(), + ], + false, + Some("deduplicated restore".into()), + ) + .await + .expect("restore minimal roots"); + assert_eq!(restored.outcomes.len(), 1); + assert_eq!(restored.outcomes[0].path, Path::new("src/deep")); + assert_eq!( + std::fs::read(root.join("src/deep/a.txt")).expect("read"), + b"a1\n" + ); + // A path absent at the checkpoint is removed. let outcome = handle .restore_path(target(v1.row_id), "new.txt".into()) .await .expect("restore absent"); - assert_eq!( - outcome.action, - acyclic_engine::rewind::RestoreAction::Removed - ); + assert_eq!(outcome.action, acyclic::rewind::RestoreAction::Removed); assert!(!root.join("new.txt").exists()); #[cfg(unix)] diff --git a/crates/acyclic/tests/restart_rewind.rs b/crates/acyclic/tests/restart_rewind.rs new file mode 100644 index 0000000..d1f9686 --- /dev/null +++ b/crates/acyclic/tests/restart_rewind.rs @@ -0,0 +1,60 @@ +//! Recovery has to run through the real CLI entry point before it can +//! canonicalize a repository name temporarily absent during Windows rewind. + +#[cfg(windows)] +#[test] +fn cli_recovers_a_missing_repo_before_loading_config() -> Result<(), Box> { + for default_repo in [false, true] { + let work = tempfile::tempdir()?; + let parent = work.path().canonicalize()?; + let repo = parent.join("repo"); + let scratch = parent.join(format!(".repo.{}-swap", acyclic::product::NAME)); + let staged = parent.join("staged"); + std::fs::create_dir(&scratch)?; + std::fs::write(scratch.join("old.txt"), b"original")?; + std::fs::create_dir(&staged)?; + let store = parent.join("custom-store"); + let trash = store.join("trash"); + std::fs::create_dir_all(&trash)?; + let journal = store.join("rewind-journal.json"); + std::fs::write( + &journal, + serde_json::json!({ + "target_generation": "00".repeat(32), + "repo_root": repo, + "tmp": staged, + "phase": "Swapping", + "carried": [], + }) + .to_string(), + )?; + let locator = parent.join(format!(".repo.{}-rewind.json", acyclic::product::NAME)); + std::fs::write( + &locator, + serde_json::json!({ + "repo_root": repo, + "journal": journal, + "trash": trash, + "trash_ttl_days": 30, + }) + .to_string(), + )?; + + let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_acyclic")); + command.env("HOME", &parent); + command.current_dir(if default_repo { &scratch } else { &parent }); + if !default_repo { + command.arg("--repo").arg("repo"); + } + let output = command.arg("policy").output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(std::fs::read(repo.join("old.txt"))?, b"original"); + assert!(!journal.exists()); + assert!(!locator.exists()); + } + Ok(()) +} diff --git a/docs/design/00-overview.md b/docs/design/00-overview.md index 76d9977..e9a3b0e 100644 --- a/docs/design/00-overview.md +++ b/docs/design/00-overview.md @@ -25,7 +25,6 @@ Each launch is one engine increment plus one coherent story, ordered by dependen | 1 | Rewind | Merkle snapshot store + host hooks | Never fear letting the agent loose | [01-rewind.md](01-rewind.md) | | 2 | Timeline | Turn-linked metadata index | The repo at any point in the conversation | [02-timeline.md](02-timeline.md) | | 3 | Forks | Copy-on-write materialization | N parallel attempts, pick the winner | [03-forks.md](03-forks.md) | -| 4 | Safe Mode | Session redirection + write interposition | Agents on the codebase, not agents' mistakes in it | [04-safe-mode.md](04-safe-mode.md) | | 5 | Monorepo | Merkle-aware content + symbol index | The repo that finally works with agents | [05-monorepo.md](05-monorepo.md) | Cross-cutting, not a launch: **[Speculation](08-speculation.md)** — the daemon computes what the agent is about to ask for (the session brief, a turn summary) in the time when nobody is waiting. Off by default. @@ -33,7 +32,6 @@ Cross-cutting, not a launch: **[Speculation](08-speculation.md)** — the daemon Sequencing rationale: - Launches 1–2 ship on the cheap engine (hooks + snapshot store) without committing to how forks work. The hard CoW decision only becomes due at Launch 3. -- Launch 4 depends on Launch 3's fork engine (dry-run and scratch trees are fork features). - Launch 5 is dependency-independent (separate index engine) — pull it forward if monorepo teams become the target buyer. ## The load-bearing decision @@ -45,14 +43,13 @@ Second load-bearing constraint, from compliance: **purge-through-history and sna ## Strategic risks (from the design review) 1. **Launch 1 collides with native host features.** Claude Code ships its own checkpoint/rewind. Differentiation must be the headline, not fine print: we capture what Bash did (installs, migrations, generated files), untracked/gitignored state, cross-session persistence, cross-host consistency — and checkpoints become forks. Expect hosts to keep commoditizing the basic rewind; the moat is the Merkle/CoW engine and Launches 3/5. -2. **Unverified host-API assumptions** — validate before public promises: (a) transparent tool interception for indexed search (hook rewrite limits differ per host); (b) session redirection for dry-run (path display, git status confusion); (c) checkpoint alignment in hosts without lifecycle hooks. +2. **Unverified host-API assumptions** — validate before public promises: (a) transparent tool interception for indexed search (hook rewrite limits differ per host); (b) checkpoint alignment in hosts without lifecycle hooks. 3. **Naming**: repo is `graphcoder-plugin`, CLI is `acyclic`, and Graphcoder is a different product on the roadmap. Resolve before launch. ## Settled since this was written -1. **Fork engine mechanism** — decided the opposite way to the lean recorded here. Mounts shipped; there is no reflink or `clonefile` path in the codebase, and the fallback when no mount provider is available is a **full copy**, not a reflink. Forks are routes inside one kernel mount rather than N mounts. See `implementation-forks.md`. +1. **Fork engine mechanism** — mounts shipped. Forks require the native provider and are routes inside one kernel mount rather than N mounts. See `implementation-forks.md`. 2. **Checkpoint alignment in hosts without lifecycle hooks** — resolved by the MCP adapter plus the `auto_checkpoint_idle_ms` timer. MCP is a second adapter shape this doc's thesis line does not yet mention. -3. **Session redirection for dry-run** — built and tested as Safe Mode, mount-only. ## Open questions (not yet settled) @@ -61,5 +58,4 @@ Second load-bearing constraint, from compliance: **purge-through-history and sna 3. **Naming**: repo is `graphcoder-plugin`, CLI is `acyclic`, and Graphcoder is a different product on the roadmap. Still unresolved. 4. **Repo visibility** — this repo is private while its `acyclic-fs` dependency is public, which blocks the curl installer, the attestations, and the open-source claim. Carried in `06-installation.md` and `07-compliance.md`; it belongs at overview level because it gates positioning, not just packaging. 5. **The upstream retention dependency.** GC, purge and enforced retention all wait on an `acyclic-fs` retention-release fact that does not exist. This is the single largest gap between the compliance story and the code, and it is not ours to close. -6. **Degrade or refuse without a mount provider.** Forks silently fall back to full copies; Safe Mode refuses to start. A repo with `dry_run = true` checked in therefore runs against the real tree on a host without mounts. Which of those two behaviours is right has never been decided as a policy. 7. **What remains unvalidated at scale.** Store performance on a real 10GB tree is still the first thing to validate, as it was when this doc was written. The latency gate runs 20k files / 256 MB, and `init` baseline capture runs ~230 s/GiB. diff --git a/docs/design/01-rewind.md b/docs/design/01-rewind.md index 6687e99..dcf23b8 100644 --- a/docs/design/01-rewind.md +++ b/docs/design/01-rewind.md @@ -116,7 +116,7 @@ leads with what it can't do: 4. **Hardcoded constants that are really policy** — mutation batch size, maximum capture paths, extent spans, watch queue depth, prompt excerpt bytes. None are configurable and none are documented. -5. **Excluded paths are invisible to forks and Safe Mode sessions.** A secret +5. **Excluded paths are invisible to forks.** A secret kept out of the store is also absent from every fork, which is either the correct safety property or a broken build, depending on the secret. *No lean.* diff --git a/docs/design/02-timeline.md b/docs/design/02-timeline.md index a132ce4..3a98b2a 100644 --- a/docs/design/02-timeline.md +++ b/docs/design/02-timeline.md @@ -35,7 +35,7 @@ Same engine, adds the time dimension. Fast-follow; bundles into Launch 1 if it l ## Status (2026-09-06): shipped -Implemented in `acyclic-engine` (index), the daemon, the CLI, and the Claude +Implemented in `acyclic` (index), the daemon, the CLI, and the Claude Code adapter; `tests/acceptance/timeline.sh` covers all four acceptance criteria and runs in `run-all.sh`. diff --git a/docs/design/03-forks.md b/docs/design/03-forks.md index c02cc38..0a9a5de 100644 --- a/docs/design/03-forks.md +++ b/docs/design/03-forks.md @@ -23,12 +23,9 @@ The fork engine. The headline demo launch. This section previously read as a forward plan. Launch 3 shipped; what follows is what exists, including where it diverged from the plan. -- **Copy-on-write materialization** — decided as **mounts**, the opposite of the - lean this doc recorded. There is no reflink or `clonefile` path in the - codebase. All forks are routes inside **one** kernel mount, not N mounts. - Without a mount provider every fork is a **full copy**, so the O(1) claim holds - in mount mode only — the degraded path is keyed on mount availability, not on - filesystem reflink support. +- **Copy-on-write materialization** — implemented as native mounts. There is no + reflink, `clonefile`, or full-copy path. All forks are routes inside one + kernel mount, and a supported native mount provider is required. - **Fork workspace management** — partially. Teardown and placement exist; per-fork port allocation and env provisioning were dropped as out of scope. - **Subagent orchestration wiring** — shipped, but as a prompt-level skill and a diff --git a/docs/design/04-safe-mode.md b/docs/design/04-safe-mode.md deleted file mode 100644 index 98f361d..0000000 --- a/docs/design/04-safe-mode.md +++ /dev/null @@ -1,122 +0,0 @@ -# Launch 4 — Safe Mode - -Fork-engine features for trust-sensitive teams. Depends on Launch 3. - -Status: **built, mount-dependent.** Claims below are tagged *shipped*, -*not built*, or *diverged* where what exists differs from what was planned. - -## Features - -11. **Dry-run mode** — the whole session runs against a fork; nothing hits the - real tree until the dev approves the final diff. *Shipped, off by default.* -12. **Ephemeral scratch trees** — disposable full-repo copies that vanish on - session end. *Diverged: the auto-drop machinery exists and runs at session - end, but nothing creates a tagged scratch tree. There is no user-facing - scratch verb and no caller sets the session id, so this feature has a - working destructor and no constructor.* -13. **Guarded paths** — writes the agent can't make, enforced at the filesystem - layer rather than by prompt hope. *Shipped for mounted forks and Safe Mode - sessions only. See the first open question — outside those, it is a no-op.* - -## User journey - -The journey as originally written is still the target. Two steps do not work -as described today: - -1. Maya's team checks in `.acyclic/config.toml`: dry-run on, `.env` and - `migrations/` guarded. Everyone who clones inherits the policy. *Shipped.* -2. A teammate starts a session rooted in a fork; paths look normal, the real - tree is untouched. *Shipped, if the host has a mount provider.* -3. The agent grabs a scratch tree for a destructive codemod and lets it vanish. - *Not built — see feature 12.* -4. The agent tries to edit `.env`; the write is refused at the interposition - layer. *Shipped inside the session. Note the macOS caveat below: the refused - write can still report success to the shell.* -5. The session ends with a final diff the teammate reviews and approves. - *Diverged: `session-resolve`, `session-apply` and `session-discard` are - hidden CLI-only verbs. No host adapter wires them, they are not MCP tools, - and session end does not resolve the session — the shadow mount stays up.* - -## What was built - -- **Session redirection** — *shipped.* The session is rooted in a shadow mount. -- **Approval-gated apply** — *shipped as CLI verbs, unwired.* See journey 5. -- **Write interposition for guarded paths** — *diverged, and in the opposite - order to the plan.* The plan was to ship hook-level blocking first and - upgrade with the mount. Hook-level was never built: the hook contract is to - never block and always exit 0, and it never reads `guarded_paths`. Only the - mount-level guard exists, so enforcement and Safe Mode arrived together. -- **Policy configuration** — *shipped, minus scratch-tree limits*, which have - no config key because there are no scratch trees. -- **Scratch-tree lifecycle** — *diverged.* There is no janitor process. - Session-owned forks are dropped best-effort in the session-end handler, and a - crashed daemon's shadow mount is reaped opportunistically on the next CLI - invocation — an on-demand sweep, not a cycle. - -### Constraints that were never written down - -- **One Safe Mode session per repo.** A second is refused outright. -- **Apply requires an unmoved mainline.** If the real tree moved during the - session, the choice is rewind to base or start again — the doc promised an - atomic apply with no such condition. -- **Mainline capture is suspended for the whole session.** A dry-run session - records no mainline checkpoints. -- **Guarded paths are excluded from the session diff entirely**, so the - approval view never shows them. -- **The guard surface is wider than "writes"**: it covers rename and hard-link - on both source and destination, and unconditionally refuses AppleDouble - sidecars. -- **Reflink acceleration is disabled whenever any guard is configured**, so - guarded repos silently fall back to read-and-write copies. - -## Acceptance criteria - -- A dry-run session is indistinguishable from a normal one to the agent. - **Unverified** — there is no live-host Safe Mode test; the suite drives the - CLI with a synthetic host name. -- Guarded-path writes are refused with a legible error the agent can act on. - **Partially met, with a known correctness caveat**: on macOS a refused write - can still return exit 0 to the shell because of NFS write-back caching, so - the agent may see success where the filesystem saw refusal. -- Rejected sessions leave zero trace on the real tree. **Met** — and the - converse is now explicit: a crash discards all session work. Zero trace and - zero recovery are the same property. -- Orphaned scratch trees collected within one janitor cycle. **Not met** — no - janitor exists. - -## Open questions (not yet settled) - -1. **Guarded paths are a no-op in the default configuration.** With - `dry_run = false` (the default) and no fork, `guarded_paths` is never - consulted and the agent can write `.env` freely. A team that checks in - guarded paths and reads the README table has every reason to believe - otherwise. This is the highest-severity gap in this launch. *No lean — it - is either a documentation fix or a feature.* -2. **Silent degradation on a host without mounts.** Safe Mode refuses to start, - the session-start op fails, and the hook prints to stderr and exits 0. A - repo with `dry_run = true` checked in therefore runs against the real tree - with no visible failure. *Current lean: this should be loud — a refusal to - proceed rather than a warning nobody reads.* -3. **Copy-mode forks are unguarded**, because the guard wraps mount routes - only. On a mount-less host guarded paths are unenforced even for explicit - forks. *No lean.* -4. **Do the approval verbs get wired into hosts?** They are the visible half of - the feature and are currently reachable only by someone who reads the hidden - CLI surface. *Current lean: yes, and Safe Mode should not be promoted until - they are.* -5. **Is the NFS write-back caveat acceptable?** It weakens the acceptance - criterion it contradicts, and the failure direction is the dangerous one — - the agent believes a guarded write succeeded. *No lean.* -6. **Should scratch trees be built or dropped from the feature list?** The - destructor exists; the constructor does not. *Current lean: drop the claim - until there is a verb.* -7. **Hardcoded constants that are really policy**: the 16-fork cap, the 5s reap - probe, and the 2s pre-tool wait after which the tool proceeds uncheckpointed. - -## Open design risks - -- Session redirection fights host assumptions (path display, git status - confusion). Still unvalidated against a real host. -- Hook-level guarded paths may not be worth shipping alone. **This was - settled by not doing it** — the mount-level guard shipped instead, which - means guarded paths inherit every one of Safe Mode's platform constraints. diff --git a/docs/design/05-monorepo.md b/docs/design/05-monorepo.md index b5d850f..2352cbc 100644 --- a/docs/design/05-monorepo.md +++ b/docs/design/05-monorepo.md @@ -73,7 +73,7 @@ which reads as though Launch 5 exists. 3. **`exclude` blinds any future index.** Excluded paths never enter a checkpoint, so a Merkle-aware index can never answer for them, and the answer it gives will be silently incomplete rather than refused. *No lean.* -4. **Forks and Safe Mode sessions do not see excluded paths either**, which +4. **Forks do not see excluded paths either**, which bears directly on "searches against a fork must answer from that tree's state". *No lean.* The converse is now a known gap: paths *created* inside a fork are captured regardless of `exclude` — see diff --git a/docs/design/06-installation.md b/docs/design/06-installation.md index 37f9232..59accf6 100644 --- a/docs/design/06-installation.md +++ b/docs/design/06-installation.md @@ -206,7 +206,7 @@ The two adapter shapes above — lifecycle hooks, and JSON-based MCP registratio ## Configuration -- Per-repo: `.acyclic/config.toml` — checked in, so teams share policy: checkpoint granularity, guarded paths, dry-run default, store size caps, retention TTLs. +- Per-repo: `.acyclic/config.toml` — checked in, so teams share policy: checkpoint granularity, guarded paths, store size caps, retention TTLs. - Per-machine: `~/.config/acyclic/` — defaults. - Zero config is a supported state — defaults are safe everywhere. @@ -348,4 +348,4 @@ of them, so they come first. ## Design commitment -This resolves the mechanism question in favor of **CLI-as-core with host adapters** (not MCP-as-core, not per-host deep builds). That choice is what makes OpenCode and future hosts nearly free. MCP wraps the CLI where a host has no other extension point — Claude Desktop's `acyclic mcp` adapter is exactly that: every MCP tool is a thin translation into the same `acyclic-proto::Op` the CLI and hooks already send the daemon, no engine logic lives in the MCP layer itself. See the Claude Desktop row above for why it's marked experimental rather than promoted to a supported host yet. +This resolves the mechanism question in favor of **CLI-as-core with host adapters** (not MCP-as-core, not per-host deep builds). That choice is what makes OpenCode and future hosts nearly free. MCP wraps the CLI where a host has no other extension point — Claude Desktop's `acyclic mcp` adapter is exactly that: every MCP tool is a thin translation into the same private `proto::Op` the CLI and hooks already send the daemon, no engine logic lives in the MCP layer itself. See the Claude Desktop row above for why it's marked experimental rather than promoted to a supported host yet. diff --git a/docs/design/07-compliance.md b/docs/design/07-compliance.md index e9a5432..1d277df 100644 --- a/docs/design/07-compliance.md +++ b/docs/design/07-compliance.md @@ -113,9 +113,8 @@ Two honest limits on that pitch: - Hosts without a prompt hook — Claude Desktop, VS Code, OpenCode — get checkpoints without turn linkage, so the headline evidence is thinner exactly where the adapter is thinnest. -- The "a human approved it before it reached the real tree" half rests on Safe - Mode, which is off by default, refuses to start without a mount provider, and - whose approval verbs are hidden CLI-only and wired into no host. +- V1 does not interpose a human approval gate on ordinary agent writes. Explicit + forks isolate work only when the user or host chooses that workflow. ## When the cloud arrives (v2) @@ -149,4 +148,4 @@ as a silent default — and the engine remains fully usable with the cloud off. claiming it.* 6. **Is `exclude` enough as the only shipped control?** It is prefix-based, start-time-bound, and non-retroactive, and excluded paths are invisible to - forks and Safe Mode sessions. *No lean.* + forks. *No lean.* diff --git a/docs/design/implementation-forks.md b/docs/design/implementation-forks.md index 8c76d4c..233c08c 100644 --- a/docs/design/implementation-forks.md +++ b/docs/design/implementation-forks.md @@ -3,8 +3,8 @@ Ships `03-forks.md`: N-way local forks, pick the winner. Built on fs's native mounts — the resolution of the plan's open CoW question is **mounts, not reflinks**: fs now ships qualified FUSE-T/FUSE/ProjFS drivers and the mount -path is the strategic asset (lazy hydration, Launch 4's filesystem-level -enforcement). Reflinks remain a fallback if mount UX disappoints. +path is the strategic asset (lazy hydration and filesystem-level guarded-path +enforcement). A native mount provider is required. ## How a fork works (the fs wiring, verified against fsd's usage) diff --git a/docs/design/implementation-merge.md b/docs/design/implementation-merge.md index 2f39f48..6dc9ff5 100644 --- a/docs/design/implementation-merge.md +++ b/docs/design/implementation-merge.md @@ -32,8 +32,7 @@ plan. **Out (later launches, listed so nobody re-litigates them mid-build):** rename detection; semantic or AST merges; markers written into the -*mainline*; Safe Mode `apply_session` onto a moved mainline (keeps the -unmoved-mainline rule); merging directory-ancestry overlaps; live rebase +*mainline*; merging directory-ancestry overlaps; live rebase of a fork while the mainline moves. ## Definitions @@ -132,10 +131,8 @@ When at least one file is `Conflicted` and nothing is refused: 1. Build **R** = M with the conflicted files replaced by their marker-bearing content. R is F rebased onto H. -2. Write R's differences from F into the fork: for a mount fork, through - the fork's `SharedLocalCheckout` (the mount serves it live); for a - copy fork, into the copy directory as well. `capture_copy` already - handles copy → overlay at promote. +2. Write R's differences from F through the mounted workspace so the mount + and kernel caches observe every change. 3. Set `fork.base = H`. Record R as `fork rebased onto (N conflict(s))`. 4. Record the open conflict on the fork: `{ base: B, ours: F, theirs: H, @@ -168,12 +165,12 @@ consistency: the fork survives every non-landing outcome. ## Where the code goes ``` -crates/acyclic-engine/src/merge.rs NEW — entry table, merge3 port, marker scan -crates/acyclic-engine/src/pipeline.rs BuildGeneration { from, entries } request -crates/acyclic-engine/src/fork.rs ForkSeed.base becomes mutable; OpenConflict -crates/acyclic-engine/src/config.rs [merge] max_file_bytes +crates/acyclic/src/merge.rs NEW — entry table, merge3 port, marker scan +crates/acyclic/src/pipeline.rs BuildGeneration { from, entries } request +crates/acyclic/src/fork.rs ForkSeed.base becomes mutable; OpenConflict +crates/acyclic/src/config.rs [merge] max_file_bytes crates/acyclic/src/server.rs replay_onto_head → merge_onto_head; rebase path -crates/acyclic-proto/src/lib.rs PromoteInfo.merged_files; ForkInfo.conflict +crates/acyclic/src/lib.rs PromoteInfo.merged_files; ForkInfo.conflict crates/acyclic/src/main.rs promote / forks output crates/acyclic/src/install.rs skill: PARTITION + conflict resolution tests/acceptance/merge.sh G4/G5-adjacent flips; G13–G26 added @@ -206,7 +203,7 @@ before anything is written. One pipeline request `BuildGeneration { from: GenerationId, entries }`: scratch checkout at `from` (`scratch_checkout` exists), write each entry through the SDK checkout API the fork tests already use, `checkpoint()` -it unpublished (as `resolve_session` does), `record_generation` it. +it unpublished, then `record_generation` it. M = `BuildGeneration(H, merged ∪ taken)`; R = `BuildGeneration(M, conflicted)`. No head movement, no tree writes. @@ -263,7 +260,7 @@ conflict state, proto fields, CLI lines, marker-scan on re-promote. Skill text update. **C3 — acceptance.** `merge.sh` changes below, run in mount mode on macOS -and copy mode on Linux, plus `forks.sh`, `safe-mode.sh`, and the journey +and copy mode on Linux, plus `forks.sh` and the journey suites unchanged. Update `spec-forks.md`'s out-of-scope line, its stale M4/M5 rows, and the c905be4 "discard" wording in the same commit. @@ -310,10 +307,9 @@ Acceptance (`merge.sh`, each from a fresh fork set): ## Risks and open questions -- **Writing into a live mount fork.** R−F goes through the fork's shared - checkout while the agent may hold the mount open. Serialize under the - checkout lock like `capture_copy` does; document that an editor with - the file open sees the markers on next read. +- **Writing into a live mount fork.** R−F goes through the mounted path while + the agent may hold it open. The driver serializes the resulting overlay + mutations; an editor with the file open sees markers on its next read. - **Fork base mutation.** `ForkSeed.base` is immutable today and the daemon's fork table copies it. Both must update atomically with the rebase record; a daemon restart already loses forks, so no persistence @@ -377,13 +373,10 @@ Deliberate divergences: drain), recorded as the single landed row. The `before promote …` row records the published head without another drain. R is built from M with the marker files on top. A rebase writes R − F into the - fork: through the shared checkout for a mount fork (route detached - during promote, re-attached after), via `restore_path_into` for a copy - fork. -- **Rebased forks land by checkpoint-and-swap.** A rebased mount fork's - checkout still sits on the head it was cut from, so an optimistic - commit against the new head would conflict. `ForkState.rebased` routes - such forks through the same resolve/apply pair copy forks use. + fork through its mounted path so driver and kernel caches remain coherent. +- **Rebased forks land through snapshot/apply.** A rebased fork's checkout + still sits on the head it was cut from, so an optimistic commit against the + new head would conflict. - **Subtree copy and removal are iterative.** The fs facade's futures are large enough that three nested levels overflowed the pipeline thread's 2 MiB stack in a debug build. Both walks use an explicit work @@ -391,8 +384,6 @@ Deliberate divergences: - **`rewind --last` is not the undo.** It targets the latest real checkpoint, which after a merge is the landed row. The undo is the `before promote (merge)` safety row, exactly as for a replay. -- **`ACYCLIC_FORCE_COPY_FORKS`** makes a mount-capable host use copy - forks so `merge.sh` runs both modes on one machine. - **Wire.** `PromoteInfo` gained `merged_files`, `conflicts`, and `fork_path`; `ForkEntry` gained `conflict_paths` and `conflict` (base/ours/theirs). A conflicting promote returns a non-zero exit @@ -419,9 +410,8 @@ Deliberate divergences: unmoved mainline used to land by whole-tree swap, which replaced the repo directory and needed the skill's `cd "$PWD"` step. Promote now always goes through the merge path: with an unmoved head the plan is - "take every fork path" and they are written in place. Safe Mode's - `session-apply` is the only remaining swap. `merge.sh` G11 asserts - the repo inode survives a promote. + "take every fork path" and they are written in place. `merge.sh` G11 + asserts the repo inode survives a promote. - **Diff output marks gitignored paths** with a `(gitignored)` suffix and a `(K gitignored)` count so the skill's smallest-diff rule can ignore cache and build noise; the paths stay listed because rewind @@ -435,8 +425,8 @@ Deliberate divergences: `invalidate` returned Ok, and the FUSE transport (Linux, FUSE-T on the macOS runner) has no invalidation at all. The daemon now writes the rebase into `//…` with plain filesystem operations - (`merge::materialize_paths`), the same way copy forks get theirs, so - every cache saw the operation. Promote also no longer detaches the + (`merge::materialize_paths`), so every cache saw the operation. Promote + also no longer detaches the route before working; it snapshots under the checkout lock and drops the route only after the fork lands, which removed the negative-entry window that hid re-attached forks on Linux. @@ -451,14 +441,11 @@ Deliberate divergences: prints the wait-path p95 as information. - **Path tracing.** `ACYCLIC_TRACE=1` makes the CLI, hook, daemon, and pipeline log every branch taken and its cost (client connect/spawn, - op dispatch and reply, WAIT vs ENQUEUE checkpoint, shadowed noop, + op dispatch and reply, WAIT vs ENQUEUE checkpoint, recovery baseline, watcher drain polls/batches/stop reason, snapshot, index, publish, promote mode and outcome, merge plan counts, root hints). Daemon lines land in the store's `daemon.log`. -- **Safe Mode S9 race.** After an empty session resolve unmounted the - shadow and installed a fresh watcher, a late mount-teardown event for - the repo root itself reached the next drain, and the engine refused a - mutation targeting the volume root. Root-targeted hints are now +- **Root-targeted watcher hints.** Root-targeted hints are now stripped before capture: metadata-only ones dropped, structural ones (root created/removed/renamed, i.e. a mount came or went) trigger a fresh watcher and a recovery baseline instead of a failed request. diff --git a/docs/design/implementation-rewind.md b/docs/design/implementation-rewind.md index ec2ac3f..544dddc 100644 --- a/docs/design/implementation-rewind.md +++ b/docs/design/implementation-rewind.md @@ -17,17 +17,14 @@ The engine is imported, not built: `acyclic-fs` + `acyclic-fs-mount` via path de ``` crates/ - acyclic-engine/ lib: store, pipeline, index, rewind, diff, config - acyclic-proto/ lib: CLI ↔ daemon message types - acyclic/ bin: CLI + hidden `__daemon` subcommand - acyclic-qual/ Phase 0 harness → standing bench suite (exists) + acyclic/ one binary package: CLI, daemon, private engine and protocol modules adapters/claude-code/ hooks, /rewind command, self-rollback skill tests/acceptance/ one script per acceptance criterion ``` --- -# Phase 1 — the engine library (`acyclic-engine`) +# Phase 1 — the engine library (`acyclic`) **Goal:** everything below the wire works and survives crashes, proven by tests. @@ -82,7 +79,7 @@ Single-file restore (`rewind --path`): copy the one file out of the target check **Goal:** the bare-CLI product works end to end. -- **Protocol** (`acyclic-proto`): newline-delimited JSON over the unix socket. Ops: `ping, status, checkpoint, timeline, rewind, diff, session_start, session_end, commit, stop`. +- **Protocol** (`acyclic/src/proto.rs`): newline-delimited JSON over the local daemon transport. Ops: `ping, status, checkpoint, timeline, rewind, diff, session_start, session_end, commit, stop`. The one latency-critical detail: `checkpoint{wait:false}` replies on *enqueue* (~ms — the PostToolUse path); `wait:true` replies when the checkpoint lands (the PreToolUse path). - **Daemon** (`acyclic __daemon`): socket listener + pipeline + janitor (trash TTL, idle-commit timer). Pidfile prevents doubles; SIGTERM drains the queue and commits. - **CLI verbs:** `init`, `checkpoint [-m] [--wait] [--durable]`, `timeline`, `rewind [--path]` (prints blast summary, confirms), `diff [--stat]`, `status`, `stop`, `install `. @@ -124,12 +121,12 @@ Built as `acyclic hook ` + `acyclic install ` (the installer embeds Shipped: - Acceptance: `exclusions.sh` (declared paths never captured, noop on excluded-only edits, restore refused, rename into an excluded prefix scrubbed, rewind carries the live copies, pre-rule history untouched) and `growth.sh` (60 distinct checkpoints cost ~52 KB each, identical on a 4x larger tree: per-checkpoint overhead is tree pages, never a tree copy). Journey/soak/crash/latency were already green; the migration-script scenario is journey.sh's Bash side-effect step. -- Snapshot exclusions (`exclude` in config): `acyclic-engine/src/exclude.rs`. Watcher hints at or under a rule are dropped before capture; renames across the boundary re-examine the uncovered side; a capture hinted at an ancestor (or the baseline) is followed by a checkout scrub before the generation is checkpointed; a full rewind (and promote / Safe Mode apply, which share `rewind::execute`) moves the live excluded paths into the new tree before the swap, journaled as a `Carrying` phase so kill -9 at any point returns them. +- Snapshot exclusions (`exclude` in config): `crates/acyclic/src/exclude.rs`. Watcher hints at or under a rule are dropped before capture; renames across the boundary re-examine the uncovered side; a capture hinted at an ancestor (or the baseline) is followed by a checkout scrub before the generation is checkpointed; a full rewind moves the live excluded paths into the new tree before the swap, journaled as a `Carrying` phase so kill -9 at any point returns them. - Retention: `status` reports store size, trash is TTL-pruned. **No fs GC, no purge in v1**, and this is a finding, not an omission: at sdk `8eced48`, `collect_local_garbage` keeps only authority heads plus retention facts (`RetentionKind::{Checkpoint,Pin,ForkBase}`), `retain_workspace_generation` is create-only (no release fact exists), and `prove_generation_closure` does not follow `GenerationRoot::parents`. Running GC would delete every checkpoint but the head; pinning every checkpoint first would make the store append-only forever. Purge cannot remove bytes from a retained generation for the same reason. **Upstream ask:** a retention-release fact (mirror of `encode_workspace_deleted` for retention authorities) honoured by the collector, plus a bulk "retain these N generations" call. With that, the plugin's policy is straightforward: pin what the TTL keeps, release the rest, collect. - Release engineering: `deny.toml` + a `deny` CI job (licenses inside a permissive allowlist, advisories, sources); an SPDX SBOM generated from `Cargo.lock` per target in `release.yml`, attested against each binary with `actions/attest-sbom`, one copy attached to the GitHub release and listed in `SHA256SUMS`; `scripts/install.sh` (verified download into `~/.local/bin`, `file://` base URL for offline tests); `scripts/install-smoke.sh` runs it on a bare Debian container and drives init → checkpoint → rewind, exclusions included. npm `@acyclic-labs/plugin` 0.0.1 is live. - Docs: README leads with the differentiators, documents `exclude`, and states the retention stance and caveats. -Still open, by decision: the public name (`acyclic` vs graphcoder), and cutting the first `v*` tag through `release.yml`; both are the owner's call. Exclusions do not reach forks or Safe Mode sessions (they are served from generations), documented as a caveat. +Still open, by decision: the public name (`acyclic` vs graphcoder), and cutting the first `v*` tag through `release.yml`; both are the owner's call. Exclusions do not reach forks (they are served from generations), documented as a caveat. --- diff --git a/docs/design/spec-forks.md b/docs/design/spec-forks.md index 24920a8..5226b7e 100644 --- a/docs/design/spec-forks.md +++ b/docs/design/spec-forks.md @@ -77,8 +77,7 @@ Unmounts, discards the overlay, removes the workspace dir. Unknown id → error. onto the real tree one at a time with the same atomic single-path restore, checkpointed and published; a `manual` row labeled `promote (N path(s) written in place)` records it. The repo - directory is never replaced by a promote (Safe Mode `session-apply` - still swaps and says so). Gitignored files included. + directory is never replaced by a promote. Gitignored files included. ### Daemon stop / crash Stop unmounts all forks and removes workspace dirs before the pipeline @@ -107,6 +106,6 @@ remove) at next start, before the store opens. | U2: stale sweep removes dirs | `fork.rs::sweep_removes_stale_directories` | unit | ✅ | Out of scope (documented): rename detection, semantic merges, conflict -markers on the mainline, Safe Mode `apply_session` onto a moved mainline, -fork persistence across daemon restarts, subagent orchestration, per-fork +markers on the mainline, fork persistence across daemon restarts, +subagent orchestration, per-fork port/env provisioning. diff --git a/docs/windows-verification.md b/docs/windows-verification.md index ff3132d..c96252e 100644 --- a/docs/windows-verification.md +++ b/docs/windows-verification.md @@ -2,8 +2,7 @@ **Status: verified on real Windows hardware.** Checks 1–8 below were run on Windows 11 (26200) with the MSVC toolchain, against a release build of this -branch. Two capabilities are deliberately *not* offered on Windows — mounted -forks and Safe Mode — for a reason recorded under [Known +branch. Mounted forks have platform-specific limits recorded under [Known limits](#known-limits); everything else behaves as it does on POSIX. `tests/acceptance/windows-smoke.sh` is the executable form of checks 3–7, @@ -16,14 +15,14 @@ lint job cannot see. | Area | Change | | --- | --- | | `crates/acyclic/src/ipc.rs` | New. The transport split: Unix domain socket vs. Windows named pipe, with an owner-only pipe DACL. | -| `crates/acyclic-engine/src/names.rs` | New. The one definition of how a host name becomes engine bytes. | +| `crates/acyclic/src/names.rs` | New. The one definition of how a host name becomes engine bytes. | | `crates/acyclic/src/client.rs` | `ipc::ClientStream`; daemon spawn no longer leaks stdio handles or stands in the repo. | | `crates/acyclic/src/server.rs` | `ipc::Listener`/`ipc::ServerStream`; fork route names carry the host encoding. | -| `crates/acyclic-engine/src/store.rs` | Volume profile and component byte budget come from `names`. | -| `crates/acyclic-engine/src/rewind.rs` | Windows directory exchange; journal write fixed; staging is retry-safe. | -| `crates/acyclic-engine/src/guard.rs` | Guarded prefixes and `AppleDouble` matching in the host encoding. | -| `crates/acyclic-engine/src/{diff,exclude}.rs` | Name decoding via `names` instead of per-file UTF-8 fallbacks. | -| `crates/acyclic-engine/src/fork.rs` | Windows is held to copy forks (see below). | +| `crates/acyclic/src/store.rs` | Volume profile and component byte budget come from `names`. | +| `crates/acyclic/src/rewind.rs` | Windows directory exchange; journal write fixed; staging is retry-safe. | +| `crates/acyclic/src/guard.rs` | Guarded prefixes and `AppleDouble` matching in the host encoding. | +| `crates/acyclic/src/{diff,exclude}.rs` | Name decoding via `names` instead of per-file UTF-8 fallbacks. | +| `crates/acyclic/src/fork.rs` | Windows forks require Projected File System. | | `crates/acyclic/src/main.rs` | The client steps out of the tree before asking for a whole-tree swap. | | `.github/workflows/{ci,release}.yml` | A `windows` CI job; `win32/x64` release matrix entry. | | `packaging/npm/*.sh`, `deny.toml` | `win32` platform package; MSVC target in the license set. | @@ -52,9 +51,9 @@ That is one decision, not two, and it reaches further than capture. The `ProjFS` provider decodes every entry name it is handed as UTF-16LE, so a name that arrives in any other encoding is *projected as mojibake rather than rejected* — fork route ids passed as raw ASCII came back as six garbage -characters. The Safe Mode guard has the sharper version of the same problem: -it compares configured prefixes byte-for-byte against mount path components, -and a guard that never matches **fails open**. `crates/acyclic-engine/src/names.rs` +characters. Guarded fork paths have the sharper version of the same problem: +they compare configured prefixes byte-for-byte against mount path components, +and a guard that never matches **fails open**. `crates/acyclic/src/names.rs` exists so there is exactly one place this can be got wrong. Because the profile is fixed for the life of a volume, a store created on @@ -169,16 +168,14 @@ correct. ## Known limits -- **Forks are always copies, and Safe Mode is off.** `ProjFS` mounts and +- **Forks require a writable native mount.** `ProjFS` mounts and projects a fork correctly — the tree appears and reads back fine — but writes into the projection stop at the `ProjFS` local cache and never reach the overlay checkout. A mounted fork therefore looked like it worked while `fork-diff` reported no changes and `promote` landed nothing: it silently - ate the work. `mount_capability()` now reports mounts unavailable on - Windows, so forks take the copy path, which is verified end to end (write, - `fork-diff`, `promote` all behave). Safe Mode needs a real mount and is - unavailable for the same reason. Revisit if the sdk's `ProjFS` provider - gains write-back. + ate the work. `mount_capability()` reports mounts unavailable on Windows + until the SDK's `ProjFS` provider gains write-back, and fork creation fails + explicitly instead of materializing a full-tree compatibility copy. - **The tree exchange is not atomic.** `RENAME_EXCHANGE` has no Windows equivalent, so `atomic_exchange` does three renames through a scratch @@ -189,11 +186,10 @@ correct. crash mid-swap is recovered rather than impossible, which is weaker than the APFS/`renameat2` guarantee. -- **No read/write deadlines.** `ClientStream::set_read_timeout` and - `set_write_timeout` are no-ops on Windows: a pipe opened as a `File` - carries no per-handle timeout. The pre-tool hook's deadline therefore does - not bound anything there. Closing this needs overlapped I/O or a watchdog - thread. **The latency gate's guarantee does not hold on Windows.** +- **Client call deadlines.** Windows named-pipe clients use Tokio's IOCP + transport. The call timeout covers the whole request and response, including + partial writes and replies. A timed-out client must reconnect before another + call so a late daemon response cannot be paired with the wrong request. - **A speculative run's timeout kills without a grace period, and a descendant can escape the job.** `spec_runner` puts each run in a job diff --git a/product.toml b/product.toml index a7662e5..f11f265 100644 --- a/product.toml +++ b/product.toml @@ -4,7 +4,7 @@ # commands, skill names, error-message prefixes, the `_TRACE` and # `_HOOK` environment variables, release asset names, and the npm bin. # -# Readers: crates/acyclic-engine/build.rs (Rust, via env!), scripts/product.sh +# Readers: crates/acyclic/build.rs (Rust, via env!), scripts/product.sh # (shell), .github/workflows (awk). scripts/check-product-name.sh fails CI if # the one self-contained file (scripts/install.sh) drifts from this. # diff --git a/scripts/check-product-name.sh b/scripts/check-product-name.sh index 22c3551..eb13eaa 100755 --- a/scripts/check-product-name.sh +++ b/scripts/check-product-name.sh @@ -3,7 +3,7 @@ # 1. scripts/install.sh is fetched standalone and mirrors `name`, # `github_repo`, and `npm_package`; they must match exactly. # 2. No user-facing Rust source spells the name out. Only crate/module -# identifiers (acyclic_fs, acyclic_engine, acyclic-fs ...) and comments +# identifiers (acyclic_fs, ...) and comments # may contain it; strings, paths, and doc templates go through # `product::NAME` / `product::render`. set -euo pipefail @@ -28,7 +28,7 @@ have_pypi="$(awk -F'"' '/^name = /{print $2; exit}' "$ROOT/packaging/pypi/pyproj # Literal uses of the current name in strings/paths of user-facing crates. # Comments (// and //!) are allowed; identifiers with '_' are crate paths. stray="$(grep -rn --include='*.rs' -E "\"[^\"]*\b${want_name}\b[^\"]*\"|\`${want_name} |\.${want_name}/" \ - "$ROOT/crates/acyclic/src" "$ROOT/crates/acyclic-engine/src" \ + "$ROOT/crates/acyclic/src" \ | grep -v -E "${want_name}[_-](fs|engine|proto|qual)|^[^:]+:[0-9]+:\s*//" || true)" if [ -n "$stray" ]; then echo "hardcoded product name in Rust source (use product::NAME / product::render):" >&2 diff --git a/scripts/ci-local.sh b/scripts/ci-local.sh index 1b56fbc..ab39c24 100755 --- a/scripts/ci-local.sh +++ b/scripts/ci-local.sh @@ -1,78 +1,6 @@ #!/usr/bin/env bash -# Runs what .github/workflows/ci.yml runs, job by job, on this machine, and -# prints one line per step at the end so a red step is easy to find. Every -# step runs even after an earlier one fails (CI's jobs are independent too); -# the exit code is non-zero if any step failed. -# -# scripts/ci-local.sh everything, including the acceptance suite (slowest) -# scripts/ci-local.sh --no-acceptance the static gates, unit tests, release build, coverage -# scripts/ci-local.sh --no-coverage skip cargo-llvm-cov (needs `cargo install cargo-llvm-cov`) -# -# Needs: stable toolchain with rustfmt + clippy, cargo-deny, node (for the -# duplication check), and unless --no-coverage: cargo-llvm-cov plus the -# `llvm-tools-preview` rustup component it drives -# (`rustup component add llvm-tools-preview`). macOS also needs -# FUSE-T for the fork/Safe Mode acceptance scripts. -set -uo pipefail +# The same bounded gate runs locally and on each supported CI host. +set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT" -# Same as CI: the pinned acyclic-fs git dependency needs the git CLI's -# credentials and protocol support, not cargo's built-in fetcher. -export CARGO_NET_GIT_FETCH_WITH_CLI=true - -run_acceptance=1 -run_coverage=1 -for arg in "$@"; do - case "$arg" in - --no-acceptance) run_acceptance=0 ;; - --no-coverage) run_coverage=0 ;; - *) echo "unknown flag: $arg" >&2; exit 2 ;; - esac -done - -results=() -failed=0 -step() { - local name="$1" - shift - echo - echo "=== $name" - local started=$SECONDS - if "$@"; then - results+=("PASS $((SECONDS - started))s $name") - else - results+=("FAIL $((SECONDS - started))s $name") - failed=1 - fi -} - -# deny job -step "product name single-sourced" bash scripts/check-product-name.sh -step "no secrets or forbidden files" bash scripts/check-no-secrets.sh -step "code quality (width, TODOs, comment blocks, duplication)" bash scripts/check-code-quality.sh -step "cargo deny" cargo deny --locked check - -# lint job -step "cargo fmt --check" cargo fmt --all --check -step "cargo clippy -D warnings" cargo clippy --workspace --all-targets --all-features -- -D warnings - -# test job -step "cargo test" cargo test --workspace -step "cargo build --release" cargo build --release -if [ "$run_acceptance" -eq 1 ]; then - step "acceptance suite" env \ - ACYCLIC_BIN="$ROOT/target/release/acyclic" \ - ACYCLIC_QUAL="$ROOT/target/release/acyclic-qual" \ - ACYCLIC_LAT_FILES=5000 ACYCLIC_LAT_MB=64 ACYCLIC_SOAK_ROUNDS=30 \ - bash tests/acceptance/run-all.sh -fi - -# coverage job -if [ "$run_coverage" -eq 1 ]; then - step "coverage (floor: see ci.yml)" cargo llvm-cov --workspace --all-features --summary-only --fail-under-lines 48 -fi - -echo -echo "=== summary" -printf '%s\n' "${results[@]}" -exit "$failed" +SDK="$(cd "$ROOT/../../../sdk" && pwd)" +exec python3 "$SDK/scripts/qualify-local.py" --plugin-root "$ROOT" "$@" diff --git a/scripts/docker-linux.sh b/scripts/docker-linux.sh index 8a534d2..08ba569 100755 --- a/scripts/docker-linux.sh +++ b/scripts/docker-linux.sh @@ -1,51 +1,24 @@ #!/usr/bin/env bash -# Linux validation without leaving the Mac: build and run the full test + -# acceptance suite inside a Linux container. acyclic-fs is fetched from its -# own public git repo (see Cargo.toml), not a sibling checkout; all build -# artifacts and test state stay on container-local filesystems (which is -# also what exercises renameat2(RENAME_EXCHANGE) and inotify on a Linux -# kernel for real). -# -# Usage: scripts/docker-linux.sh [image] +# Run the identical qualification gate on a Linux kernel in a container. set -euo pipefail - PLUGIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SDK="$(cd "$PLUGIN/../../../sdk" && pwd)" IMAGE="${1:-rust:1-bookworm}" - -# FUSE inside the container (fork mounts): pass the device + cap when the -# host offers them; forks.sh skips gracefully otherwise. FUSE_FLAGS=() -if [ -e /dev/fuse ] || [ "$(uname -s)" = "Darwin" ]; then +if [ -e /dev/fuse ]; then FUSE_FLAGS=(--device /dev/fuse --cap-add SYS_ADMIN) fi - docker run --rm \ "${FUSE_FLAGS[@]}" \ - -v "$PLUGIN:/src/graphcoder-plugin:ro" \ + -v "$PLUGIN:/lab/worktrees/graphcoder/plugin-sdk-lab:ro" \ + -v "$SDK:/lab/sdk:ro" \ -v acyclic-linux-cargo:/cargo \ -v acyclic-linux-target:/build \ -e CARGO_HOME=/cargo \ -e CARGO_TARGET_DIR=/build/target \ - -e CARGO_NET_GIT_FETCH_WITH_CLI=true \ "$IMAGE" bash -eu -o pipefail -c ' export DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null apt-get install -y -qq sqlite3 procps python3 >/dev/null - - cd /src/graphcoder-plugin - echo "=== cargo test (workspace)" - cargo test --workspace 2>&1 | grep -E "test result|error" || true - cargo test --workspace >/dev/null - - echo "=== release build" - cargo build --release - - echo "=== acceptance suite" - export TMPDIR=/tmp - export ACYCLIC_BIN=/build/target/release/acyclic - export ACYCLIC_QUAL=/build/target/release/acyclic-qual - export ACYCLIC_LAT_FILES=5000 - export ACYCLIC_LAT_MB=64 - export ACYCLIC_SOAK_ROUNDS=30 - bash tests/acceptance/run-all.sh + python3 /lab/sdk/scripts/qualify-local.py ' diff --git a/tests/acceptance/common.sh b/tests/acceptance/common.sh index f5d1d2c..25bb4c6 100755 --- a/tests/acceptance/common.sh +++ b/tests/acceptance/common.sh @@ -5,7 +5,8 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" BIN="${ACYCLIC_BIN:-$REPO_ROOT/target/debug/acyclic}" -QUAL="${ACYCLIC_QUAL:-$REPO_ROOT/target/debug/acyclic-qual}" +SDK_ROOT="$(cd "$REPO_ROOT/../../../sdk" && pwd)" +QUAL="${ACYCLIC_QUAL:-$SDK_ROOT/target/debug/qualify}" WORK="$(mktemp -d "${TMPDIR:-/tmp}/acyclic-acceptance.XXXXXX")" # Canonicalize: macOS TMPDIR ends in "/" and /var -> /private/var, so the diff --git a/tests/acceptance/crash.sh b/tests/acceptance/crash.sh index 140c78f..fb129ff 100755 --- a/tests/acceptance/crash.sh +++ b/tests/acceptance/crash.sh @@ -27,7 +27,7 @@ acy checkpoint --wait --kind post >/dev/null || fail "checkpoint after kill -9" # --- crash mid-swap: journal-driven recovery ------------------------------ # The atomic-exchange platforms (macOS/Linux) always leave the repo whole; # a crash there leaves the journal plus a stray tmp tree. The repo-missing -# two-step variant is unit-tested in acyclic-engine (recover()). +# two-step variant is unit-tested in acyclic (recover()). acy stop >/dev/null sleep 0.5 JOURNAL="$(ls -d "$STORES"/*/)"rewind-journal.json diff --git a/tests/acceptance/cursor-e2e.sh b/tests/acceptance/cursor-e2e.sh index 274ed55..ba31cb8 100755 --- a/tests/acceptance/cursor-e2e.sh +++ b/tests/acceptance/cursor-e2e.sh @@ -12,8 +12,8 @@ # # Cursor's own session id (reported here as `session_id` in --output-format # json) is the same identifier its beforeShellExecution/afterFileEdit hooks -# send as `conversation_id` — that's the fallback `hook::Payload::session()` -# resolves to `session_id` for attribution (Cursor's own `sessionStart` +# send as `conversation_id`; `hook::Payload::session()` maps that field to +# the timeline session (Cursor's own `sessionStart` # event does send `session_id` directly). If Cursor ever splits these, the # timeline/diff assertions below will fail loudly rather than silently # mis-attribute. diff --git a/tests/acceptance/latency.sh b/tests/acceptance/latency.sh index 5ec83fc..944f883 100755 --- a/tests/acceptance/latency.sh +++ b/tests/acceptance/latency.sh @@ -18,6 +18,9 @@ BUDGET_MS="${ACYCLIC_LAT_BUDGET_MS:-100}" setup_repo "$QUAL" corpus "$R" "$FILES" "$CORPUS_MB" >/dev/null || fail "corpus generation" acy init >/dev/null || fail "init (includes baseline of the corpus)" +if [ "${ACYCLIC_TRACE:-}" = 1 ]; then + grep 'baseline phases:' "$STORES"/*/daemon.log | tail -1 || true +fi # Warm-up round, then timed samples. Each round touches a file so the # checkpoint has real work queued behind the ack. diff --git a/tests/acceptance/run-all.sh b/tests/acceptance/run-all.sh index 837f438..e2c14b3 100755 --- a/tests/acceptance/run-all.sh +++ b/tests/acceptance/run-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Runs the full acceptance suite against the built binaries. -# ACYCLIC_BIN / ACYCLIC_QUAL override binary paths (default: target/debug) +# ACYCLIC_BIN / ACYCLIC_QUAL override plugin and SDK qualifier paths # Individual scripts also run standalone. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/tests/acceptance/safe-mode.sh b/tests/acceptance/safe-mode.sh deleted file mode 100755 index 22940e5..0000000 --- a/tests/acceptance/safe-mode.sh +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env bash -# Launch 4 Safe Mode acceptance (spec: docs/design/04-safe-mode.md): session -# redirection through a shadow mount at the repo root, filesystem-level -# guarded paths that hold against arbitrary shell, approval-gated -# apply/discard with zero trace, conflict legibility, and crash sweep. -# Skips (exit 0) when native mounts are unavailable unless -# ACYCLIC_FORKS_REQUIRED=1. -source "$(dirname "${BASH_SOURCE[0]}")/common.sh" - -skip() { - if [ "${ACYCLIC_FORKS_REQUIRED:-0}" = "1" ]; then - fail "$*" - fi - echo "SKIP($(basename "$0")): $*" - exit 0 -} - -# Every mount is torn down on exit even on failure: the shadow mount sits on -# the real repo root, so a leak here poisons every later script. -safe_teardown() { - acy stop >/dev/null 2>&1 || true - sleep 0.3 - if mount | grep -q " $R "; then - umount -f "$R" 2>/dev/null || diskutil unmount force "$R" >/dev/null 2>&1 || true - fi - teardown -} -trap safe_teardown EXIT - -# Environment hygiene (mirrors forks.sh): a kill -9'd daemon anywhere orphans -# its go-nfsv4 helper, and orphaned helpers wedge FUSE-T's tiny shared NFS -# port pool for every later mount — which this script does many of, including -# a crash-recovery remount. Reap acyclic helpers + mounts up front. -if [ "$(uname -s)" = "Darwin" ]; then - pkill -f 'go-nfsv4.*acyclic-fs' 2>/dev/null || true - sleep 0.5 - mount | awk '/fuse-t:\/acyclic-fs|127\.0\.0\.1:\//{print $3}' | while read -r M; do - case "$M" in */acyclic-acceptance.*) umount -f "$M" 2>/dev/null || true ;; esac - done -fi - -shadowed() { mount | grep -q " $R "; } -sha() { shasum -a 256 "$1" | awk '{print $1}'; } -tree_digest() { - (cd "$1" && find . -type f ! -path './.acyclic/*' -print0 | sort -z \ - | xargs -0 shasum -a 256 | shasum -a 256 | awk '{print $1}') -} -# A guarded command must be REFUSED, but on the macOS loopback-NFS transport -# a rejected write can still return exit 0 to the shell (client write-back -# caching acks the write before the server refuses it). So a guard assertion -# must verify the *effect* was prevented, not the command's exit status — -# `must_fail` is only for control-plane commands (session/apply) that reply -# synchronously. Guarded-write refusals are checked by content below. -must_fail() { - local label="$1"; shift - if "$@" 2>"$WORK/err"; then - fail "$label: succeeded but must be refused" - fi -} - -setup_repo -mkdir -p "$R/migrations" "$R/src/nested" -printf 'CREATE TABLE a;\n' > "$R/migrations/001.sql" -printf 'deep\n' > "$R/src/nested/deep.txt" -printf 'ORIGINAL-B\n' > "$R/src/b.rs" -cat >> "$R/.acyclic/config.toml" <<'EOF' -dry_run = true -guarded_paths = [".env", "migrations/"] -EOF -acy init >/dev/null || fail "init" -REAL_BEFORE="$(tree_digest "$R")" -INODE_BEFORE="$(inode "$R")" - -# --- S1: session start shadows the repo root; identical to the agent ------ -if ! OUT="$(acy session-start dry-1 --host acceptance 2>&1)"; then - echo "$OUT" | grep -qi 'mount' && skip "native mounts unavailable: $OUT" - fail "session-start: $OUT" -fi -shadowed || fail "S1: repo root is not shadow-mounted after session-start" -[ "$(cat "$R/src/main.rs")" = "ORIGINAL" ] || fail "S1: shadow does not serve the tree" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S1: gitignored file missing from shadow" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S1: guarded file unreadable" -[ "$(cat "$R/src/nested/deep.txt")" = "deep" ] || fail "S1: nested path missing" -[ "$(ls "$R" | sort | tr '\n' ' ')" = "$(printf 'migrations src \n' )" ] \ - || fail "S1: listing differs from real tree: $(ls "$R" | tr '\n' ' ')" - -# --- S2: a second Safe Mode session is refused while one is active -------- -must_fail "S2: second session-start" acy session-start dry-2 --host acceptance -grep -qi "already active" "$WORK/err" || fail "S2: refusal not legible: $(cat "$WORK/err")" - -# --- S3: ordinary writes land in the shadow, at every depth --------------- -printf 'MIGRATED\n' > "$R/src/main.rs" || fail "S3: write main.rs" -printf 'deeper\n' > "$R/src/nested/deep.txt" || fail "S3: nested write" -mkdir -p "$R/src/new/dir" || fail "S3: mkdir" -printf 'fresh\n' > "$R/src/new/dir/file.txt" || fail "S3: create in new dir" -rm "$R/src/b.rs" || fail "S3: rm" -head -c 4096 /dev/zero > "$R/generated.bin" || fail "S3: generate artifact" -mv "$R/src/nested/deep.txt" "$R/src/nested/moved.txt" || fail "S3: rename" -[ "$(cat "$R/src/main.rs")" = "MIGRATED" ] || fail "S3: readback main.rs" -[ "$(cat "$R/src/nested/moved.txt")" = "deeper" ] || fail "S3: readback rename" -[ ! -e "$R/src/b.rs" ] || fail "S3: rm not reflected" - -# --- S4: guarded paths refuse EVERY mutation from arbitrary shell --------- -# Each mutation is attempted through plain shell; the exit code is ignored -# (macOS NFS write-back may ack a doomed write), so refusal is proven by the -# guarded content/structure being byte-identical afterward. -try() { "$@" >/dev/null 2>&1 || true; } -try bash -c "printf 'LEAK=1\n' > '$R/.env'" -try bash -c "printf 'LEAK=1\n' >> '$R/.env'" -try bash -c ": > '$R/.env'" -try rm "$R/.env" -try mv "$R/.env" "$R/env.bak" -try bash -c "printf 'x\n' > '$R/tmp.txt' && mv '$R/tmp.txt' '$R/.env'" -try chmod 600 "$R/.env" -try bash -c "printf 'DROP TABLE a;\n' > '$R/migrations/001.sql'" -try bash -c "printf 'x\n' > '$R/migrations/002.sql'" -try mkdir "$R/migrations/sub" -try rm "$R/migrations/001.sql" -try bash -c "rm -rf '$R/migrations'" -try bash -c "mv '$R/generated.bin' '$R/migrations/'" -try mv "$R/migrations" "$R/old-migrations" -rm -f "$R/tmp.txt" -# The refusals were real: guarded content and structure are byte-identical. -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S4: .env changed: $(cat "$R/.env")" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S4: migration changed" -[ ! -e "$R/migrations/002.sql" ] || fail "S4: file created in guarded dir" -[ ! -e "$R/migrations/sub" ] || fail "S4: dir created in guarded dir" -[ -e "$R/generated.bin" ] || fail "S4: artifact vanished during refused move" -[ -e "$R/migrations" ] || fail "S4: guarded dir vanished" -[ ! -e "$R/env.bak" ] && [ ! -e "$R/old-migrations" ] || fail "S4: guarded rename escaped" -# Reads under guard keep working, and a sibling path (prefix, not guarded) does too. -printf 'ok\n' > "$R/migrations-notes.txt" || fail "S4: prefix-sibling path wrongly guarded" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S4: guarded read broken" - -# --- S5: the real tree is untouched underneath, and hooks keep working ----- -# Checkpoints taken mid-session (hooks fire as normal) must neither wedge the -# daemon nor pollute the mainline timeline with fork content. -acy checkpoint --wait --kind post --session-id dry-1 --tool-name Bash >/dev/null \ - || fail "S5: checkpoint during shadowed session" -STATUS="$(acy status)" -echo "$STATUS" | grep -q "state: ready" || fail "S5: daemon not ready mid-session: $STATUS" - -# --- S6: resolve unmounts, restores the real view, shows the exact diff --- -DIFF="$(acy session-resolve dry-1)" || fail "S6: session-resolve: $DIFF" -shadowed && fail "S6: still shadow-mounted after resolve" -[ "$(inode "$R")" = "$INODE_BEFORE" ] || fail "S6: real root inode changed" -[ "$(tree_digest "$R")" = "$REAL_BEFORE" ] || fail "S6: real tree changed before apply" -echo "$DIFF" | grep -q "^M src/main.rs" || fail "S6: diff missing edit: $DIFF" -echo "$DIFF" | grep -q "^D src/b.rs" || fail "S6: diff missing rm: $DIFF" -echo "$DIFF" | grep -q "^A generated.bin" || fail "S6: diff missing artifact: $DIFF" -echo "$DIFF" | grep -q "^A src/new/dir/file.txt" || fail "S6: diff missing nested create: $DIFF" -echo "$DIFF" | grep -q "^A src/nested/moved.txt" || fail "S6: diff missing rename target: $DIFF" -echo "$DIFF" | grep -q "^D src/nested/deep.txt" || fail "S6: diff missing rename source: $DIFF" -echo "$DIFF" | grep -q "^A migrations-notes.txt" || fail "S6: diff missing sibling: $DIFF" -echo "$DIFF" | grep -q "\.env" && fail "S6: guarded .env in diff: $DIFF" -echo "$DIFF" | grep -q "migrations/" && fail "S6: guarded dir in diff: $DIFF" -echo "$DIFF" | grep -q "session-apply dry-1" || fail "S6: no apply instruction: $DIFF" -must_fail "S6: resolve twice" acy session-resolve dry-1 -must_fail "S6: resolve unknown" acy session-resolve nope - -# --- S7: discard leaves zero trace; a fresh session starts clean ---------- -acy session-discard dry-1 >/dev/null || fail "S7: session-discard" -[ "$(tree_digest "$R")" = "$REAL_BEFORE" ] || fail "S7: discard touched the real tree" -must_fail "S7: apply after discard" acy session-apply dry-1 -acy session-start dry-2 --host acceptance >/dev/null || fail "S7: new session after discard" -shadowed || fail "S7: second session not shadowed" -[ "$(cat "$R/src/main.rs")" = "ORIGINAL" ] || fail "S7: discarded edit leaked into next session" -[ ! -e "$R/generated.bin" ] || fail "S7: discarded artifact leaked into next session" - -# --- S8: a session with no writes resolves as no changes ----------------- -OUT="$(acy session-resolve dry-2)" || fail "S8: resolve empty session" -echo "$OUT" | grep -q "no changes" || fail "S8: expected 'no changes': $OUT" -shadowed && fail "S8: still mounted after empty resolve" -[ "$(tree_digest "$R")" = "$REAL_BEFORE" ] || fail "S8: empty session changed the tree" - -# --- S9: apply lands atomically; guarded content survives ---------------- -acy session-start dry-3 --host acceptance >/dev/null || fail "S9: session-start" -printf 'APPLIED\n' > "$R/src/main.rs" -printf 'brand new\n' > "$R/new.txt" -rm "$R/src/b.rs" -acy session-resolve dry-3 >/dev/null || fail "S9: resolve" -[ "$(cat "$R/src/main.rs")" = "ORIGINAL" ] || fail "S9: real tree changed before apply" -OUT="$(acy session-apply dry-3)" || fail "S9: session-apply: $OUT" -shadowed && fail "S9: mounted after apply" -[ "$(cat "$R/src/main.rs")" = "APPLIED" ] || fail "S9: edit not applied" -[ "$(cat "$R/new.txt")" = "brand new" ] || fail "S9: new file not applied" -[ ! -e "$R/src/b.rs" ] || fail "S9: rm not applied" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S9: guarded .env changed by apply" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S9: guarded migration changed" -[ -d "$R/.acyclic" ] || fail "S9: repo config lost by apply" -must_fail "S9: apply twice" acy session-apply dry-3 -# The applied tree is the daemon's new mainline: a checkpoint/rewind cycle -# round-trips it, and the old tree is retained for the trash TTL. -acy checkpoint --wait --kind post >/dev/null || fail "S9: checkpoint after apply" -echo "$OUT" | grep -q "reload your editor" || fail "S9: no editor warning: $OUT" - -# --- S10: apply conflicts legibly when the mainline moved ----------------- -acy session-start dry-4 --host acceptance >/dev/null || fail "S10: session-start" -printf 'FORK-EDIT\n' > "$R/src/main.rs" -acy session-resolve dry-4 >/dev/null || fail "S10: resolve" -printf 'MAINLINE-EDIT\n' > "$R/src/main.rs" -settle 0.4 -acy checkpoint --wait --durable --kind post >/dev/null || fail "S10: mainline checkpoint" -set +e -OUT="$(acy session-apply dry-4 2>&1)" -CODE=$? -set -e -[ "$CODE" -ne 0 ] || fail "S10: apply succeeded over a moved mainline: $OUT" -echo "$OUT" | grep -qi "conflict\|moved\|behind" || fail "S10: conflict not legible: $OUT" -[ "$(cat "$R/src/main.rs")" = "MAINLINE-EDIT" ] || fail "S10: conflicting apply touched the tree" - -# --- S11: guarded paths also hold on explicit forks ---------------------- -# Verified via server truth (promote), not client reads: the macOS NFS -# client caches a doomed write and would echo it back on `cat`, so a -# guarded write's refusal shows only in what promote actually lands. -FORK_OUT="$(acy fork -n 1)" || fail "S11: fork" -FID="$(echo "$FORK_OUT" | awk '/^fork /{print $2}')" -F="$(dirname "$R")/.$(basename "$R").forks/mnt/$FID" -[ -f "$F/.env" ] || fail "S11: fork missing .env" -try bash -c "printf 'LEAK\n' > '$F/.env'" # guarded — must not land -try rm "$F/migrations/001.sql" # guarded — must not land -printf 'S11-FORK\n' > "$F/src/main.rs" || fail "S11: fork unguarded write" -settle 0.4 -acy promote "$FID" >/dev/null || fail "S11: promote" -# Only the unguarded edit reached the real tree; guarded paths are intact. -[ "$(cat "$R/src/main.rs")" = "S11-FORK" ] || fail "S11: unguarded edit not landed" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S11: guarded .env landed a change" -[ "$(cat "$R/migrations/001.sql")" = "CREATE TABLE a;" ] || fail "S11: guarded migration changed" - -# --- S12: stop unmounts an active shadow; nothing is left on the root ----- -acy session-start dry-5 --host acceptance >/dev/null || fail "S12: session-start" -printf 'LOST\n' > "$R/src/main.rs" -shadowed || fail "S12: not shadowed" -acy stop >/dev/null || fail "S12: stop" -sleep 0.5 -shadowed && fail "S12: shadow left mounted after stop" -[ "$(cat "$R/src/main.rs")" = "S11-FORK" ] || fail "S12: stop leaked shadow writes: $(cat "$R/src/main.rs")" - -# --- S13: kill -9 mid-session: the next daemon sweeps the dead shadow ----- -acy status >/dev/null || fail "S13: respawn" -acy session-start dry-6 --host acceptance >/dev/null || fail "S13: session-start" -printf 'LOST-2\n' > "$R/src/main.rs" -shadowed || fail "S13: not shadowed" -PID="$(daemon_pid)" -[ -n "$PID" ] || fail "S13: no daemon pid" -kill -9 "$PID" -sleep 0.5 -# A dead NFS server behind the mount: the real tree must come back on the -# next daemon start, without a manual umount. -acy status >/dev/null || fail "S13: daemon restart with dead shadow" -sleep 0.3 -shadowed && fail "S13: dead shadow not swept" -[ "$(cat "$R/src/main.rs")" = "S11-FORK" ] || fail "S13: crash leaked shadow writes: $(cat "$R/src/main.rs")" -[ "$(cat "$R/.env")" = "SECRET=1" ] || fail "S13: .env after crash" -must_fail "S13: resolve dead session" acy session-resolve dry-6 -# And the repo is fully usable again: a new session works end to end. -acy session-start dry-7 --host acceptance >/dev/null || fail "S13: session after sweep" -printf 'AFTER-CRASH\n' > "$R/src/main.rs" -acy session-resolve dry-7 | grep -q "^M src/main.rs" || fail "S13: resolve after sweep" -acy session-apply dry-7 >/dev/null || fail "S13: apply after sweep" -[ "$(cat "$R/src/main.rs")" = "AFTER-CRASH" ] || fail "S13: apply after sweep" - -pass "safe-mode: shadow session, guarded paths vs shell, resolve/discard/apply, conflict, stop + crash sweep" diff --git a/tests/acceptance/windows-smoke.sh b/tests/acceptance/windows-smoke.sh index fe60dad..b024c89 100644 --- a/tests/acceptance/windows-smoke.sh +++ b/tests/acceptance/windows-smoke.sh @@ -10,7 +10,7 @@ # (a daemon that inherits it never lets the caller see EOF) # - capture and diff under UTF-16LE names, non-ASCII included # - rewind, which renames the repo root and so trips every open handle -# - copy-mode forks and promote, the fork path Windows actually uses +# - required ProjFS-accelerated forks # # Runs under Git Bash on a GitHub windows runner. ACYCLIC_BIN overrides the # binary (default: target/release/acyclic.exe). @@ -21,20 +21,36 @@ ROOT="$(cd "$HERE/../.." && pwd)" ACYCLIC="${ACYCLIC_BIN:-$ROOT/target/release/acyclic.exe}" [ -x "$ACYCLIC" ] || { echo "windows-smoke: no binary at $ACYCLIC" >&2; exit 1; } -WORK="$(mktemp -d)" +WORK="$(mktemp -d -t acyclic-windows-smoke.XXXXXXXX)" +WORK="$(cd "$WORK" && pwd -P)" REPO="$WORK/repo" cleanup() { + local status=$? + if [ "$status" -ne 0 ] && [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ]; then + /usr/bin/find "$WORK/stores" -name daemon.log -exec tail -n 80 {} \; 2>/dev/null || true + fi "$ACYCLIC" --repo "$REPO" stop >/dev/null 2>&1 || true - # A daemon that outlives the run holds the store open and fails the next one. - powershell -NoProfile -Command \ - "Stop-Process -Name acyclic -Force -ErrorAction SilentlyContinue" >/dev/null 2>&1 || true - rm -rf "$WORK" 2>/dev/null || true + if [[ "$WORK" == */acyclic-windows-smoke.* && -d "$WORK" ]]; then + rm -rf -- "$WORK" 2>/dev/null || true + fi } trap cleanup EXIT fail() { echo "windows-smoke: $1" >&2; exit 1; } +metric() { + [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ] || return 0 + local now + now="$(date +%s%N)" + echo "windows-smoke metric $1 ms=$(((now - metric_start) / 1000000))" + metric_start="$now" +} + mkdir -p "$REPO/src" +mkdir -p "$REPO/.acyclic" +# Keep the daemon's durable store inside this disposable fixture, not under +# the runner account's persistent HOME. +printf 'store_dir = "%s"\n' "$(cygpath -m "$WORK/stores")" > "$REPO/.acyclic/config.toml" cd "$REPO" git init -q . git config user.email smoke@example.com @@ -46,33 +62,34 @@ printf 'base\n' > a.txt printf 'unicode\n' > "ünïcøde.txt" git add -A git commit -qm init +metric_start="$(date +%s%N)" echo "--- init (stdout through a pipe: must not hang)" # `| cat` is the whole point: a daemon holding an inherited stdout handle # leaves this blocked forever rather than returning. "$ACYCLIC" init < /dev/null | cat > /dev/null || fail "init failed" +metric init echo "--- the daemon pipe is reachable only by this account" powershell -NoProfile -ExecutionPolicy Bypass -File "$HERE/windows-pipe-acl.ps1" \ || fail "the daemon pipe is not owner-only" +metric pipe_acl echo "--- checkpoint and timeline" -# Let the baseline settle before editing. `init` returning does not guarantee -# the baseline predates a write landing microseconds later: the change gets -# folded into checkpoint #1 rather than appearing as a later one, and the diff -# below then has nothing to report. That race is not Windows-specific, and is -# not what this script is here to test. -sleep 3 +# `init` pings the pipeline, whose reply is queued behind its baseline. +"$ACYCLIC" timeline < /dev/null | grep -q baseline \ + || fail "init returned before the baseline was ready" printf 'DIFFERENT AND LONGER CONTENT\n' > src/main.rs printf 'extra\n' > added.txt -sleep 2 "$ACYCLIC" checkpoint < /dev/null > /dev/null || fail "checkpoint failed" "$ACYCLIC" timeline < /dev/null | grep -q baseline || fail "timeline has no baseline" +metric checkpoint echo "--- diff names the changed paths" diff_out="$("$ACYCLIC" diff 1 2 < /dev/null)" grep -q 'added.txt' <<< "$diff_out" || fail "diff missed added.txt: $diff_out" grep -qi 'main.rs' <<< "$diff_out" || fail "diff missed main.rs: $diff_out" +metric diff echo "--- rewind restores content and drops added files" # `rewind` asks for confirmation on a tty; feed it the answer rather than @@ -83,24 +100,53 @@ grep -q 'ORIGINAL CONTENT LINE' src/main.rs || fail "rewind did not restore main [ ! -e added.txt ] || fail "rewind left added.txt behind" [ -e "ünïcøde.txt" ] || fail "rewind lost the non-ASCII path" [ -d .git ] || fail "rewind lost .git" +metric rewind -echo "--- forks are copies here, and promote lands them" +echo "--- fork writes and promote land" "$ACYCLIC" fork -n 1 < /dev/null > /dev/null || fail "fork failed" -fork_id="$("$ACYCLIC" forks < /dev/null | head -1 | awk '{print $1}')" +fork_row="$("$ACYCLIC" forks < /dev/null | head -1)" +fork_id="$(awk '{print $1}' <<< "$fork_row")" +fork_mode="$(awk '{print $2}' <<< "$fork_row")" [ -n "$fork_id" ] || fail "no fork id" -fork_dir="$(dirname "$REPO")/.$(basename "$REPO").forks/copy/$fork_id" -[ -d "$fork_dir" ] || fail "no copy-fork directory at $fork_dir" +case "$fork_mode" in + mount) fork_dir="$(dirname "$REPO")/.$(basename "$REPO").forks/mnt/$fork_id" ;; + copy) fork_dir="$(dirname "$REPO")/.$(basename "$REPO").forks/copy/$fork_id" ;; + *) fail "Windows fork reported an unknown mode: $fork_row" ;; +esac +[ -d "$fork_dir" ] || fail "no $fork_mode fork directory at $fork_dir" +metric "${fork_mode}_fork" printf 'FORK WORK\n' > "$fork_dir/note.txt" printf 'edited in fork\n' > "$fork_dir/a.txt" "$ACYCLIC" fork-diff "$fork_id" < /dev/null | grep -q 'note.txt' \ || fail "fork-diff did not see the fork's write" +metric fork_diff "$ACYCLIC" promote "$fork_id" < /dev/null > /dev/null || fail "promote failed" grep -q 'FORK WORK' note.txt || fail "promote did not land note.txt" grep -q 'edited in fork' a.txt || fail "promote did not land a.txt" +metric promote + +if [ "${ACYCLIC_SMOKE_METRICS:-0}" = 1 ]; then + pid_file="$(/usr/bin/find "$WORK/stores" -name daemon.pid -print -quit)" + if [ -n "$pid_file" ]; then + ACYCLIC_SMOKE_PID="$(< "$pid_file")" powershell -NoProfile -Command \ + '$p = Get-Process -Id $env:ACYCLIC_SMOKE_PID; "windows-smoke resource cpu_ms={0} working_set_bytes={1} private_bytes={2}" -f [int64]($p.CPU * 1000), $p.WorkingSet64, $p.PrivateMemorySize64' + fi + echo "windows-smoke resource store_kib=$(du -sk "$WORK/stores" | awk '{print $1}')" + metric_start="$(date +%s%N)" +fi + +echo "--- daemon restart retains promoted files and timeline" +"$ACYCLIC" stop < /dev/null > /dev/null || fail "stop failed" +"$ACYCLIC" timeline < /dev/null | grep -q baseline \ + || fail "timeline did not recover after restart" +grep -q 'FORK WORK' note.txt || fail "restart lost promoted note.txt" +grep -q 'edited in fork' a.txt || fail "restart lost promoted a.txt" +metric restart echo "--- a second daemon refuses the store" if "$ACYCLIC" __daemon "$REPO" < /dev/null > /dev/null 2>&1; then fail "a second daemon started against a live store" fi +metric second_daemon_refusal echo "windows-smoke: green"