From 65671193a6a4f9e8c2ebffd8f88a90eb0fdee83c Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sat, 7 Nov 2026 22:20:31 -0500 Subject: [PATCH 1/5] feat: vm_sync / boxery sync push a local git checkout into a VM Commits go as a git bundle over vm_copy's streaming transport; neither side contacts a git remote. The VM checks out local HEAD (detached) and switches its working directory to a snapshot of HEAD plus staged, unstaged and untracked changes, keeping ignored files. Refuses (changing nothing) on VM-side edits since the last sync (refs/boxery/synced) and on credential files. Staged swap with a pending record, rollback and crash recovery. Reports a tree hash computed on both sides and lockfile changes. --- ARCHITECTURE.md | 2 +- controller/cli.mjs | 30 +++ controller/sync.mjs | 27 +++ core/boxery-core.mjs | 2 +- core/boxery-tools.mjs | 18 +- core/sync-guest.sh | 152 +++++++++++++++ core/sync.mjs | 267 ++++++++++++++++++++++++++ docs/README.md | 1 + docs/cli.md | 3 + docs/copy.md | 2 + docs/mcp.md | 8 +- docs/pi-extension.md | 1 + docs/sync.md | 60 ++++++ extension/index.ts | 2 +- mcp/boxery-mcp.mjs | 20 +- test/fixtures/boxery-help-all.txt | 1 + test/fixtures/boxery-help.txt | 1 + test/fixtures/sync-worker.mjs | 18 ++ test/lint.sh | 2 +- test/parity.test.mjs | 1 + test/sync.test.mjs | 298 ++++++++++++++++++++++++++++++ 21 files changed, 903 insertions(+), 13 deletions(-) create mode 100644 controller/sync.mjs create mode 100644 core/sync-guest.sh create mode 100644 core/sync.mjs create mode 100644 docs/sync.md create mode 100644 test/fixtures/sync-worker.mjs create mode 100644 test/sync.test.mjs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2cb25d6a..158e3eb9 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,7 +61,7 @@ VM (Ubuntu 24.04; XFCE desktop unless --headless) | Component | Runs on | Role | |---|---|---| | `core/boxery-tools.mjs` | agent machine (or host, for HTTP MCP) | **The `vm_*` tool table**: name, description, JSON Schema and runner for every tool, plus `vm_manage`/`vm_info`/`vm_lease` action logic (hosts included) over front-end session hooks, and `runTool()`, which resolves each call's host and VM. MCP and pi are both built from it and run tools through it, so they can't drift (`test/frontends.test.mjs`: a tool only one front-end has must say why). `test/parity.test.mjs` maps every tool and action to its `boxery` command (or why it has none) and every command back, so the CLI can't silently fall behind either | -| `core/boxery-core.mjs`, `core/copy-stream.mjs`, `core/atomic-edit.mjs`, `core/shell.mjs` | same | Transport and tool implementation: `sshRun`/`execRemote` (remote process-group kill on cancel), path mapping, read/write/edit (atomic, detects concurrent changes), streaming tar `vm_copy` (local⇄VM⇄VM, across hosts), computer use via xdotool, Playwright over CDP, `LeaseKeeper`, host-profile resolution with `AsyncLocalStorage` | +| `core/boxery-core.mjs`, `core/copy-stream.mjs`, `core/sync.mjs` + `core/sync-guest.sh`, `core/atomic-edit.mjs`, `core/shell.mjs` | same | Transport and tool implementation: `sshRun`/`execRemote` (remote process-group kill on cancel), path mapping, read/write/edit (atomic, detects concurrent changes), streaming tar `vm_copy` (local⇄VM⇄VM, across hosts), `vm_sync` (a local git checkout → VM: a bundle over the copy transport, then a staged swap in the VM), computer use via xdotool, Playwright over CDP, `LeaseKeeper`, host-profile resolution with `AsyncLocalStorage` | | `core/grants.mjs`, `core/grant-operations.mjs`, `core/grant-approval.mjs`, `core/grants-guest.sh` | agent machine | Shared spec/deny-directory validation and prompt text; grant store locking/edits, edge-route application and tmpfs reconciliation. Agent front-ends approve through elicitation/pi dialogs, then call operations in-process, without spawning the CLI. The guest script is a single reconciliation template with values inserted at apply time | | `core/boxery-jobs.ts`, `core/boxery-job-runner.sh` | piped into the VM per call | Background jobs (`run --bg`, `jobs`, `logs`, `wait`, `kill`; `vm_jobs`). The helper is erasable TypeScript run by the VM's Node (type stripping, ≥ 22.18), never installed, so a VM always runs the caller's version; it answers in JSON. The runner is plain bash, sent with each start: it runs the command in its own process group, independent of tmux, and records the real exit code. | | `backends/index.mjs` | agent machine and host | What each backend declares: supported and tested boxery-host platforms, runtimes, prerequisites and capabilities, as data (no probes). `host/capabilities.mjs` checks against it, `boxery-host backends`/`capabilities` print it, and `vm_info` lists it; it imports nothing. A backend without a declaration (one added as a `.sh` file) runs unchecked ([backends](docs/backends.md)) | diff --git a/controller/cli.mjs b/controller/cli.mjs index b8dd3f52..3df38380 100644 --- a/controller/cli.mjs +++ b/controller/cli.mjs @@ -262,6 +262,23 @@ async function cmdCp(s, args) { await copy(s.host, args); } +/** sync NAME [--from DIR] [--to PATH] [--json]: vm_sync's own code. Needs only the profile's ssh_config. @param {Selected} s @param {string[]} args @returns {Promise} */ +async function cmdSync(s, args) { + let name = "", from, to, json = false; + for (let i = 0; i < args.length; i++) { + const a = args[i], v = args[i + 1]; + if (a === "--from") { if (v == null) die("--from needs a directory"); from = v; i++; } + else if (a === "--to") { if (v == null) die("--to needs a path"); to = v; i++; } + else if (a === "--json") json = true; + else if (a.startsWith("-")) die(`unknown flag ${a}`); + else if (name) usage("sync"); + else name = a; + } + needVm(name); + const { sync } = await import("./sync.mjs"); + await sync(s.host, name, { from, to, json }); +} + /** version [--json] @param {Selected | null} s @param {string[]} args */ async function cmdVersion(s, args) { const json = args[0] === "--json"; @@ -523,6 +540,19 @@ export const COMMANDS = { status: {}, run: cmdCp, }, + sync: { + help: { + group: "use", summary: "push a local git checkout into a VM, so its tree matches exactly", usage: ["sync NAME [--from DIR] [--to PATH] [--json]"], + details: [ + "Sends the commits the VM lacks as a git bundle, checks out the same commit (detached) and applies staged, unstaged and untracked changes, deletions included. Ignored files in the VM (node_modules, builds) stay. No git remote or GitHub access is needed on either side.", + "Refuses, changing nothing, when the VM checkout has changes the last sync didn't make, or the checkout holds a credential file (.env, .npmrc, auth files).", + ], + flags: [["--from DIR", "Local checkout (default: the current directory)"], ["--to PATH", "VM directory (default ~/project; relative paths are under it)"], ["--json", "Print the result as JSON"]], + examples: [["This checkout into demo's ~/project", "boxery sync demo"], ["Another checkout, into ~/project/api", "boxery sync demo --from ../api --to api"]], + seeAlso: ["cp"], docs: "sync.md", + }, + run: cmdSync, + }, desktop: { help: { group: "use", summary: "open a VM's desktop in your browser (noVNC over ssh)", usage: ["desktop NAME [--no-open]"], flags: [["--no-open", "Print the URL instead of opening it"]], docs: "cli.md#desktop-and-ports" }, run: async (s, args) => { needHost(s); await cmdDesktop(s, args); }, diff --git a/controller/sync.mjs b/controller/sync.mjs new file mode 100644 index 00000000..4a7648e1 --- /dev/null +++ b/controller/sync.mjs @@ -0,0 +1,27 @@ +// boxery sync NAME: the CLI front-end of vm_sync. It runs the same TOOLS entry as the MCP server and the +// pi extension, so the transfer, refusals and result cannot drift (docs/sync.md). +import { constants } from "node:os"; +import * as core from "../core/boxery-core.mjs"; +import { TOOLS } from "../core/boxery-tools.mjs"; +import { die, quit, say } from "./lib.mjs"; +/** @import { Tool } from "../core/boxery-tools.mjs" */ +/** @import { HostProfile } from "../core/boxery-core.mjs" */ + +/** @param {HostProfile} host @param {string} vm @param {{ from?: string, to?: string, json?: boolean }} opts */ +export async function sync(host, vm, { from, to, json = false }) { + // Ctrl-C cancels like an MCP cancellation: before the VM applies anything, nothing changes there. + const abort = new AbortController(); + let interrupted = 0; + const handlers = /** @type {const} */ (["SIGINT", "SIGTERM", "SIGHUP"]).map((sig) => /** @type {[NodeJS.Signals, () => void]} */ ([sig, () => { interrupted = 128 + constants.signals[sig]; abort.abort(); }])); + for (const [sig, h] of handlers) process.once(sig, h); + const tool = /** @type {Tool} */ (TOOLS.find((t) => t.name === "vm_sync")); + try { + const r = await core.withHost(host, () => tool.run(vm, { from, to }, /** @type {any} */ ({ localCwd: process.cwd(), signal: abort.signal }) /* no session: vm_sync needs none */)); + say(json ? JSON.stringify(r.structured) : r.text); + if (r.isError) quit(1); + } catch (/** @type {any} */ e) { + if (e?.exitCode !== undefined) throw e; + if (interrupted) { process.stderr.write(`boxery: ${e.message}\n`); quit(interrupted); } + die(e.message); + } finally { for (const [sig, h] of handlers) process.off(sig, h); } +} diff --git a/core/boxery-core.mjs b/core/boxery-core.mjs index c8cf30f8..5531343e 100644 --- a/core/boxery-core.mjs +++ b/core/boxery-core.mjs @@ -890,7 +890,7 @@ export function copyRoot(dir, { deny = [] } = {}) { * unless it is inside the root and neither inside nor containing a denied path. * @param {LocalConfinement} confinement */ -function confineLocal({ root, deny = [] }) { +export function confineLocal({ root, deny = [] }) { const denied = copyDenied(deny); const check = (/** @type {string} */ real, shown = real) => { if (!isWithin(real, root)) throw new Error(`local:${shown} is outside the copy root ${root}`); diff --git a/core/boxery-tools.mjs b/core/boxery-tools.mjs index 36af6be6..8caaabdc 100644 --- a/core/boxery-tools.mjs +++ b/core/boxery-tools.mjs @@ -7,7 +7,7 @@ // Front-ends call runTool(), which resolves the call's host and VM against their Session and runs the // tool on that host. ctx: { signal, session, host, current, localCwd, localCopy?, progress?, approve? }. // progress(message) reports a status line while the tool runs (MCP progress notifications); front-ends -// without it pass nothing. localCopy limits vm_copy's local: endpoints (see copyTool); front-ends that run +// without it pass nothing. localCopy limits vm_copy's local: endpoints (see copyTool) and vm_sync's local checkout; front-ends that run // on the user's own machine pass nothing. // engine: pi runs this tool with its own built-in engine (same rendering/truncation as bash, read, …) // noVm: the front-end does not resolve a VM; the tool gets every argument, `vm` included @@ -16,6 +16,7 @@ // them only through the Session hooks. import * as core from "./boxery-core.mjs"; import { BACKENDS as BACKEND_DECLARATIONS } from "../backends/index.mjs"; +import { SYNC_OUTPUT, syncTool } from "./sync.mjs"; /** @import { Approve, HostProfile, LocalConfinement, ManageArgs, Progress, ToolResult } from "./boxery-core.mjs" */ /** A JSON Schema fragment. @typedef {{ type?: string | string[], description?: string, enum?: readonly string[], properties?: Record, items?: JsonSchema, required?: string[], anyOf?: JsonSchema[], minimum?: number, maximum?: number }} JsonSchema */ /** A tool's input schema: always an object with named properties. @typedef {JsonSchema & { type: "object", properties: Record, required: string[] }} ObjectSchema */ @@ -179,6 +180,21 @@ export const TOOLS = [ annotations: { destructiveHint: true, openWorldHint: false }, run: (_vm, a, x) => core.copyTool(a, { current: a.vm ?? x.current, localCwd: x.localCwd, local: x.localCopy, signal: x.signal }), }, + { + name: "vm_sync", + title: "VM sync", + description: + "Push a local git checkout into a VM so the VM's tree matches it exactly: same commit (detached HEAD), plus staged, unstaged and untracked changes, deletions included; the VM's ignored files (node_modules, builds) stay. " + + "Use it instead of vm_copy for a repository: it sends only what changed, needs no git remote or GitHub access, refuses VM-side edits it would overwrite, and reports whether lockfiles changed. " + + "It never syncs credentials: it refuses .env, .npmrc and other auth files (ask with vm_grant).", + parameters: obj({ + from: { type: "string", description: "Local checkout (default: this agent's working directory)" }, + to: { type: "string", description: `VM directory (default ${core.PROJECT}; relative paths are under it)` }, + }), + annotations: { destructiveHint: true, idempotentHint: true, openWorldHint: false }, + outputSchema: SYNC_OUTPUT, + run: (vm, a, x) => syncTool(vm, a, { localCwd: x.localCwd, local: x.localCopy, signal: x.signal }), + }, { name: "vm_screenshot", title: "VM screenshot", diff --git a/core/sync-guest.sh b/core/sync-guest.sh new file mode 100644 index 00000000..8099c709 --- /dev/null +++ b/core/sync-guest.sh @@ -0,0 +1,152 @@ +# shellcheck shell=bash +# vm_sync, the VM side. core/sync.mjs sends this text with every call (it is never installed), after +# assignments to MODE (probe | apply), DIR, for probe WANT, and for apply: FORMAT BUNDLE SNAP HEAD_SHA EXPECT_SYNCED EXPECT_HEAD. +# It never contacts a git remote. Output: key=value lines on stdout. Exit 3: refused, nothing changed +# (the reason on stderr). Exit 1: failed; apply has restored the checkout it found. +# +# State kept in the VM repo: refs/boxery/synced (the snapshot commit of the last sync: local HEAD plus +# the working changes, as one commit), refs/boxery/base (local HEAD then), and while a swap runs, +# $G/boxery-sync-pending ("OLD_TREE NEW_TREE") so a later sync can finish or undo an interrupted one. +set -u +: "${MODE:?}" "${DIR:?}" +export LC_ALL=C GIT_TERMINAL_PROMPT=0 GIT_OPTIONAL_LOCKS=0 +LOCKFILES=(':(glob)**/pnpm-lock.yaml' ':(glob)**/package-lock.json' ':(glob)**/yarn.lock' ':(glob)**/bun.lock') +refuse() { printf '%s\n' "$*" >&2; exit 3; } +die() { printf '%s\n' "$*" >&2; exit 1; } +rev() { git rev-parse -q --verify "$1^{commit}" 2>/dev/null || true; } +is_repo_root() { local top; top=$(git rev-parse --show-toplevel 2>/dev/null) && [ "$top" = "$(pwd -P)" ]; } +setup_git() { + G=$(git rev-parse --absolute-git-dir) || die "git: cannot find the repository in $DIR" + OBJ=$(git rev-parse --path-format=absolute --git-path objects) || die "git is too old (needs 2.31+)" + IDX=$(git rev-parse --path-format=absolute --git-path index) + EMPTY=$(git hash-object -t tree /dev/null) +} +# The tree to compare the working directory with: the last sync's, else HEAD's, else nothing. +baseline() { + local s h; s=$(rev refs/boxery/synced); h=$(rev HEAD) + if [ -n "$s" ]; then git rev-parse "$s^{tree}"; elif [ -n "$h" ]; then git rev-parse "$h^{tree}"; else echo "$EMPTY"; fi +} +# The working directory as a tree, as `git add -A` sees it (tracked files plus untracked ones that +# aren't ignored), starting from tree $1. Uses a scratch index and object directory, so neither the +# repo's index nor its object store changes. With $2 = diff, also lists up to 20 changes against $1. +wt() { + local t rc + t=$(mktemp -d) || return 1 + mkdir "$t/o" + [ ! -f "$IDX" ] || cp "$IDX" "$t/i" + ( + export GIT_INDEX_FILE="$t/i" GIT_OBJECT_DIRECTORY="$t/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" + # One-tree --reset keeps the copied index's stat data for entries it doesn't change, so unchanged + # files aren't hashed again, and (unlike -m) doesn't refuse entries that aren't up to date. + git read-tree --reset "$1" || exit 1 + git add -A 2>/dev/null || exit 1 + tree=$(git write-tree) || exit 1 + echo "$tree" + if [ "${2:-}" = diff ] && [ "$tree" != "$1" ]; then + git diff-tree -r --no-renames --name-status "$1" "$tree" | head -n 20 | sed 's/^/changed=/' + fi + ) + rc=$?; rm -rf "$t"; return "$rc" +} + +probe() { + if [ ! -e "$DIR" ]; then echo state=absent; return; fi + [ -d "$DIR" ] || refuse "$DIR exists and is not a directory" + cd "$DIR" || die "cannot enter $DIR" + if ! is_repo_root; then + if [ -n "$(ls -A)" ]; then echo state=dir; ls -A | head -n 5 | sed 's/^/entry=/'; else echo state=empty; fi + return + fi + setup_git + echo state=repo + echo "format=$(git rev-parse --show-object-format)" + echo "synced=$(rev refs/boxery/synced)" + echo "base=$(rev refs/boxery/base)" + echo "head=$(rev HEAD)" + [ -z "${WANT:-}" ] || ! git cat-file -e "$WANT^{commit}" 2>/dev/null || echo has=1 + if [ -f "$G/boxery-sync-pending" ]; then echo pending=1; return; fi + local b; b=$(baseline) + echo "baseline=$b" + wt "$b" diff | sed '1s/^/worktree=/' || die "cannot read the working directory of $DIR" +} + +# Put back tree $1 where tree $2 was swapped in (or half swapped in), and the index found before. +rollback() { + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$2" && + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree --reset -u "$2" "$1" || return 1 + if [ -f "$G/boxery-sync-index.bak" ]; then mv -f "$G/boxery-sync-index.bak" "$IDX"; else rm -f "$IDX"; fi + git update-ref -d refs/boxery/incoming 2>/dev/null + rm -f "$G/boxery-sync-index" "$G/boxery-sync-pending" +} +# A swap a crash interrupted: done if its refs were written, otherwise undone. +recover() { + [ -f "$G/boxery-sync-pending" ] || return 0 + local old new s + read -r old new <"$G/boxery-sync-pending" || die "unreadable $G/boxery-sync-pending" + s=$(rev refs/boxery/synced) + if [ -n "$s" ] && [ "$(git rev-parse "$s^{tree}")" = "$new" ]; then + rm -f "$G/boxery-sync-pending" "$G/boxery-sync-index" "$G/boxery-sync-index.bak" + else + rollback "$old" "$new" || die "an interrupted sync left $DIR half updated and it could not be undone; see $G/boxery-sync-pending" + fi +} +# Swap tree $1 for $2 in the working directory, then point the index, HEAD and our refs at the new state. +swap() { + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$1" || return 1 + GIT_INDEX_FILE="$G/boxery-sync-index" git update-index -q --refresh >/dev/null 2>&1 + # Two-tree switch: rewrites only the paths that differ and deletes what $2 lacks. Ignored files stay, + # unless $2 tracks that very path. --reset overwrites untracked files, but apply checked there are none. + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree --reset -u "$1" "$2" || return 1 + # The index holds local HEAD, so `git status` in the VM shows the working changes, as it does locally. + git read-tree "$HEAD_SHA" || return 1 + git update-index -q --refresh >/dev/null 2>&1 + printf '%s\n' 'option no-deref' "update HEAD $HEAD_SHA" "update refs/boxery/synced $SNAP" "update refs/boxery/base $HEAD_SHA" \ + 'delete refs/boxery/incoming' | git update-ref -m "boxery sync" --stdin || return 1 +} + +apply() { + : "${FORMAT:?}" "${SNAP:?}" "${HEAD_SHA:?}" "${BUNDLE?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" + # Past this point the caller has committed: finish or roll back, even if it cancels or disconnects. + trap '' INT TERM HUP PIPE + [ -z "$BUNDLE" ] || trap 'rm -f "$BUNDLE"' EXIT + mkdir -p "$DIR" && cd "$DIR" || die "cannot create $DIR" + if ! is_repo_root; then + [ -z "$(ls -A)" ] || refuse "$DIR is not a git repository and is not empty" + git init -q --object-format="$FORMAT" || die "git init failed in $DIR" + fi + setup_git + exec 9>"$G/boxery-sync.lock" + if command -v flock >/dev/null; then flock -w 60 9 || refuse "another sync into $DIR is running"; fi + recover + [ "$(rev refs/boxery/synced)" = "$EXPECT_SYNCED" ] && [ "$(rev HEAD)" = "$EXPECT_HEAD" ] || + refuse "the checkout in $DIR changed during the sync; run it again" + local b w new prev lock=0 + b=$(baseline) + w=$(wt "$b") || die "cannot read the working directory of $DIR" + [ "$w" = "$b" ] || refuse "uncommitted changes appeared in $DIR during the sync; run it again" + if [ -n "$BUNDLE" ]; then + git bundle verify -q "$BUNDLE" >/dev/null 2>&1 || die "the bundle doesn't apply to the VM's repository" + git fetch -q --no-tags --no-write-fetch-head "$BUNDLE" "+refs/heads/sync:refs/boxery/incoming" || die "git fetch from the bundle failed" + rm -f "$BUNDLE" + fi + git cat-file -e "$SNAP^{commit}" 2>/dev/null || die "the VM lacks commit $SNAP after the transfer" + new=$(git rev-parse "$SNAP^{tree}") || die "cannot read $SNAP" + prev=$(rev refs/boxery/synced); [ -n "$prev" ] || prev=$(rev HEAD) + git diff --quiet "${prev:-$EMPTY}" "$SNAP" -- "${LOCKFILES[@]}" || lock=1 + rm -f "$G/boxery-sync-index.bak" + [ ! -f "$IDX" ] || cp "$IDX" "$G/boxery-sync-index.bak" || die "cannot back up the index" + printf '%s %s\n' "$b" "$new" >"$G/boxery-sync-pending" || die "cannot write $G/boxery-sync-pending" + if ! swap "$b" "$new"; then + rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" + die "the sync failed; the checkout in $DIR was restored" + fi + rm -f "$G/boxery-sync-pending" "$G/boxery-sync-index" "$G/boxery-sync-index.bak" + echo "lockfile_changed=$lock" + wt "$new" | sed '1s/^/tree=/' || die "synced, but cannot read the working directory of $DIR back" +} + +case "$MODE" in +probe) probe ;; +apply) apply ;; +*) die "unknown mode $MODE" ;; +esac diff --git a/core/sync.mjs b/core/sync.mjs new file mode 100644 index 00000000..75797582 --- /dev/null +++ b/core/sync.mjs @@ -0,0 +1,267 @@ +// vm_sync: push the caller's local git checkout into a VM, so the VM's tree matches it exactly (docs/sync.md). +// Commits travel as a git bundle over vm_copy's streaming transport (copy-stream.mjs); neither side ever +// contacts a git remote. The VM side is core/sync-guest.sh, sent with every call. The local repository +// is only read: the snapshot of its working changes is built in a scratch index and object directory. +import { execFile } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import path from "node:path"; +import { q } from "./shell.mjs"; +import { streamCopy } from "./copy-stream.mjs"; +import { PROJECT, REMOTE_HOME, assertName, confineLocal, currentHost, deniedDirs, isWithin, sshRun, vmPath } from "./boxery-core.mjs"; +/** @import { LocalConfinement, ToolResult } from "./boxery-core.mjs" */ +/** @typedef {{ from?: string, to?: string }} SyncParams */ +/** @typedef {{ localCwd?: string, local?: LocalConfinement | false, signal?: AbortSignal }} SyncOpts */ +/** What the VM reported (sync-guest.sh probe, or apply's tree and lockfile_changed). @typedef {{ state: string, format?: string, synced?: string, base?: string, head?: string, has?: string, pending?: string, baseline?: string, worktree?: string, tree?: string, lockfile_changed?: string, changed: string[], entry: string[] }} Probe */ + +const guestTemplate = readFileSync(new URL("./sync-guest.sh", import.meta.url), "utf8"); + +export const SYNC_DISABLED = + "vm_sync is disabled on this server: it runs on the boxery control plane, so a local checkout would be that machine's files, not yours. " + + "Ask the user to start the server with --copy-root DIR to allow syncing checkouts inside DIR, or sync from a stdio MCP server or pi on your own machine."; + +/** Lockfiles whose change means the VM's dependencies need installing again. */ +export const LOCKFILES = ["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lock"]; + +// Credentials vm_sync never puts on a VM's disk (and so into its snapshots): the agent asks with vm_grant instead. +// Templates (.env.example, .env.sample, .env.template) hold no values and are allowed. +const DENY_NAME = /^(\.env(?!\.(example|sample|template)$)(\..*|rc)?|\.npmrc|\.pypirc|\.netrc|_netrc|\.git-credentials|\.pgpass|\.dockercfg|\.s3cfg|\.boto|auth\.json|credentials(\.json)?|id_(rsa|dsa|ecdsa|ed25519)(_sk)?)$/; +const DENY_PATH = /(^|\/)(\.docker\/config\.json|\.kube\/config|gh\/hosts\.yml)$/; +/** Is a repository path a credential file vm_sync refuses? @param {string} p */ +export const deniedSyncPath = (p) => DENY_NAME.test(path.posix.basename(p)) || DENY_PATH.test(p); + +const IDENTITY = { GIT_AUTHOR_NAME: "boxery sync", GIT_AUTHOR_EMAIL: "sync@boxery.invalid", GIT_AUTHOR_DATE: "@0 +0000" }; +const COMMITTER = { GIT_COMMITTER_NAME: IDENTITY.GIT_AUTHOR_NAME, GIT_COMMITTER_EMAIL: IDENTITY.GIT_AUTHOR_EMAIL, GIT_COMMITTER_DATE: IDENTITY.GIT_AUTHOR_DATE }; +// Repository-locating variables from the caller's environment would point git elsewhere. +const GIT_LOCATION = ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_NAMESPACE"]; +const baseEnv = () => { + /** @type {NodeJS.ProcessEnv} */ + const env = { ...process.env, LC_ALL: "C", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0" }; + for (const k of GIT_LOCATION) delete env[k]; + return env; +}; +const aborted = () => Object.assign(new Error("sync aborted; the VM was not changed"), { name: "AbortError" }); + +/** Run git locally → stdout. @param {string[]} args @param {{ cwd?: string, env?: NodeJS.ProcessEnv }} [opts] @returns {Promise} */ +function git(args, { cwd, env = baseEnv() } = {}) { + return new Promise((resolve, reject) => { + execFile("git", args, { cwd, env, maxBuffer: 256 * 1024 * 1024, encoding: "utf8" }, (error, stdout, stderr) => { + if (!error) return resolve(stdout); + const e = /** @type {NodeJS.ErrnoException} */ (error); + reject(new Error(e.code === "ENOENT" ? "git is not installed on this machine" : `git ${args[0]}: ${stderr.trim() || e.message}`)); + }); + }); +} +const tryGit = (/** @type {string[]} */ args, /** @type {{ cwd?: string, env?: NodeJS.ProcessEnv }} */ opts) => git(args, opts).then((s) => s.trim(), () => null); + +/** key=value lines from sync-guest.sh. Repeated keys (changed, entry) collect into arrays. @param {string} out @returns {Probe} */ +function parseLines(out) { + /** @type {any} */ + const r = { changed: [], entry: [] }; + for (const line of out.split("\n")) { + const i = line.indexOf("="); + if (i <= 0) continue; + const k = line.slice(0, i), v = line.slice(i + 1); + if (Array.isArray(r[k])) r[k].push(v); + else r[k] = v; + } + return r; +} + +/** Run sync-guest.sh in `vm` with `vars`. Exit 3 is a refusal: nothing changed. @param {string} vm @param {Record} vars */ +async function guest(vm, vars) { + const assignments = Object.entries(vars).map(([k, v]) => `${k}=${q(v)}`).join("\n"); + // No signal: probe is short, and apply must finish or roll back once started (it ignores signals itself). + const r = await sshRun(vm, `${assignments}\n${guestTemplate}`); + const err = r.stderr.trim(); + if (r.code === 3) throw new Error(`vm_sync refused: ${err}; nothing in the VM was changed`); + if (r.code !== 0) { + if (/git: (command )?not found/.test(err)) throw new Error(`git is not installed in VM "${vm}"`); + throw new Error(`vm_sync failed in VM "${vm}": ${err || `exit ${r.code}`}`); + } + return parseLines(r.stdout.toString("utf8")); +} + +/** + * The local checkout: repository root, HEAD, and a snapshot commit of HEAD plus every working change + * (staged, unstaged, untracked but not ignored), built without changing the repository. + * @param {string} from @param {string} scratch + */ +async function snapshot(from, scratch) { + const top = await tryGit(["rev-parse", "--show-toplevel"], { cwd: from }); + if (!top) { + const bare = await tryGit(["rev-parse", "--is-bare-repository"], { cwd: from }); + if (bare === "true") throw new Error(`${from} is a bare git repository; vm_sync pushes a checkout with a working tree`); + throw new Error(`${from} is not a git checkout (no repository found there or above it). vm_sync pushes a git checkout; use vm_copy for other files.`); + } + const root = await realpath(top); + const head = await tryGit(["rev-parse", "-q", "--verify", "HEAD^{commit}"], { cwd: root }); + if (!head) throw new Error(`${root} has no commits yet; commit once before syncing (vm_sync needs a HEAD to check out)`); + const format = (await git(["rev-parse", "--show-object-format"], { cwd: root })).trim(); + const objects = (await git(["rev-parse", "--path-format=absolute", "--git-path", "objects"], { cwd: root })).trim(); + const index = (await git(["rev-parse", "--path-format=absolute", "--git-path", "index"], { cwd: root })).trim(); + const scratchObjects = path.join(scratch, "objects"); + await mkdir(scratchObjects); + const scratchIndex = path.join(scratch, "index"); + let haveIndex = true; + try { await writeFile(scratchIndex, readFileSync(index)); } catch (e) { if (/** @type {any} */ (e).code !== "ENOENT") throw e; haveIndex = false; } + const env = { ...baseEnv(), GIT_INDEX_FILE: scratchIndex, GIT_OBJECT_DIRECTORY: scratchObjects, GIT_ALTERNATE_OBJECT_DIRECTORIES: objects }; + if (!haveIndex) await git(["read-tree", "HEAD"], { cwd: root, env }); + await git(["add", "-A"], { cwd: root, env }); + const tree = (await git(["write-tree"], { cwd: root, env })).trim(); + const headTree = (await git(["rev-parse", "HEAD^{tree}"], { cwd: root })).trim(); + // Deterministic (fixed identity and date), so the same working changes give the same commit every time. + const snap = tree === headTree ? head + : (await git(["commit-tree", tree, "-p", head, "-m", "boxery sync: working changes"], { cwd: root, env: { ...env, ...IDENTITY, ...COMMITTER } })).trim(); + const counts = { modified: 0, untracked: 0, deleted: 0 }; + if (tree !== headTree) { + const tracked = new Set((await git(["ls-files", "-z"], { cwd: root })).split("\0")); + const fields = (await git(["diff-tree", "-r", "-z", "-M", "--name-status", headTree, tree], { cwd: root, env })).split("\0").filter(Boolean); + for (let i = 0; i < fields.length;) { + const status = fields[i++]; + const paths = status.startsWith("R") || status.startsWith("C") ? [fields[i++], fields[i++]] : [fields[i++]]; + if (status === "D") counts.deleted++; + else if (status === "A" && !tracked.has(paths[0])) counts.untracked++; + else counts.modified++; + } + } + const files = (await git(["ls-tree", "-r", "-z", "--name-only", tree], { cwd: root, env })).split("\0").filter(Boolean); + return { root, head, format, objects, scratchObjects, env, tree, snap, counts, files }; +} + +/** + * Refuse credentials: a deny-listed file name anywhere in the tree, or anything in boxery's own state or + * the user's key directories (what vm_copy refuses), even where the checkout contains one of those. + * @param {string} root @param {string[]} files @param {string[]} [deny] + */ +function refuseCredentials(root, files, deny) { + const named = files.filter(deniedSyncPath); + if (named.length) + throw new Error(`vm_sync refused: ${named.slice(0, 5).join(", ")}${named.length > 5 ? ` (+${named.length - 5} more)` : ""} in ${root} looks like a credential file, which vm_sync never puts on a VM's disk (it would persist in its snapshots). ` + + "Ignore it in .gitignore (or remove it) to sync the rest, and ask for the credential with vm_grant. Nothing in the VM was changed."); + for (const { dir } of deniedDirs(deny)) { + if (isWithin(root, dir)) throw new Error(`vm_sync refused: ${root} is inside ${dir} (boxery's own configuration or the user's keys), which vm_sync never copies; use vm_grant for credentials`); + if (!isWithin(dir, root)) continue; + const rel = path.relative(root, dir).split(path.sep).join("/"); + const hit = files.find((f) => isWithin(f, rel)); + if (hit) throw new Error(`vm_sync refused: ${path.join(root, hit)} is inside ${dir} (boxery's own configuration or the user's keys), which vm_sync never copies; ignore it in .gitignore, and use vm_grant for credentials`); + } +} + +/** + * A bundle of the commits the VM lacks (null when it has the snapshot already), excluding what it has. + * @param {Awaited>} s @param {Probe} probe @param {string} scratch @param {string} id + */ +async function bundle(s, probe, scratch, id) { + if (probe.has === "1") return null; + // A bare repository whose alternates are the scratch objects and the checkout's: refs/heads/sync can + // be written there without touching the checkout's refs. + const bare = path.join(scratch, "bundle.git"); + const env = baseEnv(); + await git(["init", "-q", "--bare", `--object-format=${s.format}`, bare], { env }); + await writeFile(path.join(bare, "objects", "info", "alternates"), `${s.scratchObjects}\n${s.objects}\n`); + await git(["--git-dir", bare, "update-ref", "refs/heads/sync", s.snap], { env }); + const have = []; + for (const c of new Set([probe.synced, probe.base, probe.head].filter(Boolean))) + if (await tryGit(["--git-dir", bare, "cat-file", "-e", `${c}^{commit}`], { env }) !== null) have.push(`^${c}`); + const file = path.join(scratch, "out", `sync-${id}.bundle`); + await mkdir(path.dirname(file)); + try { + await git(["--git-dir", bare, "bundle", "create", "-q", file, "refs/heads/sync", ...have], { env }); + } catch (e) { + // "empty bundle": everything is reachable from what the VM has; send the snapshot whole instead. + if (!have.length || !/empty bundle/.test(/** @type {Error} */ (e).message)) throw e; + await git(["--git-dir", bare, "bundle", "create", "-q", file, "refs/heads/sync"], { env }); + } + return file; +} + +/** + * vm_sync / boxery sync: make the VM checkout at `to` (default ~/project) match the local checkout at + * `from` (default the caller's cwd) exactly: HEAD detached at the same commit, plus every working change. + * `local` is vm_copy's local: policy (undefined: unrestricted; false: refused; {root, deny}: confined). + * @param {string} vm @param {SyncParams} params @param {SyncOpts} [opts] @returns {Promise} + */ +export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd(), local, signal } = {}) { + const started = Date.now(); + assertName(vm); + if (local === false) throw new Error(SYNC_DISABLED); + const fromSpec = String(from ?? ".").replace(/^local:/, "") || "."; + const confined = local ? confineLocal(local) : null; + const fromPath = confined ? confined.source(fromSpec) : path.resolve(localCwd, fromSpec.replace(/^~(?=\/|$)/, homedir())); + const dest = vmPath(to ?? PROJECT); + if (dest === "/") throw new Error("vm_sync can't sync into the VM's filesystem root"); + const host = currentHost(); + const label = `${vm}:${dest}`; + if (signal?.aborted) throw aborted(); + const id = randomBytes(8).toString("hex"); + const scratch = await mkdtemp(path.join(tmpdir(), "boxery-sync-")); + const remoteBundle = `${REMOTE_HOME}/.cache/boxery/sync-${id}.bundle`; + let sent = false; + try { + const s = await snapshot(fromPath, scratch); + confined?.check(s.root); + refuseCredentials(s.root, s.files, local ? local.deny : undefined); + if (signal?.aborted) throw aborted(); + const probe = await guest(vm, { MODE: "probe", DIR: dest, WANT: s.snap }); + if (probe.state === "dir") + throw new Error(`vm_sync refused: ${label} is not a git repository and is not empty (${probe.entry.join(", ")}…); sync into an empty or new directory (to), or move those files away. Nothing in the VM was changed.`); + if (probe.state === "repo" && probe.format && probe.format !== s.format) + throw new Error(`vm_sync refused: the repository at ${label} uses ${probe.format} object ids and the local one ${s.format}`); + if (probe.state === "repo" && probe.pending !== "1" && probe.worktree !== probe.baseline) { + const what = probe.synced ? "since the last sync" : "against its HEAD"; + throw new Error(`vm_sync refused: ${label} has changes ${what} that vm_sync didn't make:\n${probe.changed.map((c) => ` ${c.replace("\t", " ")}`).join("\n")}\n` + + "Commit them in the VM and bring them back yourself, or discard them (git -C DIR stash -u, or checkout/clean), then sync again. Nothing in the VM was changed."); + } + const file = await bundle(s, probe, scratch, id); + let bytes = 0; + if (file) { + sent = true; + bytes = await streamCopy({ path: file, label: "local" }, { path: remoteBundle, vm, sshConf: host.sshConf, label: `${host.name}/${vm}` }, { signal }); + } + // The last point to stop: past here the VM applies the snapshot completely or not at all. + if (signal?.aborted) throw aborted(); + const applied = await guest(vm, { + MODE: "apply", DIR: dest, FORMAT: s.format, BUNDLE: file ? remoteBundle : "", SNAP: s.snap, HEAD_SHA: s.head, + EXPECT_SYNCED: probe.synced ?? "", EXPECT_HEAD: probe.head ?? "", + }); + sent = false; // apply removed the bundle + const match = applied.tree === s.tree; + const lockfileChanged = applied.lockfile_changed === "1"; + const short = s.head.slice(0, 7), { modified, untracked, deleted } = s.counts; + const text = `synced ${short} + ${modified} modified, ${untracked} untracked, ${deleted} deleted → ${label} · ` + + (match ? "tree matches" : `TREE MISMATCH (local ${s.tree.slice(0, 12)}, VM ${String(applied.tree).slice(0, 12)})`) + + (lockfileChanged ? " · lockfile changed" : ""); + return { + text, + isError: !match, + structured: { + vm, host: host.name, from: s.root, to: dest, sha: s.head, snapshot: s.snap, + counts: s.counts, tree: { local: s.tree, vm: applied.tree ?? null, match }, + lockfile_changed: lockfileChanged, transferred_bytes: bytes, elapsed_ms: Date.now() - started, + }, + }; + } finally { + // A bundle the transfer left but apply never consumed (cancelled or refused before it ran) is removed. + if (sent) await sshRun(vm, `rm -f ${q(remoteBundle)}`, { timeoutMs: 15_000 }).catch(() => {}); + await rm(scratch, { recursive: true, force: true }); + } +} + +export const SYNC_OUTPUT = { + type: "object", + properties: { + vm: { type: "string" }, host: { type: "string" }, + from: { type: "string", description: "the local repository root" }, + to: { type: "string", description: "the VM checkout" }, + sha: { type: "string", description: "local HEAD, checked out (detached) in the VM" }, + snapshot: { type: "string", description: "commit of HEAD plus the working changes (= sha when there are none); the VM's refs/boxery/synced" }, + counts: { type: "object", properties: { modified: { type: "integer" }, untracked: { type: "integer" }, deleted: { type: "integer" } } }, + tree: { type: "object", description: "git tree of the whole working directory (tracked + untracked, not ignored), computed on each side", properties: { local: { type: "string" }, vm: { type: ["string", "null"] }, match: { type: "boolean" } } }, + lockfile_changed: { type: "boolean", description: `a lockfile (${LOCKFILES.join(", ")}) differs from the previous sync: dependencies need installing` }, + transferred_bytes: { type: "integer", description: "bytes of bundle sent (0: the VM had every commit)" }, + elapsed_ms: { type: "integer" }, + }, +}; diff --git a/docs/README.md b/docs/README.md index ec811325..f53ebc1d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ - **[Shared pool & leases](pool.md)**: several agents share a pool of VMs safely. - **[GitHub Actions runners](github-runners.md)**: ephemeral self-hosted runners, one job per freshly reset pool VM. - **[File transfers](copy.md)**: streaming `vm_copy`, destination semantics and cancellation. +- **[Checkout sync](sync.md)**: `vm_sync` pushes a local git checkout into a VM so its tree matches exactly. ## Reference diff --git a/docs/cli.md b/docs/cli.md index f495efc7..7c74311d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -146,6 +146,8 @@ boxery --host hornet cp local:notes.md ci-1:~/notes.md `cp` is the `vm_copy` tool's own code, so it takes the same locations, `local:PATH`, `NAME:PATH` (a VM on the selected host) and `HOST/NAME:PATH`, and behaves the same way: streaming tar, staging, destination and cancellation rules are in [file transfers](copy.md). The one difference: the CLI has no current VM, so a bare path is on this machine, like `cp`/`scp` (`vm_copy` reads it as the current VM). A bare path that looks like a location with an invalid VM name (`Demo:x`) is refused instead of being written locally. Ctrl-C cancels and cleans up staging. `cp` runs on Node (≥ 20), which the npm package already needs. +`boxery sync demo [--from DIR] [--to PATH] [--json]` pushes a local git checkout into a VM so its tree matches exactly (commits as a git bundle, plus uncommitted and untracked changes); it is the `vm_sync` tool's own code ([sync](sync.md)). + `attach` starts pi, opencode or a shell (nothing else) and needs a terminal; without one it says to use `prompt` or `run --bg`, which work headless. `prompt` logs to `~/project/.boxery/task-*.log` in the VM. Attach to a task with `boxery attach demo pi task-…`. pi needs the `llm:` grant for its provider (`llm:chatgpt` unless you changed pi's settings in the VM), and a new VM has none: `prompt` checks first and, if it's missing, starts nothing and prints the `boxery grant` command to run (the older spelling of `boxery access grant`). ### Background command jobs @@ -308,6 +310,7 @@ Generated from the help `boxery help --all` shows (`controller/cli.mjs`); edit i | `boxery ssh NAME[@HOST] [CMD...]` | Shell into a VM (also: ssh NAME.boxery) | | `boxery run [--bg] NAME [--cwd DIR] [--] CMD [ARG...]` | Run a command in ~/project, args as given (like docker exec) [→](#shell-and-in-vm-agents) | | `boxery cp FROM TO` | Copy files and directories to, from and between VMs [→](copy.md) | +| `boxery sync NAME [--from DIR] [--to PATH] [--json]` | Push a local git checkout into a VM, so its tree matches exactly [→](sync.md) | | `boxery desktop NAME [--no-open]` | Open a VM's desktop in your browser (noVNC over ssh) [→](#desktop-and-ports) | | `boxery forward NAME PORT [LOCAL_PORT]` | Forward a VM port to localhost | | `boxery attach NAME [pi\|opencode\|shell] [SESSION]` | Attach to an agent in a persistent tmux session (= agent attach) | diff --git a/docs/copy.md b/docs/copy.md index f5a4862b..1a91a1ea 100644 --- a/docs/copy.md +++ b/docs/copy.md @@ -4,6 +4,8 @@ `boxery cp FROM TO` is the same code from the CLI, with the same locations, except that a bare path is local there (the CLI has no current VM). See the [CLI reference](cli.md#files). +For a git checkout, use [`vm_sync`](sync.md): it sends only what changed, keeps the VM's ignored files and checks that the trees match. + `vm_copy` transfers a file or directory **when explicitly called**. It does not continuously synchronize workspaces, and ordinary shell/file/browser tools do not copy the project back and forth. VM forks use a separate disk-image mechanism. ## Locations diff --git a/docs/mcp.md b/docs/mcp.md index fd19dc17..ab11713a 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -9,7 +9,7 @@ There are two ways to run it: | | Runs on | Transport | Good for | |---|---|---|---| | **A. stdio** (recommended) | your laptop | stdin/stdout | Full features. Grants and `vm_copy local:` use your laptop. | -| **B. HTTP** | the VM host | Streamable HTTP over Tailscale | Nothing to install on clients. `vm_copy local:` is off unless confined with `--copy-root`. | +| **B. HTTP** | the VM host | Streamable HTTP over Tailscale | Nothing to install on clients. `vm_copy local:` and `vm_sync` are off unless confined with `--copy-root`. | ## A. stdio on your laptop @@ -75,7 +75,9 @@ With `--copy-root DIR`, `local:` paths are confined to `DIR`: - Boxery's own state and the user's keys are refused even inside `DIR`, as is any path that contains them: `$BOXERY_CONF` and other `~/.config/boxery*` profiles, `~/.boxery` and registered profile directories, `~/.ssh`, `~/.gnupg`, `~/.pi`, and the `--token-file`. The server refuses to start if `DIR` is inside one of them. - A VM → local copy can create a new path or replace a file, but never merges into or replaces an existing directory: copy to a new path instead. It is extracted in a private staging directory (tar refuses absolute and `..` entries and writes through links within the archive), refused if it hard-links a file outside itself, and moved into place with `rename`, which replaces a symlink rather than following it. Symlinks inside a copied tree arrive as symlinks; later `local:` paths through them are checked like any other. -The `initialize` instructions and the `vm_copy` description tell the agent which of the two applies. stdio mode is unchanged: there `local:` is your own machine. `--copy-root` confines it there too, if you want that. +`vm_sync` ([sync](sync.md)) reads a local checkout, so it follows the same rule: refused over HTTP without `--copy-root`; with it, `from` resolves inside `DIR` and is checked the same way. + +The `initialize` instructions and the `vm_copy` and `vm_sync` descriptions tell the agent which of the two applies. stdio mode is unchanged: there `local:` is your own machine. `--copy-root` confines it there too, if you want that. ## Protocol features @@ -177,7 +179,7 @@ commit outcome**, not rollback: reread before retrying. | `--vm NAME` | Default VM. Without it, the model picks one with `vm_manage {action:"use"}`, and tools fail closed when no `vm` is given. | | `--http ADDR:PORT` | Serve Streamable HTTP at `/mcp` instead of stdio. Each MCP session keeps its own current VM. | | `--token-file PATH` | Bearer token from a file instead of `$BOXERY_MCP_TOKEN` | -| `--copy-root DIR` | Allow `vm_copy` `local:` paths, confined to `DIR` (see [B](#b-streamable-http-on-the-host-over-tailscale)). Over HTTP, `local:` is refused without it. | +| `--copy-root DIR` | Allow `vm_copy` `local:` paths and `vm_sync` checkouts, confined to `DIR` (see [B](#b-streamable-http-on-the-host-over-tailscale)). Over HTTP, both are refused without it. | | `--allow-user LOGIN` | Also require this `Tailscale-User-Login` header (set by `tailscale serve`). Loopback binds only. Can be repeated. | | `--allow-origin ORIGIN` | Accept requests with this `Origin`. Foreign origins are rejected by default. | | `--no-auth` | Skip the bearer token. **Local testing only:** refused unless `--http` binds loopback (`127.0.0.1`, `::1`, `localhost`). | diff --git a/docs/pi-extension.md b/docs/pi-extension.md index 06123bba..b61f496f 100644 --- a/docs/pi-extension.md +++ b/docs/pi-extension.md @@ -48,6 +48,7 @@ The current VM is saved in the session, so `pi --continue` picks it back up. Thr |---|---| | `vm_bash`, `vm_read`, `vm_write`, `vm_edit` | Shell and files in the VM. Same engines as the built-ins, with the same truncation, edit semantics and image handling. Relative paths resolve to `/home/agent/project`. | | `vm_copy` | Streaming [file transfers](copy.md) between `local:path`, `NAME:path`, or a bare path (= current VM). Works VM to VM too. | +| `vm_sync` | Push the local git checkout into the VM so its tree matches exactly ([sync](sync.md)) | | `vm_jobs` | Background jobs (`start` · `list` · `logs` · `wait` · `kill`) for long commands like test suites. Same as `boxery run --bg` ([CLI](cli.md#background-command-jobs)); details in [MCP](mcp.md#background-jobs). | | `vm_screenshot` | Screenshot of the XFCE desktop, downscaled to 1280 px (clicks are scaled back) | | `vm_click`, `vm_type`, `vm_key`, `vm_scroll` | Computer use via xdotool on `:1` | diff --git a/docs/sync.md b/docs/sync.md new file mode 100644 index 00000000..f08caafb --- /dev/null +++ b/docs/sync.md @@ -0,0 +1,60 @@ +# Pushing a checkout with `vm_sync` and `boxery sync` + +← [Docs](README.md) + +`vm_sync` makes a VM's copy of a git checkout match your local one exactly: the same commit, plus your staged, unstaged and untracked changes. Agents edit locally and run code in the VM; one sync before each run keeps the VM current, without giving the VM GitHub access or carrying patches by hand. It only pushes: nothing comes back from the VM, and there is no watch mode. + +```bash +boxery sync demo # the checkout you're in → demo:~/project +boxery sync demo --from ../api --to api # another checkout → demo:~/project/api +boxery sync demo --json # the result as JSON +``` + +```json +{"vm":"demo"} +{"vm":"demo","from":"../api","to":"api"} +``` + +`from` is a local directory inside the checkout (default: the agent's working directory, or the CLI's current directory); the whole repository is synced from its root. `to` is the VM directory (default `~/project`; relative paths are under it). `boxery sync` runs `vm_sync`'s own code ([CLI](cli.md#files)). + +## What it does + +1. **Snapshot.** It records local HEAD plus every working change (what `git add -A` would stage: tracked files, and untracked files `.gitignore` doesn't exclude, deletions and renames included) as one commit. It uses a scratch index and object directory: your index, refs and stash are untouched. The commit has a fixed author and date, so the same working state always gives the same commit. +2. **Transfer.** The commits the VM lacks go over as a `git bundle`, on the same streaming transport as [`vm_copy`](copy.md). Neither side contacts a git remote, so the VM needs no GitHub grant. If the VM already has the snapshot, nothing is sent. If `to` doesn't exist or is empty, a repository is initialized there. +3. **Swap.** In the VM, HEAD is detached at local HEAD's commit, and the working directory is switched to the snapshot: only the paths that differ are rewritten, and files the snapshot lacks are deleted, including ones an earlier sync put there. Ignored files (`node_modules`, build output) stay: there is no `git clean -x`. The VM's index holds HEAD, so `git status` and `git diff` in the VM show the same working changes as they do locally. +4. **Check.** Both sides hash the whole working directory (tracked plus untracked, not ignored) as a git tree, the same way, and compare. The result says `tree matches`, or `TREE MISMATCH` (an error). + +```text +synced 3f2a9c1 + 2 modified, 1 untracked, 0 deleted → demo:/home/agent/project · tree matches +``` + +Structured fields: `sha` (local HEAD), `snapshot`, `counts` (`modified`, `untracked`, `deleted`), `tree` (`local`, `vm`, `match`), `lockfile_changed`, `transferred_bytes`, `elapsed_ms`. + +`lockfile_changed` is true when `pnpm-lock.yaml`, `package-lock.json`, `yarn.lock` or `bun.lock` (at any depth) differs from the previous sync, or from the VM's HEAD on the first one. vm_sync installs nothing; run your install step when it says so. + +A second sync with no local changes sends nothing and still reports `tree matches`. + +## When it refuses + +It refuses, and changes nothing in the VM, when: + +- **The VM checkout has changes the last sync didn't make.** The VM records each sync in `refs/boxery/synced` (and local HEAD in `refs/boxery/base`). Any difference between the VM's working directory and that snapshot (edited, new or deleted files, not ignored ones) is listed in the error. Commit them in the VM and bring them back yourself, or discard them, then sync again. A VM repository that was never synced is compared with its HEAD. +- **The checkout holds a credential file.** `.env` and `.env.*` (except `.env.example`, `.env.sample`, `.env.template`), `.envrc`, `.npmrc`, `.pypirc`, `.netrc`, `.git-credentials`, `.pgpass`, `auth.json`, `credentials`, private ssh keys (`id_ed25519`, …), `.docker/config.json`, `.kube/config`, `gh/hosts.yml`, and anything inside boxery's own state or `~/.ssh`, `~/.gnupg`, `~/.pi` (what `vm_copy` refuses). The error names the path. Ignore it in `.gitignore` to sync the rest, and get the credential to the VM with a [grant](grants.md), which keeps it off the VM's disk and out of its snapshots. Ignored files are never synced, so an ignored `.env` is fine. +- **`to` is a non-empty directory that isn't a git repository**, or one whose object format (sha1/sha256) differs. +- **The local path isn't a git checkout**, or has no commits yet. Use `vm_copy` for plain files. + +## Cancellation and failure + +Cancelling (an MCP cancellation, pi's Escape, Ctrl-C for `boxery sync`) before the transfer finishes leaves the VM unchanged: the partial bundle is removed with `vm_copy`'s staging. Once the VM starts applying, it ignores signals and finishes or rolls back; a lost connection doesn't interrupt it. + +The swap is staged: the VM fetches the bundle into its object store (which changes no files), then writes a pending record, switches the working directory, and finally updates HEAD and its refs in one transaction. If a step fails, it switches back and restores the index. If the VM crashes mid-swap, the next sync finds the pending record and finishes it (if the refs were written) or undoes it, before checking for VM-side changes. Fetched objects stay in the VM's repository either way; they're harmless and `git gc` collects them. + +A sync takes a lock in the VM repository, so two syncs into one checkout run one after the other. If the checkout changes between the check and the swap, the sync refuses. + +## Limits + +- Needs git (2.31+) on both sides; boxery's images have it. +- Submodules are synced as gitlinks (their commit ids), not their contents. +- Executable bits and symlinks follow git's rules; other file modes, owners and timestamps aren't carried. +- The VM keeps every synced snapshot's objects until `git gc`. +- On an HTTP MCP server, `vm_sync` reads checkouts on that server, so it follows `vm_copy`'s `local:` rule: refused without `--copy-root`, and confined to `DIR` with it ([MCP server](mcp.md#b-streamable-http-on-the-host-over-tailscale)). diff --git a/extension/index.ts b/extension/index.ts index ff0b3f4d..ee758359 100644 --- a/extension/index.ts +++ b/extension/index.ts @@ -4,7 +4,7 @@ * Built-in tools (bash, read, write, edit, …) are NOT touched: they keep acting on this machine, so * other extensions that wrap them keep working. VM work goes through clearly named vm_* tools: * - * vm_bash vm_read vm_write vm_edit vm_copy files and shell in a VM + * vm_bash vm_read vm_write vm_edit vm_copy vm_sync files and shell in a VM * vm_jobs background command jobs (start / list / logs / wait / kill) * vm_screenshot vm_click vm_type vm_key vm_scroll computer use on the VM desktop * vm_browser Playwright → Chrome on the VM desktop diff --git a/mcp/boxery-mcp.mjs b/mcp/boxery-mcp.mjs index 377ebbd3..0a195a25 100755 --- a/mcp/boxery-mcp.mjs +++ b/mcp/boxery-mcp.mjs @@ -7,10 +7,10 @@ // auth: bearer token from $BOXERY_MCP_TOKEN or --token-file (required unless --no-auth, loopback only), // optional --allow-user a@b.c (checks Tailscale-User-Login from `tailscale serve`) // --ui offer vm_view (live desktop view) even to clients that don't advertise MCP Apps support -// --copy-root DIR allow vm_copy local: paths only inside DIR (never boxery's config, ~/.ssh, ~/.pi) +// --copy-root DIR allow vm_copy local: paths and vm_sync checkouts only inside DIR (never boxery's config, ~/.ssh, ~/.pi) // // "local" (vm_copy) and grant sources are the machine this server runs on. Over HTTP that is the -// control plane, so vm_copy local: endpoints are refused unless --copy-root confines them. +// control plane, so vm_copy local: endpoints and vm_sync are refused unless --copy-root confines them. import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; import { readFileSync } from "node:fs"; import http from "node:http"; @@ -79,9 +79,19 @@ const LOCAL_COPY_NOTE = // ---------------------------------------------------------------- tools // Built from the shared table (core/boxery-tools.mjs), host selection included. MCP adds only -// vm_copy's note on what `local:` means for this server, and the live desktop view below. +// vm_copy's and vm_sync's notes on what `local:` means for this server, and the live desktop view below. +// vm_sync reads a local checkout, so the same local: policy applies to it (core/sync.mjs). +const LOCAL_SYNC_NOTE = + LOCAL_COPY === false + ? "On this server vm_sync is disabled: it runs on the boxery control plane, not your machine." + : LOCAL_COPY + ? `On this server \`from\` is a checkout on the machine it runs on, confined to ${COPY_ROOT} (relative paths resolve there).` + : null; /** @type {Record Tool>} */ -const MCP_TOOL = { vm_copy: (t) => (LOCAL_COPY_NOTE ? { ...t, description: `${t.description} ${LOCAL_COPY_NOTE}` } : t) }; +const MCP_TOOL = { + vm_copy: (t) => (LOCAL_COPY_NOTE ? { ...t, description: `${t.description} ${LOCAL_COPY_NOTE}` } : t), + vm_sync: (t) => (LOCAL_SYNC_NOTE ? { ...t, description: `${t.description} ${LOCAL_SYNC_NOTE}` } : t), +}; /** @type {McpTool[]} */ const TOOLS = SHARED_TOOLS.map((t) => MCP_TOOL[t.name]?.(t) ?? t).map((t) => ({ name: t.name, @@ -226,7 +236,7 @@ const instructions = (/** @type {Session} */ s) => host: s.host.name, localToolsNote: "Your other tools (shell, files) act on this machine, not a VM; use them only when you mean to work here.", - }) + (LOCAL_COPY_NOTE ? `\nvm_copy: ${LOCAL_COPY_NOTE}` : ""); + }) + (LOCAL_COPY_NOTE ? `\nvm_copy: ${LOCAL_COPY_NOTE}` : "") + (LOCAL_SYNC_NOTE ? `\nvm_sync: ${LOCAL_SYNC_NOTE}` : ""); /** Send the client a request on this call's channel and resolve with its result. Abort sends notifications/cancelled. * @param {Session} session @param {Notify} send @param {string} method @param {object} params @param {AbortSignal} [signal] @returns {Promise} */ diff --git a/test/fixtures/boxery-help-all.txt b/test/fixtures/boxery-help-all.txt index 16ae7de0..54cc52be 100644 --- a/test/fixtures/boxery-help-all.txt +++ b/test/fixtures/boxery-help-all.txt @@ -21,6 +21,7 @@ Work in a VM: ssh NAME[@HOST] [CMD...] Shell into a VM (also: ssh NAME.boxery) run NAME CMD [ARG...] Run a command in ~/project, args as given (like docker exec) cp FROM TO Copy files and directories to, from and between VMs + sync NAME Push a local git checkout into a VM, so its tree matches exactly desktop NAME Open a VM's desktop in your browser (noVNC over ssh) forward NAME PORT [LOCAL] Forward a VM port to localhost attach NAME [AGENT] Attach to an agent in a persistent tmux session (= agent attach) diff --git a/test/fixtures/boxery-help.txt b/test/fixtures/boxery-help.txt index da6f1737..f3091a1c 100644 --- a/test/fixtures/boxery-help.txt +++ b/test/fixtures/boxery-help.txt @@ -21,6 +21,7 @@ Work in a VM: ssh NAME[@HOST] [CMD...] Shell into a VM (also: ssh NAME.boxery) run NAME CMD [ARG...] Run a command in ~/project, args as given (like docker exec) cp FROM TO Copy files and directories to, from and between VMs + sync NAME Push a local git checkout into a VM, so its tree matches exactly desktop NAME Open a VM's desktop in your browser (noVNC over ssh) forward NAME PORT [LOCAL] Forward a VM port to localhost attach NAME [AGENT] Attach to an agent in a persistent tmux session (= agent attach) diff --git a/test/fixtures/sync-worker.mjs b/test/fixtures/sync-worker.mjs new file mode 100644 index 00000000..cc74e093 --- /dev/null +++ b/test/fixtures/sync-worker.mjs @@ -0,0 +1,18 @@ +// One vm_sync call in its own process (core reads BOXERY_HOME at import), against the fake SSH endpoints +// of test/fixtures/copy-ssh.mjs. argv: VM, then JSON {from, to, local?}. IPC "abort" cancels. +import { withHost } from "../../core/boxery-core.mjs"; +import { syncTool } from "../../core/sync.mjs"; +const [vm, settings = "{}"] = process.argv.slice(2); +const { local, ...params } = JSON.parse(settings); +const ac = new AbortController(); +process.on("message", (m) => { if (m === "abort") ac.abort(); }); +process.once("disconnect", () => ac.abort()); +/** @type {any} */ +let result; +try { + result = { ok: true, ...(await withHost("alpha", () => syncTool(vm, params, { signal: ac.signal, local }))) }; +} catch (e) { + result = { ok: false, error: /** @type {Error} */ (e).message, name: /** @type {Error} */ (e).name }; +} +process.stdout.write(`${JSON.stringify(result)}\n`); +if (process.connected) process.disconnect(); diff --git a/test/lint.sh b/test/lint.sh index 743b75bc..71aface7 100755 --- a/test/lint.sh +++ b/test/lint.sh @@ -2,7 +2,7 @@ # Static checks: shellcheck (warnings are errors), node syntax checks. set -euo pipefail cd "$(dirname "$0")/.." -shellcheck -S warning controller/boxery test/fixtures/bash-profile.sh test/fixtures/ssh-config-golden.sh core/boxery-job-runner.sh host/boxery-host host/backend.sh host/backends/*.sh host/backends/lib/*.sh image/layers/*.sh e2e.sh mcp/test-mcp.sh test/shell/*.sh test/shell/*/*.sh test/bench/*.sh +shellcheck -S warning controller/boxery test/fixtures/bash-profile.sh test/fixtures/ssh-config-golden.sh core/boxery-job-runner.sh core/sync-guest.sh host/boxery-host host/backend.sh host/backends/*.sh host/backends/lib/*.sh image/layers/*.sh e2e.sh mcp/test-mcp.sh test/shell/*.sh test/shell/*/*.sh test/bench/*.sh for f in controller/*.mjs core/*.mjs backends/*.mjs host/backends/lib/*.mjs guests/*.mjs help/*.mjs host/*.mjs mcp/*.mjs proxy/*.mjs test/*.mjs test/fixtures/*.mjs evals/*.mjs evals/lib/*.mjs scripts/*.mjs; do node --check "$f"; done # VM-side TypeScript runs under type stripping: it must parse (types are checked by `npm run typecheck`) # (only where this Node can strip types: 22.18+/23.6+; older Node skips it, like the job tests) diff --git a/test/parity.test.mjs b/test/parity.test.mjs index b4fd6c5e..c91a900a 100644 --- a/test/parity.test.mjs +++ b/test/parity.test.mjs @@ -19,6 +19,7 @@ const CLI_FOR_TOOL = { vm_write: "cp", vm_edit: none("people edit in their editor, over boxery ssh or after boxery cp"), vm_copy: "cp", + vm_sync: "sync", vm_screenshot: DESKTOP, vm_click: DESKTOP, vm_type: DESKTOP, diff --git a/test/sync.test.mjs b/test/sync.test.mjs new file mode 100644 index 00000000..5506df74 --- /dev/null +++ b/test/sync.test.mjs @@ -0,0 +1,298 @@ +// vm_sync with real git and tar against temporary fake SSH endpoints (test/fixtures/copy-ssh.mjs): each +// "VM" is a directory, and its ssh runs scripts there with /home/agent mapped to it. No real VMs. +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, test as nodeTest } from "node:test"; +import { q } from "../core/shell.mjs"; +import { deniedSyncPath } from "../core/sync.mjs"; +const test = (/** @type {string} */ name, /** @type {() => Promise | void} */ fn) => nodeTest(name, { timeout: 60_000 }, fn); + +const FIXTURES = path.resolve(import.meta.dirname, "fixtures"); +/** @type {string} */ let dir; +/** @type {NodeJS.ProcessEnv} */ let env; +/** @type {string} */ let repo; +const vmProject = () => path.join(dir, "vms", "alpha", "vm", "project"); +const GIT_ENV = { GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t", GIT_CONFIG_NOSYSTEM: "1" }; +const git = (/** @type {string} */ cwd, /** @type {string[]} */ ...args) => execFileSync("git", args, { cwd, env, encoding: "utf8" }).trim(); +const write = (/** @type {string} */ root, /** @type {string} */ rel, /** @type {string | Buffer} */ data) => { + mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + writeFileSync(path.join(root, rel), data); +}; +const read = (/** @type {string} */ root, /** @type {string} */ rel) => readFileSync(path.join(root, rel), "utf8"); + +/** Run vm_sync in a worker; abortWhen(dir) → abort once it returns true. @param {object} [params] @param {{ abortWhen?: () => boolean, vm?: string }} [opts] */ +function sync(params = {}, { abortWhen, vm = "vm" } = {}) { + const child = spawn(process.execPath, [path.join(FIXTURES, "sync-worker.mjs"), vm, JSON.stringify({ from: repo, ...params })], { + env, cwd: dir, stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + let out = "", err = ""; + child.stdout?.on("data", (d) => { out += d; }); + child.stderr?.on("data", (d) => { err += d; }); + let timer; + if (abortWhen) timer = setInterval(() => { if (abortWhen() && child.connected) { child.send("abort"); clearInterval(timer); } }, 5); + return new Promise((resolve, reject) => { + child.on("close", () => { + clearInterval(timer); + try { resolve(JSON.parse(out)); } catch { reject(new Error(`sync worker produced no result: ${err} ${out}`)); } + }); + }); +} +async function ok(params, opts) { + const r = /** @type {any} */ (await sync(params, opts)); + assert.equal(r.ok, true, r.error); + return r; +} +/** Every tracked and untracked (not ignored) file with its content, relative to root. @param {string} root */ +function tree(root) { + const files = git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0").filter(Boolean) + .filter((f) => existsSync(path.join(root, f))); + return Object.fromEntries([...new Set(files)].sort().map((f) => [f, readFileSync(path.join(root, f)).toString("base64")])); +} + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), "boxery-sync-")); + mkdirSync(path.join(dir, "bin")); + const home = path.join(dir, "home"); + mkdirSync(path.join(home, "host-registry"), { recursive: true }); + const profile = path.join(home, "profiles", "alpha"); + mkdirSync(profile, { recursive: true }); + writeFileSync(path.join(profile, "config"), "# fake\n"); + writeFileSync(path.join(profile, "ssh_config"), "# fake\n"); + writeFileSync(path.join(home, "host-registry", "alpha"), `${profile}\n`); + writeFileSync(path.join(home, "default-host"), "alpha\n"); + for (const vm of ["vm", "other"]) mkdirSync(path.join(dir, "vms", "alpha", vm, "tmp"), { recursive: true }); + writeFileSync(path.join(dir, "calls"), ""); + writeFileSync(path.join(dir, "bin", "ssh"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "copy-ssh.mjs"))} "$@"\n`, { mode: 0o755 }); + writeFileSync(path.join(dir, "bin", "tar"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "copy-tar.mjs"))} "$@"\n`, { mode: 0o755 }); + writeFileSync(path.join(dir, "bin", "setsid"), "#!/usr/bin/env bash\n[ \"$1\" != -w ] || shift\nexec perl -e 'setpgrp(0,0); exec @ARGV or die' -- \"$@\"\n", { mode: 0o755 }); + env = { + ...process.env, ...GIT_ENV, HOME: home, PATH: `${path.join(dir, "bin")}:${process.env.PATH}`, BOXERY_HOME: home, + COPY_ROOT: dir, COPY_LOG: path.join(dir, "calls"), REAL_TAR: execFileSync("sh", ["-c", "command -v tar"], { encoding: "utf8" }).trim(), + }; + repo = path.join(dir, "local", "repo"); + mkdirSync(repo, { recursive: true }); + git(repo, "init", "-q", "-b", "main"); + write(repo, ".gitignore", "node_modules/\ndist/\n"); + write(repo, "README.md", "hello\n"); + write(repo, "src/app.js", "console.log(1);\n"); + write(repo, "src/old name.txt", "rename me\n"); + write(repo, "gone.txt", "delete me\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "first"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +test("first sync into an empty directory: a detached checkout that matches", async () => { + const r = await ok(); + const head = git(repo, "rev-parse", "HEAD"); + assert.match(r.text, new RegExp(`^synced ${head.slice(0, 7)} \\+ 0 modified, 0 untracked, 0 deleted → vm:/home/agent/project · tree matches$`)); + assert.equal(r.structured.sha, head); + assert.equal(r.structured.tree.match, true); + assert.equal(r.structured.tree.local, r.structured.tree.vm); + assert.ok(r.structured.transferred_bytes > 0); + assert.equal(typeof r.structured.elapsed_ms, "number"); + assert.equal(git(vmProject(), "rev-parse", "HEAD"), head); + assert.throws(() => git(vmProject(), "symbolic-ref", "-q", "HEAD"), "detached"); + assert.deepEqual(tree(vmProject()), tree(repo)); + assert.equal(git(vmProject(), "status", "--porcelain"), "", "a clean checkout in the VM"); + assert.equal(git(vmProject(), "remote"), "", "no remote configured"); +}); + +test("new commits, dirty, untracked, deleted and renamed files, a binary and spaces in names", async () => { + await ok(); + write(repo, "src/app.js", "console.log(2);\n"); + git(repo, "commit", "-qam", "second"); + write(repo, "README.md", "hello, changed\n"); // unstaged + write(repo, "staged.txt", "staged\n"); + git(repo, "add", "staged.txt"); // staged, new + write(repo, "notes with spaces.md", "untracked\n"); + write(repo, "bin/blob.bin", Buffer.from([0, 255, 1, 254, 0, 10, 13, 0])); + unlinkSync(path.join(repo, "gone.txt")); + renameSync(path.join(repo, "src/old name.txt"), path.join(repo, "src/new name.txt")); + const r = await ok(); + assert.equal(r.structured.sha, git(repo, "rev-parse", "HEAD")); + assert.equal(git(vmProject(), "rev-parse", "HEAD"), r.structured.sha); + assert.equal(r.structured.tree.match, true); + assert.deepEqual(r.structured.counts, { modified: 3, untracked: 2, deleted: 1 }, "README, staged.txt, the rename; two untracked; gone.txt"); + assert.deepEqual(tree(vmProject()), tree(repo)); + assert.deepEqual(readFileSync(path.join(vmProject(), "bin/blob.bin")), Buffer.from([0, 255, 1, 254, 0, 10, 13, 0])); + assert.ok(!existsSync(path.join(vmProject(), "gone.txt"))); + assert.ok(!existsSync(path.join(vmProject(), "src/old name.txt"))); + // The VM shows the same working changes against HEAD as the local checkout. + assert.match(git(vmProject(), "status", "--porcelain"), /notes with spaces/); + assert.equal(git(repo, "status", "--porcelain").length > 0, true, "the local checkout is untouched"); + assert.equal(git(repo, "for-each-ref").includes("boxery"), false, "no refs written locally"); + + // Reverting local changes removes the earlier sync's leftovers. + git(repo, "checkout", "-q", "--", "."); + git(repo, "reset", "-q", "--hard"); + git(repo, "clean", "-qfd"); + const r2 = await ok(); + assert.deepEqual(r2.structured.counts, { modified: 0, untracked: 0, deleted: 0 }); + assert.deepEqual(tree(vmProject()), tree(repo)); + assert.ok(!existsSync(path.join(vmProject(), "notes with spaces.md"))); +}); + +test("ignored files in the VM survive a sync", async () => { + await ok(); + write(vmProject(), "node_modules/dep/index.js", "module.exports = 1;\n"); + write(vmProject(), "dist/out.js", "built\n"); + write(repo, "src/app.js", "console.log(3);\n"); + const r = await ok(); + assert.equal(r.structured.tree.match, true); + assert.equal(read(vmProject(), "node_modules/dep/index.js"), "module.exports = 1;\n"); + assert.equal(read(vmProject(), "dist/out.js"), "built\n"); + assert.equal(read(vmProject(), "src/app.js"), "console.log(3);\n"); +}); + +test("a VM-side edit blocks the sync and changes nothing", async () => { + await ok(); + write(vmProject(), "src/app.js", "edited in the VM\n"); + write(vmProject(), "scratch.txt", "new in the VM\n"); + write(repo, "README.md", "local change\n"); + const before = tree(vmProject()); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /refused: vm:\/home\/agent\/project has changes since the last sync that vm_sync didn't make/); + assert.match(r.error, /M src\/app\.js/); + assert.match(r.error, /A scratch\.txt/); + assert.deepEqual(tree(vmProject()), before); + assert.equal(read(vmProject(), "README.md"), "hello\n"); +}); + +test("a dirty VM checkout that was never synced blocks the sync", async () => { + mkdirSync(vmProject(), { recursive: true }); + git(vmProject(), "init", "-q"); + write(vmProject(), "mine.txt", "x\n"); + git(vmProject(), "add", "-A"); + git(vmProject(), "commit", "-qm", "vm"); + write(vmProject(), "mine.txt", "dirty\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /changes against its HEAD/); + assert.equal(read(vmProject(), "mine.txt"), "dirty\n"); +}); + +test("a non-empty directory that isn't a repository is refused", async () => { + write(vmProject(), "stray.txt", "x\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /is not a git repository and is not empty \(stray\.txt/); + assert.ok(!existsSync(path.join(vmProject(), ".git"))); +}); + +test("deny-listed files block the sync, naming the path and vm_grant", async () => { + await ok(); + write(repo, "config/.env.local", "SECRET=1\n"); + write(repo, "README.md", "changed\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /config\/\.env\.local/); + assert.match(r.error, /vm_grant/); + assert.equal(read(vmProject(), "README.md"), "hello\n", "nothing changed in the VM"); + // Once ignored, the rest syncs. + write(repo, ".gitignore", "node_modules/\ndist/\n.env*\n"); + assert.equal((await ok()).structured.tree.match, true); + assert.ok(!existsSync(path.join(vmProject(), "config/.env.local"))); +}); + +test("the deny list", () => { + for (const p of [".env", ".env.local", "a/.env.production", ".envrc", ".npmrc", "x/.pypirc", ".netrc", ".git-credentials", "auth.json", ".docker/config.json", "home/.kube/config", "credentials", "id_ed25519"]) + assert.ok(deniedSyncPath(p), p); + for (const p of [".env.example", ".env.sample", "environment.ts", "src/auth.js", "npmrc.md", "docs/credentials.md", "id_ed25519.pub"]) + assert.ok(!deniedSyncPath(p), p); +}); + +test("a second sync with no local changes transfers nothing and still matches", async () => { + write(repo, "wip.txt", "uncommitted\n"); + const first = await ok(); + const calls = readFileSync(path.join(dir, "calls"), "utf8").split("\n").filter(Boolean).length; + const second = await ok(); + assert.equal(second.structured.transferred_bytes, 0); + assert.equal(second.structured.tree.match, true); + assert.equal(second.structured.lockfile_changed, false); + assert.equal(second.structured.snapshot, first.structured.snapshot, "the same working changes give the same snapshot commit"); + assert.match(second.text, /· tree matches$/); + const tars = readFileSync(path.join(dir, "calls"), "utf8").split("\n").filter(Boolean).slice(calls).map((l) => JSON.parse(l)).filter((c) => c.kind === "tar"); + assert.deepEqual(tars, [], "no tar transfer the second time"); +}); + +test("a lockfile change is flagged, once", async () => { + write(repo, "package-lock.json", "{\"v\":1}\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "lock"); + assert.equal((await ok()).structured.lockfile_changed, true, "new on the first sync"); + assert.equal((await ok()).structured.lockfile_changed, false); + write(repo, "packages/web/pnpm-lock.yaml", "lockfileVersion: 9\n"); // untracked, nested + const r = await ok(); + assert.equal(r.structured.lockfile_changed, true); + assert.match(r.text, /lockfile changed/); + write(repo, "README.md", "unrelated\n"); + assert.equal((await ok()).structured.lockfile_changed, false); +}); + +test("cancellation mid-transfer leaves the VM unchanged", async () => { + await ok(); + const before = tree(vmProject()); + const head = git(vmProject(), "rev-parse", "HEAD"); + write(repo, "big.bin", Buffer.alloc(8 * 1024 * 1024, 7)); + write(repo, "README.md", "changed\n"); + env.COPY_SLOW_SOURCE = "local"; + const calls = () => readFileSync(path.join(dir, "calls"), "utf8"); + const mark = calls().length; + const r = /** @type {any} */ (await sync({}, { abortWhen: () => calls().slice(mark).includes("\"role\":\"destination\"") })); + assert.equal(r.ok, false); + assert.equal(r.name, "AbortError"); + assert.deepEqual(tree(vmProject()), before); + assert.equal(git(vmProject(), "rev-parse", "HEAD"), head); + const cache = path.join(dir, "vms", "alpha", "vm", ".cache", "boxery"); + assert.deepEqual(existsSync(cache) ? execFileSync("ls", ["-A", cache], { encoding: "utf8" }).trim() : "", "", "no bundle or staging left"); + delete env.COPY_SLOW_SOURCE; + assert.equal((await ok()).structured.tree.match, true, "a later sync completes"); +}); + +test("an interrupted swap is undone by the next sync", async () => { + await ok(); + write(repo, "README.md", "next\n"); + // Simulate a crash mid-swap: the pending record is there, the working directory half updated. + const g = path.join(vmProject(), ".git"); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced^{tree}"); + write(vmProject(), "README.md", "half written\n"); + const half = git(vmProject(), "rev-parse", `${git(vmProject(), "stash", "create")}^{tree}`); + writeFileSync(path.join(g, "boxery-sync-pending"), `${synced} ${half}\n`); + writeFileSync(path.join(g, "boxery-sync-index.bak"), readFileSync(path.join(g, "index"))); + const r = await ok(); + assert.equal(r.structured.tree.match, true); + assert.equal(read(vmProject(), "README.md"), "next\n"); + assert.ok(!existsSync(path.join(g, "boxery-sync-pending"))); +}); + +test("a local path that isn't a git checkout gives a clear error", async () => { + const plain = path.join(dir, "plain"); + mkdirSync(plain); + const r = /** @type {any} */ (await sync({ from: plain })); + assert.equal(r.ok, false); + assert.match(r.error, /is not a git checkout .*use vm_copy for other files/); + assert.ok(!existsSync(vmProject()), "the VM was not touched"); +}); + +test("a subdirectory syncs its whole repository; to picks the VM directory", async () => { + const r = await ok({ from: path.join(repo, "src"), to: "elsewhere/repo" }); + assert.equal(r.structured.from, repo); + assert.equal(r.structured.to, "/home/agent/project/elsewhere/repo"); + assert.deepEqual(tree(path.join(vmProject(), "elsewhere/repo")), tree(repo)); +}); + +test("local: refused or confined like vm_copy", async () => { + const refused = /** @type {any} */ (await sync({ local: false })); + assert.equal(refused.ok, false); + assert.match(refused.error, /vm_sync is disabled on this server/); + const outside = /** @type {any} */ (await sync({ local: { root: path.join(dir, "vms") } })); + assert.equal(outside.ok, false); + assert.match(outside.error, /outside the copy root/); + const inside = await ok({ from: "repo", local: { root: path.join(dir, "local") } }); + assert.equal(inside.structured.tree.match, true); +}); From 8dbd5ecef92cbc7b5f45867bfa133adabce5289a Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sat, 7 Nov 2026 22:54:04 -0500 Subject: [PATCH 2/5] =?UTF-8?q?fix(sync):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20history=20scan,=20confinement,=20safe=20recovery,=20ignored-?= =?UTF-8?q?file=20clobbering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Refuse credentials anywhere in the commits the bundle sends, not just the tree. - Confined (--copy-root): no repo discovery above the root, git dir/objects/alternates must be inside it, and no fsmonitor, hooks or filter drivers run. - Crash recovery refuses rather than overwrite files edited since the crash. - Refuse when the new tree would overwrite or delete ignored VM files, or when new ignore rules expose VM files (rolled back, nothing recorded). - Refuse a VM target inside another checkout; clear errors for shallow clones. - Case-insensitive, broader deny list; pipefail; mkdir lock without flock; two-tree read-tree -m so a file changed mid-swap aborts before any write; no LFS smudge or lazy fetch; Ctrl-C waits for an in-progress apply. --- controller/cli.mjs | 2 +- controller/sync.mjs | 8 +- core/boxery-core.mjs | 5 +- core/sync-guest.sh | 179 +++++++++++++++++++++++++++++++++---------- core/sync.mjs | 98 +++++++++++++++++------ docs/sync.md | 26 ++++--- test/sync.test.mjs | 134 +++++++++++++++++++++++++++++++- 7 files changed, 371 insertions(+), 81 deletions(-) diff --git a/controller/cli.mjs b/controller/cli.mjs index 3df38380..0e799af6 100644 --- a/controller/cli.mjs +++ b/controller/cli.mjs @@ -548,7 +548,7 @@ export const COMMANDS = { "Refuses, changing nothing, when the VM checkout has changes the last sync didn't make, or the checkout holds a credential file (.env, .npmrc, auth files).", ], flags: [["--from DIR", "Local checkout (default: the current directory)"], ["--to PATH", "VM directory (default ~/project; relative paths are under it)"], ["--json", "Print the result as JSON"]], - examples: [["This checkout into demo's ~/project", "boxery sync demo"], ["Another checkout, into ~/project/api", "boxery sync demo --from ../api --to api"]], + examples: [["This checkout into demo's ~/project", "boxery sync demo"], ["Another checkout, into ~/api (not inside ~/project, which is a checkout itself)", "boxery sync demo --from ../api --to ~/api"]], seeAlso: ["cp"], docs: "sync.md", }, run: cmdSync, diff --git a/controller/sync.mjs b/controller/sync.mjs index 4a7648e1..522a57b1 100644 --- a/controller/sync.mjs +++ b/controller/sync.mjs @@ -12,8 +12,12 @@ export async function sync(host, vm, { from, to, json = false }) { // Ctrl-C cancels like an MCP cancellation: before the VM applies anything, nothing changes there. const abort = new AbortController(); let interrupted = 0; - const handlers = /** @type {const} */ (["SIGINT", "SIGTERM", "SIGHUP"]).map((sig) => /** @type {[NodeJS.Signals, () => void]} */ ([sig, () => { interrupted = 128 + constants.signals[sig]; abort.abort(); }])); - for (const [sig, h] of handlers) process.once(sig, h); + // Once the VM has started applying, the sync finishes (or rolls back) there regardless; wait for its result. + const handlers = /** @type {const} */ (["SIGINT", "SIGTERM", "SIGHUP"]).map((sig) => /** @type {[NodeJS.Signals, () => void]} */ ([sig, () => { + if (!interrupted) process.stderr.write("boxery: stopping; if the VM is already applying the sync, it finishes or rolls back first\n"); + interrupted = 128 + constants.signals[sig]; abort.abort(); + }])); + for (const [sig, h] of handlers) process.on(sig, h); const tool = /** @type {Tool} */ (TOOLS.find((t) => t.name === "vm_sync")); try { const r = await core.withHost(host, () => tool.run(vm, { from, to }, /** @type {any} */ ({ localCwd: process.cwd(), signal: abort.signal }) /* no session: vm_sync needs none */)); diff --git a/core/boxery-core.mjs b/core/boxery-core.mjs index 5531343e..fd3a1db3 100644 --- a/core/boxery-core.mjs +++ b/core/boxery-core.mjs @@ -29,7 +29,8 @@ export { q } from "./shell.mjs"; /** @typedef {{ name: string, dir: string, default: boolean, destination: string | null, backend: string | null, same_as: string[] }} HostDescription */ /** @typedef {{ host?: HostProfile, vm: string | null, path: string }} Loc */ /** env: ssh's environment (default: this process's); batch: false lets ssh ask (a terminal's passphrase). - * @typedef {{ input?: string | Buffer, signal?: AbortSignal, timeoutMs?: number, onData?: (d: Buffer) => void, maxBytes?: number, closePipes?: boolean, env?: NodeJS.ProcessEnv, batch?: boolean }} SshRunOpts */ + * detached: ssh in its own process group, so a terminal's Ctrl-C doesn't cut off a remote step that must finish. + * @typedef {{ input?: string | Buffer, signal?: AbortSignal, timeoutMs?: number, onData?: (d: Buffer) => void, maxBytes?: number, closePipes?: boolean, env?: NodeJS.ProcessEnv, batch?: boolean, detached?: boolean }} SshRunOpts */ /** @typedef {{ code: number | null, stdout: Buffer, stderr: string }} SshResult */ /** @typedef {{ onData?: (d: Buffer) => void, signal?: AbortSignal, timeout?: number }} ExecRemoteOpts */ /** @typedef {{ exitCode: number | null }} ExecResult */ @@ -352,7 +353,7 @@ export function sshRun(vm, script, opts = {}) { if (!existsSync(sshConf)) return reject(new Error(`boxery not set up (${sshConf} missing); run: boxery setup user@host`)); // stdin is a pipe whenever it is used (opts.input != null). const child = /** @type {ChildProcessWithoutNullStreams} */ (spawn("ssh", ["-F", sshConf, ...(opts.batch === false ? [] : ["-o", "BatchMode=yes"]), `${vm}.boxery`, `bash -c ${q(script)}`], { - stdio: [opts.input != null ? "pipe" : "ignore", "pipe", "pipe"], ...(opts.env ? { env: opts.env } : {}), + stdio: [opts.input != null ? "pipe" : "ignore", "pipe", "pipe"], ...(opts.env ? { env: opts.env } : {}), ...(opts.detached ? { detached: true } : {}), })); /** @type {Buffer[]} */ const out = []; diff --git a/core/sync-guest.sh b/core/sync-guest.sh index 8099c709..2c0a8afc 100644 --- a/core/sync-guest.sh +++ b/core/sync-guest.sh @@ -1,20 +1,27 @@ # shellcheck shell=bash # vm_sync, the VM side. core/sync.mjs sends this text with every call (it is never installed), after -# assignments to MODE (probe | apply), DIR, for probe WANT, and for apply: FORMAT BUNDLE SNAP HEAD_SHA EXPECT_SYNCED EXPECT_HEAD. -# It never contacts a git remote. Output: key=value lines on stdout. Exit 3: refused, nothing changed -# (the reason on stderr). Exit 1: failed; apply has restored the checkout it found. +# assignments to MODE (probe | apply), DIR, for probe WANT, and for apply: FORMAT BUNDLE SNAP HEAD_SHA +# TREE EXPECT_SYNCED EXPECT_HEAD. It never contacts a git remote. Output: key=value lines on stdout. +# Exit 3: refused (the reason on stderr); the checkout is as it was. Exit 1: failed; apply has restored +# the checkout it found, or says it couldn't. # # State kept in the VM repo: refs/boxery/synced (the snapshot commit of the last sync: local HEAD plus # the working changes, as one commit), refs/boxery/base (local HEAD then), and while a swap runs, # $G/boxery-sync-pending ("OLD_TREE NEW_TREE") so a later sync can finish or undo an interrupted one. -set -u +set -u -o pipefail : "${MODE:?}" "${DIR:?}" -export LC_ALL=C GIT_TERMINAL_PROMPT=0 GIT_OPTIONAL_LOCKS=0 +export LC_ALL=C GIT_TERMINAL_PROMPT=0 GIT_OPTIONAL_LOCKS=0 GIT_NO_LAZY_FETCH=1 GIT_LFS_SKIP_SMUDGE=1 LOCKFILES=(':(glob)**/pnpm-lock.yaml' ':(glob)**/package-lock.json' ':(glob)**/yarn.lock' ':(glob)**/bun.lock') refuse() { printf '%s\n' "$*" >&2; exit 3; } die() { printf '%s\n' "$*" >&2; exit 1; } rev() { git rev-parse -q --verify "$1^{commit}" 2>/dev/null || true; } is_repo_root() { local top; top=$(git rev-parse --show-toplevel 2>/dev/null) && [ "$top" = "$(pwd -P)" ]; } +# A directory inside another repository's working tree would show up there as a new nested repository. +inside_other_repo() { + local top + top=$(cd "$1" 2>/dev/null && git rev-parse --show-toplevel 2>/dev/null) || return 1 + [ -n "$top" ] && [ "$top" != "$2" ] && printf '%s' "$top" +} setup_git() { G=$(git rev-parse --absolute-git-dir) || die "git: cannot find the repository in $DIR" OBJ=$(git rev-parse --path-format=absolute --git-path objects) || die "git is too old (needs 2.31+)" @@ -28,7 +35,7 @@ baseline() { } # The working directory as a tree, as `git add -A` sees it (tracked files plus untracked ones that # aren't ignored), starting from tree $1. Uses a scratch index and object directory, so neither the -# repo's index nor its object store changes. With $2 = diff, also lists up to 20 changes against $1. +# repo's index nor its object store changes. wt() { local t rc t=$(mktemp -d) || return 1 @@ -38,25 +45,39 @@ wt() { export GIT_INDEX_FILE="$t/i" GIT_OBJECT_DIRECTORY="$t/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" # One-tree --reset keeps the copied index's stat data for entries it doesn't change, so unchanged # files aren't hashed again, and (unlike -m) doesn't refuse entries that aren't up to date. - git read-tree --reset "$1" || exit 1 - git add -A 2>/dev/null || exit 1 - tree=$(git write-tree) || exit 1 - echo "$tree" - if [ "${2:-}" = diff ] && [ "$tree" != "$1" ]; then - git diff-tree -r --no-renames --name-status "$1" "$tree" | head -n 20 | sed 's/^/changed=/' - fi + git read-tree --reset "$1" && git add -A 2>/dev/null && git write-tree ) rc=$?; rm -rf "$t"; return "$rc" } +# Up to 20 changes from tree $1 to tree $2 (both trees' objects must exist). +changes() { { git diff-tree -r --no-renames --name-status "$1" "$2" 2>/dev/null || true; } | head -n 20; } +# Up to 20 changes from tree $1 to the working directory. +worktree_changes() { + local t + t=$(mktemp -d) || return 1 + mkdir "$t/o" + ( + export GIT_INDEX_FILE="$t/i" GIT_OBJECT_DIRECTORY="$t/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" + git read-tree "$1" && git add -A 2>/dev/null && w=$(git write-tree) && changes "$1" "$w" + ) + rm -rf "$t" +} probe() { - if [ ! -e "$DIR" ]; then echo state=absent; return; fi + if [ ! -e "$DIR" ]; then + local up; up=$(dirname "$DIR"); while [ ! -d "$up" ]; do up=$(dirname "$up"); done + local other; other=$(inside_other_repo "$up" "") && refuse "$DIR is inside the git checkout $other; sync into a directory outside it" + echo state=absent; return + fi [ -d "$DIR" ] || refuse "$DIR exists and is not a directory" cd "$DIR" || die "cannot enter $DIR" if ! is_repo_root; then - if [ -n "$(ls -A)" ]; then echo state=dir; ls -A | head -n 5 | sed 's/^/entry=/'; else echo state=empty; fi + local other; other=$(inside_other_repo . "") && refuse "$DIR is inside the git checkout $other; sync into a directory outside it" + local entries; entries=$(ls -A) + if [ -n "$entries" ]; then echo state=dir; printf '%s\n' "$entries" | head -n 5 | sed 's/^/entry=/'; else echo state=empty; fi return fi + local outer; outer=$(inside_other_repo .. "") && refuse "$DIR is inside the git checkout $outer; sync into a directory outside it" setup_git echo state=repo echo "format=$(git rev-parse --show-object-format)" @@ -65,63 +86,112 @@ probe() { echo "head=$(rev HEAD)" [ -z "${WANT:-}" ] || ! git cat-file -e "$WANT^{commit}" 2>/dev/null || echo has=1 if [ -f "$G/boxery-sync-pending" ]; then echo pending=1; return; fi - local b; b=$(baseline) + local b w + b=$(baseline) || die "cannot read the last sync's state in $DIR" echo "baseline=$b" - wt "$b" diff | sed '1s/^/worktree=/' || die "cannot read the working directory of $DIR" + w=$(wt "$b") || die "cannot read the working directory of $DIR" + echo "worktree=$w" + [ "$w" = "$b" ] || worktree_changes "$b" | sed 's/^/changed=/' } # Put back tree $1 where tree $2 was swapped in (or half swapped in), and the index found before. rollback() { GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$2" && GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree --reset -u "$2" "$1" || return 1 + restore_index +} +restore_index() { if [ -f "$G/boxery-sync-index.bak" ]; then mv -f "$G/boxery-sync-index.bak" "$IDX"; else rm -f "$IDX"; fi git update-ref -d refs/boxery/incoming 2>/dev/null rm -f "$G/boxery-sync-index" "$G/boxery-sync-pending" } -# A swap a crash interrupted: done if its refs were written, otherwise undone. +# A swap a crash interrupted: done if its refs were written, otherwise undone, but only if every file it +# would put back is still as the swap left it (old, new or absent); anything else is a VM edit. recover() { [ -f "$G/boxery-sync-pending" ] || return 0 - local old new s + local old new s cur foreign read -r old new <"$G/boxery-sync-pending" || die "unreadable $G/boxery-sync-pending" s=$(rev refs/boxery/synced) if [ -n "$s" ] && [ "$(git rev-parse "$s^{tree}")" = "$new" ]; then rm -f "$G/boxery-sync-pending" "$G/boxery-sync-index" "$G/boxery-sync-index.bak" - else - rollback "$old" "$new" || die "an interrupted sync left $DIR half updated and it could not be undone; see $G/boxery-sync-pending" + return 0 fi + cur=$(mktemp -d) || die "mktemp failed" + mkdir "$cur/o" + foreign=$( + export GIT_INDEX_FILE="$cur/i" GIT_OBJECT_DIRECTORY="$cur/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" + git read-tree "$old" && git add -A 2>/dev/null && w=$(git write-tree) || exit 1 + # A path that differs from both the old and the new tree is neither ours nor untouched. + comm -12 <(git diff-tree -r --no-renames --name-only "$old" "$w" | sort) <(git diff-tree -r --no-renames --name-only "$new" "$w" | sort) + ) || { rm -rf "$cur"; die "cannot read the working directory of $DIR"; } + rm -rf "$cur" + [ -z "$foreign" ] || refuse "an interrupted earlier sync left $DIR half updated, and these files were changed since: $(printf '%s\n' "$foreign" | head -n 20 | tr '\n' ' ') +Save or discard them in the VM, then sync again (the interrupted sync is undone first)" + rollback "$old" "$new" || die "an interrupted sync left $DIR half updated and it could not be undone; see $G/boxery-sync-pending" } -# Swap tree $1 for $2 in the working directory, then point the index, HEAD and our refs at the new state. +# Paths the new tree adds that already exist in the VM: ignored files (apply checked nothing else is +# untracked) the swap would overwrite or delete. A directory counts if it holds ignored files. +clobbered() { + local p d + while IFS= read -r -d '' p; do + if [ -L "$p" ] || [ -f "$p" ]; then printf '%s\n' "$p"; continue; fi + if [ -d "$p" ]; then + [ -z "$(git ls-files -z --others --ignored --exclude-standard --directory -- "$p" | head -c1)" ] || printf '%s/\n' "$p" + continue + fi + d=$(dirname "$p") + while [ "$d" != . ]; do + if [ -L "$d" ] || { [ -e "$d" ] && [ ! -d "$d" ]; }; then printf '%s\n' "$d"; break; fi + [ ! -d "$d" ] || break + d=$(dirname "$d") + done + done < <(git diff-tree -r -z --no-renames --diff-filter=A --name-only "$1" "$2") +} +# Swap tree $1 for $2 in the working directory. 2: git refused before writing anything (a file changed +# under it). 4: done, but the result isn't $TREE (new ignore rules expose VM files). 1: other failure. swap() { GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$1" || return 1 GIT_INDEX_FILE="$G/boxery-sync-index" git update-index -q --refresh >/dev/null 2>&1 - # Two-tree switch: rewrites only the paths that differ and deletes what $2 lacks. Ignored files stay, - # unless $2 tracks that very path. --reset overwrites untracked files, but apply checked there are none. - GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree --reset -u "$1" "$2" || return 1 - # The index holds local HEAD, so `git status` in the VM shows the working changes, as it does locally. - git read-tree "$HEAD_SHA" || return 1 - git update-index -q --refresh >/dev/null 2>&1 - printf '%s\n' 'option no-deref' "update HEAD $HEAD_SHA" "update refs/boxery/synced $SNAP" "update refs/boxery/base $HEAD_SHA" \ - 'delete refs/boxery/incoming' | git update-ref -m "boxery sync" --stdin || return 1 + # Two-tree merge: rewrites only the paths that differ and deletes what $2 lacks; ignored files stay. + # git checks every path first and writes nothing if one changed since apply looked. + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree -m -u "$1" "$2" 2>"$G/boxery-sync-err" || return 2 + rm -f "$G/boxery-sync-err" + RESULT=$(wt "$2") || return 1 + [ "$RESULT" = "$TREE" ] || return 4 + return 0 } apply() { - : "${FORMAT:?}" "${SNAP:?}" "${HEAD_SHA:?}" "${BUNDLE?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" + : "${FORMAT:?}" "${SNAP:?}" "${HEAD_SHA:?}" "${TREE:?}" "${BUNDLE?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" # Past this point the caller has committed: finish or roll back, even if it cancels or disconnects. trap '' INT TERM HUP PIPE - [ -z "$BUNDLE" ] || trap 'rm -f "$BUNDLE"' EXIT - mkdir -p "$DIR" && cd "$DIR" || die "cannot create $DIR" - if ! is_repo_root; then - [ -z "$(ls -A)" ] || refuse "$DIR is not a git repository and is not empty" + LOCKDIR="" + trap 'rm -f "${BUNDLE:-/nonexistent}"; [ -z "$LOCKDIR" ] || rmdir "$LOCKDIR" 2>/dev/null' EXIT + if [ ! -d "$DIR" ] || [ -z "$(ls -A "$DIR")" ]; then + [ -z "$EXPECT_HEAD$EXPECT_SYNCED" ] || refuse "the checkout in $DIR changed during the sync; run it again" + mkdir -p "$DIR" && cd "$DIR" || die "cannot create $DIR" git init -q --object-format="$FORMAT" || die "git init failed in $DIR" + else + cd "$DIR" || die "cannot enter $DIR" + is_repo_root || refuse "$DIR is not a git repository and is not empty" fi setup_git - exec 9>"$G/boxery-sync.lock" - if command -v flock >/dev/null; then flock -w 60 9 || refuse "another sync into $DIR is running"; fi + if command -v flock >/dev/null; then + exec 9>"$G/boxery-sync.lock" + flock -w 60 9 || refuse "another sync into $DIR is running" + else + local i=0 + until mkdir "$G/boxery-sync.lock.d" 2>/dev/null; do + i=$((i + 1)); [ "$i" -lt 600 ] || refuse "another sync into $DIR is running (or $G/boxery-sync.lock.d is stale: remove it)" + sleep 0.1 + done + LOCKDIR="$G/boxery-sync.lock.d" + fi recover [ "$(rev refs/boxery/synced)" = "$EXPECT_SYNCED" ] && [ "$(rev HEAD)" = "$EXPECT_HEAD" ] || refuse "the checkout in $DIR changed during the sync; run it again" - local b w new prev lock=0 - b=$(baseline) + local b w new prev lock=0 hit rc + b=$(baseline) || die "cannot read the last sync's state in $DIR" w=$(wt "$b") || die "cannot read the working directory of $DIR" [ "$w" = "$b" ] || refuse "uncommitted changes appeared in $DIR during the sync; run it again" if [ -n "$BUNDLE" ]; then @@ -131,18 +201,43 @@ apply() { fi git cat-file -e "$SNAP^{commit}" 2>/dev/null || die "the VM lacks commit $SNAP after the transfer" new=$(git rev-parse "$SNAP^{tree}") || die "cannot read $SNAP" + hit=$(clobbered "$b" "$new" | head -n 20) || die "cannot compare the new tree with $DIR" + if [ -n "$hit" ]; then + git update-ref -d refs/boxery/incoming 2>/dev/null + refuse "the local checkout now tracks paths that exist in the VM as ignored files (build output, dependencies), which the sync would overwrite or delete: $(printf '%s\n' "$hit" | tr '\n' ' ') +Remove them in the VM (or stop tracking them locally), then sync again" + fi prev=$(rev refs/boxery/synced); [ -n "$prev" ] || prev=$(rev HEAD) git diff --quiet "${prev:-$EMPTY}" "$SNAP" -- "${LOCKFILES[@]}" || lock=1 rm -f "$G/boxery-sync-index.bak" [ ! -f "$IDX" ] || cp "$IDX" "$G/boxery-sync-index.bak" || die "cannot back up the index" printf '%s %s\n' "$b" "$new" >"$G/boxery-sync-pending" || die "cannot write $G/boxery-sync-pending" - if ! swap "$b" "$new"; then + swap "$b" "$new"; rc=$? + case $rc in + 0) ;; + 2) + local why; why=$(head -c 2000 "$G/boxery-sync-err" 2>/dev/null); rm -f "$G/boxery-sync-err" + restore_index + refuse "files in $DIR changed during the sync; run it again ($why)" ;; + 4) + local extra; extra=$(worktree_changes "$new") + rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" + refuse "with the new .gitignore, files in the VM that were ignored no longer are: $(printf '%s\n' "$extra" | tr '\t\n' ' ') +Remove them in the VM (or ignore them locally), then sync again" ;; + *) rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" - die "the sync failed; the checkout in $DIR was restored" + die "the sync failed; the checkout in $DIR was restored" ;; + esac + # The index holds local HEAD, so `git status` in the VM shows the working changes, as it does locally. + if ! { git read-tree "$HEAD_SHA" && printf '%s\n' 'option no-deref' "update HEAD $HEAD_SHA" "update refs/boxery/synced $SNAP" \ + "update refs/boxery/base $HEAD_SHA" 'delete refs/boxery/incoming' | git update-ref -m "boxery sync" --stdin; }; then + rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" + die "the sync failed writing refs; the checkout in $DIR was restored" fi + git update-index -q --refresh >/dev/null 2>&1 rm -f "$G/boxery-sync-pending" "$G/boxery-sync-index" "$G/boxery-sync-index.bak" echo "lockfile_changed=$lock" - wt "$new" | sed '1s/^/tree=/' || die "synced, but cannot read the working directory of $DIR back" + echo "tree=$RESULT" } case "$MODE" in diff --git a/core/sync.mjs b/core/sync.mjs index 75797582..c85e1017 100644 --- a/core/sync.mjs +++ b/core/sync.mjs @@ -26,28 +26,41 @@ export const SYNC_DISABLED = export const LOCKFILES = ["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lock"]; // Credentials vm_sync never puts on a VM's disk (and so into its snapshots): the agent asks with vm_grant instead. -// Templates (.env.example, .env.sample, .env.template) hold no values and are allowed. -const DENY_NAME = /^(\.env(?!\.(example|sample|template)$)(\..*|rc)?|\.npmrc|\.pypirc|\.netrc|_netrc|\.git-credentials|\.pgpass|\.dockercfg|\.s3cfg|\.boto|auth\.json|credentials(\.json)?|id_(rsa|dsa|ecdsa|ed25519)(_sk)?)$/; -const DENY_PATH = /(^|\/)(\.docker\/config\.json|\.kube\/config|gh\/hosts\.yml)$/; +// Case-insensitive. Templates (.env.example, .env.sample, .env.template, .env.dist) hold no values and are allowed. +const ENV_TEMPLATE = /^\.env[._-](example|sample|template|dist|defaults)$/i; +const DENY_NAME = new RegExp("^(" + [ + "\\.env([._-].*|rc)?", "\\.npmrc", "\\.yarnrc\\.yml", "\\.pypirc", "\\.netrc", "_netrc", "\\.git-credentials", "\\.pgpass", "\\.my\\.cnf", + "\\.dockercfg", "\\.s3cfg", "\\.boto", "\\.terraformrc", "auth\\.json", "credentials(\\.json)?", "application_default_credentials\\.json", + "id_[a-z0-9_-]+", ".*\\.(pem|key|p12|pfx|jks|keystore)", +].join("|") + ")$", "i"); +const DENY_PATH = /(^|\/)(\.docker\/config\.json|\.kube\/config|gh\/hosts\.yml|\.aws\/config)$/i; /** Is a repository path a credential file vm_sync refuses? @param {string} p */ -export const deniedSyncPath = (p) => DENY_NAME.test(path.posix.basename(p)) || DENY_PATH.test(p); +export function deniedSyncPath(p) { + const name = path.posix.basename(p); + if (ENV_TEMPLATE.test(name) || /\.pub$/i.test(name)) return false; + return DENY_NAME.test(name) || DENY_PATH.test(p); +} const IDENTITY = { GIT_AUTHOR_NAME: "boxery sync", GIT_AUTHOR_EMAIL: "sync@boxery.invalid", GIT_AUTHOR_DATE: "@0 +0000" }; const COMMITTER = { GIT_COMMITTER_NAME: IDENTITY.GIT_AUTHOR_NAME, GIT_COMMITTER_EMAIL: IDENTITY.GIT_AUTHOR_EMAIL, GIT_COMMITTER_DATE: IDENTITY.GIT_AUTHOR_DATE }; // Repository-locating variables from the caller's environment would point git elsewhere. const GIT_LOCATION = ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_NAMESPACE"]; +// No lazy fetches from a partial clone's promisor remote: vm_sync never contacts a remote. const baseEnv = () => { /** @type {NodeJS.ProcessEnv} */ - const env = { ...process.env, LC_ALL: "C", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0" }; + const env = { ...process.env, LC_ALL: "C", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0", GIT_NO_LAZY_FETCH: "1" }; for (const k of GIT_LOCATION) delete env[k]; return env; }; +// A confined server (HTTP MCP with --copy-root) reads a checkout an agent may have written: git must not +// run anything that checkout's config names (fsmonitor, hooks, filter drivers) or look outside the root. +const SAFE_CONFIG = ["-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "protocol.allow=never"]; const aborted = () => Object.assign(new Error("sync aborted; the VM was not changed"), { name: "AbortError" }); /** Run git locally → stdout. @param {string[]} args @param {{ cwd?: string, env?: NodeJS.ProcessEnv }} [opts] @returns {Promise} */ function git(args, { cwd, env = baseEnv() } = {}) { return new Promise((resolve, reject) => { - execFile("git", args, { cwd, env, maxBuffer: 256 * 1024 * 1024, encoding: "utf8" }, (error, stdout, stderr) => { + execFile("git", [...SAFE_CONFIG, ...args], { cwd, env, maxBuffer: 256 * 1024 * 1024, encoding: "utf8" }, (error, stdout, stderr) => { if (!error) return resolve(stdout); const e = /** @type {NodeJS.ErrnoException} */ (error); reject(new Error(e.code === "ENOENT" ? "git is not installed on this machine" : `git ${args[0]}: ${stderr.trim() || e.message}`)); @@ -74,7 +87,8 @@ function parseLines(out) { async function guest(vm, vars) { const assignments = Object.entries(vars).map(([k, v]) => `${k}=${q(v)}`).join("\n"); // No signal: probe is short, and apply must finish or roll back once started (it ignores signals itself). - const r = await sshRun(vm, `${assignments}\n${guestTemplate}`); + // Detached: a terminal's Ctrl-C reaches the whole foreground group; ssh must survive it to report the result. + const r = await sshRun(vm, `${assignments}\n${guestTemplate}`, { detached: true }); const err = r.stderr.trim(); if (r.code === 3) throw new Error(`vm_sync refused: ${err}; nothing in the VM was changed`); if (r.code !== 0) { @@ -84,40 +98,73 @@ async function guest(vm, vars) { return parseLines(r.stdout.toString("utf8")); } +/** + * Confined (--copy-root): the repository's root, git directory, common directory, object store and its + * alternates must all be inside the root and clear of denied directories, and its config must name no + * filter driver (git add would run it). Checked before git reads the working tree. + * @param {string} from @param {ReturnType} confined @param {NodeJS.ProcessEnv} env + */ +async function checkConfinedRepo(from, confined, env) { + const out = await tryGit(["rev-parse", "--path-format=absolute", "--show-toplevel", "--absolute-git-dir", "--git-common-dir", "--git-path", "objects"], { cwd: from, env }); + if (!out) return; + const dirs = out.split("\n").filter(Boolean); + const objects = dirs[dirs.length - 1]; + let alternates = ""; + try { alternates = readFileSync(path.join(objects, "info", "alternates"), "utf8"); } catch {} + for (const a of alternates.split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"))) dirs.push(path.resolve(objects, a)); + for (const d of dirs) { + let real; + try { real = await realpath(d); } catch { continue; } + confined.check(real, `${from} (its repository uses ${d})`); + } + const filters = await tryGit(["config", "--get-regexp", "^filter\\..*\\.(clean|smudge|process)$"], { cwd: from, env }); + if (filters) throw new Error(`vm_sync refused: the repository at ${from} configures a git filter driver (${filters.split("\n")[0]}), which this server won't run`); +} + /** * The local checkout: repository root, HEAD, and a snapshot commit of HEAD plus every working change * (staged, unstaged, untracked but not ignored), built without changing the repository. - * @param {string} from @param {string} scratch + * @param {string} from @param {string} scratch @param {ReturnType | null} confined */ -async function snapshot(from, scratch) { - const top = await tryGit(["rev-parse", "--show-toplevel"], { cwd: from }); +async function snapshot(from, scratch, confined) { + const env0 = baseEnv(); + if (confined) { + // Don't discover a repository above the root. + env0.GIT_CEILING_DIRECTORIES = path.dirname(confined.root); + await checkConfinedRepo(from, confined, env0); + } + const g0 = { env: env0 }; + const top = await tryGit(["rev-parse", "--show-toplevel"], { cwd: from, ...g0 }); if (!top) { - const bare = await tryGit(["rev-parse", "--is-bare-repository"], { cwd: from }); + const bare = await tryGit(["rev-parse", "--is-bare-repository"], { cwd: from, ...g0 }); if (bare === "true") throw new Error(`${from} is a bare git repository; vm_sync pushes a checkout with a working tree`); throw new Error(`${from} is not a git checkout (no repository found there or above it). vm_sync pushes a git checkout; use vm_copy for other files.`); } const root = await realpath(top); - const head = await tryGit(["rev-parse", "-q", "--verify", "HEAD^{commit}"], { cwd: root }); + confined?.check(root); + const head = await tryGit(["rev-parse", "-q", "--verify", "HEAD^{commit}"], { cwd: root, ...g0 }); if (!head) throw new Error(`${root} has no commits yet; commit once before syncing (vm_sync needs a HEAD to check out)`); - const format = (await git(["rev-parse", "--show-object-format"], { cwd: root })).trim(); - const objects = (await git(["rev-parse", "--path-format=absolute", "--git-path", "objects"], { cwd: root })).trim(); - const index = (await git(["rev-parse", "--path-format=absolute", "--git-path", "index"], { cwd: root })).trim(); + if (await tryGit(["rev-parse", "--is-shallow-repository"], { cwd: root, ...g0 }) === "true") + throw new Error(`${root} is a shallow clone; vm_sync sends history the VM lacks and can't from a shallow one. Run git fetch --unshallow first`); + const format = (await git(["rev-parse", "--show-object-format"], { cwd: root, ...g0 })).trim(); + const objects = (await git(["rev-parse", "--path-format=absolute", "--git-path", "objects"], { cwd: root, ...g0 })).trim(); + const index = (await git(["rev-parse", "--path-format=absolute", "--git-path", "index"], { cwd: root, ...g0 })).trim(); const scratchObjects = path.join(scratch, "objects"); await mkdir(scratchObjects); const scratchIndex = path.join(scratch, "index"); let haveIndex = true; try { await writeFile(scratchIndex, readFileSync(index)); } catch (e) { if (/** @type {any} */ (e).code !== "ENOENT") throw e; haveIndex = false; } - const env = { ...baseEnv(), GIT_INDEX_FILE: scratchIndex, GIT_OBJECT_DIRECTORY: scratchObjects, GIT_ALTERNATE_OBJECT_DIRECTORIES: objects }; + const env = { ...env0, GIT_INDEX_FILE: scratchIndex, GIT_OBJECT_DIRECTORY: scratchObjects, GIT_ALTERNATE_OBJECT_DIRECTORIES: objects }; if (!haveIndex) await git(["read-tree", "HEAD"], { cwd: root, env }); await git(["add", "-A"], { cwd: root, env }); const tree = (await git(["write-tree"], { cwd: root, env })).trim(); - const headTree = (await git(["rev-parse", "HEAD^{tree}"], { cwd: root })).trim(); + const headTree = (await git(["rev-parse", "HEAD^{tree}"], { cwd: root, ...g0 })).trim(); // Deterministic (fixed identity and date), so the same working changes give the same commit every time. const snap = tree === headTree ? head : (await git(["commit-tree", tree, "-p", head, "-m", "boxery sync: working changes"], { cwd: root, env: { ...env, ...IDENTITY, ...COMMITTER } })).trim(); const counts = { modified: 0, untracked: 0, deleted: 0 }; if (tree !== headTree) { - const tracked = new Set((await git(["ls-files", "-z"], { cwd: root })).split("\0")); + const tracked = new Set((await git(["ls-files", "-z"], { cwd: root, ...g0 })).split("\0")); const fields = (await git(["diff-tree", "-r", "-z", "-M", "--name-status", headTree, tree], { cwd: root, env })).split("\0").filter(Boolean); for (let i = 0; i < fields.length;) { const status = fields[i++]; @@ -128,7 +175,7 @@ async function snapshot(from, scratch) { } } const files = (await git(["ls-tree", "-r", "-z", "--name-only", tree], { cwd: root, env })).split("\0").filter(Boolean); - return { root, head, format, objects, scratchObjects, env, tree, snap, counts, files }; + return { root, head, format, objects, scratchObjects, env, tree, snap, counts, files, headTree }; } /** @@ -140,7 +187,7 @@ function refuseCredentials(root, files, deny) { const named = files.filter(deniedSyncPath); if (named.length) throw new Error(`vm_sync refused: ${named.slice(0, 5).join(", ")}${named.length > 5 ? ` (+${named.length - 5} more)` : ""} in ${root} looks like a credential file, which vm_sync never puts on a VM's disk (it would persist in its snapshots). ` + - "Ignore it in .gitignore (or remove it) to sync the rest, and ask for the credential with vm_grant. Nothing in the VM was changed."); + "Ignore it in .gitignore (or remove it, and git rm it if tracked) to sync the rest, and ask for the credential with vm_grant. Nothing in the VM was changed."); for (const { dir } of deniedDirs(deny)) { if (isWithin(root, dir)) throw new Error(`vm_sync refused: ${root} is inside ${dir} (boxery's own configuration or the user's keys), which vm_sync never copies; use vm_grant for credentials`); if (!isWithin(dir, root)) continue; @@ -166,6 +213,12 @@ async function bundle(s, probe, scratch, id) { const have = []; for (const c of new Set([probe.synced, probe.base, probe.head].filter(Boolean))) if (await tryGit(["--git-dir", bare, "cat-file", "-e", `${c}^{commit}`], { env }) !== null) have.push(`^${c}`); + // The bundle carries every commit the VM lacks, not just the snapshot: refuse a credential anywhere in them. + const history = (await git(["--git-dir", bare, "log", "--format=", "--name-only", "--no-renames", "-z", "refs/heads/sync", ...have], { env })).split("\0").map((p) => p.trim()).filter(Boolean); + const leaked = [...new Set(history.filter(deniedSyncPath))]; + if (leaked.length) + throw new Error(`vm_sync refused: ${leaked.slice(0, 5).join(", ")} is in the history this sync would send (a commit the VM lacks has it), and vm_sync never puts credential files on a VM's disk. ` + + "It's not in your working tree, but its old contents are in that commit. Remove it from history (git rebase / git filter-repo) before syncing, and ask for the credential with vm_grant. Nothing in the VM was changed."); const file = path.join(scratch, "out", `sync-${id}.bundle`); await mkdir(path.dirname(file)); try { @@ -201,8 +254,7 @@ export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd() const remoteBundle = `${REMOTE_HOME}/.cache/boxery/sync-${id}.bundle`; let sent = false; try { - const s = await snapshot(fromPath, scratch); - confined?.check(s.root); + const s = await snapshot(fromPath, scratch, confined); refuseCredentials(s.root, s.files, local ? local.deny : undefined); if (signal?.aborted) throw aborted(); const probe = await guest(vm, { MODE: "probe", DIR: dest, WANT: s.snap }); @@ -224,7 +276,7 @@ export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd() // The last point to stop: past here the VM applies the snapshot completely or not at all. if (signal?.aborted) throw aborted(); const applied = await guest(vm, { - MODE: "apply", DIR: dest, FORMAT: s.format, BUNDLE: file ? remoteBundle : "", SNAP: s.snap, HEAD_SHA: s.head, + MODE: "apply", DIR: dest, FORMAT: s.format, BUNDLE: file ? remoteBundle : "", SNAP: s.snap, HEAD_SHA: s.head, TREE: s.tree, EXPECT_SYNCED: probe.synced ?? "", EXPECT_HEAD: probe.head ?? "", }); sent = false; // apply removed the bundle diff --git a/docs/sync.md b/docs/sync.md index f08caafb..d1bbe377 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -6,13 +6,13 @@ ```bash boxery sync demo # the checkout you're in → demo:~/project -boxery sync demo --from ../api --to api # another checkout → demo:~/project/api +boxery sync demo --from ../api --to ~/api # another checkout → demo:~/api boxery sync demo --json # the result as JSON ``` ```json {"vm":"demo"} -{"vm":"demo","from":"../api","to":"api"} +{"vm":"demo","from":"../api","to":"~/api"} ``` `from` is a local directory inside the checkout (default: the agent's working directory, or the CLI's current directory); the whole repository is synced from its root. `to` is the VM directory (default `~/project`; relative paths are under it). `boxery sync` runs `vm_sync`'s own code ([CLI](cli.md#files)). @@ -21,7 +21,7 @@ boxery sync demo --json # the result as JSON 1. **Snapshot.** It records local HEAD plus every working change (what `git add -A` would stage: tracked files, and untracked files `.gitignore` doesn't exclude, deletions and renames included) as one commit. It uses a scratch index and object directory: your index, refs and stash are untouched. The commit has a fixed author and date, so the same working state always gives the same commit. 2. **Transfer.** The commits the VM lacks go over as a `git bundle`, on the same streaming transport as [`vm_copy`](copy.md). Neither side contacts a git remote, so the VM needs no GitHub grant. If the VM already has the snapshot, nothing is sent. If `to` doesn't exist or is empty, a repository is initialized there. -3. **Swap.** In the VM, HEAD is detached at local HEAD's commit, and the working directory is switched to the snapshot: only the paths that differ are rewritten, and files the snapshot lacks are deleted, including ones an earlier sync put there. Ignored files (`node_modules`, build output) stay: there is no `git clean -x`. The VM's index holds HEAD, so `git status` and `git diff` in the VM show the same working changes as they do locally. +3. **Swap.** In the VM, HEAD is detached at local HEAD's commit, and the working directory is switched to the snapshot: only the paths that differ are rewritten, and files the snapshot lacks are deleted, including ones an earlier sync put there. Ignored files (`node_modules`, build output) stay: there is no `git clean -x`. If the snapshot would put a tracked file where the VM has an ignored one, the sync refuses instead (below). The VM's index holds HEAD, so `git status` and `git diff` in the VM show the same working changes as they do locally. 4. **Check.** Both sides hash the whole working directory (tracked plus untracked, not ignored) as a git tree, the same way, and compare. The result says `tree matches`, or `TREE MISMATCH` (an error). ```text @@ -39,22 +39,28 @@ A second sync with no local changes sends nothing and still reports `tree matche It refuses, and changes nothing in the VM, when: - **The VM checkout has changes the last sync didn't make.** The VM records each sync in `refs/boxery/synced` (and local HEAD in `refs/boxery/base`). Any difference between the VM's working directory and that snapshot (edited, new or deleted files, not ignored ones) is listed in the error. Commit them in the VM and bring them back yourself, or discard them, then sync again. A VM repository that was never synced is compared with its HEAD. -- **The checkout holds a credential file.** `.env` and `.env.*` (except `.env.example`, `.env.sample`, `.env.template`), `.envrc`, `.npmrc`, `.pypirc`, `.netrc`, `.git-credentials`, `.pgpass`, `auth.json`, `credentials`, private ssh keys (`id_ed25519`, …), `.docker/config.json`, `.kube/config`, `gh/hosts.yml`, and anything inside boxery's own state or `~/.ssh`, `~/.gnupg`, `~/.pi` (what `vm_copy` refuses). The error names the path. Ignore it in `.gitignore` to sync the rest, and get the credential to the VM with a [grant](grants.md), which keeps it off the VM's disk and out of its snapshots. Ignored files are never synced, so an ignored `.env` is fine. -- **`to` is a non-empty directory that isn't a git repository**, or one whose object format (sha1/sha256) differs. -- **The local path isn't a git checkout**, or has no commits yet. Use `vm_copy` for plain files. +- **The checkout holds a credential file.** Matched case-insensitively: `.env` and `.env.*`/`.env-*`/`.env_*` (except `.env.example`, `.env.sample`, `.env.template`, `.env.dist`, `.env.defaults`), `.envrc`, `.npmrc`, `.yarnrc.yml`, `.pypirc`, `.netrc`, `.git-credentials`, `.pgpass`, `.my.cnf`, `.terraformrc`, `auth.json`, `credentials`, `application_default_credentials.json`, private keys (`id_*` except `.pub`, `*.pem`, `*.key`, `*.p12`, `*.pfx`, `*.jks`, `*.keystore`), `.docker/config.json`, `.kube/config`, `.aws/config`, `gh/hosts.yml`, and anything inside boxery's own state or `~/.ssh`, `~/.gnupg`, `~/.pi` (what `vm_copy` refuses). The error names the path. Ignore it in `.gitignore` to sync the rest, and get the credential to the VM with a [grant](grants.md), which keeps it off the VM's disk and out of its snapshots. Ignored files are never synced, so an ignored `.env` is fine. +- **The commits it would send contain one.** The bundle carries every commit the VM lacks (on a first sync, the whole history), so a credential file committed once and deleted later is refused too. Remove it from history first. +- **The local checkout now tracks a path the VM has as an ignored file** (a force-added build file, or `dist/` un-ignored and replaced by a file): the swap would overwrite or delete the VM's file. Or, with the new `.gitignore`, **files the VM has stop being ignored**, so the trees can't match. The error lists the paths; remove them in the VM (or change the local checkout), and sync again. +- **`to` is a non-empty directory that isn't a git repository**, is inside another checkout (`~/project/api` when `~/project` is a repository: use `~/api`), or its object format (sha1/sha256) differs. +- **The local path isn't a git checkout**, has no commits yet, or is a shallow clone. Use `vm_copy` for plain files. + +"Nothing changed" means the checkout: a refused first sync into an empty or missing directory may leave it as an empty repository. ## Cancellation and failure -Cancelling (an MCP cancellation, pi's Escape, Ctrl-C for `boxery sync`) before the transfer finishes leaves the VM unchanged: the partial bundle is removed with `vm_copy`'s staging. Once the VM starts applying, it ignores signals and finishes or rolls back; a lost connection doesn't interrupt it. +Cancelling (an MCP cancellation, pi's Escape, Ctrl-C for `boxery sync`) before the transfer finishes leaves the VM unchanged: the partial bundle is removed with `vm_copy`'s staging. Once the VM starts applying, it ignores signals and finishes or rolls back; a lost connection doesn't interrupt it, and `boxery sync` waits for its result after Ctrl-C. -The swap is staged: the VM fetches the bundle into its object store (which changes no files), then writes a pending record, switches the working directory, and finally updates HEAD and its refs in one transaction. If a step fails, it switches back and restores the index. If the VM crashes mid-swap, the next sync finds the pending record and finishes it (if the refs were written) or undoes it, before checking for VM-side changes. Fetched objects stay in the VM's repository either way; they're harmless and `git gc` collects them. +The swap is staged: the VM fetches the bundle into its object store (which changes no files), then writes a pending record, switches the working directory, and finally updates HEAD and its refs in one transaction. If a step fails, it switches back and restores the index. If the VM crashes mid-swap, the next sync finds the pending record and finishes it (if the refs were written) or undoes it, before checking for VM-side changes. It undoes it only if every file is as the swap left it; a file edited since is reported, and the record stays. Fetched objects stay in the VM's repository either way; they're harmless and `git gc` collects them. -A sync takes a lock in the VM repository, so two syncs into one checkout run one after the other. If the checkout changes between the check and the swap, the sync refuses. +A sync takes a lock in the VM repository (`flock`, or a lock directory without it), so two syncs into one checkout run one after the other. If the checkout changes between the check and the swap, the sync refuses: git checks every path before writing any. ## Limits - Needs git (2.31+) on both sides; boxery's images have it. - Submodules are synced as gitlinks (their commit ids), not their contents. +- Git LFS files arrive as pointer files: the VM doesn't download them, since vm_sync never contacts a remote. +- Untracked files are hashed into a temporary directory on the local machine, so a very large one needs that much space there. - Executable bits and symlinks follow git's rules; other file modes, owners and timestamps aren't carried. - The VM keeps every synced snapshot's objects until `git gc`. -- On an HTTP MCP server, `vm_sync` reads checkouts on that server, so it follows `vm_copy`'s `local:` rule: refused without `--copy-root`, and confined to `DIR` with it ([MCP server](mcp.md#b-streamable-http-on-the-host-over-tailscale)). +- On an HTTP MCP server, `vm_sync` reads checkouts on that server, so it follows `vm_copy`'s `local:` rule: refused without `--copy-root`, and confined to `DIR` with it ([MCP server](mcp.md#b-streamable-http-on-the-host-over-tailscale)). Confined, the repository's git directory, object store and alternates must be inside `DIR` too, git doesn't look for a repository above `DIR`, and it runs none of the checkout's hooks, fsmonitor or filter drivers. diff --git a/test/sync.test.mjs b/test/sync.test.mjs index 5506df74..0a1435b3 100644 --- a/test/sync.test.mjs +++ b/test/sync.test.mjs @@ -202,7 +202,9 @@ test("deny-listed files block the sync, naming the path and vm_grant", async () test("the deny list", () => { for (const p of [".env", ".env.local", "a/.env.production", ".envrc", ".npmrc", "x/.pypirc", ".netrc", ".git-credentials", "auth.json", ".docker/config.json", "home/.kube/config", "credentials", "id_ed25519"]) assert.ok(deniedSyncPath(p), p); - for (const p of [".env.example", ".env.sample", "environment.ts", "src/auth.js", "npmrc.md", "docs/credentials.md", "id_ed25519.pub"]) + for (const p of [".ENV", ".env-prod", ".env_local", "ID_RSA", "id_rsa_work", "Credentials.json", ".yarnrc.yml", "certs/server.pem", "tls.key", "store.p12", ".terraformrc"]) + assert.ok(deniedSyncPath(p), p); + for (const p of [".env.example", ".env.sample", ".env-template", "environment.ts", "src/auth.js", "npmrc.md", "docs/credentials.md", "id_ed25519.pub", "ID_RSA.PUB", "keys.ts"]) assert.ok(!deniedSyncPath(p), p); }); @@ -296,3 +298,133 @@ test("local: refused or confined like vm_copy", async () => { const inside = await ok({ from: "repo", local: { root: path.join(dir, "local") } }); assert.equal(inside.structured.tree.match, true); }); + +test("a credential in history the VM lacks blocks the sync, even when it's gone from the tree", async () => { + write(repo, ".env", "SECRET=1\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "oops"); + git(repo, "rm", "-q", ".env"); + git(repo, "commit", "-qm", "remove it"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /\.env is in the history this sync would send/); + assert.ok(!existsSync(vmProject()), "nothing sent"); +}); + +test("a VM edit made after an interrupted swap isn't lost", async () => { + await ok(); + write(repo, "README.md", "next\n"); + const g = path.join(vmProject(), ".git"); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced^{tree}"); + write(vmProject(), "README.md", "half written\n"); + const half = git(vmProject(), "rev-parse", `${git(vmProject(), "stash", "create")}^{tree}`); + writeFileSync(path.join(g, "boxery-sync-pending"), `${synced} ${half}\n`); + write(vmProject(), "src/app.js", "edited in the VM after the crash\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /interrupted earlier sync .* src\/app\.js/s); + assert.equal(read(vmProject(), "src/app.js"), "edited in the VM after the crash\n"); + assert.ok(existsSync(path.join(g, "boxery-sync-pending")), "the record stays for the next attempt"); +}); + +test("an interrupted swap whose refs were written is finished, not undone", async () => { + await ok(); + const g = path.join(vmProject(), ".git"); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced^{tree}"); + writeFileSync(path.join(g, "boxery-sync-pending"), `${"0".repeat(40)} ${synced}\n`); + const r = await ok(); + assert.equal(r.structured.tree.match, true); + assert.ok(!existsSync(path.join(g, "boxery-sync-pending"))); +}); + +test("a path the VM has as an ignored file that local now tracks is refused, not overwritten", async () => { + await ok(); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced"); + write(vmProject(), "dist/out.js", "the VM's own build\n"); + write(repo, "dist/out.js", "force-added locally\n"); + git(repo, "add", "-f", "dist/out.js"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /tracks paths that exist in the VM as ignored files .*dist\/out\.js/s); + assert.equal(read(vmProject(), "dist/out.js"), "the VM's own build\n"); + assert.equal(git(vmProject(), "rev-parse", "refs/boxery/synced"), synced, "nothing recorded"); + assert.throws(() => git(vmProject(), "rev-parse", "-q", "--verify", "refs/boxery/incoming"), "fetched ref dropped"); +}); + +test("an ignored VM directory replaced by a tracked file is refused", async () => { + await ok(); + write(vmProject(), "dist/keep.js", "built\n"); + write(repo, ".gitignore", "node_modules/\n"); + write(repo, "dist", "now a file\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /dist/); + assert.equal(read(vmProject(), "dist/keep.js"), "built\n"); +}); + +test("un-ignoring a path with VM files refuses cleanly, and the next sync isn't stuck", async () => { + await ok(); + write(vmProject(), "dist/out.js", "built\n"); + write(repo, ".gitignore", "node_modules/\n"); + const before = git(vmProject(), "rev-parse", "refs/boxery/synced"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /ignored no longer are: .*dist\/out\.js/s); + assert.equal(git(vmProject(), "rev-parse", "refs/boxery/synced"), before, "nothing recorded"); + assert.equal(read(vmProject(), ".gitignore"), "node_modules/\ndist/\n", "rolled back"); + rmSync(path.join(vmProject(), "dist"), { recursive: true }); + assert.equal((await ok()).structured.tree.match, true); +}); + +test("a VM directory inside another checkout is refused", async () => { + await ok(); + const r = /** @type {any} */ (await sync({ to: "api" })); + assert.equal(r.ok, false); + assert.match(r.error, /is inside the git checkout \S*project/); + assert.ok(!existsSync(path.join(vmProject(), "api"))); +}); + +test("a shallow clone gives a clear error", async () => { + git(repo, "commit", "-q", "--allow-empty", "-m", "second"); + const shallow = path.join(dir, "local", "shallow"); + git(dir, "clone", "-q", "--depth", "1", `file://${repo}`, shallow); + const r = /** @type {any} */ (await sync({ from: shallow })); + assert.equal(r.ok, false); + assert.match(r.error, /shallow clone/); +}); + +test("confined: no repository outside the root through a gitfile, and none of its config runs", async () => { + const root = path.join(dir, "share"); + const outside = path.join(dir, "secret-repo"); + mkdirSync(outside); + git(outside, "init", "-q"); + write(outside, "private.txt", "private\n"); + git(outside, "add", "-A"); + git(outside, "commit", "-qm", "private"); + const marker = path.join(dir, "fsmonitor-ran"); + git(outside, "config", "core.fsmonitor", `touch ${marker}; false`); + mkdirSync(path.join(root, "checkout"), { recursive: true }); + writeFileSync(path.join(root, "checkout", ".git"), `gitdir: ${path.join(outside, ".git")}\n`); + const r = /** @type {any} */ (await sync({ from: "checkout", local: { root } })); + assert.equal(r.ok, false); + assert.match(r.error, /outside the copy root/); + assert.ok(!existsSync(marker), "fsmonitor never ran"); + assert.ok(!existsSync(vmProject())); + + // A repository above the root isn't discovered from inside it either. + git(dir, "init", "-q"); + git(dir, "config", "core.fsmonitor", `touch ${marker}; false`); + mkdirSync(path.join(root, "plain")); + const above = /** @type {any} */ (await sync({ from: "plain", local: { root } })); + assert.equal(above.ok, false); + assert.match(above.error, /not a git checkout/); + assert.ok(!existsSync(marker), "fsmonitor never ran"); +}); + +test("confined: a filter driver in the checkout's config is refused", async () => { + const root = path.join(dir, "local"); + git(repo, "config", "filter.x.clean", "touch /tmp/never"); + const r = /** @type {any} */ (await sync({ from: "repo", local: { root } })); + assert.equal(r.ok, false); + assert.match(r.error, /filter driver/); +}); From b220dabc9fd219811cf5c44a03c34d54550cd5d5 Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sun, 8 Nov 2026 10:21:39 -0500 Subject: [PATCH 3/5] refactor(sync): send commits with git push instead of a bundle over tar git push over the VM's own ssh_config streams only the objects the VM lacks (negotiated against all its refs) into receive-pack, to a hidden refs/boxery/incoming/ ref. No bundle file, tar or staging directory; receive-pack's quarantine makes a cancelled push leave nothing. No hooks run on either side. The history credential check now excludes everything reachable from the VM's ref tips. The VM repo is created (in a missing or empty dir only) after the checks, just before the push. Result fields: transferred_bytes -> pushed, objects_sent. --- ARCHITECTURE.md | 2 +- controller/cli.mjs | 2 +- core/sync-guest.sh | 51 +++++++++------- core/sync.mjs | 119 +++++++++++++++++++++++-------------- docs/cli.md | 2 +- docs/sync.md | 10 ++-- test/fixtures/copy-ssh.mjs | 4 +- test/fixtures/sync-git.mjs | 17 ++++++ test/sync.test.mjs | 50 +++++++++++----- 9 files changed, 167 insertions(+), 90 deletions(-) create mode 100644 test/fixtures/sync-git.mjs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 158e3eb9..61837d9e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,7 +61,7 @@ VM (Ubuntu 24.04; XFCE desktop unless --headless) | Component | Runs on | Role | |---|---|---| | `core/boxery-tools.mjs` | agent machine (or host, for HTTP MCP) | **The `vm_*` tool table**: name, description, JSON Schema and runner for every tool, plus `vm_manage`/`vm_info`/`vm_lease` action logic (hosts included) over front-end session hooks, and `runTool()`, which resolves each call's host and VM. MCP and pi are both built from it and run tools through it, so they can't drift (`test/frontends.test.mjs`: a tool only one front-end has must say why). `test/parity.test.mjs` maps every tool and action to its `boxery` command (or why it has none) and every command back, so the CLI can't silently fall behind either | -| `core/boxery-core.mjs`, `core/copy-stream.mjs`, `core/sync.mjs` + `core/sync-guest.sh`, `core/atomic-edit.mjs`, `core/shell.mjs` | same | Transport and tool implementation: `sshRun`/`execRemote` (remote process-group kill on cancel), path mapping, read/write/edit (atomic, detects concurrent changes), streaming tar `vm_copy` (local⇄VM⇄VM, across hosts), `vm_sync` (a local git checkout → VM: a bundle over the copy transport, then a staged swap in the VM), computer use via xdotool, Playwright over CDP, `LeaseKeeper`, host-profile resolution with `AsyncLocalStorage` | +| `core/boxery-core.mjs`, `core/copy-stream.mjs`, `core/sync.mjs` + `core/sync-guest.sh`, `core/atomic-edit.mjs`, `core/shell.mjs` | same | Transport and tool implementation: `sshRun`/`execRemote` (remote process-group kill on cancel), path mapping, read/write/edit (atomic, detects concurrent changes), streaming tar `vm_copy` (local⇄VM⇄VM, across hosts), `vm_sync` (a local git checkout → VM: `git push` over the VM's ssh connection, then a staged swap in the VM), computer use via xdotool, Playwright over CDP, `LeaseKeeper`, host-profile resolution with `AsyncLocalStorage` | | `core/grants.mjs`, `core/grant-operations.mjs`, `core/grant-approval.mjs`, `core/grants-guest.sh` | agent machine | Shared spec/deny-directory validation and prompt text; grant store locking/edits, edge-route application and tmpfs reconciliation. Agent front-ends approve through elicitation/pi dialogs, then call operations in-process, without spawning the CLI. The guest script is a single reconciliation template with values inserted at apply time | | `core/boxery-jobs.ts`, `core/boxery-job-runner.sh` | piped into the VM per call | Background jobs (`run --bg`, `jobs`, `logs`, `wait`, `kill`; `vm_jobs`). The helper is erasable TypeScript run by the VM's Node (type stripping, ≥ 22.18), never installed, so a VM always runs the caller's version; it answers in JSON. The runner is plain bash, sent with each start: it runs the command in its own process group, independent of tmux, and records the real exit code. | | `backends/index.mjs` | agent machine and host | What each backend declares: supported and tested boxery-host platforms, runtimes, prerequisites and capabilities, as data (no probes). `host/capabilities.mjs` checks against it, `boxery-host backends`/`capabilities` print it, and `vm_info` lists it; it imports nothing. A backend without a declaration (one added as a `.sh` file) runs unchecked ([backends](docs/backends.md)) | diff --git a/controller/cli.mjs b/controller/cli.mjs index 0e799af6..aaeaaf92 100644 --- a/controller/cli.mjs +++ b/controller/cli.mjs @@ -544,7 +544,7 @@ export const COMMANDS = { help: { group: "use", summary: "push a local git checkout into a VM, so its tree matches exactly", usage: ["sync NAME [--from DIR] [--to PATH] [--json]"], details: [ - "Sends the commits the VM lacks as a git bundle, checks out the same commit (detached) and applies staged, unstaged and untracked changes, deletions included. Ignored files in the VM (node_modules, builds) stay. No git remote or GitHub access is needed on either side.", + "git pushes the commits the VM lacks over its ssh connection, checks out the same commit (detached) and applies staged, unstaged and untracked changes, deletions included. Ignored files in the VM (node_modules, builds) stay. No git remote or GitHub access is needed on either side.", "Refuses, changing nothing, when the VM checkout has changes the last sync didn't make, or the checkout holds a credential file (.env, .npmrc, auth files).", ], flags: [["--from DIR", "Local checkout (default: the current directory)"], ["--to PATH", "VM directory (default ~/project; relative paths are under it)"], ["--json", "Print the result as JSON"]], diff --git a/core/sync-guest.sh b/core/sync-guest.sh index 2c0a8afc..3e19c548 100644 --- a/core/sync-guest.sh +++ b/core/sync-guest.sh @@ -1,7 +1,8 @@ # shellcheck shell=bash # vm_sync, the VM side. core/sync.mjs sends this text with every call (it is never installed), after -# assignments to MODE (probe | apply), DIR, for probe WANT, and for apply: FORMAT BUNDLE SNAP HEAD_SHA -# TREE EXPECT_SYNCED EXPECT_HEAD. It never contacts a git remote. Output: key=value lines on stdout. +# assignments to MODE (probe | init | apply), DIR, for probe WANT, for init FORMAT, and for apply: SNAP +# HEAD_SHA TREE EXPECT_SYNCED EXPECT_HEAD. Commits arrive between probe and apply by git push (to +# refs/boxery/incoming/*), never through this script. It never contacts a git remote. Output: key=value lines on stdout. # Exit 3: refused (the reason on stderr); the checkout is as it was. Exit 1: failed; apply has restored # the checkout it found, or says it couldn't. # @@ -85,6 +86,8 @@ probe() { echo "base=$(rev refs/boxery/base)" echo "head=$(rev HEAD)" [ -z "${WANT:-}" ] || ! git cat-file -e "$WANT^{commit}" 2>/dev/null || echo has=1 + # What the VM has, so the caller can tell which commits a push would send (and check them). + git for-each-ref --count=500 --format='tip=%(objectname)' 2>/dev/null if [ -f "$G/boxery-sync-pending" ]; then echo pending=1; return; fi local b w b=$(baseline) || die "cannot read the last sync's state in $DIR" @@ -94,15 +97,30 @@ probe() { [ "$w" = "$b" ] || worktree_changes "$b" | sed 's/^/changed=/' } +# A repository for git push to land in, in a missing or empty directory only. +init() { + : "${FORMAT:?}" + if [ -d "$DIR" ] && [ -n "$(ls -A "$DIR")" ]; then + cd "$DIR" && is_repo_root && return 0 + refuse "$DIR is not a git repository and is not empty" + fi + mkdir -p "$DIR" && cd "$DIR" || die "cannot create $DIR" + git init -q --object-format="$FORMAT" || die "git init failed in $DIR" +} + # Put back tree $1 where tree $2 was swapped in (or half swapped in), and the index found before. rollback() { GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$2" && GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree --reset -u "$2" "$1" || return 1 restore_index } +# Refs git push left (this sync's, and any an earlier cancelled one did). +drop_incoming() { + git for-each-ref --format='delete %(refname)' refs/boxery/incoming/ 2>/dev/null | git update-ref --stdin 2>/dev/null || true +} restore_index() { if [ -f "$G/boxery-sync-index.bak" ]; then mv -f "$G/boxery-sync-index.bak" "$IDX"; else rm -f "$IDX"; fi - git update-ref -d refs/boxery/incoming 2>/dev/null + drop_incoming rm -f "$G/boxery-sync-index" "$G/boxery-sync-pending" } # A swap a crash interrupted: done if its refs were written, otherwise undone, but only if every file it @@ -162,19 +180,13 @@ swap() { } apply() { - : "${FORMAT:?}" "${SNAP:?}" "${HEAD_SHA:?}" "${TREE:?}" "${BUNDLE?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" + : "${SNAP:?}" "${HEAD_SHA:?}" "${TREE:?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" # Past this point the caller has committed: finish or roll back, even if it cancels or disconnects. trap '' INT TERM HUP PIPE LOCKDIR="" - trap 'rm -f "${BUNDLE:-/nonexistent}"; [ -z "$LOCKDIR" ] || rmdir "$LOCKDIR" 2>/dev/null' EXIT - if [ ! -d "$DIR" ] || [ -z "$(ls -A "$DIR")" ]; then - [ -z "$EXPECT_HEAD$EXPECT_SYNCED" ] || refuse "the checkout in $DIR changed during the sync; run it again" - mkdir -p "$DIR" && cd "$DIR" || die "cannot create $DIR" - git init -q --object-format="$FORMAT" || die "git init failed in $DIR" - else - cd "$DIR" || die "cannot enter $DIR" - is_repo_root || refuse "$DIR is not a git repository and is not empty" - fi + trap '[ -z "$LOCKDIR" ] || rmdir "$LOCKDIR" 2>/dev/null' EXIT + cd "$DIR" 2>/dev/null || refuse "$DIR is gone" + is_repo_root || refuse "$DIR is not a git repository" setup_git if command -v flock >/dev/null; then exec 9>"$G/boxery-sync.lock" @@ -194,16 +206,11 @@ apply() { b=$(baseline) || die "cannot read the last sync's state in $DIR" w=$(wt "$b") || die "cannot read the working directory of $DIR" [ "$w" = "$b" ] || refuse "uncommitted changes appeared in $DIR during the sync; run it again" - if [ -n "$BUNDLE" ]; then - git bundle verify -q "$BUNDLE" >/dev/null 2>&1 || die "the bundle doesn't apply to the VM's repository" - git fetch -q --no-tags --no-write-fetch-head "$BUNDLE" "+refs/heads/sync:refs/boxery/incoming" || die "git fetch from the bundle failed" - rm -f "$BUNDLE" - fi - git cat-file -e "$SNAP^{commit}" 2>/dev/null || die "the VM lacks commit $SNAP after the transfer" + git cat-file -e "$SNAP^{commit}" 2>/dev/null || die "the VM lacks commit $SNAP after the push" new=$(git rev-parse "$SNAP^{tree}") || die "cannot read $SNAP" hit=$(clobbered "$b" "$new" | head -n 20) || die "cannot compare the new tree with $DIR" if [ -n "$hit" ]; then - git update-ref -d refs/boxery/incoming 2>/dev/null + drop_incoming refuse "the local checkout now tracks paths that exist in the VM as ignored files (build output, dependencies), which the sync would overwrite or delete: $(printf '%s\n' "$hit" | tr '\n' ' ') Remove them in the VM (or stop tracking them locally), then sync again" fi @@ -230,11 +237,12 @@ Remove them in the VM (or ignore them locally), then sync again" ;; esac # The index holds local HEAD, so `git status` in the VM shows the working changes, as it does locally. if ! { git read-tree "$HEAD_SHA" && printf '%s\n' 'option no-deref' "update HEAD $HEAD_SHA" "update refs/boxery/synced $SNAP" \ - "update refs/boxery/base $HEAD_SHA" 'delete refs/boxery/incoming' | git update-ref -m "boxery sync" --stdin; }; then + "update refs/boxery/base $HEAD_SHA" | git update-ref -m "boxery sync" --stdin; }; then rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" die "the sync failed writing refs; the checkout in $DIR was restored" fi git update-index -q --refresh >/dev/null 2>&1 + drop_incoming rm -f "$G/boxery-sync-pending" "$G/boxery-sync-index" "$G/boxery-sync-index.bak" echo "lockfile_changed=$lock" echo "tree=$RESULT" @@ -242,6 +250,7 @@ Remove them in the VM (or ignore them locally), then sync again" ;; case "$MODE" in probe) probe ;; +init) init ;; apply) apply ;; *) die "unknown mode $MODE" ;; esac diff --git a/core/sync.mjs b/core/sync.mjs index c85e1017..9b21044b 100644 --- a/core/sync.mjs +++ b/core/sync.mjs @@ -1,20 +1,20 @@ // vm_sync: push the caller's local git checkout into a VM, so the VM's tree matches it exactly (docs/sync.md). -// Commits travel as a git bundle over vm_copy's streaming transport (copy-stream.mjs); neither side ever -// contacts a git remote. The VM side is core/sync-guest.sh, sent with every call. The local repository +// Commits travel by `git push` over the VM's ssh connection (the profile's ssh_config, pinned host key), +// straight into git receive-pack in the VM; neither side ever contacts a git remote. The VM side is core/sync-guest.sh, sent with every call. The local repository // is only read: the snapshot of its working changes is built in a scratch index and object directory. -import { execFile } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; import { readFileSync } from "node:fs"; import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import path from "node:path"; import { q } from "./shell.mjs"; -import { streamCopy } from "./copy-stream.mjs"; -import { PROJECT, REMOTE_HOME, assertName, confineLocal, currentHost, deniedDirs, isWithin, sshRun, vmPath } from "./boxery-core.mjs"; +import { PROJECT, assertName, confineLocal, currentHost, deniedDirs, isWithin, sshRun, vmPath } from "./boxery-core.mjs"; /** @import { LocalConfinement, ToolResult } from "./boxery-core.mjs" */ /** @typedef {{ from?: string, to?: string }} SyncParams */ /** @typedef {{ localCwd?: string, local?: LocalConfinement | false, signal?: AbortSignal }} SyncOpts */ -/** What the VM reported (sync-guest.sh probe, or apply's tree and lockfile_changed). @typedef {{ state: string, format?: string, synced?: string, base?: string, head?: string, has?: string, pending?: string, baseline?: string, worktree?: string, tree?: string, lockfile_changed?: string, changed: string[], entry: string[] }} Probe */ +/** What the VM reported (sync-guest.sh probe, or apply's tree and lockfile_changed). tip: the VM repo's ref tips. + * @typedef {{ state: string, format?: string, synced?: string, base?: string, head?: string, has?: string, pending?: string, baseline?: string, worktree?: string, tree?: string, lockfile_changed?: string, changed: string[], entry: string[], tip: string[] }} Probe */ const guestTemplate = readFileSync(new URL("./sync-guest.sh", import.meta.url), "utf8"); @@ -57,14 +57,16 @@ const baseEnv = () => { const SAFE_CONFIG = ["-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "protocol.allow=never"]; const aborted = () => Object.assign(new Error("sync aborted; the VM was not changed"), { name: "AbortError" }); -/** Run git locally → stdout. @param {string[]} args @param {{ cwd?: string, env?: NodeJS.ProcessEnv }} [opts] @returns {Promise} */ -function git(args, { cwd, env = baseEnv() } = {}) { +/** Run git locally → stdout. @param {string[]} args @param {{ cwd?: string, env?: NodeJS.ProcessEnv, input?: string }} [opts] @returns {Promise} */ +function git(args, { cwd, env = baseEnv(), input } = {}) { return new Promise((resolve, reject) => { - execFile("git", [...SAFE_CONFIG, ...args], { cwd, env, maxBuffer: 256 * 1024 * 1024, encoding: "utf8" }, (error, stdout, stderr) => { + const child = execFile("git", [...SAFE_CONFIG, ...args], { cwd, env, maxBuffer: 256 * 1024 * 1024, encoding: "utf8" }, (error, stdout, stderr) => { if (!error) return resolve(stdout); const e = /** @type {NodeJS.ErrnoException} */ (error); - reject(new Error(e.code === "ENOENT" ? "git is not installed on this machine" : `git ${args[0]}: ${stderr.trim() || e.message}`)); + reject(new Error(e.code === "ENOENT" ? "git is not installed on this machine" : `git ${args.find((a) => !a.startsWith("-") && !a.includes("=") && a !== "--git-dir") ?? args[0]}: ${stderr.trim() || e.message}`)); }); + child.stdin?.on("error", () => {}); + child.stdin?.end(input ?? ""); }); } const tryGit = (/** @type {string[]} */ args, /** @type {{ cwd?: string, env?: NodeJS.ProcessEnv }} */ opts) => git(args, opts).then((s) => s.trim(), () => null); @@ -72,7 +74,7 @@ const tryGit = (/** @type {string[]} */ args, /** @type {{ cwd?: string, env?: N /** key=value lines from sync-guest.sh. Repeated keys (changed, entry) collect into arrays. @param {string} out @returns {Probe} */ function parseLines(out) { /** @type {any} */ - const r = { changed: [], entry: [] }; + const r = { changed: [], entry: [], tip: [] }; for (const line of out.split("\n")) { const i = line.indexOf("="); if (i <= 0) continue; @@ -198,37 +200,61 @@ function refuseCredentials(root, files, deny) { } /** - * A bundle of the commits the VM lacks (null when it has the snapshot already), excluding what it has. - * @param {Awaited>} s @param {Probe} probe @param {string} scratch @param {string} id + * A bare scratch repository to push from: its alternates are the scratch objects and the checkout's, so + * refs/heads/sync can point at the snapshot without touching the checkout's refs. Refuses a credential + * anywhere in the commits the VM may lack (those not reachable from its ref tips), not just the snapshot. + * @param {Awaited>} s @param {Probe} probe @param {string} scratch */ -async function bundle(s, probe, scratch, id) { - if (probe.has === "1") return null; - // A bare repository whose alternates are the scratch objects and the checkout's: refs/heads/sync can - // be written there without touching the checkout's refs. - const bare = path.join(scratch, "bundle.git"); +async function pushSource(s, probe, scratch) { + const bare = path.join(scratch, "push.git"); const env = baseEnv(); await git(["init", "-q", "--bare", `--object-format=${s.format}`, bare], { env }); await writeFile(path.join(bare, "objects", "info", "alternates"), `${s.scratchObjects}\n${s.objects}\n`); await git(["--git-dir", bare, "update-ref", "refs/heads/sync", s.snap], { env }); - const have = []; - for (const c of new Set([probe.synced, probe.base, probe.head].filter(Boolean))) - if (await tryGit(["--git-dir", bare, "cat-file", "-e", `${c}^{commit}`], { env }) !== null) have.push(`^${c}`); - // The bundle carries every commit the VM lacks, not just the snapshot: refuse a credential anywhere in them. - const history = (await git(["--git-dir", bare, "log", "--format=", "--name-only", "--no-renames", "-z", "refs/heads/sync", ...have], { env })).split("\0").map((p) => p.trim()).filter(Boolean); + // Tips this repository doesn't have are skipped: then nothing they reach is excluded, which only errs + // towards refusing. + const tips = [...new Set([probe.synced, probe.base, probe.head, ...probe.tip].filter((t) => t && /^[0-9a-f]{40,64}$/.test(t)))]; + const history = (await git(["--git-dir", bare, "log", "--format=", "--name-only", "--no-renames", "-z", "--ignore-missing", "--stdin"], { env, input: `refs/heads/sync\n--not\n${tips.join("\n")}\n` })) + .split("\0").map((p) => p.trim()).filter(Boolean); const leaked = [...new Set(history.filter(deniedSyncPath))]; if (leaked.length) throw new Error(`vm_sync refused: ${leaked.slice(0, 5).join(", ")} is in the history this sync would send (a commit the VM lacks has it), and vm_sync never puts credential files on a VM's disk. ` + "It's not in your working tree, but its old contents are in that commit. Remove it from history (git rebase / git filter-repo) before syncing, and ask for the credential with vm_grant. Nothing in the VM was changed."); - const file = path.join(scratch, "out", `sync-${id}.bundle`); - await mkdir(path.dirname(file)); - try { - await git(["--git-dir", bare, "bundle", "create", "-q", file, "refs/heads/sync", ...have], { env }); - } catch (e) { - // "empty bundle": everything is reachable from what the VM has; send the snapshot whole instead. - if (!have.length || !/empty bundle/.test(/** @type {Error} */ (e).message)) throw e; - await git(["--git-dir", bare, "bundle", "create", "-q", file, "refs/heads/sync"], { env }); - } - return file; + return bare; +} + +/** + * git push the snapshot into the VM repository at `dest`, to a hidden ref. receive-pack quarantines + * incoming objects until the ref update succeeds, so a cancelled push leaves nothing behind. No hooks run + * on either side, and the push goes only to this VM, over its own ssh_config. + * → objects sent (null if git didn't say: nothing to send, or no progress line). + * @param {string} bare @param {string} vm @param {string} sshConf @param {string} dest @param {string} ref @param {AbortSignal} [signal] + * @returns {Promise} + */ +function push(bare, vm, sshConf, dest, ref, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(aborted()); + const env = { ...baseEnv(), GIT_SSH_COMMAND: `ssh -F ${q(sshConf)} -o BatchMode=yes`, GIT_SSH_VARIANT: "ssh" }; + const args = [...SAFE_CONFIG, "-c", "protocol.ssh.allow=always", "--git-dir", bare, "push", "--progress", "--no-verify", "--no-signed", "--no-recurse-submodules", "--no-follow-tags", + "--receive-pack=git -c core.hooksPath=/dev/null receive-pack", `${vm}.boxery:${dest}`, `+refs/heads/sync:${ref}`]; + // Its own process group, so cancelling stops git, its ssh and pack-objects together. + const child = spawn("git", args, { env, detached: true, stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (d) => { stderr = (stderr + d).slice(-64 * 1024); }); + const stop = () => { try { process.kill(-(/** @type {number} */ (child.pid)), "SIGTERM"); } catch {} }; + signal?.addEventListener("abort", stop, { once: true }); + child.on("error", (e) => { signal?.removeEventListener("abort", stop); reject(new Error(`git push: ${e.message}`)); }); + child.on("close", (code) => { + signal?.removeEventListener("abort", stop); + if (signal?.aborted) return reject(aborted()); + if (code !== 0) { + const lines = stderr.split(/[\r\n]+/).filter((l) => l && !/^(Enumerating|Counting|Delta|Compressing|Writing|Total) /.test(l)); + return reject(new Error(`vm_sync: git push to VM "${vm}" failed: ${lines.join("\n").trim() || `exit ${code}`}; nothing in the checkout was changed`)); + } + const m = /Writing objects: 100% \((\d+)\/\d+\)/.exec(stderr); + resolve(m ? Number(m[1]) : null); + }); + }); } /** @@ -251,8 +277,7 @@ export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd() if (signal?.aborted) throw aborted(); const id = randomBytes(8).toString("hex"); const scratch = await mkdtemp(path.join(tmpdir(), "boxery-sync-")); - const remoteBundle = `${REMOTE_HOME}/.cache/boxery/sync-${id}.bundle`; - let sent = false; + const ref = `refs/boxery/incoming/${id}`; try { const s = await snapshot(fromPath, scratch, confined); refuseCredentials(s.root, s.files, local ? local.deny : undefined); @@ -267,19 +292,21 @@ export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd() throw new Error(`vm_sync refused: ${label} has changes ${what} that vm_sync didn't make:\n${probe.changed.map((c) => ` ${c.replace("\t", " ")}`).join("\n")}\n` + "Commit them in the VM and bring them back yourself, or discard them (git -C DIR stash -u, or checkout/clean), then sync again. Nothing in the VM was changed."); } - const file = await bundle(s, probe, scratch, id); - let bytes = 0; - if (file) { - sent = true; - bytes = await streamCopy({ path: file, label: "local" }, { path: remoteBundle, vm, sshConf: host.sshConf, label: `${host.name}/${vm}` }, { signal }); + const pushed = probe.has !== "1"; + let objects = 0; + if (pushed) { + const bare = await pushSource(s, probe, scratch); + if (signal?.aborted) throw aborted(); + // receive-pack needs a repository: create one in a missing or empty directory (only that). + if (probe.state !== "repo") await guest(vm, { MODE: "init", DIR: dest, FORMAT: s.format }); + objects = (await push(bare, vm, host.sshConf, dest, ref, signal)) ?? 0; } // The last point to stop: past here the VM applies the snapshot completely or not at all. if (signal?.aborted) throw aborted(); const applied = await guest(vm, { - MODE: "apply", DIR: dest, FORMAT: s.format, BUNDLE: file ? remoteBundle : "", SNAP: s.snap, HEAD_SHA: s.head, TREE: s.tree, + MODE: "apply", DIR: dest, SNAP: s.snap, HEAD_SHA: s.head, TREE: s.tree, EXPECT_SYNCED: probe.synced ?? "", EXPECT_HEAD: probe.head ?? "", }); - sent = false; // apply removed the bundle const match = applied.tree === s.tree; const lockfileChanged = applied.lockfile_changed === "1"; const short = s.head.slice(0, 7), { modified, untracked, deleted } = s.counts; @@ -292,12 +319,11 @@ export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd() structured: { vm, host: host.name, from: s.root, to: dest, sha: s.head, snapshot: s.snap, counts: s.counts, tree: { local: s.tree, vm: applied.tree ?? null, match }, - lockfile_changed: lockfileChanged, transferred_bytes: bytes, elapsed_ms: Date.now() - started, + lockfile_changed: lockfileChanged, pushed, objects_sent: objects, elapsed_ms: Date.now() - started, }, }; } finally { - // A bundle the transfer left but apply never consumed (cancelled or refused before it ran) is removed. - if (sent) await sshRun(vm, `rm -f ${q(remoteBundle)}`, { timeoutMs: 15_000 }).catch(() => {}); + // (A pushed ref apply never reached is harmless; the next successful apply drops every incoming ref.) await rm(scratch, { recursive: true, force: true }); } } @@ -313,7 +339,8 @@ export const SYNC_OUTPUT = { counts: { type: "object", properties: { modified: { type: "integer" }, untracked: { type: "integer" }, deleted: { type: "integer" } } }, tree: { type: "object", description: "git tree of the whole working directory (tracked + untracked, not ignored), computed on each side", properties: { local: { type: "string" }, vm: { type: ["string", "null"] }, match: { type: "boolean" } } }, lockfile_changed: { type: "boolean", description: `a lockfile (${LOCKFILES.join(", ")}) differs from the previous sync: dependencies need installing` }, - transferred_bytes: { type: "integer", description: "bytes of bundle sent (0: the VM had every commit)" }, + pushed: { type: "boolean", description: "false: the VM already had the snapshot, so nothing was pushed" }, + objects_sent: { type: "integer", description: "git objects the push sent (0 when nothing was pushed)" }, elapsed_ms: { type: "integer" }, }, }; diff --git a/docs/cli.md b/docs/cli.md index 7c74311d..87701e9b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -146,7 +146,7 @@ boxery --host hornet cp local:notes.md ci-1:~/notes.md `cp` is the `vm_copy` tool's own code, so it takes the same locations, `local:PATH`, `NAME:PATH` (a VM on the selected host) and `HOST/NAME:PATH`, and behaves the same way: streaming tar, staging, destination and cancellation rules are in [file transfers](copy.md). The one difference: the CLI has no current VM, so a bare path is on this machine, like `cp`/`scp` (`vm_copy` reads it as the current VM). A bare path that looks like a location with an invalid VM name (`Demo:x`) is refused instead of being written locally. Ctrl-C cancels and cleans up staging. `cp` runs on Node (≥ 20), which the npm package already needs. -`boxery sync demo [--from DIR] [--to PATH] [--json]` pushes a local git checkout into a VM so its tree matches exactly (commits as a git bundle, plus uncommitted and untracked changes); it is the `vm_sync` tool's own code ([sync](sync.md)). +`boxery sync demo [--from DIR] [--to PATH] [--json]` pushes a local git checkout into a VM so its tree matches exactly (commits by `git push` over the VM's ssh connection, plus uncommitted and untracked changes); it is the `vm_sync` tool's own code ([sync](sync.md)). `attach` starts pi, opencode or a shell (nothing else) and needs a terminal; without one it says to use `prompt` or `run --bg`, which work headless. `prompt` logs to `~/project/.boxery/task-*.log` in the VM. Attach to a task with `boxery attach demo pi task-…`. pi needs the `llm:` grant for its provider (`llm:chatgpt` unless you changed pi's settings in the VM), and a new VM has none: `prompt` checks first and, if it's missing, starts nothing and prints the `boxery grant` command to run (the older spelling of `boxery access grant`). diff --git a/docs/sync.md b/docs/sync.md index d1bbe377..8077a2ee 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -20,7 +20,7 @@ boxery sync demo --json # the result as JSON ## What it does 1. **Snapshot.** It records local HEAD plus every working change (what `git add -A` would stage: tracked files, and untracked files `.gitignore` doesn't exclude, deletions and renames included) as one commit. It uses a scratch index and object directory: your index, refs and stash are untouched. The commit has a fixed author and date, so the same working state always gives the same commit. -2. **Transfer.** The commits the VM lacks go over as a `git bundle`, on the same streaming transport as [`vm_copy`](copy.md). Neither side contacts a git remote, so the VM needs no GitHub grant. If the VM already has the snapshot, nothing is sent. If `to` doesn't exist or is empty, a repository is initialized there. +2. **Push.** The snapshot is `git push`ed straight into the VM repository, to a hidden ref (`refs/boxery/incoming/…`), over the VM's own ssh connection (the host profile's `ssh_config`, pinned host key). git works out which commits the VM lacks from all of its refs and streams only those into `git receive-pack`: no archive, no temporary file. No hooks run on either side. Nothing contacts a git remote, so the VM needs no GitHub grant. If the VM already has the snapshot, nothing is pushed. If `to` doesn't exist or is empty, a repository is initialized there first (after the checks pass). 3. **Swap.** In the VM, HEAD is detached at local HEAD's commit, and the working directory is switched to the snapshot: only the paths that differ are rewritten, and files the snapshot lacks are deleted, including ones an earlier sync put there. Ignored files (`node_modules`, build output) stay: there is no `git clean -x`. If the snapshot would put a tracked file where the VM has an ignored one, the sync refuses instead (below). The VM's index holds HEAD, so `git status` and `git diff` in the VM show the same working changes as they do locally. 4. **Check.** Both sides hash the whole working directory (tracked plus untracked, not ignored) as a git tree, the same way, and compare. The result says `tree matches`, or `TREE MISMATCH` (an error). @@ -28,7 +28,7 @@ boxery sync demo --json # the result as JSON synced 3f2a9c1 + 2 modified, 1 untracked, 0 deleted → demo:/home/agent/project · tree matches ``` -Structured fields: `sha` (local HEAD), `snapshot`, `counts` (`modified`, `untracked`, `deleted`), `tree` (`local`, `vm`, `match`), `lockfile_changed`, `transferred_bytes`, `elapsed_ms`. +Structured fields: `sha` (local HEAD), `snapshot`, `counts` (`modified`, `untracked`, `deleted`), `tree` (`local`, `vm`, `match`), `lockfile_changed`, `pushed` (false when the VM already had the snapshot), `objects_sent`, `elapsed_ms`. `lockfile_changed` is true when `pnpm-lock.yaml`, `package-lock.json`, `yarn.lock` or `bun.lock` (at any depth) differs from the previous sync, or from the VM's HEAD on the first one. vm_sync installs nothing; run your install step when it says so. @@ -40,7 +40,7 @@ It refuses, and changes nothing in the VM, when: - **The VM checkout has changes the last sync didn't make.** The VM records each sync in `refs/boxery/synced` (and local HEAD in `refs/boxery/base`). Any difference between the VM's working directory and that snapshot (edited, new or deleted files, not ignored ones) is listed in the error. Commit them in the VM and bring them back yourself, or discard them, then sync again. A VM repository that was never synced is compared with its HEAD. - **The checkout holds a credential file.** Matched case-insensitively: `.env` and `.env.*`/`.env-*`/`.env_*` (except `.env.example`, `.env.sample`, `.env.template`, `.env.dist`, `.env.defaults`), `.envrc`, `.npmrc`, `.yarnrc.yml`, `.pypirc`, `.netrc`, `.git-credentials`, `.pgpass`, `.my.cnf`, `.terraformrc`, `auth.json`, `credentials`, `application_default_credentials.json`, private keys (`id_*` except `.pub`, `*.pem`, `*.key`, `*.p12`, `*.pfx`, `*.jks`, `*.keystore`), `.docker/config.json`, `.kube/config`, `.aws/config`, `gh/hosts.yml`, and anything inside boxery's own state or `~/.ssh`, `~/.gnupg`, `~/.pi` (what `vm_copy` refuses). The error names the path. Ignore it in `.gitignore` to sync the rest, and get the credential to the VM with a [grant](grants.md), which keeps it off the VM's disk and out of its snapshots. Ignored files are never synced, so an ignored `.env` is fine. -- **The commits it would send contain one.** The bundle carries every commit the VM lacks (on a first sync, the whole history), so a credential file committed once and deleted later is refused too. Remove it from history first. +- **The commits it would send contain one.** The push carries every commit the VM lacks (on a first sync, the whole history), so a credential file committed once and deleted later is refused too. Remove it from history first. - **The local checkout now tracks a path the VM has as an ignored file** (a force-added build file, or `dist/` un-ignored and replaced by a file): the swap would overwrite or delete the VM's file. Or, with the new `.gitignore`, **files the VM has stop being ignored**, so the trees can't match. The error lists the paths; remove them in the VM (or change the local checkout), and sync again. - **`to` is a non-empty directory that isn't a git repository**, is inside another checkout (`~/project/api` when `~/project` is a repository: use `~/api`), or its object format (sha1/sha256) differs. - **The local path isn't a git checkout**, has no commits yet, or is a shallow clone. Use `vm_copy` for plain files. @@ -49,9 +49,9 @@ It refuses, and changes nothing in the VM, when: ## Cancellation and failure -Cancelling (an MCP cancellation, pi's Escape, Ctrl-C for `boxery sync`) before the transfer finishes leaves the VM unchanged: the partial bundle is removed with `vm_copy`'s staging. Once the VM starts applying, it ignores signals and finishes or rolls back; a lost connection doesn't interrupt it, and `boxery sync` waits for its result after Ctrl-C. +Cancelling (an MCP cancellation, pi's Escape, Ctrl-C for `boxery sync`) before the push finishes leaves the VM unchanged: `git receive-pack` keeps incoming objects in a quarantine directory until the ref update succeeds, and removes it when the push dies. Once the VM starts applying, it ignores signals and finishes or rolls back; a lost connection doesn't interrupt it, and `boxery sync` waits for its result after Ctrl-C. -The swap is staged: the VM fetches the bundle into its object store (which changes no files), then writes a pending record, switches the working directory, and finally updates HEAD and its refs in one transaction. If a step fails, it switches back and restores the index. If the VM crashes mid-swap, the next sync finds the pending record and finishes it (if the refs were written) or undoes it, before checking for VM-side changes. It undoes it only if every file is as the swap left it; a file edited since is reported, and the record stays. Fetched objects stay in the VM's repository either way; they're harmless and `git gc` collects them. +The swap is staged: the pushed objects are already in the VM's object store (which changes no files); apply writes a pending record, switches the working directory, and finally updates HEAD and its refs in one transaction. If a step fails, it switches back and restores the index. If the VM crashes mid-swap, the next sync finds the pending record and finishes it (if the refs were written) or undoes it, before checking for VM-side changes. It undoes it only if every file is as the swap left it; a file edited since is reported, and the record stays. Pushed objects stay in the VM's repository either way; they're harmless and `git gc` collects them. A sync takes a lock in the VM repository (`flock`, or a lock directory without it), so two syncs into one checkout run one after the other. If the checkout changes between the check and the swap, the sync refuses: git checks every path before writing any. diff --git a/test/fixtures/copy-ssh.mjs b/test/fixtures/copy-ssh.mjs index d7514470..550943f1 100644 --- a/test/fixtures/copy-ssh.mjs +++ b/test/fixtures/copy-ssh.mjs @@ -35,7 +35,9 @@ if (preparing && process.env.COPY_LATE_PREPARE === endpoint) { } if (preparing && process.env.COPY_SLOW_PREPARE === endpoint) await new Promise((r) => setTimeout(r, 200)); /** @type {NodeJS.ProcessEnv} */ -const env = { ...process.env, COPY_ENDPOINT: endpoint, HOME: root }; +// A real sshd starts the remote command with a fresh environment; git push's ssh inherits git's +// exec-path at the front of PATH, which FAKE_VM_PATH (when set) undoes. +const env = { ...process.env, COPY_ENDPOINT: endpoint, HOME: root, ...(process.env.FAKE_VM_PATH ? { PATH: process.env.FAKE_VM_PATH } : {}) }; if (process.env.COPY_SKIP_EXIT_SNAPSHOT === endpoint) env.BASH_ENV = path.join(/** @type {string} */ (process.env.COPY_ROOT), "no-exit-trap.sh"); const child = spawn("bash", ["-c", script], { env, cwd: root, stdio: "inherit" }); child.on("error", () => { process.exitCode = 255; }); diff --git a/test/fixtures/sync-git.mjs b/test/fixtures/sync-git.mjs new file mode 100644 index 00000000..3cdd04ca --- /dev/null +++ b/test/fixtures/sync-git.mjs @@ -0,0 +1,17 @@ +#!/usr/bin/env node +// git double for test/sync.test.mjs: runs the real git (REAL_GIT), except that with SYNC_SLOW_RECEIVE set, +// receive-pack (the VM end of git push) reads its input slowly and marks that it started, so a test can +// cancel a push mid-transfer. +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { Transform } from "node:stream"; +const args = process.argv.slice(2); +const slow = process.env.SYNC_SLOW_RECEIVE && args.includes("receive-pack"); +const child = spawn(/** @type {string} */ (process.env.REAL_GIT), args, { stdio: [slow ? "pipe" : "inherit", "inherit", "inherit"] }); +if (slow) { + writeFileSync(/** @type {string} */ (process.env.SYNC_SLOW_RECEIVE), String(process.pid)); + const throttle = new Transform({ transform(chunk, _e, cb) { setTimeout(() => cb(null, chunk), 20); } }); + process.stdin.pipe(throttle).pipe(/** @type {import("node:stream").Writable} */ (child.stdin)); +} +for (const sig of /** @type {const} */ (["SIGTERM", "SIGINT", "SIGHUP"])) process.on(sig, () => child.kill(sig)); +child.on("close", (code, signal) => { process.exitCode = code ?? (signal ? 1 : 0); }); diff --git a/test/sync.test.mjs b/test/sync.test.mjs index 0a1435b3..6d76856f 100644 --- a/test/sync.test.mjs +++ b/test/sync.test.mjs @@ -2,7 +2,8 @@ // "VM" is a directory, and its ssh runs scripts there with /home/agent mapped to it. No real VMs. import assert from "node:assert/strict"; import { execFileSync, spawn } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, test as nodeTest } from "node:test"; @@ -71,6 +72,7 @@ beforeEach(() => { env = { ...process.env, ...GIT_ENV, HOME: home, PATH: `${path.join(dir, "bin")}:${process.env.PATH}`, BOXERY_HOME: home, COPY_ROOT: dir, COPY_LOG: path.join(dir, "calls"), REAL_TAR: execFileSync("sh", ["-c", "command -v tar"], { encoding: "utf8" }).trim(), + REAL_GIT: execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim(), }; repo = path.join(dir, "local", "repo"); mkdirSync(repo, { recursive: true }); @@ -92,7 +94,10 @@ test("first sync into an empty directory: a detached checkout that matches", asy assert.equal(r.structured.sha, head); assert.equal(r.structured.tree.match, true); assert.equal(r.structured.tree.local, r.structured.tree.vm); - assert.ok(r.structured.transferred_bytes > 0); + assert.equal(r.structured.pushed, true); + assert.ok(r.structured.objects_sent > 0); + assert.ok(readFileSync(path.join(dir, "calls"), "utf8").includes("receive-pack"), "commits went by git push"); + assert.equal(git(vmProject(), "for-each-ref", "refs/boxery/incoming/"), "", "the pushed ref is dropped after apply"); assert.equal(typeof r.structured.elapsed_ms, "number"); assert.equal(git(vmProject(), "rev-parse", "HEAD"), head); assert.throws(() => git(vmProject(), "symbolic-ref", "-q", "HEAD"), "detached"); @@ -213,13 +218,14 @@ test("a second sync with no local changes transfers nothing and still matches", const first = await ok(); const calls = readFileSync(path.join(dir, "calls"), "utf8").split("\n").filter(Boolean).length; const second = await ok(); - assert.equal(second.structured.transferred_bytes, 0); + assert.equal(second.structured.pushed, false); + assert.equal(second.structured.objects_sent, 0); assert.equal(second.structured.tree.match, true); assert.equal(second.structured.lockfile_changed, false); assert.equal(second.structured.snapshot, first.structured.snapshot, "the same working changes give the same snapshot commit"); assert.match(second.text, /· tree matches$/); - const tars = readFileSync(path.join(dir, "calls"), "utf8").split("\n").filter(Boolean).slice(calls).map((l) => JSON.parse(l)).filter((c) => c.kind === "tar"); - assert.deepEqual(tars, [], "no tar transfer the second time"); + const pushes = readFileSync(path.join(dir, "calls"), "utf8").split("\n").filter(Boolean).slice(calls).map((l) => JSON.parse(l)).filter((c) => String(c.script).includes("receive-pack")); + assert.deepEqual(pushes, [], "no push the second time"); }); test("a lockfile change is flagged, once", async () => { @@ -236,26 +242,42 @@ test("a lockfile change is flagged, once", async () => { assert.equal((await ok()).structured.lockfile_changed, false); }); -test("cancellation mid-transfer leaves the VM unchanged", async () => { +test("cancellation mid-push leaves the VM unchanged", async () => { await ok(); const before = tree(vmProject()); const head = git(vmProject(), "rev-parse", "HEAD"); - write(repo, "big.bin", Buffer.alloc(8 * 1024 * 1024, 7)); + const refs = git(vmProject(), "for-each-ref"); + write(repo, "big.bin", randomBytes(4 * 1024 * 1024)); write(repo, "README.md", "changed\n"); - env.COPY_SLOW_SOURCE = "local"; - const calls = () => readFileSync(path.join(dir, "calls"), "utf8"); - const mark = calls().length; - const r = /** @type {any} */ (await sync({}, { abortWhen: () => calls().slice(mark).includes("\"role\":\"destination\"") })); + const started = path.join(dir, "receive-started"); + // Only here: a git double whose receive-pack (the VM end of the push) is slow. + writeFileSync(path.join(dir, "bin", "git"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "sync-git.mjs"))} "$@"\n`, { mode: 0o755 }); + env.SYNC_SLOW_RECEIVE = started; + env.FAKE_VM_PATH = env.PATH; + const r = /** @type {any} */ (await sync({}, { abortWhen: () => existsSync(started) })); + delete env.SYNC_SLOW_RECEIVE; + delete env.FAKE_VM_PATH; + unlinkSync(path.join(dir, "bin", "git")); + assert.ok(existsSync(started), `the push was in flight (${r.error ?? "it finished"})`); assert.equal(r.ok, false); assert.equal(r.name, "AbortError"); assert.deepEqual(tree(vmProject()), before); assert.equal(git(vmProject(), "rev-parse", "HEAD"), head); - const cache = path.join(dir, "vms", "alpha", "vm", ".cache", "boxery"); - assert.deepEqual(existsSync(cache) ? execFileSync("ls", ["-A", cache], { encoding: "utf8" }).trim() : "", "", "no bundle or staging left"); - delete env.COPY_SLOW_SOURCE; + assert.equal(git(vmProject(), "for-each-ref"), refs, "no ref was written"); + // receive-pack quarantines incoming objects and removes them when the push dies. + const objects = path.join(vmProject(), ".git", "objects"); + for (let i = 0; i < 100 && readdirSync(objects).some((n) => n.startsWith("tmp_objdir")); i++) await new Promise((res) => setTimeout(res, 50)); + assert.deepEqual(readdirSync(objects).filter((n) => n.startsWith("tmp_objdir")), [], "no quarantined objects left"); assert.equal((await ok()).structured.tree.match, true, "a later sync completes"); }); +test("a push into a missing directory creates the repository only there", async () => { + const r = await ok({ to: "~/fresh/repo" }); + assert.equal(r.structured.tree.match, true); + const vmHome = path.join(dir, "vms", "alpha", "vm"); + assert.deepEqual(tree(path.join(vmHome, "fresh", "repo")), tree(repo)); +}); + test("an interrupted swap is undone by the next sync", async () => { await ok(); write(repo, "README.md", "next\n"); From 377741f5c38ef089721a03d0f77dcba47a0bf606 Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sun, 8 Nov 2026 15:46:13 -0500 Subject: [PATCH 4/5] =?UTF-8?q?fix(sync):=20second=20review=20=E2=80=94=20?= =?UTF-8?q?racy=20index=20copies,=20merge=20credentials,=20config=20isolat?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Copy the index with its mtime (both sides): a newer copy hid same-second, same-size edits, so local changes went unsynced and VM edits were missed. - History scan uses rev-list --objects (what the push sends), so merge commits and roots count; deny-dir check covers history too. - The push and its scratch repo read no user/system config and no GIT_* env: url.(push)insteadOf can't redirect it to another VM. - Push ssh uses ControlMaster=no; receive.autogc=false; cancel escalates TERM to KILL and doesn't wait forever on stderr. - Guest: no hooks or fsmonitor from the VM repo's config. - Undo init when the push into a new dir fails; drop only this sync's incoming ref plus abandoned ones over an hour old. - core.splitIndex=false for scratch-index ops; refuse partial clones. --- core/sync-guest.sh | 38 ++++++++++++--- core/sync.mjs | 94 ++++++++++++++++++++++++++------------ docs/sync.md | 9 ++-- test/fixtures/copy-ssh.mjs | 3 +- test/sync.test.mjs | 94 +++++++++++++++++++++++++++++++++++++- 5 files changed, 196 insertions(+), 42 deletions(-) diff --git a/core/sync-guest.sh b/core/sync-guest.sh index 3e19c548..d4227544 100644 --- a/core/sync-guest.sh +++ b/core/sync-guest.sh @@ -1,7 +1,7 @@ # shellcheck shell=bash # vm_sync, the VM side. core/sync.mjs sends this text with every call (it is never installed), after -# assignments to MODE (probe | init | apply), DIR, for probe WANT, for init FORMAT, and for apply: SNAP -# HEAD_SHA TREE EXPECT_SYNCED EXPECT_HEAD. Commits arrive between probe and apply by git push (to +# assignments to MODE (probe | init | uninit | apply), DIR, for probe WANT, for init FORMAT, and for apply: +# SNAP HEAD_SHA TREE REF EXPECT_SYNCED EXPECT_HEAD. Commits arrive between probe and apply by git push (to # refs/boxery/incoming/*), never through this script. It never contacts a git remote. Output: key=value lines on stdout. # Exit 3: refused (the reason on stderr); the checkout is as it was. Exit 1: failed; apply has restored # the checkout it found, or says it couldn't. @@ -12,6 +12,8 @@ set -u -o pipefail : "${MODE:?}" "${DIR:?}" export LC_ALL=C GIT_TERMINAL_PROMPT=0 GIT_OPTIONAL_LOCKS=0 GIT_NO_LAZY_FETCH=1 GIT_LFS_SKIP_SMUDGE=1 +# No hooks (update-ref runs reference-transaction) and no fsmonitor from the VM repository's config. +export GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0=/dev/null GIT_CONFIG_KEY_1=core.fsmonitor GIT_CONFIG_VALUE_1=false LOCKFILES=(':(glob)**/pnpm-lock.yaml' ':(glob)**/package-lock.json' ':(glob)**/yarn.lock' ':(glob)**/bun.lock') refuse() { printf '%s\n' "$*" >&2; exit 3; } die() { printf '%s\n' "$*" >&2; exit 1; } @@ -41,7 +43,9 @@ wt() { local t rc t=$(mktemp -d) || return 1 mkdir "$t/o" - [ ! -f "$IDX" ] || cp "$IDX" "$t/i" + # -p keeps the index's mtime: git re-checks entries changed in the same second as the index was written + # by comparing with it, and a newer copy would make such an edit look clean. + [ ! -f "$IDX" ] || cp -p "$IDX" "$t/i" ( export GIT_INDEX_FILE="$t/i" GIT_OBJECT_DIRECTORY="$t/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" # One-tree --reset keeps the copied index's stat data for entries it doesn't change, so unchanged @@ -108,15 +112,36 @@ init() { git init -q --object-format="$FORMAT" || die "git init failed in $DIR" } +# Undo init after a push that didn't land: remove the repository init made, if it is still just that (no +# refs, no objects), and the directory if it is then empty. +uninit() { + cd "$DIR" 2>/dev/null && is_repo_root || return 0 + setup_git + [ -z "$(git for-each-ref --count=1)" ] || return 0 + [ -z "$(find "$OBJ" -type f ! -path '*/info/*' 2>/dev/null | head -n 1)" ] || return 0 + [ "$(ls -A)" = .git ] || return 0 + rm -rf "$G" && cd / && rmdir "$DIR" 2>/dev/null + return 0 +} + # Put back tree $1 where tree $2 was swapped in (or half swapped in), and the index found before. rollback() { GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$2" && GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree --reset -u "$2" "$1" || return 1 restore_index } -# Refs git push left (this sync's, and any an earlier cancelled one did). +# The ref this sync's push wrote, and those of earlier syncs that never applied (over an hour old: not a +# concurrent sync's). drop_incoming() { - git for-each-ref --format='delete %(refname)' refs/boxery/incoming/ 2>/dev/null | git update-ref --stdin 2>/dev/null || true + local cutoff; cutoff=$(( $(date +%s) - 3600 )) + { + [ -z "${REF:-}" ] || printf 'delete %s\n' "$REF" + git for-each-ref --format='%(refname)' refs/boxery/incoming/ 2>/dev/null | while read -r r; do + t=${r##*/}; t=${t%%-*} + case "$t" in '' | *[!0-9]*) t=0 ;; esac + [ "$r" = "${REF:-}" ] || [ "$t" -ge "$cutoff" ] || printf 'delete %s\n' "$r" + done + } | git update-ref --stdin 2>/dev/null || true } restore_index() { if [ -f "$G/boxery-sync-index.bak" ]; then mv -f "$G/boxery-sync-index.bak" "$IDX"; else rm -f "$IDX"; fi @@ -180,7 +205,7 @@ swap() { } apply() { - : "${SNAP:?}" "${HEAD_SHA:?}" "${TREE:?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" + : "${SNAP:?}" "${HEAD_SHA:?}" "${TREE:?}" "${REF?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" # Past this point the caller has committed: finish or roll back, even if it cancels or disconnects. trap '' INT TERM HUP PIPE LOCKDIR="" @@ -251,6 +276,7 @@ Remove them in the VM (or ignore them locally), then sync again" ;; case "$MODE" in probe) probe ;; init) init ;; +uninit) uninit ;; apply) apply ;; *) die "unknown mode $MODE" ;; esac diff --git a/core/sync.mjs b/core/sync.mjs index 9b21044b..f1d54178 100644 --- a/core/sync.mjs +++ b/core/sync.mjs @@ -5,7 +5,7 @@ import { execFile, spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; import { readFileSync } from "node:fs"; -import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { copyFile, mkdir, mkdtemp, realpath, rm, stat, utimes, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; import path from "node:path"; import { q } from "./shell.mjs"; @@ -43,19 +43,22 @@ export function deniedSyncPath(p) { const IDENTITY = { GIT_AUTHOR_NAME: "boxery sync", GIT_AUTHOR_EMAIL: "sync@boxery.invalid", GIT_AUTHOR_DATE: "@0 +0000" }; const COMMITTER = { GIT_COMMITTER_NAME: IDENTITY.GIT_AUTHOR_NAME, GIT_COMMITTER_EMAIL: IDENTITY.GIT_AUTHOR_EMAIL, GIT_COMMITTER_DATE: IDENTITY.GIT_AUTHOR_DATE }; -// Repository-locating variables from the caller's environment would point git elsewhere. -const GIT_LOCATION = ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_COMMON_DIR", "GIT_NAMESPACE"]; -// No lazy fetches from a partial clone's promisor remote: vm_sync never contacts a remote. +// No GIT_* variable from the caller's environment reaches git: they relocate the repository, inject config +// (GIT_CONFIG_COUNT/PARAMETERS, so url rewrites), or change which transports are allowed. No lazy fetches +// from a partial clone's promisor remote either: vm_sync never contacts a remote. +/** @returns {NodeJS.ProcessEnv} */ const baseEnv = () => { - /** @type {NodeJS.ProcessEnv} */ - const env = { ...process.env, LC_ALL: "C", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0", GIT_NO_LAZY_FETCH: "1" }; - for (const k of GIT_LOCATION) delete env[k]; - return env; + const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith("GIT_"))); + return { ...env, LC_ALL: "C", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0", GIT_NO_LAZY_FETCH: "1" }; }; -// A confined server (HTTP MCP with --copy-root) reads a checkout an agent may have written: git must not -// run anything that checkout's config names (fsmonitor, hooks, filter drivers) or look outside the root. -const SAFE_CONFIG = ["-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "protocol.allow=never"]; -const aborted = () => Object.assign(new Error("sync aborted; the VM was not changed"), { name: "AbortError" }); +// The push and its scratch repository read no user or system config: a url.insteadOf/pushInsteadOf would +// send the snapshot somewhere else (another VM), push.* options change the push. (The snapshot itself keeps +// the user's config: core.excludesFile decides what's untracked.) +const isolatedEnv = () => ({ ...baseEnv(), GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }); +// git must not run anything a checkout's config names (fsmonitor, hooks; filter drivers are refused when +// confined) or write a split index into the checkout's .git. +const SAFE_CONFIG = ["-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "core.splitIndex=false", "-c", "protocol.allow=never"]; +const aborted = () => Object.assign(new Error("sync aborted; the VM checkout was not changed"), { name: "AbortError" }); /** Run git locally → stdout. @param {string[]} args @param {{ cwd?: string, env?: NodeJS.ProcessEnv, input?: string }} [opts] @returns {Promise} */ function git(args, { cwd, env = baseEnv(), input } = {}) { @@ -148,6 +151,8 @@ async function snapshot(from, scratch, confined) { if (!head) throw new Error(`${root} has no commits yet; commit once before syncing (vm_sync needs a HEAD to check out)`); if (await tryGit(["rev-parse", "--is-shallow-repository"], { cwd: root, ...g0 }) === "true") throw new Error(`${root} is a shallow clone; vm_sync sends history the VM lacks and can't from a shallow one. Run git fetch --unshallow first`); + if (await tryGit(["config", "--get-regexp", "^(extensions\\.partialclone|remote\\..*\\.promisor)$"], { cwd: root, ...g0 })) + throw new Error(`${root} is a partial clone (objects are missing until fetched from its remote); vm_sync never contacts a remote and can't send objects it doesn't have`); const format = (await git(["rev-parse", "--show-object-format"], { cwd: root, ...g0 })).trim(); const objects = (await git(["rev-parse", "--path-format=absolute", "--git-path", "objects"], { cwd: root, ...g0 })).trim(); const index = (await git(["rev-parse", "--path-format=absolute", "--git-path", "index"], { cwd: root, ...g0 })).trim(); @@ -155,7 +160,13 @@ async function snapshot(from, scratch, confined) { await mkdir(scratchObjects); const scratchIndex = path.join(scratch, "index"); let haveIndex = true; - try { await writeFile(scratchIndex, readFileSync(index)); } catch (e) { if (/** @type {any} */ (e).code !== "ENOENT") throw e; haveIndex = false; } + try { + // Keep the index's mtime: git re-checks entries modified in the same second as the index was + // written ("racy" entries) by comparing with it. A newer copy would make such an edit look clean. + await copyFile(index, scratchIndex); + const st = await stat(index); + await utimes(scratchIndex, st.atime, st.mtime); + } catch (e) { if (/** @type {any} */ (e).code !== "ENOENT") throw e; haveIndex = false; } const env = { ...env0, GIT_INDEX_FILE: scratchIndex, GIT_OBJECT_DIRECTORY: scratchObjects, GIT_ALTERNATE_OBJECT_DIRECTORIES: objects }; if (!haveIndex) await git(["read-tree", "HEAD"], { cwd: root, env }); await git(["add", "-A"], { cwd: root, env }); @@ -203,23 +214,26 @@ function refuseCredentials(root, files, deny) { * A bare scratch repository to push from: its alternates are the scratch objects and the checkout's, so * refs/heads/sync can point at the snapshot without touching the checkout's refs. Refuses a credential * anywhere in the commits the VM may lack (those not reachable from its ref tips), not just the snapshot. - * @param {Awaited>} s @param {Probe} probe @param {string} scratch + * @param {Awaited>} s @param {Probe} probe @param {string} scratch @param {string[]} [deny] */ -async function pushSource(s, probe, scratch) { +async function pushSource(s, probe, scratch, deny) { const bare = path.join(scratch, "push.git"); - const env = baseEnv(); + const env = isolatedEnv(); await git(["init", "-q", "--bare", `--object-format=${s.format}`, bare], { env }); await writeFile(path.join(bare, "objects", "info", "alternates"), `${s.scratchObjects}\n${s.objects}\n`); await git(["--git-dir", bare, "update-ref", "refs/heads/sync", s.snap], { env }); - // Tips this repository doesn't have are skipped: then nothing they reach is excluded, which only errs - // towards refusing. + // What the push will send: every object reachable from the snapshot and not from what the VM has (by + // path, so merges and roots count too). Tips this repository doesn't have are skipped: then nothing they + // reach is excluded, which only errs towards refusing. The tips are the VM's own report (probe), so this + // trusts the VM to say what it has; see docs/sync.md. const tips = [...new Set([probe.synced, probe.base, probe.head, ...probe.tip].filter((t) => t && /^[0-9a-f]{40,64}$/.test(t)))]; - const history = (await git(["--git-dir", bare, "log", "--format=", "--name-only", "--no-renames", "-z", "--ignore-missing", "--stdin"], { env, input: `refs/heads/sync\n--not\n${tips.join("\n")}\n` })) - .split("\0").map((p) => p.trim()).filter(Boolean); - const leaked = [...new Set(history.filter(deniedSyncPath))]; + const listed = await git(["--git-dir", bare, "rev-list", "--objects", "--ignore-missing", "--stdin"], { env, input: `refs/heads/sync\n--not\n${tips.join("\n")}\n` }); + const history = [...new Set(listed.split("\n").map((l) => l.slice(l.indexOf(" ") + 1)).filter((p) => p && !/^[0-9a-f]{40,64}$/.test(p)))]; + const leaked = history.filter(deniedSyncPath); if (leaked.length) throw new Error(`vm_sync refused: ${leaked.slice(0, 5).join(", ")} is in the history this sync would send (a commit the VM lacks has it), and vm_sync never puts credential files on a VM's disk. ` + "It's not in your working tree, but its old contents are in that commit. Remove it from history (git rebase / git filter-repo) before syncing, and ask for the credential with vm_grant. Nothing in the VM was changed."); + refuseCredentials(s.root, history, deny); return bare; } @@ -234,17 +248,28 @@ async function pushSource(s, probe, scratch) { function push(bare, vm, sshConf, dest, ref, signal) { return new Promise((resolve, reject) => { if (signal?.aborted) return reject(aborted()); - const env = { ...baseEnv(), GIT_SSH_COMMAND: `ssh -F ${q(sshConf)} -o BatchMode=yes`, GIT_SSH_VARIANT: "ssh" }; + // ControlMaster=no, as vm_copy: reuse a shared connection, but never become the master inside the + // process group a cancel kills (other boxery calls would lose it). + const env = { ...isolatedEnv(), GIT_SSH_COMMAND: `ssh -F ${q(sshConf)} -o BatchMode=yes -o ControlMaster=no`, GIT_SSH_VARIANT: "ssh" }; const args = [...SAFE_CONFIG, "-c", "protocol.ssh.allow=always", "--git-dir", bare, "push", "--progress", "--no-verify", "--no-signed", "--no-recurse-submodules", "--no-follow-tags", - "--receive-pack=git -c core.hooksPath=/dev/null receive-pack", `${vm}.boxery:${dest}`, `+refs/heads/sync:${ref}`]; + "--receive-pack=git -c core.hooksPath=/dev/null -c receive.autogc=false receive-pack", `${vm}.boxery:${dest}`, `+refs/heads/sync:${ref}`]; // Its own process group, so cancelling stops git, its ssh and pack-objects together. const child = spawn("git", args, { env, detached: true, stdio: ["ignore", "ignore", "pipe"] }); let stderr = ""; child.stderr.on("data", (d) => { stderr = (stderr + d).slice(-64 * 1024); }); - const stop = () => { try { process.kill(-(/** @type {number} */ (child.pid)), "SIGTERM"); } catch {} }; + /** @type {NodeJS.Timeout[]} */ + const timers = []; + const group = (/** @type {NodeJS.Signals} */ sig) => { try { process.kill(-(/** @type {number} */ (child.pid)), sig); } catch {} }; + // TERM, then KILL; and don't wait forever for stderr a descendant (a ProxyCommand) may still hold. + const stop = () => { + group("SIGTERM"); + timers.push(setTimeout(() => group("SIGKILL"), 1000), setTimeout(() => child.stderr.destroy(), 3000)); + }; signal?.addEventListener("abort", stop, { once: true }); child.on("error", (e) => { signal?.removeEventListener("abort", stop); reject(new Error(`git push: ${e.message}`)); }); child.on("close", (code) => { + for (const t of timers) clearTimeout(t); + if (signal?.aborted) group("SIGKILL"); signal?.removeEventListener("abort", stop); if (signal?.aborted) return reject(aborted()); if (code !== 0) { @@ -277,7 +302,9 @@ export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd() if (signal?.aborted) throw aborted(); const id = randomBytes(8).toString("hex"); const scratch = await mkdtemp(path.join(tmpdir(), "boxery-sync-")); - const ref = `refs/boxery/incoming/${id}`; + // The time in the name lets a later sync tell an abandoned ref from a concurrent one (snapshot commits + // carry a fixed date, so the commit can't say). + const ref = `refs/boxery/incoming/${Math.floor(Date.now() / 1000)}-${id}`; try { const s = await snapshot(fromPath, scratch, confined); refuseCredentials(s.root, s.files, local ? local.deny : undefined); @@ -295,16 +322,23 @@ export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd() const pushed = probe.has !== "1"; let objects = 0; if (pushed) { - const bare = await pushSource(s, probe, scratch); + const bare = await pushSource(s, probe, scratch, local ? local.deny : undefined); if (signal?.aborted) throw aborted(); - // receive-pack needs a repository: create one in a missing or empty directory (only that). - if (probe.state !== "repo") await guest(vm, { MODE: "init", DIR: dest, FORMAT: s.format }); - objects = (await push(bare, vm, host.sshConf, dest, ref, signal)) ?? 0; + // receive-pack needs a repository: create one in a missing or empty directory (only that), and + // remove it again if the push doesn't land. + const created = probe.state !== "repo"; + if (created) await guest(vm, { MODE: "init", DIR: dest, FORMAT: s.format }); + try { + objects = (await push(bare, vm, host.sshConf, dest, ref, signal)) ?? 0; + } catch (e) { + if (created) await guest(vm, { MODE: "uninit", DIR: dest }).catch(() => {}); + throw e; + } } // The last point to stop: past here the VM applies the snapshot completely or not at all. if (signal?.aborted) throw aborted(); const applied = await guest(vm, { - MODE: "apply", DIR: dest, SNAP: s.snap, HEAD_SHA: s.head, TREE: s.tree, + MODE: "apply", DIR: dest, SNAP: s.snap, HEAD_SHA: s.head, TREE: s.tree, REF: pushed ? ref : "", EXPECT_SYNCED: probe.synced ?? "", EXPECT_HEAD: probe.head ?? "", }); const match = applied.tree === s.tree; diff --git a/docs/sync.md b/docs/sync.md index 8077a2ee..4908add9 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -20,7 +20,7 @@ boxery sync demo --json # the result as JSON ## What it does 1. **Snapshot.** It records local HEAD plus every working change (what `git add -A` would stage: tracked files, and untracked files `.gitignore` doesn't exclude, deletions and renames included) as one commit. It uses a scratch index and object directory: your index, refs and stash are untouched. The commit has a fixed author and date, so the same working state always gives the same commit. -2. **Push.** The snapshot is `git push`ed straight into the VM repository, to a hidden ref (`refs/boxery/incoming/…`), over the VM's own ssh connection (the host profile's `ssh_config`, pinned host key). git works out which commits the VM lacks from all of its refs and streams only those into `git receive-pack`: no archive, no temporary file. No hooks run on either side. Nothing contacts a git remote, so the VM needs no GitHub grant. If the VM already has the snapshot, nothing is pushed. If `to` doesn't exist or is empty, a repository is initialized there first (after the checks pass). +2. **Push.** The snapshot is `git push`ed straight into the VM repository, to a hidden ref (`refs/boxery/incoming/…`), over the VM's own ssh connection (the host profile's `ssh_config`, pinned host key). git works out which commits the VM lacks from all of its refs and streams only those into `git receive-pack`: no archive, no temporary file. The push reads none of your global or system git config and no `GIT_*` environment variables, so a `url.insteadOf` can't redirect it. No hooks run on either side. Nothing contacts a git remote, so the VM needs no GitHub grant. If the VM already has the snapshot, nothing is pushed. If `to` doesn't exist or is empty, a repository is initialized there first (after the checks pass), and removed again if the push fails. 3. **Swap.** In the VM, HEAD is detached at local HEAD's commit, and the working directory is switched to the snapshot: only the paths that differ are rewritten, and files the snapshot lacks are deleted, including ones an earlier sync put there. Ignored files (`node_modules`, build output) stay: there is no `git clean -x`. If the snapshot would put a tracked file where the VM has an ignored one, the sync refuses instead (below). The VM's index holds HEAD, so `git status` and `git diff` in the VM show the same working changes as they do locally. 4. **Check.** Both sides hash the whole working directory (tracked plus untracked, not ignored) as a git tree, the same way, and compare. The result says `tree matches`, or `TREE MISMATCH` (an error). @@ -40,12 +40,12 @@ It refuses, and changes nothing in the VM, when: - **The VM checkout has changes the last sync didn't make.** The VM records each sync in `refs/boxery/synced` (and local HEAD in `refs/boxery/base`). Any difference between the VM's working directory and that snapshot (edited, new or deleted files, not ignored ones) is listed in the error. Commit them in the VM and bring them back yourself, or discard them, then sync again. A VM repository that was never synced is compared with its HEAD. - **The checkout holds a credential file.** Matched case-insensitively: `.env` and `.env.*`/`.env-*`/`.env_*` (except `.env.example`, `.env.sample`, `.env.template`, `.env.dist`, `.env.defaults`), `.envrc`, `.npmrc`, `.yarnrc.yml`, `.pypirc`, `.netrc`, `.git-credentials`, `.pgpass`, `.my.cnf`, `.terraformrc`, `auth.json`, `credentials`, `application_default_credentials.json`, private keys (`id_*` except `.pub`, `*.pem`, `*.key`, `*.p12`, `*.pfx`, `*.jks`, `*.keystore`), `.docker/config.json`, `.kube/config`, `.aws/config`, `gh/hosts.yml`, and anything inside boxery's own state or `~/.ssh`, `~/.gnupg`, `~/.pi` (what `vm_copy` refuses). The error names the path. Ignore it in `.gitignore` to sync the rest, and get the credential to the VM with a [grant](grants.md), which keeps it off the VM's disk and out of its snapshots. Ignored files are never synced, so an ignored `.env` is fine. -- **The commits it would send contain one.** The push carries every commit the VM lacks (on a first sync, the whole history), so a credential file committed once and deleted later is refused too. Remove it from history first. +- **The commits it would send contain one.** The push carries every commit the VM lacks (on a first sync, the whole history), so a credential file committed once and deleted later (merges included) is refused too. Remove it from history first. Which commits the VM lacks is worked out from the refs the VM reports, so this check trusts the VM's report. - **The local checkout now tracks a path the VM has as an ignored file** (a force-added build file, or `dist/` un-ignored and replaced by a file): the swap would overwrite or delete the VM's file. Or, with the new `.gitignore`, **files the VM has stop being ignored**, so the trees can't match. The error lists the paths; remove them in the VM (or change the local checkout), and sync again. - **`to` is a non-empty directory that isn't a git repository**, is inside another checkout (`~/project/api` when `~/project` is a repository: use `~/api`), or its object format (sha1/sha256) differs. -- **The local path isn't a git checkout**, has no commits yet, or is a shallow clone. Use `vm_copy` for plain files. +- **The local path isn't a git checkout**, has no commits yet, or is a shallow or partial clone. Use `vm_copy` for plain files. -"Nothing changed" means the checkout: a refused first sync into an empty or missing directory may leave it as an empty repository. +"Nothing changed" means the checkout. A first sync into a missing directory can leave its parent directories behind. ## Cancellation and failure @@ -61,6 +61,7 @@ A sync takes a lock in the VM repository (`flock`, or a lock directory without i - Submodules are synced as gitlinks (their commit ids), not their contents. - Git LFS files arrive as pointer files: the VM doesn't download them, since vm_sync never contacts a remote. - Untracked files are hashed into a temporary directory on the local machine, so a very large one needs that much space there. +- Ignore rules that exist only on your machine (`core.excludesFile`, `.git/info/exclude`) aren't sent: files the VM produces that only those rules ignore show up as VM changes, and the sync refuses. Put such patterns in `.gitignore`. - Executable bits and symlinks follow git's rules; other file modes, owners and timestamps aren't carried. - The VM keeps every synced snapshot's objects until `git gc`. - On an HTTP MCP server, `vm_sync` reads checkouts on that server, so it follows `vm_copy`'s `local:` rule: refused without `--copy-root`, and confined to `DIR` with it ([MCP server](mcp.md#b-streamable-http-on-the-host-over-tailscale)). Confined, the repository's git directory, object store and alternates must be inside `DIR` too, git doesn't look for a repository above `DIR`, and it runs none of the checkout's hooks, fsmonitor or filter drivers. diff --git a/test/fixtures/copy-ssh.mjs b/test/fixtures/copy-ssh.mjs index 550943f1..ecf70e37 100644 --- a/test/fixtures/copy-ssh.mjs +++ b/test/fixtures/copy-ssh.mjs @@ -6,6 +6,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; const HELPER = fileURLToPath(new URL("./copy-helper.mjs", import.meta.url)); const args = process.argv.slice(2); +const argv = [...args]; let conf, vm; while (args.length > 1) { const flag = /** @type {string} */ (args.shift()); @@ -18,7 +19,7 @@ if (!["alpha", "beta", "gamma"].includes(host) || !["vm", "other"].includes(/** const endpoint = `${host}/${vm}`; const root = path.join(/** @type {string} */ (process.env.COPY_ROOT), "vms", host, /** @type {string} */ (vm)); const script = args[0].replaceAll("/home/agent", root).replaceAll("/tmp/.boxery-copy-", `${root}/tmp/.boxery-copy-`); -appendFileSync(/** @type {string} */ (process.env.COPY_LOG), JSON.stringify({ kind: "ssh", endpoint, script, pid: process.pid }) + "\n"); +appendFileSync(/** @type {string} */ (process.env.COPY_LOG), JSON.stringify({ kind: "ssh", endpoint, script, pid: process.pid, argv }) + "\n"); const cleanup = script.includes('for record in "$t"/pgid.') && !script.includes("setsid -w"); if (process.env.COPY_FAIL_CLEANUP === endpoint && cleanup) { process.stderr.write("injected cleanup connection failure\n"); process.exit(255); diff --git a/test/sync.test.mjs b/test/sync.test.mjs index 6d76856f..16d77356 100644 --- a/test/sync.test.mjs +++ b/test/sync.test.mjs @@ -2,7 +2,7 @@ // "VM" is a directory, and its ssh runs scripts there with /home/agent mapped to it. No real VMs. import assert from "node:assert/strict"; import { execFileSync, spawn } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; import { randomBytes } from "node:crypto"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -450,3 +450,95 @@ test("confined: a filter driver in the checkout's config is refused", async () = assert.equal(r.ok, false); assert.match(r.error, /filter driver/); }); + +test("an edit in the same second as the index was written, same size, is still seen on both sides", async () => { + // git trusts an entry's stat data unless the file changed in the same second as the index was written; + // a copy of the index with a newer mtime would hide such an edit. Pin both to one second to force it. + const T = new Date("2026-01-01T00:00:00Z"); + const racy = (/** @type {string} */ root, /** @type {string} */ rel, /** @type {string} */ text) => { + git(root, "update-index", "-q", "--refresh"); + utimesSync(path.join(root, rel), T, T); + git(root, "update-index", "-q", "--really-refresh"); + utimesSync(path.join(root, ".git", "index"), T, T); + writeFileSync(path.join(root, rel), text); + utimesSync(path.join(root, rel), T, T); + }; + await ok(); + racy(vmProject(), "README.md", "HELLO\n"); // an agent edit in the VM + const blocked = /** @type {any} */ (await sync()); + assert.equal(blocked.ok, false, "the VM edit was overlooked"); + assert.match(blocked.error, /M README\.md/); + writeFileSync(path.join(vmProject(), "README.md"), "hello\n"); + racy(repo, "README.md", "jello\n"); // a local edit + const r = await ok(); + assert.equal(r.structured.counts.modified, 1, "the local edit was overlooked"); + assert.equal(read(vmProject(), "README.md"), "jello\n"); +}); + +test("a credential added and removed in merge commits blocks the sync", async () => { + git(repo, "checkout", "-q", "-b", "side"); + write(repo, "side.txt", "x\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "side"); + git(repo, "checkout", "-q", "main"); + git(repo, "merge", "-q", "--no-ff", "--no-commit", "side"); + write(repo, ".env", "SECRET=1\n"); + git(repo, "add", "-f", ".env"); + git(repo, "commit", "-qm", "merge with a secret"); + git(repo, "rm", "-q", ".env"); + git(repo, "commit", "-qm", "drop it"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /\.env is in the history this sync would send/); +}); + +test("the push ignores the user's git config and GIT_* environment, and goes only to the VM", async () => { + await ok(); + const hostile = path.join(dir, "hostile.gitconfig"); + writeFileSync(hostile, '[url "other.boxery:"]\n\tpushInsteadOf = vm.boxery:\n\tinsteadOf = vm.boxery:\n[push]\n\tpushOption = x\n\tnegotiate = true\n'); + Object.assign(env, { GIT_CONFIG_GLOBAL: hostile, GIT_ALLOW_PROTOCOL: "https", GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "url.other.boxery:.pushInsteadOf", GIT_CONFIG_VALUE_0: "vm.boxery:" }); + write(repo, "README.md", "changed\n"); + const mark = readFileSync(path.join(dir, "calls"), "utf8").length; + let r; + try { r = await ok(); } finally { + for (const k of ["GIT_CONFIG_GLOBAL", "GIT_ALLOW_PROTOCOL", "GIT_CONFIG_COUNT", "GIT_CONFIG_KEY_0", "GIT_CONFIG_VALUE_0"]) delete env[k]; + } + assert.equal(r.structured.tree.match, true); + assert.ok(!existsSync(path.join(dir, "vms", "alpha", "other", "project")), "nothing went to another VM"); + const pushes = readFileSync(path.join(dir, "calls"), "utf8").slice(mark).split("\n").filter(Boolean).map((l) => JSON.parse(l)).filter((c) => String(c.script).includes("receive-pack")); + assert.equal(pushes.length, 1); + assert.equal(pushes[0].endpoint, "alpha/vm"); + assert.ok(pushes[0].argv.includes("ControlMaster=no"), "never becomes the shared connection's master"); + assert.ok(pushes[0].argv.includes("BatchMode=yes")); + assert.match(pushes[0].script, /core\.hooksPath=\/dev\/null .*receive\.autogc=false receive-pack/); +}); + +test("a push that fails into a new directory leaves no repository behind", async () => { + writeFileSync(path.join(dir, "bin", "git"), `#!/usr/bin/env bash\nfor a in "$@"; do [ "$a" != receive-pack ] || { echo "injected receive-pack failure" >&2; exit 1; }; done\nexec ${q(env.REAL_GIT ?? "git")} "$@"\n`, { mode: 0o755 }); + env.FAKE_VM_PATH = env.PATH; + let r; + try { r = /** @type {any} */ (await sync({ to: "~/fresh/repo" })); } finally { unlinkSync(path.join(dir, "bin", "git")); delete env.FAKE_VM_PATH; } + assert.equal(r.ok, false); + assert.match(r.error, /git push to VM "vm" failed/); + assert.ok(!existsSync(path.join(dir, "vms", "alpha", "vm", "fresh", "repo")), "init undone"); +}); + +test("a stale incoming ref from an abandoned sync is dropped; a recent one is kept", async () => { + await ok(); + const head = git(vmProject(), "rev-parse", "HEAD"); + git(vmProject(), "update-ref", "refs/boxery/incoming/1000-old", head); + git(vmProject(), "update-ref", `refs/boxery/incoming/${Math.floor(Date.now() / 1000)}-busy`, head); + write(repo, "README.md", "again\n"); + await ok(); + const refs = git(vmProject(), "for-each-ref", "--format=%(refname)", "refs/boxery/incoming/"); + assert.doesNotMatch(refs, /1000-old/); + assert.match(refs, /-busy/); +}); + +test("a partial clone gives a clear error", async () => { + git(repo, "config", "extensions.partialClone", "origin"); + git(repo, "config", "core.repositoryformatversion", "1"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /partial clone/); +}); From 2a7bf48b9e80101f4690d79ff457cc16293d8050 Mon Sep 17 00:00:00 2001 From: Ben Vinegar Date: Sun, 8 Nov 2026 21:04:19 -0500 Subject: [PATCH 5/5] fix(sync): isolate CI fixtures and declare terminal status --- controller/cli.mjs | 1 + test/sync.test.mjs | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/controller/cli.mjs b/controller/cli.mjs index aaeaaf92..8098b25d 100644 --- a/controller/cli.mjs +++ b/controller/cli.mjs @@ -551,6 +551,7 @@ export const COMMANDS = { examples: [["This checkout into demo's ~/project", "boxery sync demo"], ["Another checkout, into ~/api (not inside ~/project, which is a checkout itself)", "boxery sync demo --from ../api --to ~/api"]], seeAlso: ["cp"], docs: "sync.md", }, + status: { vm: true }, run: cmdSync, }, desktop: { diff --git a/test/sync.test.mjs b/test/sync.test.mjs index 16d77356..850b38e3 100644 --- a/test/sync.test.mjs +++ b/test/sync.test.mjs @@ -2,7 +2,7 @@ // "VM" is a directory, and its ssh runs scripts there with /home/agent mapped to it. No real VMs. import assert from "node:assert/strict"; import { execFileSync, spawn } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; import { randomBytes } from "node:crypto"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -54,7 +54,8 @@ function tree(root) { } beforeEach(() => { - dir = mkdtempSync(path.join(tmpdir(), "boxery-sync-")); + // Match the canonical --copy-root used by servers (macOS's temp directory is under /var → /private/var). + dir = realpathSync(mkdtempSync(path.join(tmpdir(), "boxery-sync-"))); mkdirSync(path.join(dir, "bin")); const home = path.join(dir, "home"); mkdirSync(path.join(home, "host-registry"), { recursive: true }); @@ -68,9 +69,11 @@ beforeEach(() => { writeFileSync(path.join(dir, "calls"), ""); writeFileSync(path.join(dir, "bin", "ssh"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "copy-ssh.mjs"))} "$@"\n`, { mode: 0o755 }); writeFileSync(path.join(dir, "bin", "tar"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "copy-tar.mjs"))} "$@"\n`, { mode: 0o755 }); + // sync deliberately strips caller GIT_* overrides; isolate system config at the executable boundary. + writeFileSync(path.join(dir, "bin", "git"), `#!/usr/bin/env bash\nexport GIT_CONFIG_NOSYSTEM=1\nexec ${q(execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim())} "$@"\n`, { mode: 0o755 }); writeFileSync(path.join(dir, "bin", "setsid"), "#!/usr/bin/env bash\n[ \"$1\" != -w ] || shift\nexec perl -e 'setpgrp(0,0); exec @ARGV or die' -- \"$@\"\n", { mode: 0o755 }); env = { - ...process.env, ...GIT_ENV, HOME: home, PATH: `${path.join(dir, "bin")}:${process.env.PATH}`, BOXERY_HOME: home, + ...process.env, ...GIT_ENV, HOME: home, XDG_CONFIG_HOME: path.join(home, ".config"), PATH: `${path.join(dir, "bin")}:${process.env.PATH}`, BOXERY_HOME: home, COPY_ROOT: dir, COPY_LOG: path.join(dir, "calls"), REAL_TAR: execFileSync("sh", ["-c", "command -v tar"], { encoding: "utf8" }).trim(), REAL_GIT: execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim(), };