diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2cb25d6a..61837d9e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -61,7 +61,7 @@ VM (Ubuntu 24.04; XFCE desktop unless --headless) | Component | Runs on | Role | |---|---|---| | `core/boxery-tools.mjs` | agent machine (or host, for HTTP MCP) | **The `vm_*` tool table**: name, description, JSON Schema and runner for every tool, plus `vm_manage`/`vm_info`/`vm_lease` action logic (hosts included) over front-end session hooks, and `runTool()`, which resolves each call's host and VM. MCP and pi are both built from it and run tools through it, so they can't drift (`test/frontends.test.mjs`: a tool only one front-end has must say why). `test/parity.test.mjs` maps every tool and action to its `boxery` command (or why it has none) and every command back, so the CLI can't silently fall behind either | -| `core/boxery-core.mjs`, `core/copy-stream.mjs`, `core/atomic-edit.mjs`, `core/shell.mjs` | same | Transport and tool implementation: `sshRun`/`execRemote` (remote process-group kill on cancel), path mapping, read/write/edit (atomic, detects concurrent changes), streaming tar `vm_copy` (local⇄VM⇄VM, across hosts), computer use via xdotool, Playwright over CDP, `LeaseKeeper`, host-profile resolution with `AsyncLocalStorage` | +| `core/boxery-core.mjs`, `core/copy-stream.mjs`, `core/sync.mjs` + `core/sync-guest.sh`, `core/atomic-edit.mjs`, `core/shell.mjs` | same | Transport and tool implementation: `sshRun`/`execRemote` (remote process-group kill on cancel), path mapping, read/write/edit (atomic, detects concurrent changes), streaming tar `vm_copy` (local⇄VM⇄VM, across hosts), `vm_sync` (a local git checkout → VM: `git push` over the VM's ssh connection, then a staged swap in the VM), computer use via xdotool, Playwright over CDP, `LeaseKeeper`, host-profile resolution with `AsyncLocalStorage` | | `core/grants.mjs`, `core/grant-operations.mjs`, `core/grant-approval.mjs`, `core/grants-guest.sh` | agent machine | Shared spec/deny-directory validation and prompt text; grant store locking/edits, edge-route application and tmpfs reconciliation. Agent front-ends approve through elicitation/pi dialogs, then call operations in-process, without spawning the CLI. The guest script is a single reconciliation template with values inserted at apply time | | `core/boxery-jobs.ts`, `core/boxery-job-runner.sh` | piped into the VM per call | Background jobs (`run --bg`, `jobs`, `logs`, `wait`, `kill`; `vm_jobs`). The helper is erasable TypeScript run by the VM's Node (type stripping, ≥ 22.18), never installed, so a VM always runs the caller's version; it answers in JSON. The runner is plain bash, sent with each start: it runs the command in its own process group, independent of tmux, and records the real exit code. | | `backends/index.mjs` | agent machine and host | What each backend declares: supported and tested boxery-host platforms, runtimes, prerequisites and capabilities, as data (no probes). `host/capabilities.mjs` checks against it, `boxery-host backends`/`capabilities` print it, and `vm_info` lists it; it imports nothing. A backend without a declaration (one added as a `.sh` file) runs unchecked ([backends](docs/backends.md)) | diff --git a/controller/cli.mjs b/controller/cli.mjs index b8dd3f52..8098b25d 100644 --- a/controller/cli.mjs +++ b/controller/cli.mjs @@ -262,6 +262,23 @@ async function cmdCp(s, args) { await copy(s.host, args); } +/** sync NAME [--from DIR] [--to PATH] [--json]: vm_sync's own code. Needs only the profile's ssh_config. @param {Selected} s @param {string[]} args @returns {Promise} */ +async function cmdSync(s, args) { + let name = "", from, to, json = false; + for (let i = 0; i < args.length; i++) { + const a = args[i], v = args[i + 1]; + if (a === "--from") { if (v == null) die("--from needs a directory"); from = v; i++; } + else if (a === "--to") { if (v == null) die("--to needs a path"); to = v; i++; } + else if (a === "--json") json = true; + else if (a.startsWith("-")) die(`unknown flag ${a}`); + else if (name) usage("sync"); + else name = a; + } + needVm(name); + const { sync } = await import("./sync.mjs"); + await sync(s.host, name, { from, to, json }); +} + /** version [--json] @param {Selected | null} s @param {string[]} args */ async function cmdVersion(s, args) { const json = args[0] === "--json"; @@ -523,6 +540,20 @@ export const COMMANDS = { status: {}, run: cmdCp, }, + sync: { + help: { + group: "use", summary: "push a local git checkout into a VM, so its tree matches exactly", usage: ["sync NAME [--from DIR] [--to PATH] [--json]"], + details: [ + "git pushes the commits the VM lacks over its ssh connection, checks out the same commit (detached) and applies staged, unstaged and untracked changes, deletions included. Ignored files in the VM (node_modules, builds) stay. No git remote or GitHub access is needed on either side.", + "Refuses, changing nothing, when the VM checkout has changes the last sync didn't make, or the checkout holds a credential file (.env, .npmrc, auth files).", + ], + flags: [["--from DIR", "Local checkout (default: the current directory)"], ["--to PATH", "VM directory (default ~/project; relative paths are under it)"], ["--json", "Print the result as JSON"]], + examples: [["This checkout into demo's ~/project", "boxery sync demo"], ["Another checkout, into ~/api (not inside ~/project, which is a checkout itself)", "boxery sync demo --from ../api --to ~/api"]], + seeAlso: ["cp"], docs: "sync.md", + }, + status: { vm: true }, + run: cmdSync, + }, desktop: { help: { group: "use", summary: "open a VM's desktop in your browser (noVNC over ssh)", usage: ["desktop NAME [--no-open]"], flags: [["--no-open", "Print the URL instead of opening it"]], docs: "cli.md#desktop-and-ports" }, run: async (s, args) => { needHost(s); await cmdDesktop(s, args); }, diff --git a/controller/sync.mjs b/controller/sync.mjs new file mode 100644 index 00000000..522a57b1 --- /dev/null +++ b/controller/sync.mjs @@ -0,0 +1,31 @@ +// boxery sync NAME: the CLI front-end of vm_sync. It runs the same TOOLS entry as the MCP server and the +// pi extension, so the transfer, refusals and result cannot drift (docs/sync.md). +import { constants } from "node:os"; +import * as core from "../core/boxery-core.mjs"; +import { TOOLS } from "../core/boxery-tools.mjs"; +import { die, quit, say } from "./lib.mjs"; +/** @import { Tool } from "../core/boxery-tools.mjs" */ +/** @import { HostProfile } from "../core/boxery-core.mjs" */ + +/** @param {HostProfile} host @param {string} vm @param {{ from?: string, to?: string, json?: boolean }} opts */ +export async function sync(host, vm, { from, to, json = false }) { + // Ctrl-C cancels like an MCP cancellation: before the VM applies anything, nothing changes there. + const abort = new AbortController(); + let interrupted = 0; + // Once the VM has started applying, the sync finishes (or rolls back) there regardless; wait for its result. + const handlers = /** @type {const} */ (["SIGINT", "SIGTERM", "SIGHUP"]).map((sig) => /** @type {[NodeJS.Signals, () => void]} */ ([sig, () => { + if (!interrupted) process.stderr.write("boxery: stopping; if the VM is already applying the sync, it finishes or rolls back first\n"); + interrupted = 128 + constants.signals[sig]; abort.abort(); + }])); + for (const [sig, h] of handlers) process.on(sig, h); + const tool = /** @type {Tool} */ (TOOLS.find((t) => t.name === "vm_sync")); + try { + const r = await core.withHost(host, () => tool.run(vm, { from, to }, /** @type {any} */ ({ localCwd: process.cwd(), signal: abort.signal }) /* no session: vm_sync needs none */)); + say(json ? JSON.stringify(r.structured) : r.text); + if (r.isError) quit(1); + } catch (/** @type {any} */ e) { + if (e?.exitCode !== undefined) throw e; + if (interrupted) { process.stderr.write(`boxery: ${e.message}\n`); quit(interrupted); } + die(e.message); + } finally { for (const [sig, h] of handlers) process.off(sig, h); } +} diff --git a/core/boxery-core.mjs b/core/boxery-core.mjs index c8cf30f8..fd3a1db3 100644 --- a/core/boxery-core.mjs +++ b/core/boxery-core.mjs @@ -29,7 +29,8 @@ export { q } from "./shell.mjs"; /** @typedef {{ name: string, dir: string, default: boolean, destination: string | null, backend: string | null, same_as: string[] }} HostDescription */ /** @typedef {{ host?: HostProfile, vm: string | null, path: string }} Loc */ /** env: ssh's environment (default: this process's); batch: false lets ssh ask (a terminal's passphrase). - * @typedef {{ input?: string | Buffer, signal?: AbortSignal, timeoutMs?: number, onData?: (d: Buffer) => void, maxBytes?: number, closePipes?: boolean, env?: NodeJS.ProcessEnv, batch?: boolean }} SshRunOpts */ + * detached: ssh in its own process group, so a terminal's Ctrl-C doesn't cut off a remote step that must finish. + * @typedef {{ input?: string | Buffer, signal?: AbortSignal, timeoutMs?: number, onData?: (d: Buffer) => void, maxBytes?: number, closePipes?: boolean, env?: NodeJS.ProcessEnv, batch?: boolean, detached?: boolean }} SshRunOpts */ /** @typedef {{ code: number | null, stdout: Buffer, stderr: string }} SshResult */ /** @typedef {{ onData?: (d: Buffer) => void, signal?: AbortSignal, timeout?: number }} ExecRemoteOpts */ /** @typedef {{ exitCode: number | null }} ExecResult */ @@ -352,7 +353,7 @@ export function sshRun(vm, script, opts = {}) { if (!existsSync(sshConf)) return reject(new Error(`boxery not set up (${sshConf} missing); run: boxery setup user@host`)); // stdin is a pipe whenever it is used (opts.input != null). const child = /** @type {ChildProcessWithoutNullStreams} */ (spawn("ssh", ["-F", sshConf, ...(opts.batch === false ? [] : ["-o", "BatchMode=yes"]), `${vm}.boxery`, `bash -c ${q(script)}`], { - stdio: [opts.input != null ? "pipe" : "ignore", "pipe", "pipe"], ...(opts.env ? { env: opts.env } : {}), + stdio: [opts.input != null ? "pipe" : "ignore", "pipe", "pipe"], ...(opts.env ? { env: opts.env } : {}), ...(opts.detached ? { detached: true } : {}), })); /** @type {Buffer[]} */ const out = []; @@ -890,7 +891,7 @@ export function copyRoot(dir, { deny = [] } = {}) { * unless it is inside the root and neither inside nor containing a denied path. * @param {LocalConfinement} confinement */ -function confineLocal({ root, deny = [] }) { +export function confineLocal({ root, deny = [] }) { const denied = copyDenied(deny); const check = (/** @type {string} */ real, shown = real) => { if (!isWithin(real, root)) throw new Error(`local:${shown} is outside the copy root ${root}`); diff --git a/core/boxery-tools.mjs b/core/boxery-tools.mjs index 36af6be6..8caaabdc 100644 --- a/core/boxery-tools.mjs +++ b/core/boxery-tools.mjs @@ -7,7 +7,7 @@ // Front-ends call runTool(), which resolves the call's host and VM against their Session and runs the // tool on that host. ctx: { signal, session, host, current, localCwd, localCopy?, progress?, approve? }. // progress(message) reports a status line while the tool runs (MCP progress notifications); front-ends -// without it pass nothing. localCopy limits vm_copy's local: endpoints (see copyTool); front-ends that run +// without it pass nothing. localCopy limits vm_copy's local: endpoints (see copyTool) and vm_sync's local checkout; front-ends that run // on the user's own machine pass nothing. // engine: pi runs this tool with its own built-in engine (same rendering/truncation as bash, read, …) // noVm: the front-end does not resolve a VM; the tool gets every argument, `vm` included @@ -16,6 +16,7 @@ // them only through the Session hooks. import * as core from "./boxery-core.mjs"; import { BACKENDS as BACKEND_DECLARATIONS } from "../backends/index.mjs"; +import { SYNC_OUTPUT, syncTool } from "./sync.mjs"; /** @import { Approve, HostProfile, LocalConfinement, ManageArgs, Progress, ToolResult } from "./boxery-core.mjs" */ /** A JSON Schema fragment. @typedef {{ type?: string | string[], description?: string, enum?: readonly string[], properties?: Record, items?: JsonSchema, required?: string[], anyOf?: JsonSchema[], minimum?: number, maximum?: number }} JsonSchema */ /** A tool's input schema: always an object with named properties. @typedef {JsonSchema & { type: "object", properties: Record, required: string[] }} ObjectSchema */ @@ -179,6 +180,21 @@ export const TOOLS = [ annotations: { destructiveHint: true, openWorldHint: false }, run: (_vm, a, x) => core.copyTool(a, { current: a.vm ?? x.current, localCwd: x.localCwd, local: x.localCopy, signal: x.signal }), }, + { + name: "vm_sync", + title: "VM sync", + description: + "Push a local git checkout into a VM so the VM's tree matches it exactly: same commit (detached HEAD), plus staged, unstaged and untracked changes, deletions included; the VM's ignored files (node_modules, builds) stay. " + + "Use it instead of vm_copy for a repository: it sends only what changed, needs no git remote or GitHub access, refuses VM-side edits it would overwrite, and reports whether lockfiles changed. " + + "It never syncs credentials: it refuses .env, .npmrc and other auth files (ask with vm_grant).", + parameters: obj({ + from: { type: "string", description: "Local checkout (default: this agent's working directory)" }, + to: { type: "string", description: `VM directory (default ${core.PROJECT}; relative paths are under it)` }, + }), + annotations: { destructiveHint: true, idempotentHint: true, openWorldHint: false }, + outputSchema: SYNC_OUTPUT, + run: (vm, a, x) => syncTool(vm, a, { localCwd: x.localCwd, local: x.localCopy, signal: x.signal }), + }, { name: "vm_screenshot", title: "VM screenshot", diff --git a/core/sync-guest.sh b/core/sync-guest.sh new file mode 100644 index 00000000..d4227544 --- /dev/null +++ b/core/sync-guest.sh @@ -0,0 +1,282 @@ +# shellcheck shell=bash +# vm_sync, the VM side. core/sync.mjs sends this text with every call (it is never installed), after +# assignments to MODE (probe | init | uninit | apply), DIR, for probe WANT, for init FORMAT, and for apply: +# SNAP HEAD_SHA TREE REF EXPECT_SYNCED EXPECT_HEAD. Commits arrive between probe and apply by git push (to +# refs/boxery/incoming/*), never through this script. It never contacts a git remote. Output: key=value lines on stdout. +# Exit 3: refused (the reason on stderr); the checkout is as it was. Exit 1: failed; apply has restored +# the checkout it found, or says it couldn't. +# +# State kept in the VM repo: refs/boxery/synced (the snapshot commit of the last sync: local HEAD plus +# the working changes, as one commit), refs/boxery/base (local HEAD then), and while a swap runs, +# $G/boxery-sync-pending ("OLD_TREE NEW_TREE") so a later sync can finish or undo an interrupted one. +set -u -o pipefail +: "${MODE:?}" "${DIR:?}" +export LC_ALL=C GIT_TERMINAL_PROMPT=0 GIT_OPTIONAL_LOCKS=0 GIT_NO_LAZY_FETCH=1 GIT_LFS_SKIP_SMUDGE=1 +# No hooks (update-ref runs reference-transaction) and no fsmonitor from the VM repository's config. +export GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0=/dev/null GIT_CONFIG_KEY_1=core.fsmonitor GIT_CONFIG_VALUE_1=false +LOCKFILES=(':(glob)**/pnpm-lock.yaml' ':(glob)**/package-lock.json' ':(glob)**/yarn.lock' ':(glob)**/bun.lock') +refuse() { printf '%s\n' "$*" >&2; exit 3; } +die() { printf '%s\n' "$*" >&2; exit 1; } +rev() { git rev-parse -q --verify "$1^{commit}" 2>/dev/null || true; } +is_repo_root() { local top; top=$(git rev-parse --show-toplevel 2>/dev/null) && [ "$top" = "$(pwd -P)" ]; } +# A directory inside another repository's working tree would show up there as a new nested repository. +inside_other_repo() { + local top + top=$(cd "$1" 2>/dev/null && git rev-parse --show-toplevel 2>/dev/null) || return 1 + [ -n "$top" ] && [ "$top" != "$2" ] && printf '%s' "$top" +} +setup_git() { + G=$(git rev-parse --absolute-git-dir) || die "git: cannot find the repository in $DIR" + OBJ=$(git rev-parse --path-format=absolute --git-path objects) || die "git is too old (needs 2.31+)" + IDX=$(git rev-parse --path-format=absolute --git-path index) + EMPTY=$(git hash-object -t tree /dev/null) +} +# The tree to compare the working directory with: the last sync's, else HEAD's, else nothing. +baseline() { + local s h; s=$(rev refs/boxery/synced); h=$(rev HEAD) + if [ -n "$s" ]; then git rev-parse "$s^{tree}"; elif [ -n "$h" ]; then git rev-parse "$h^{tree}"; else echo "$EMPTY"; fi +} +# The working directory as a tree, as `git add -A` sees it (tracked files plus untracked ones that +# aren't ignored), starting from tree $1. Uses a scratch index and object directory, so neither the +# repo's index nor its object store changes. +wt() { + local t rc + t=$(mktemp -d) || return 1 + mkdir "$t/o" + # -p keeps the index's mtime: git re-checks entries changed in the same second as the index was written + # by comparing with it, and a newer copy would make such an edit look clean. + [ ! -f "$IDX" ] || cp -p "$IDX" "$t/i" + ( + export GIT_INDEX_FILE="$t/i" GIT_OBJECT_DIRECTORY="$t/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" + # One-tree --reset keeps the copied index's stat data for entries it doesn't change, so unchanged + # files aren't hashed again, and (unlike -m) doesn't refuse entries that aren't up to date. + git read-tree --reset "$1" && git add -A 2>/dev/null && git write-tree + ) + rc=$?; rm -rf "$t"; return "$rc" +} +# Up to 20 changes from tree $1 to tree $2 (both trees' objects must exist). +changes() { { git diff-tree -r --no-renames --name-status "$1" "$2" 2>/dev/null || true; } | head -n 20; } +# Up to 20 changes from tree $1 to the working directory. +worktree_changes() { + local t + t=$(mktemp -d) || return 1 + mkdir "$t/o" + ( + export GIT_INDEX_FILE="$t/i" GIT_OBJECT_DIRECTORY="$t/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" + git read-tree "$1" && git add -A 2>/dev/null && w=$(git write-tree) && changes "$1" "$w" + ) + rm -rf "$t" +} + +probe() { + if [ ! -e "$DIR" ]; then + local up; up=$(dirname "$DIR"); while [ ! -d "$up" ]; do up=$(dirname "$up"); done + local other; other=$(inside_other_repo "$up" "") && refuse "$DIR is inside the git checkout $other; sync into a directory outside it" + echo state=absent; return + fi + [ -d "$DIR" ] || refuse "$DIR exists and is not a directory" + cd "$DIR" || die "cannot enter $DIR" + if ! is_repo_root; then + local other; other=$(inside_other_repo . "") && refuse "$DIR is inside the git checkout $other; sync into a directory outside it" + local entries; entries=$(ls -A) + if [ -n "$entries" ]; then echo state=dir; printf '%s\n' "$entries" | head -n 5 | sed 's/^/entry=/'; else echo state=empty; fi + return + fi + local outer; outer=$(inside_other_repo .. "") && refuse "$DIR is inside the git checkout $outer; sync into a directory outside it" + setup_git + echo state=repo + echo "format=$(git rev-parse --show-object-format)" + echo "synced=$(rev refs/boxery/synced)" + echo "base=$(rev refs/boxery/base)" + echo "head=$(rev HEAD)" + [ -z "${WANT:-}" ] || ! git cat-file -e "$WANT^{commit}" 2>/dev/null || echo has=1 + # What the VM has, so the caller can tell which commits a push would send (and check them). + git for-each-ref --count=500 --format='tip=%(objectname)' 2>/dev/null + if [ -f "$G/boxery-sync-pending" ]; then echo pending=1; return; fi + local b w + b=$(baseline) || die "cannot read the last sync's state in $DIR" + echo "baseline=$b" + w=$(wt "$b") || die "cannot read the working directory of $DIR" + echo "worktree=$w" + [ "$w" = "$b" ] || worktree_changes "$b" | sed 's/^/changed=/' +} + +# A repository for git push to land in, in a missing or empty directory only. +init() { + : "${FORMAT:?}" + if [ -d "$DIR" ] && [ -n "$(ls -A "$DIR")" ]; then + cd "$DIR" && is_repo_root && return 0 + refuse "$DIR is not a git repository and is not empty" + fi + mkdir -p "$DIR" && cd "$DIR" || die "cannot create $DIR" + git init -q --object-format="$FORMAT" || die "git init failed in $DIR" +} + +# Undo init after a push that didn't land: remove the repository init made, if it is still just that (no +# refs, no objects), and the directory if it is then empty. +uninit() { + cd "$DIR" 2>/dev/null && is_repo_root || return 0 + setup_git + [ -z "$(git for-each-ref --count=1)" ] || return 0 + [ -z "$(find "$OBJ" -type f ! -path '*/info/*' 2>/dev/null | head -n 1)" ] || return 0 + [ "$(ls -A)" = .git ] || return 0 + rm -rf "$G" && cd / && rmdir "$DIR" 2>/dev/null + return 0 +} + +# Put back tree $1 where tree $2 was swapped in (or half swapped in), and the index found before. +rollback() { + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$2" && + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree --reset -u "$2" "$1" || return 1 + restore_index +} +# The ref this sync's push wrote, and those of earlier syncs that never applied (over an hour old: not a +# concurrent sync's). +drop_incoming() { + local cutoff; cutoff=$(( $(date +%s) - 3600 )) + { + [ -z "${REF:-}" ] || printf 'delete %s\n' "$REF" + git for-each-ref --format='%(refname)' refs/boxery/incoming/ 2>/dev/null | while read -r r; do + t=${r##*/}; t=${t%%-*} + case "$t" in '' | *[!0-9]*) t=0 ;; esac + [ "$r" = "${REF:-}" ] || [ "$t" -ge "$cutoff" ] || printf 'delete %s\n' "$r" + done + } | git update-ref --stdin 2>/dev/null || true +} +restore_index() { + if [ -f "$G/boxery-sync-index.bak" ]; then mv -f "$G/boxery-sync-index.bak" "$IDX"; else rm -f "$IDX"; fi + drop_incoming + rm -f "$G/boxery-sync-index" "$G/boxery-sync-pending" +} +# A swap a crash interrupted: done if its refs were written, otherwise undone, but only if every file it +# would put back is still as the swap left it (old, new or absent); anything else is a VM edit. +recover() { + [ -f "$G/boxery-sync-pending" ] || return 0 + local old new s cur foreign + read -r old new <"$G/boxery-sync-pending" || die "unreadable $G/boxery-sync-pending" + s=$(rev refs/boxery/synced) + if [ -n "$s" ] && [ "$(git rev-parse "$s^{tree}")" = "$new" ]; then + rm -f "$G/boxery-sync-pending" "$G/boxery-sync-index" "$G/boxery-sync-index.bak" + return 0 + fi + cur=$(mktemp -d) || die "mktemp failed" + mkdir "$cur/o" + foreign=$( + export GIT_INDEX_FILE="$cur/i" GIT_OBJECT_DIRECTORY="$cur/o" GIT_ALTERNATE_OBJECT_DIRECTORIES="$OBJ" + git read-tree "$old" && git add -A 2>/dev/null && w=$(git write-tree) || exit 1 + # A path that differs from both the old and the new tree is neither ours nor untouched. + comm -12 <(git diff-tree -r --no-renames --name-only "$old" "$w" | sort) <(git diff-tree -r --no-renames --name-only "$new" "$w" | sort) + ) || { rm -rf "$cur"; die "cannot read the working directory of $DIR"; } + rm -rf "$cur" + [ -z "$foreign" ] || refuse "an interrupted earlier sync left $DIR half updated, and these files were changed since: $(printf '%s\n' "$foreign" | head -n 20 | tr '\n' ' ') +Save or discard them in the VM, then sync again (the interrupted sync is undone first)" + rollback "$old" "$new" || die "an interrupted sync left $DIR half updated and it could not be undone; see $G/boxery-sync-pending" +} +# Paths the new tree adds that already exist in the VM: ignored files (apply checked nothing else is +# untracked) the swap would overwrite or delete. A directory counts if it holds ignored files. +clobbered() { + local p d + while IFS= read -r -d '' p; do + if [ -L "$p" ] || [ -f "$p" ]; then printf '%s\n' "$p"; continue; fi + if [ -d "$p" ]; then + [ -z "$(git ls-files -z --others --ignored --exclude-standard --directory -- "$p" | head -c1)" ] || printf '%s/\n' "$p" + continue + fi + d=$(dirname "$p") + while [ "$d" != . ]; do + if [ -L "$d" ] || { [ -e "$d" ] && [ ! -d "$d" ]; }; then printf '%s\n' "$d"; break; fi + [ ! -d "$d" ] || break + d=$(dirname "$d") + done + done < <(git diff-tree -r -z --no-renames --diff-filter=A --name-only "$1" "$2") +} +# Swap tree $1 for $2 in the working directory. 2: git refused before writing anything (a file changed +# under it). 4: done, but the result isn't $TREE (new ignore rules expose VM files). 1: other failure. +swap() { + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree "$1" || return 1 + GIT_INDEX_FILE="$G/boxery-sync-index" git update-index -q --refresh >/dev/null 2>&1 + # Two-tree merge: rewrites only the paths that differ and deletes what $2 lacks; ignored files stay. + # git checks every path first and writes nothing if one changed since apply looked. + GIT_INDEX_FILE="$G/boxery-sync-index" git read-tree -m -u "$1" "$2" 2>"$G/boxery-sync-err" || return 2 + rm -f "$G/boxery-sync-err" + RESULT=$(wt "$2") || return 1 + [ "$RESULT" = "$TREE" ] || return 4 + return 0 +} + +apply() { + : "${SNAP:?}" "${HEAD_SHA:?}" "${TREE:?}" "${REF?}" "${EXPECT_SYNCED?}" "${EXPECT_HEAD?}" + # Past this point the caller has committed: finish or roll back, even if it cancels or disconnects. + trap '' INT TERM HUP PIPE + LOCKDIR="" + trap '[ -z "$LOCKDIR" ] || rmdir "$LOCKDIR" 2>/dev/null' EXIT + cd "$DIR" 2>/dev/null || refuse "$DIR is gone" + is_repo_root || refuse "$DIR is not a git repository" + setup_git + if command -v flock >/dev/null; then + exec 9>"$G/boxery-sync.lock" + flock -w 60 9 || refuse "another sync into $DIR is running" + else + local i=0 + until mkdir "$G/boxery-sync.lock.d" 2>/dev/null; do + i=$((i + 1)); [ "$i" -lt 600 ] || refuse "another sync into $DIR is running (or $G/boxery-sync.lock.d is stale: remove it)" + sleep 0.1 + done + LOCKDIR="$G/boxery-sync.lock.d" + fi + recover + [ "$(rev refs/boxery/synced)" = "$EXPECT_SYNCED" ] && [ "$(rev HEAD)" = "$EXPECT_HEAD" ] || + refuse "the checkout in $DIR changed during the sync; run it again" + local b w new prev lock=0 hit rc + b=$(baseline) || die "cannot read the last sync's state in $DIR" + w=$(wt "$b") || die "cannot read the working directory of $DIR" + [ "$w" = "$b" ] || refuse "uncommitted changes appeared in $DIR during the sync; run it again" + git cat-file -e "$SNAP^{commit}" 2>/dev/null || die "the VM lacks commit $SNAP after the push" + new=$(git rev-parse "$SNAP^{tree}") || die "cannot read $SNAP" + hit=$(clobbered "$b" "$new" | head -n 20) || die "cannot compare the new tree with $DIR" + if [ -n "$hit" ]; then + drop_incoming + refuse "the local checkout now tracks paths that exist in the VM as ignored files (build output, dependencies), which the sync would overwrite or delete: $(printf '%s\n' "$hit" | tr '\n' ' ') +Remove them in the VM (or stop tracking them locally), then sync again" + fi + prev=$(rev refs/boxery/synced); [ -n "$prev" ] || prev=$(rev HEAD) + git diff --quiet "${prev:-$EMPTY}" "$SNAP" -- "${LOCKFILES[@]}" || lock=1 + rm -f "$G/boxery-sync-index.bak" + [ ! -f "$IDX" ] || cp "$IDX" "$G/boxery-sync-index.bak" || die "cannot back up the index" + printf '%s %s\n' "$b" "$new" >"$G/boxery-sync-pending" || die "cannot write $G/boxery-sync-pending" + swap "$b" "$new"; rc=$? + case $rc in + 0) ;; + 2) + local why; why=$(head -c 2000 "$G/boxery-sync-err" 2>/dev/null); rm -f "$G/boxery-sync-err" + restore_index + refuse "files in $DIR changed during the sync; run it again ($why)" ;; + 4) + local extra; extra=$(worktree_changes "$new") + rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" + refuse "with the new .gitignore, files in the VM that were ignored no longer are: $(printf '%s\n' "$extra" | tr '\t\n' ' ') +Remove them in the VM (or ignore them locally), then sync again" ;; + *) + rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" + die "the sync failed; the checkout in $DIR was restored" ;; + esac + # The index holds local HEAD, so `git status` in the VM shows the working changes, as it does locally. + if ! { git read-tree "$HEAD_SHA" && printf '%s\n' 'option no-deref' "update HEAD $HEAD_SHA" "update refs/boxery/synced $SNAP" \ + "update refs/boxery/base $HEAD_SHA" | git update-ref -m "boxery sync" --stdin; }; then + rollback "$b" "$new" || die "the sync failed and the checkout in $DIR could not be restored; see $G/boxery-sync-pending" + die "the sync failed writing refs; the checkout in $DIR was restored" + fi + git update-index -q --refresh >/dev/null 2>&1 + drop_incoming + rm -f "$G/boxery-sync-pending" "$G/boxery-sync-index" "$G/boxery-sync-index.bak" + echo "lockfile_changed=$lock" + echo "tree=$RESULT" +} + +case "$MODE" in +probe) probe ;; +init) init ;; +uninit) uninit ;; +apply) apply ;; +*) die "unknown mode $MODE" ;; +esac diff --git a/core/sync.mjs b/core/sync.mjs new file mode 100644 index 00000000..f1d54178 --- /dev/null +++ b/core/sync.mjs @@ -0,0 +1,380 @@ +// vm_sync: push the caller's local git checkout into a VM, so the VM's tree matches it exactly (docs/sync.md). +// Commits travel by `git push` over the VM's ssh connection (the profile's ssh_config, pinned host key), +// straight into git receive-pack in the VM; neither side ever contacts a git remote. The VM side is core/sync-guest.sh, sent with every call. The local repository +// is only read: the snapshot of its working changes is built in a scratch index and object directory. +import { execFile, spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { copyFile, mkdir, mkdtemp, realpath, rm, stat, utimes, writeFile } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import path from "node:path"; +import { q } from "./shell.mjs"; +import { PROJECT, assertName, confineLocal, currentHost, deniedDirs, isWithin, sshRun, vmPath } from "./boxery-core.mjs"; +/** @import { LocalConfinement, ToolResult } from "./boxery-core.mjs" */ +/** @typedef {{ from?: string, to?: string }} SyncParams */ +/** @typedef {{ localCwd?: string, local?: LocalConfinement | false, signal?: AbortSignal }} SyncOpts */ +/** What the VM reported (sync-guest.sh probe, or apply's tree and lockfile_changed). tip: the VM repo's ref tips. + * @typedef {{ state: string, format?: string, synced?: string, base?: string, head?: string, has?: string, pending?: string, baseline?: string, worktree?: string, tree?: string, lockfile_changed?: string, changed: string[], entry: string[], tip: string[] }} Probe */ + +const guestTemplate = readFileSync(new URL("./sync-guest.sh", import.meta.url), "utf8"); + +export const SYNC_DISABLED = + "vm_sync is disabled on this server: it runs on the boxery control plane, so a local checkout would be that machine's files, not yours. " + + "Ask the user to start the server with --copy-root DIR to allow syncing checkouts inside DIR, or sync from a stdio MCP server or pi on your own machine."; + +/** Lockfiles whose change means the VM's dependencies need installing again. */ +export const LOCKFILES = ["pnpm-lock.yaml", "package-lock.json", "yarn.lock", "bun.lock"]; + +// Credentials vm_sync never puts on a VM's disk (and so into its snapshots): the agent asks with vm_grant instead. +// Case-insensitive. Templates (.env.example, .env.sample, .env.template, .env.dist) hold no values and are allowed. +const ENV_TEMPLATE = /^\.env[._-](example|sample|template|dist|defaults)$/i; +const DENY_NAME = new RegExp("^(" + [ + "\\.env([._-].*|rc)?", "\\.npmrc", "\\.yarnrc\\.yml", "\\.pypirc", "\\.netrc", "_netrc", "\\.git-credentials", "\\.pgpass", "\\.my\\.cnf", + "\\.dockercfg", "\\.s3cfg", "\\.boto", "\\.terraformrc", "auth\\.json", "credentials(\\.json)?", "application_default_credentials\\.json", + "id_[a-z0-9_-]+", ".*\\.(pem|key|p12|pfx|jks|keystore)", +].join("|") + ")$", "i"); +const DENY_PATH = /(^|\/)(\.docker\/config\.json|\.kube\/config|gh\/hosts\.yml|\.aws\/config)$/i; +/** Is a repository path a credential file vm_sync refuses? @param {string} p */ +export function deniedSyncPath(p) { + const name = path.posix.basename(p); + if (ENV_TEMPLATE.test(name) || /\.pub$/i.test(name)) return false; + return DENY_NAME.test(name) || DENY_PATH.test(p); +} + +const IDENTITY = { GIT_AUTHOR_NAME: "boxery sync", GIT_AUTHOR_EMAIL: "sync@boxery.invalid", GIT_AUTHOR_DATE: "@0 +0000" }; +const COMMITTER = { GIT_COMMITTER_NAME: IDENTITY.GIT_AUTHOR_NAME, GIT_COMMITTER_EMAIL: IDENTITY.GIT_AUTHOR_EMAIL, GIT_COMMITTER_DATE: IDENTITY.GIT_AUTHOR_DATE }; +// No GIT_* variable from the caller's environment reaches git: they relocate the repository, inject config +// (GIT_CONFIG_COUNT/PARAMETERS, so url rewrites), or change which transports are allowed. No lazy fetches +// from a partial clone's promisor remote either: vm_sync never contacts a remote. +/** @returns {NodeJS.ProcessEnv} */ +const baseEnv = () => { + const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !k.startsWith("GIT_"))); + return { ...env, LC_ALL: "C", GIT_OPTIONAL_LOCKS: "0", GIT_TERMINAL_PROMPT: "0", GIT_NO_LAZY_FETCH: "1" }; +}; +// The push and its scratch repository read no user or system config: a url.insteadOf/pushInsteadOf would +// send the snapshot somewhere else (another VM), push.* options change the push. (The snapshot itself keeps +// the user's config: core.excludesFile decides what's untracked.) +const isolatedEnv = () => ({ ...baseEnv(), GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }); +// git must not run anything a checkout's config names (fsmonitor, hooks; filter drivers are refused when +// confined) or write a split index into the checkout's .git. +const SAFE_CONFIG = ["-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", "core.splitIndex=false", "-c", "protocol.allow=never"]; +const aborted = () => Object.assign(new Error("sync aborted; the VM checkout was not changed"), { name: "AbortError" }); + +/** Run git locally → stdout. @param {string[]} args @param {{ cwd?: string, env?: NodeJS.ProcessEnv, input?: string }} [opts] @returns {Promise} */ +function git(args, { cwd, env = baseEnv(), input } = {}) { + return new Promise((resolve, reject) => { + const child = execFile("git", [...SAFE_CONFIG, ...args], { cwd, env, maxBuffer: 256 * 1024 * 1024, encoding: "utf8" }, (error, stdout, stderr) => { + if (!error) return resolve(stdout); + const e = /** @type {NodeJS.ErrnoException} */ (error); + reject(new Error(e.code === "ENOENT" ? "git is not installed on this machine" : `git ${args.find((a) => !a.startsWith("-") && !a.includes("=") && a !== "--git-dir") ?? args[0]}: ${stderr.trim() || e.message}`)); + }); + child.stdin?.on("error", () => {}); + child.stdin?.end(input ?? ""); + }); +} +const tryGit = (/** @type {string[]} */ args, /** @type {{ cwd?: string, env?: NodeJS.ProcessEnv }} */ opts) => git(args, opts).then((s) => s.trim(), () => null); + +/** key=value lines from sync-guest.sh. Repeated keys (changed, entry) collect into arrays. @param {string} out @returns {Probe} */ +function parseLines(out) { + /** @type {any} */ + const r = { changed: [], entry: [], tip: [] }; + for (const line of out.split("\n")) { + const i = line.indexOf("="); + if (i <= 0) continue; + const k = line.slice(0, i), v = line.slice(i + 1); + if (Array.isArray(r[k])) r[k].push(v); + else r[k] = v; + } + return r; +} + +/** Run sync-guest.sh in `vm` with `vars`. Exit 3 is a refusal: nothing changed. @param {string} vm @param {Record} vars */ +async function guest(vm, vars) { + const assignments = Object.entries(vars).map(([k, v]) => `${k}=${q(v)}`).join("\n"); + // No signal: probe is short, and apply must finish or roll back once started (it ignores signals itself). + // Detached: a terminal's Ctrl-C reaches the whole foreground group; ssh must survive it to report the result. + const r = await sshRun(vm, `${assignments}\n${guestTemplate}`, { detached: true }); + const err = r.stderr.trim(); + if (r.code === 3) throw new Error(`vm_sync refused: ${err}; nothing in the VM was changed`); + if (r.code !== 0) { + if (/git: (command )?not found/.test(err)) throw new Error(`git is not installed in VM "${vm}"`); + throw new Error(`vm_sync failed in VM "${vm}": ${err || `exit ${r.code}`}`); + } + return parseLines(r.stdout.toString("utf8")); +} + +/** + * Confined (--copy-root): the repository's root, git directory, common directory, object store and its + * alternates must all be inside the root and clear of denied directories, and its config must name no + * filter driver (git add would run it). Checked before git reads the working tree. + * @param {string} from @param {ReturnType} confined @param {NodeJS.ProcessEnv} env + */ +async function checkConfinedRepo(from, confined, env) { + const out = await tryGit(["rev-parse", "--path-format=absolute", "--show-toplevel", "--absolute-git-dir", "--git-common-dir", "--git-path", "objects"], { cwd: from, env }); + if (!out) return; + const dirs = out.split("\n").filter(Boolean); + const objects = dirs[dirs.length - 1]; + let alternates = ""; + try { alternates = readFileSync(path.join(objects, "info", "alternates"), "utf8"); } catch {} + for (const a of alternates.split("\n").map((l) => l.trim()).filter((l) => l && !l.startsWith("#"))) dirs.push(path.resolve(objects, a)); + for (const d of dirs) { + let real; + try { real = await realpath(d); } catch { continue; } + confined.check(real, `${from} (its repository uses ${d})`); + } + const filters = await tryGit(["config", "--get-regexp", "^filter\\..*\\.(clean|smudge|process)$"], { cwd: from, env }); + if (filters) throw new Error(`vm_sync refused: the repository at ${from} configures a git filter driver (${filters.split("\n")[0]}), which this server won't run`); +} + +/** + * The local checkout: repository root, HEAD, and a snapshot commit of HEAD plus every working change + * (staged, unstaged, untracked but not ignored), built without changing the repository. + * @param {string} from @param {string} scratch @param {ReturnType | null} confined + */ +async function snapshot(from, scratch, confined) { + const env0 = baseEnv(); + if (confined) { + // Don't discover a repository above the root. + env0.GIT_CEILING_DIRECTORIES = path.dirname(confined.root); + await checkConfinedRepo(from, confined, env0); + } + const g0 = { env: env0 }; + const top = await tryGit(["rev-parse", "--show-toplevel"], { cwd: from, ...g0 }); + if (!top) { + const bare = await tryGit(["rev-parse", "--is-bare-repository"], { cwd: from, ...g0 }); + if (bare === "true") throw new Error(`${from} is a bare git repository; vm_sync pushes a checkout with a working tree`); + throw new Error(`${from} is not a git checkout (no repository found there or above it). vm_sync pushes a git checkout; use vm_copy for other files.`); + } + const root = await realpath(top); + confined?.check(root); + const head = await tryGit(["rev-parse", "-q", "--verify", "HEAD^{commit}"], { cwd: root, ...g0 }); + if (!head) throw new Error(`${root} has no commits yet; commit once before syncing (vm_sync needs a HEAD to check out)`); + if (await tryGit(["rev-parse", "--is-shallow-repository"], { cwd: root, ...g0 }) === "true") + throw new Error(`${root} is a shallow clone; vm_sync sends history the VM lacks and can't from a shallow one. Run git fetch --unshallow first`); + if (await tryGit(["config", "--get-regexp", "^(extensions\\.partialclone|remote\\..*\\.promisor)$"], { cwd: root, ...g0 })) + throw new Error(`${root} is a partial clone (objects are missing until fetched from its remote); vm_sync never contacts a remote and can't send objects it doesn't have`); + const format = (await git(["rev-parse", "--show-object-format"], { cwd: root, ...g0 })).trim(); + const objects = (await git(["rev-parse", "--path-format=absolute", "--git-path", "objects"], { cwd: root, ...g0 })).trim(); + const index = (await git(["rev-parse", "--path-format=absolute", "--git-path", "index"], { cwd: root, ...g0 })).trim(); + const scratchObjects = path.join(scratch, "objects"); + await mkdir(scratchObjects); + const scratchIndex = path.join(scratch, "index"); + let haveIndex = true; + try { + // Keep the index's mtime: git re-checks entries modified in the same second as the index was + // written ("racy" entries) by comparing with it. A newer copy would make such an edit look clean. + await copyFile(index, scratchIndex); + const st = await stat(index); + await utimes(scratchIndex, st.atime, st.mtime); + } catch (e) { if (/** @type {any} */ (e).code !== "ENOENT") throw e; haveIndex = false; } + const env = { ...env0, GIT_INDEX_FILE: scratchIndex, GIT_OBJECT_DIRECTORY: scratchObjects, GIT_ALTERNATE_OBJECT_DIRECTORIES: objects }; + if (!haveIndex) await git(["read-tree", "HEAD"], { cwd: root, env }); + await git(["add", "-A"], { cwd: root, env }); + const tree = (await git(["write-tree"], { cwd: root, env })).trim(); + const headTree = (await git(["rev-parse", "HEAD^{tree}"], { cwd: root, ...g0 })).trim(); + // Deterministic (fixed identity and date), so the same working changes give the same commit every time. + const snap = tree === headTree ? head + : (await git(["commit-tree", tree, "-p", head, "-m", "boxery sync: working changes"], { cwd: root, env: { ...env, ...IDENTITY, ...COMMITTER } })).trim(); + const counts = { modified: 0, untracked: 0, deleted: 0 }; + if (tree !== headTree) { + const tracked = new Set((await git(["ls-files", "-z"], { cwd: root, ...g0 })).split("\0")); + const fields = (await git(["diff-tree", "-r", "-z", "-M", "--name-status", headTree, tree], { cwd: root, env })).split("\0").filter(Boolean); + for (let i = 0; i < fields.length;) { + const status = fields[i++]; + const paths = status.startsWith("R") || status.startsWith("C") ? [fields[i++], fields[i++]] : [fields[i++]]; + if (status === "D") counts.deleted++; + else if (status === "A" && !tracked.has(paths[0])) counts.untracked++; + else counts.modified++; + } + } + const files = (await git(["ls-tree", "-r", "-z", "--name-only", tree], { cwd: root, env })).split("\0").filter(Boolean); + return { root, head, format, objects, scratchObjects, env, tree, snap, counts, files, headTree }; +} + +/** + * Refuse credentials: a deny-listed file name anywhere in the tree, or anything in boxery's own state or + * the user's key directories (what vm_copy refuses), even where the checkout contains one of those. + * @param {string} root @param {string[]} files @param {string[]} [deny] + */ +function refuseCredentials(root, files, deny) { + const named = files.filter(deniedSyncPath); + if (named.length) + throw new Error(`vm_sync refused: ${named.slice(0, 5).join(", ")}${named.length > 5 ? ` (+${named.length - 5} more)` : ""} in ${root} looks like a credential file, which vm_sync never puts on a VM's disk (it would persist in its snapshots). ` + + "Ignore it in .gitignore (or remove it, and git rm it if tracked) to sync the rest, and ask for the credential with vm_grant. Nothing in the VM was changed."); + for (const { dir } of deniedDirs(deny)) { + if (isWithin(root, dir)) throw new Error(`vm_sync refused: ${root} is inside ${dir} (boxery's own configuration or the user's keys), which vm_sync never copies; use vm_grant for credentials`); + if (!isWithin(dir, root)) continue; + const rel = path.relative(root, dir).split(path.sep).join("/"); + const hit = files.find((f) => isWithin(f, rel)); + if (hit) throw new Error(`vm_sync refused: ${path.join(root, hit)} is inside ${dir} (boxery's own configuration or the user's keys), which vm_sync never copies; ignore it in .gitignore, and use vm_grant for credentials`); + } +} + +/** + * A bare scratch repository to push from: its alternates are the scratch objects and the checkout's, so + * refs/heads/sync can point at the snapshot without touching the checkout's refs. Refuses a credential + * anywhere in the commits the VM may lack (those not reachable from its ref tips), not just the snapshot. + * @param {Awaited>} s @param {Probe} probe @param {string} scratch @param {string[]} [deny] + */ +async function pushSource(s, probe, scratch, deny) { + const bare = path.join(scratch, "push.git"); + const env = isolatedEnv(); + await git(["init", "-q", "--bare", `--object-format=${s.format}`, bare], { env }); + await writeFile(path.join(bare, "objects", "info", "alternates"), `${s.scratchObjects}\n${s.objects}\n`); + await git(["--git-dir", bare, "update-ref", "refs/heads/sync", s.snap], { env }); + // What the push will send: every object reachable from the snapshot and not from what the VM has (by + // path, so merges and roots count too). Tips this repository doesn't have are skipped: then nothing they + // reach is excluded, which only errs towards refusing. The tips are the VM's own report (probe), so this + // trusts the VM to say what it has; see docs/sync.md. + const tips = [...new Set([probe.synced, probe.base, probe.head, ...probe.tip].filter((t) => t && /^[0-9a-f]{40,64}$/.test(t)))]; + const listed = await git(["--git-dir", bare, "rev-list", "--objects", "--ignore-missing", "--stdin"], { env, input: `refs/heads/sync\n--not\n${tips.join("\n")}\n` }); + const history = [...new Set(listed.split("\n").map((l) => l.slice(l.indexOf(" ") + 1)).filter((p) => p && !/^[0-9a-f]{40,64}$/.test(p)))]; + const leaked = history.filter(deniedSyncPath); + if (leaked.length) + throw new Error(`vm_sync refused: ${leaked.slice(0, 5).join(", ")} is in the history this sync would send (a commit the VM lacks has it), and vm_sync never puts credential files on a VM's disk. ` + + "It's not in your working tree, but its old contents are in that commit. Remove it from history (git rebase / git filter-repo) before syncing, and ask for the credential with vm_grant. Nothing in the VM was changed."); + refuseCredentials(s.root, history, deny); + return bare; +} + +/** + * git push the snapshot into the VM repository at `dest`, to a hidden ref. receive-pack quarantines + * incoming objects until the ref update succeeds, so a cancelled push leaves nothing behind. No hooks run + * on either side, and the push goes only to this VM, over its own ssh_config. + * → objects sent (null if git didn't say: nothing to send, or no progress line). + * @param {string} bare @param {string} vm @param {string} sshConf @param {string} dest @param {string} ref @param {AbortSignal} [signal] + * @returns {Promise} + */ +function push(bare, vm, sshConf, dest, ref, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) return reject(aborted()); + // ControlMaster=no, as vm_copy: reuse a shared connection, but never become the master inside the + // process group a cancel kills (other boxery calls would lose it). + const env = { ...isolatedEnv(), GIT_SSH_COMMAND: `ssh -F ${q(sshConf)} -o BatchMode=yes -o ControlMaster=no`, GIT_SSH_VARIANT: "ssh" }; + const args = [...SAFE_CONFIG, "-c", "protocol.ssh.allow=always", "--git-dir", bare, "push", "--progress", "--no-verify", "--no-signed", "--no-recurse-submodules", "--no-follow-tags", + "--receive-pack=git -c core.hooksPath=/dev/null -c receive.autogc=false receive-pack", `${vm}.boxery:${dest}`, `+refs/heads/sync:${ref}`]; + // Its own process group, so cancelling stops git, its ssh and pack-objects together. + const child = spawn("git", args, { env, detached: true, stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (d) => { stderr = (stderr + d).slice(-64 * 1024); }); + /** @type {NodeJS.Timeout[]} */ + const timers = []; + const group = (/** @type {NodeJS.Signals} */ sig) => { try { process.kill(-(/** @type {number} */ (child.pid)), sig); } catch {} }; + // TERM, then KILL; and don't wait forever for stderr a descendant (a ProxyCommand) may still hold. + const stop = () => { + group("SIGTERM"); + timers.push(setTimeout(() => group("SIGKILL"), 1000), setTimeout(() => child.stderr.destroy(), 3000)); + }; + signal?.addEventListener("abort", stop, { once: true }); + child.on("error", (e) => { signal?.removeEventListener("abort", stop); reject(new Error(`git push: ${e.message}`)); }); + child.on("close", (code) => { + for (const t of timers) clearTimeout(t); + if (signal?.aborted) group("SIGKILL"); + signal?.removeEventListener("abort", stop); + if (signal?.aborted) return reject(aborted()); + if (code !== 0) { + const lines = stderr.split(/[\r\n]+/).filter((l) => l && !/^(Enumerating|Counting|Delta|Compressing|Writing|Total) /.test(l)); + return reject(new Error(`vm_sync: git push to VM "${vm}" failed: ${lines.join("\n").trim() || `exit ${code}`}; nothing in the checkout was changed`)); + } + const m = /Writing objects: 100% \((\d+)\/\d+\)/.exec(stderr); + resolve(m ? Number(m[1]) : null); + }); + }); +} + +/** + * vm_sync / boxery sync: make the VM checkout at `to` (default ~/project) match the local checkout at + * `from` (default the caller's cwd) exactly: HEAD detached at the same commit, plus every working change. + * `local` is vm_copy's local: policy (undefined: unrestricted; false: refused; {root, deny}: confined). + * @param {string} vm @param {SyncParams} params @param {SyncOpts} [opts] @returns {Promise} + */ +export async function syncTool(vm, { from, to } = {}, { localCwd = process.cwd(), local, signal } = {}) { + const started = Date.now(); + assertName(vm); + if (local === false) throw new Error(SYNC_DISABLED); + const fromSpec = String(from ?? ".").replace(/^local:/, "") || "."; + const confined = local ? confineLocal(local) : null; + const fromPath = confined ? confined.source(fromSpec) : path.resolve(localCwd, fromSpec.replace(/^~(?=\/|$)/, homedir())); + const dest = vmPath(to ?? PROJECT); + if (dest === "/") throw new Error("vm_sync can't sync into the VM's filesystem root"); + const host = currentHost(); + const label = `${vm}:${dest}`; + if (signal?.aborted) throw aborted(); + const id = randomBytes(8).toString("hex"); + const scratch = await mkdtemp(path.join(tmpdir(), "boxery-sync-")); + // The time in the name lets a later sync tell an abandoned ref from a concurrent one (snapshot commits + // carry a fixed date, so the commit can't say). + const ref = `refs/boxery/incoming/${Math.floor(Date.now() / 1000)}-${id}`; + try { + const s = await snapshot(fromPath, scratch, confined); + refuseCredentials(s.root, s.files, local ? local.deny : undefined); + if (signal?.aborted) throw aborted(); + const probe = await guest(vm, { MODE: "probe", DIR: dest, WANT: s.snap }); + if (probe.state === "dir") + throw new Error(`vm_sync refused: ${label} is not a git repository and is not empty (${probe.entry.join(", ")}…); sync into an empty or new directory (to), or move those files away. Nothing in the VM was changed.`); + if (probe.state === "repo" && probe.format && probe.format !== s.format) + throw new Error(`vm_sync refused: the repository at ${label} uses ${probe.format} object ids and the local one ${s.format}`); + if (probe.state === "repo" && probe.pending !== "1" && probe.worktree !== probe.baseline) { + const what = probe.synced ? "since the last sync" : "against its HEAD"; + throw new Error(`vm_sync refused: ${label} has changes ${what} that vm_sync didn't make:\n${probe.changed.map((c) => ` ${c.replace("\t", " ")}`).join("\n")}\n` + + "Commit them in the VM and bring them back yourself, or discard them (git -C DIR stash -u, or checkout/clean), then sync again. Nothing in the VM was changed."); + } + const pushed = probe.has !== "1"; + let objects = 0; + if (pushed) { + const bare = await pushSource(s, probe, scratch, local ? local.deny : undefined); + if (signal?.aborted) throw aborted(); + // receive-pack needs a repository: create one in a missing or empty directory (only that), and + // remove it again if the push doesn't land. + const created = probe.state !== "repo"; + if (created) await guest(vm, { MODE: "init", DIR: dest, FORMAT: s.format }); + try { + objects = (await push(bare, vm, host.sshConf, dest, ref, signal)) ?? 0; + } catch (e) { + if (created) await guest(vm, { MODE: "uninit", DIR: dest }).catch(() => {}); + throw e; + } + } + // The last point to stop: past here the VM applies the snapshot completely or not at all. + if (signal?.aborted) throw aborted(); + const applied = await guest(vm, { + MODE: "apply", DIR: dest, SNAP: s.snap, HEAD_SHA: s.head, TREE: s.tree, REF: pushed ? ref : "", + EXPECT_SYNCED: probe.synced ?? "", EXPECT_HEAD: probe.head ?? "", + }); + const match = applied.tree === s.tree; + const lockfileChanged = applied.lockfile_changed === "1"; + const short = s.head.slice(0, 7), { modified, untracked, deleted } = s.counts; + const text = `synced ${short} + ${modified} modified, ${untracked} untracked, ${deleted} deleted → ${label} · ` + + (match ? "tree matches" : `TREE MISMATCH (local ${s.tree.slice(0, 12)}, VM ${String(applied.tree).slice(0, 12)})`) + + (lockfileChanged ? " · lockfile changed" : ""); + return { + text, + isError: !match, + structured: { + vm, host: host.name, from: s.root, to: dest, sha: s.head, snapshot: s.snap, + counts: s.counts, tree: { local: s.tree, vm: applied.tree ?? null, match }, + lockfile_changed: lockfileChanged, pushed, objects_sent: objects, elapsed_ms: Date.now() - started, + }, + }; + } finally { + // (A pushed ref apply never reached is harmless; the next successful apply drops every incoming ref.) + await rm(scratch, { recursive: true, force: true }); + } +} + +export const SYNC_OUTPUT = { + type: "object", + properties: { + vm: { type: "string" }, host: { type: "string" }, + from: { type: "string", description: "the local repository root" }, + to: { type: "string", description: "the VM checkout" }, + sha: { type: "string", description: "local HEAD, checked out (detached) in the VM" }, + snapshot: { type: "string", description: "commit of HEAD plus the working changes (= sha when there are none); the VM's refs/boxery/synced" }, + counts: { type: "object", properties: { modified: { type: "integer" }, untracked: { type: "integer" }, deleted: { type: "integer" } } }, + tree: { type: "object", description: "git tree of the whole working directory (tracked + untracked, not ignored), computed on each side", properties: { local: { type: "string" }, vm: { type: ["string", "null"] }, match: { type: "boolean" } } }, + lockfile_changed: { type: "boolean", description: `a lockfile (${LOCKFILES.join(", ")}) differs from the previous sync: dependencies need installing` }, + pushed: { type: "boolean", description: "false: the VM already had the snapshot, so nothing was pushed" }, + objects_sent: { type: "integer", description: "git objects the push sent (0 when nothing was pushed)" }, + elapsed_ms: { type: "integer" }, + }, +}; diff --git a/docs/README.md b/docs/README.md index ec811325..f53ebc1d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ - **[Shared pool & leases](pool.md)**: several agents share a pool of VMs safely. - **[GitHub Actions runners](github-runners.md)**: ephemeral self-hosted runners, one job per freshly reset pool VM. - **[File transfers](copy.md)**: streaming `vm_copy`, destination semantics and cancellation. +- **[Checkout sync](sync.md)**: `vm_sync` pushes a local git checkout into a VM so its tree matches exactly. ## Reference diff --git a/docs/cli.md b/docs/cli.md index f495efc7..87701e9b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -146,6 +146,8 @@ boxery --host hornet cp local:notes.md ci-1:~/notes.md `cp` is the `vm_copy` tool's own code, so it takes the same locations, `local:PATH`, `NAME:PATH` (a VM on the selected host) and `HOST/NAME:PATH`, and behaves the same way: streaming tar, staging, destination and cancellation rules are in [file transfers](copy.md). The one difference: the CLI has no current VM, so a bare path is on this machine, like `cp`/`scp` (`vm_copy` reads it as the current VM). A bare path that looks like a location with an invalid VM name (`Demo:x`) is refused instead of being written locally. Ctrl-C cancels and cleans up staging. `cp` runs on Node (≥ 20), which the npm package already needs. +`boxery sync demo [--from DIR] [--to PATH] [--json]` pushes a local git checkout into a VM so its tree matches exactly (commits by `git push` over the VM's ssh connection, plus uncommitted and untracked changes); it is the `vm_sync` tool's own code ([sync](sync.md)). + `attach` starts pi, opencode or a shell (nothing else) and needs a terminal; without one it says to use `prompt` or `run --bg`, which work headless. `prompt` logs to `~/project/.boxery/task-*.log` in the VM. Attach to a task with `boxery attach demo pi task-…`. pi needs the `llm:` grant for its provider (`llm:chatgpt` unless you changed pi's settings in the VM), and a new VM has none: `prompt` checks first and, if it's missing, starts nothing and prints the `boxery grant` command to run (the older spelling of `boxery access grant`). ### Background command jobs @@ -308,6 +310,7 @@ Generated from the help `boxery help --all` shows (`controller/cli.mjs`); edit i | `boxery ssh NAME[@HOST] [CMD...]` | Shell into a VM (also: ssh NAME.boxery) | | `boxery run [--bg] NAME [--cwd DIR] [--] CMD [ARG...]` | Run a command in ~/project, args as given (like docker exec) [→](#shell-and-in-vm-agents) | | `boxery cp FROM TO` | Copy files and directories to, from and between VMs [→](copy.md) | +| `boxery sync NAME [--from DIR] [--to PATH] [--json]` | Push a local git checkout into a VM, so its tree matches exactly [→](sync.md) | | `boxery desktop NAME [--no-open]` | Open a VM's desktop in your browser (noVNC over ssh) [→](#desktop-and-ports) | | `boxery forward NAME PORT [LOCAL_PORT]` | Forward a VM port to localhost | | `boxery attach NAME [pi\|opencode\|shell] [SESSION]` | Attach to an agent in a persistent tmux session (= agent attach) | diff --git a/docs/copy.md b/docs/copy.md index f5a4862b..1a91a1ea 100644 --- a/docs/copy.md +++ b/docs/copy.md @@ -4,6 +4,8 @@ `boxery cp FROM TO` is the same code from the CLI, with the same locations, except that a bare path is local there (the CLI has no current VM). See the [CLI reference](cli.md#files). +For a git checkout, use [`vm_sync`](sync.md): it sends only what changed, keeps the VM's ignored files and checks that the trees match. + `vm_copy` transfers a file or directory **when explicitly called**. It does not continuously synchronize workspaces, and ordinary shell/file/browser tools do not copy the project back and forth. VM forks use a separate disk-image mechanism. ## Locations diff --git a/docs/mcp.md b/docs/mcp.md index fd19dc17..ab11713a 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -9,7 +9,7 @@ There are two ways to run it: | | Runs on | Transport | Good for | |---|---|---|---| | **A. stdio** (recommended) | your laptop | stdin/stdout | Full features. Grants and `vm_copy local:` use your laptop. | -| **B. HTTP** | the VM host | Streamable HTTP over Tailscale | Nothing to install on clients. `vm_copy local:` is off unless confined with `--copy-root`. | +| **B. HTTP** | the VM host | Streamable HTTP over Tailscale | Nothing to install on clients. `vm_copy local:` and `vm_sync` are off unless confined with `--copy-root`. | ## A. stdio on your laptop @@ -75,7 +75,9 @@ With `--copy-root DIR`, `local:` paths are confined to `DIR`: - Boxery's own state and the user's keys are refused even inside `DIR`, as is any path that contains them: `$BOXERY_CONF` and other `~/.config/boxery*` profiles, `~/.boxery` and registered profile directories, `~/.ssh`, `~/.gnupg`, `~/.pi`, and the `--token-file`. The server refuses to start if `DIR` is inside one of them. - A VM → local copy can create a new path or replace a file, but never merges into or replaces an existing directory: copy to a new path instead. It is extracted in a private staging directory (tar refuses absolute and `..` entries and writes through links within the archive), refused if it hard-links a file outside itself, and moved into place with `rename`, which replaces a symlink rather than following it. Symlinks inside a copied tree arrive as symlinks; later `local:` paths through them are checked like any other. -The `initialize` instructions and the `vm_copy` description tell the agent which of the two applies. stdio mode is unchanged: there `local:` is your own machine. `--copy-root` confines it there too, if you want that. +`vm_sync` ([sync](sync.md)) reads a local checkout, so it follows the same rule: refused over HTTP without `--copy-root`; with it, `from` resolves inside `DIR` and is checked the same way. + +The `initialize` instructions and the `vm_copy` and `vm_sync` descriptions tell the agent which of the two applies. stdio mode is unchanged: there `local:` is your own machine. `--copy-root` confines it there too, if you want that. ## Protocol features @@ -177,7 +179,7 @@ commit outcome**, not rollback: reread before retrying. | `--vm NAME` | Default VM. Without it, the model picks one with `vm_manage {action:"use"}`, and tools fail closed when no `vm` is given. | | `--http ADDR:PORT` | Serve Streamable HTTP at `/mcp` instead of stdio. Each MCP session keeps its own current VM. | | `--token-file PATH` | Bearer token from a file instead of `$BOXERY_MCP_TOKEN` | -| `--copy-root DIR` | Allow `vm_copy` `local:` paths, confined to `DIR` (see [B](#b-streamable-http-on-the-host-over-tailscale)). Over HTTP, `local:` is refused without it. | +| `--copy-root DIR` | Allow `vm_copy` `local:` paths and `vm_sync` checkouts, confined to `DIR` (see [B](#b-streamable-http-on-the-host-over-tailscale)). Over HTTP, both are refused without it. | | `--allow-user LOGIN` | Also require this `Tailscale-User-Login` header (set by `tailscale serve`). Loopback binds only. Can be repeated. | | `--allow-origin ORIGIN` | Accept requests with this `Origin`. Foreign origins are rejected by default. | | `--no-auth` | Skip the bearer token. **Local testing only:** refused unless `--http` binds loopback (`127.0.0.1`, `::1`, `localhost`). | diff --git a/docs/pi-extension.md b/docs/pi-extension.md index 06123bba..b61f496f 100644 --- a/docs/pi-extension.md +++ b/docs/pi-extension.md @@ -48,6 +48,7 @@ The current VM is saved in the session, so `pi --continue` picks it back up. Thr |---|---| | `vm_bash`, `vm_read`, `vm_write`, `vm_edit` | Shell and files in the VM. Same engines as the built-ins, with the same truncation, edit semantics and image handling. Relative paths resolve to `/home/agent/project`. | | `vm_copy` | Streaming [file transfers](copy.md) between `local:path`, `NAME:path`, or a bare path (= current VM). Works VM to VM too. | +| `vm_sync` | Push the local git checkout into the VM so its tree matches exactly ([sync](sync.md)) | | `vm_jobs` | Background jobs (`start` · `list` · `logs` · `wait` · `kill`) for long commands like test suites. Same as `boxery run --bg` ([CLI](cli.md#background-command-jobs)); details in [MCP](mcp.md#background-jobs). | | `vm_screenshot` | Screenshot of the XFCE desktop, downscaled to 1280 px (clicks are scaled back) | | `vm_click`, `vm_type`, `vm_key`, `vm_scroll` | Computer use via xdotool on `:1` | diff --git a/docs/sync.md b/docs/sync.md new file mode 100644 index 00000000..4908add9 --- /dev/null +++ b/docs/sync.md @@ -0,0 +1,67 @@ +# Pushing a checkout with `vm_sync` and `boxery sync` + +← [Docs](README.md) + +`vm_sync` makes a VM's copy of a git checkout match your local one exactly: the same commit, plus your staged, unstaged and untracked changes. Agents edit locally and run code in the VM; one sync before each run keeps the VM current, without giving the VM GitHub access or carrying patches by hand. It only pushes: nothing comes back from the VM, and there is no watch mode. + +```bash +boxery sync demo # the checkout you're in → demo:~/project +boxery sync demo --from ../api --to ~/api # another checkout → demo:~/api +boxery sync demo --json # the result as JSON +``` + +```json +{"vm":"demo"} +{"vm":"demo","from":"../api","to":"~/api"} +``` + +`from` is a local directory inside the checkout (default: the agent's working directory, or the CLI's current directory); the whole repository is synced from its root. `to` is the VM directory (default `~/project`; relative paths are under it). `boxery sync` runs `vm_sync`'s own code ([CLI](cli.md#files)). + +## What it does + +1. **Snapshot.** It records local HEAD plus every working change (what `git add -A` would stage: tracked files, and untracked files `.gitignore` doesn't exclude, deletions and renames included) as one commit. It uses a scratch index and object directory: your index, refs and stash are untouched. The commit has a fixed author and date, so the same working state always gives the same commit. +2. **Push.** The snapshot is `git push`ed straight into the VM repository, to a hidden ref (`refs/boxery/incoming/…`), over the VM's own ssh connection (the host profile's `ssh_config`, pinned host key). git works out which commits the VM lacks from all of its refs and streams only those into `git receive-pack`: no archive, no temporary file. The push reads none of your global or system git config and no `GIT_*` environment variables, so a `url.insteadOf` can't redirect it. No hooks run on either side. Nothing contacts a git remote, so the VM needs no GitHub grant. If the VM already has the snapshot, nothing is pushed. If `to` doesn't exist or is empty, a repository is initialized there first (after the checks pass), and removed again if the push fails. +3. **Swap.** In the VM, HEAD is detached at local HEAD's commit, and the working directory is switched to the snapshot: only the paths that differ are rewritten, and files the snapshot lacks are deleted, including ones an earlier sync put there. Ignored files (`node_modules`, build output) stay: there is no `git clean -x`. If the snapshot would put a tracked file where the VM has an ignored one, the sync refuses instead (below). The VM's index holds HEAD, so `git status` and `git diff` in the VM show the same working changes as they do locally. +4. **Check.** Both sides hash the whole working directory (tracked plus untracked, not ignored) as a git tree, the same way, and compare. The result says `tree matches`, or `TREE MISMATCH` (an error). + +```text +synced 3f2a9c1 + 2 modified, 1 untracked, 0 deleted → demo:/home/agent/project · tree matches +``` + +Structured fields: `sha` (local HEAD), `snapshot`, `counts` (`modified`, `untracked`, `deleted`), `tree` (`local`, `vm`, `match`), `lockfile_changed`, `pushed` (false when the VM already had the snapshot), `objects_sent`, `elapsed_ms`. + +`lockfile_changed` is true when `pnpm-lock.yaml`, `package-lock.json`, `yarn.lock` or `bun.lock` (at any depth) differs from the previous sync, or from the VM's HEAD on the first one. vm_sync installs nothing; run your install step when it says so. + +A second sync with no local changes sends nothing and still reports `tree matches`. + +## When it refuses + +It refuses, and changes nothing in the VM, when: + +- **The VM checkout has changes the last sync didn't make.** The VM records each sync in `refs/boxery/synced` (and local HEAD in `refs/boxery/base`). Any difference between the VM's working directory and that snapshot (edited, new or deleted files, not ignored ones) is listed in the error. Commit them in the VM and bring them back yourself, or discard them, then sync again. A VM repository that was never synced is compared with its HEAD. +- **The checkout holds a credential file.** Matched case-insensitively: `.env` and `.env.*`/`.env-*`/`.env_*` (except `.env.example`, `.env.sample`, `.env.template`, `.env.dist`, `.env.defaults`), `.envrc`, `.npmrc`, `.yarnrc.yml`, `.pypirc`, `.netrc`, `.git-credentials`, `.pgpass`, `.my.cnf`, `.terraformrc`, `auth.json`, `credentials`, `application_default_credentials.json`, private keys (`id_*` except `.pub`, `*.pem`, `*.key`, `*.p12`, `*.pfx`, `*.jks`, `*.keystore`), `.docker/config.json`, `.kube/config`, `.aws/config`, `gh/hosts.yml`, and anything inside boxery's own state or `~/.ssh`, `~/.gnupg`, `~/.pi` (what `vm_copy` refuses). The error names the path. Ignore it in `.gitignore` to sync the rest, and get the credential to the VM with a [grant](grants.md), which keeps it off the VM's disk and out of its snapshots. Ignored files are never synced, so an ignored `.env` is fine. +- **The commits it would send contain one.** The push carries every commit the VM lacks (on a first sync, the whole history), so a credential file committed once and deleted later (merges included) is refused too. Remove it from history first. Which commits the VM lacks is worked out from the refs the VM reports, so this check trusts the VM's report. +- **The local checkout now tracks a path the VM has as an ignored file** (a force-added build file, or `dist/` un-ignored and replaced by a file): the swap would overwrite or delete the VM's file. Or, with the new `.gitignore`, **files the VM has stop being ignored**, so the trees can't match. The error lists the paths; remove them in the VM (or change the local checkout), and sync again. +- **`to` is a non-empty directory that isn't a git repository**, is inside another checkout (`~/project/api` when `~/project` is a repository: use `~/api`), or its object format (sha1/sha256) differs. +- **The local path isn't a git checkout**, has no commits yet, or is a shallow or partial clone. Use `vm_copy` for plain files. + +"Nothing changed" means the checkout. A first sync into a missing directory can leave its parent directories behind. + +## Cancellation and failure + +Cancelling (an MCP cancellation, pi's Escape, Ctrl-C for `boxery sync`) before the push finishes leaves the VM unchanged: `git receive-pack` keeps incoming objects in a quarantine directory until the ref update succeeds, and removes it when the push dies. Once the VM starts applying, it ignores signals and finishes or rolls back; a lost connection doesn't interrupt it, and `boxery sync` waits for its result after Ctrl-C. + +The swap is staged: the pushed objects are already in the VM's object store (which changes no files); apply writes a pending record, switches the working directory, and finally updates HEAD and its refs in one transaction. If a step fails, it switches back and restores the index. If the VM crashes mid-swap, the next sync finds the pending record and finishes it (if the refs were written) or undoes it, before checking for VM-side changes. It undoes it only if every file is as the swap left it; a file edited since is reported, and the record stays. Pushed objects stay in the VM's repository either way; they're harmless and `git gc` collects them. + +A sync takes a lock in the VM repository (`flock`, or a lock directory without it), so two syncs into one checkout run one after the other. If the checkout changes between the check and the swap, the sync refuses: git checks every path before writing any. + +## Limits + +- Needs git (2.31+) on both sides; boxery's images have it. +- Submodules are synced as gitlinks (their commit ids), not their contents. +- Git LFS files arrive as pointer files: the VM doesn't download them, since vm_sync never contacts a remote. +- Untracked files are hashed into a temporary directory on the local machine, so a very large one needs that much space there. +- Ignore rules that exist only on your machine (`core.excludesFile`, `.git/info/exclude`) aren't sent: files the VM produces that only those rules ignore show up as VM changes, and the sync refuses. Put such patterns in `.gitignore`. +- Executable bits and symlinks follow git's rules; other file modes, owners and timestamps aren't carried. +- The VM keeps every synced snapshot's objects until `git gc`. +- On an HTTP MCP server, `vm_sync` reads checkouts on that server, so it follows `vm_copy`'s `local:` rule: refused without `--copy-root`, and confined to `DIR` with it ([MCP server](mcp.md#b-streamable-http-on-the-host-over-tailscale)). Confined, the repository's git directory, object store and alternates must be inside `DIR` too, git doesn't look for a repository above `DIR`, and it runs none of the checkout's hooks, fsmonitor or filter drivers. diff --git a/extension/index.ts b/extension/index.ts index ff0b3f4d..ee758359 100644 --- a/extension/index.ts +++ b/extension/index.ts @@ -4,7 +4,7 @@ * Built-in tools (bash, read, write, edit, …) are NOT touched: they keep acting on this machine, so * other extensions that wrap them keep working. VM work goes through clearly named vm_* tools: * - * vm_bash vm_read vm_write vm_edit vm_copy files and shell in a VM + * vm_bash vm_read vm_write vm_edit vm_copy vm_sync files and shell in a VM * vm_jobs background command jobs (start / list / logs / wait / kill) * vm_screenshot vm_click vm_type vm_key vm_scroll computer use on the VM desktop * vm_browser Playwright → Chrome on the VM desktop diff --git a/mcp/boxery-mcp.mjs b/mcp/boxery-mcp.mjs index 377ebbd3..0a195a25 100755 --- a/mcp/boxery-mcp.mjs +++ b/mcp/boxery-mcp.mjs @@ -7,10 +7,10 @@ // auth: bearer token from $BOXERY_MCP_TOKEN or --token-file (required unless --no-auth, loopback only), // optional --allow-user a@b.c (checks Tailscale-User-Login from `tailscale serve`) // --ui offer vm_view (live desktop view) even to clients that don't advertise MCP Apps support -// --copy-root DIR allow vm_copy local: paths only inside DIR (never boxery's config, ~/.ssh, ~/.pi) +// --copy-root DIR allow vm_copy local: paths and vm_sync checkouts only inside DIR (never boxery's config, ~/.ssh, ~/.pi) // // "local" (vm_copy) and grant sources are the machine this server runs on. Over HTTP that is the -// control plane, so vm_copy local: endpoints are refused unless --copy-root confines them. +// control plane, so vm_copy local: endpoints and vm_sync are refused unless --copy-root confines them. import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; import { readFileSync } from "node:fs"; import http from "node:http"; @@ -79,9 +79,19 @@ const LOCAL_COPY_NOTE = // ---------------------------------------------------------------- tools // Built from the shared table (core/boxery-tools.mjs), host selection included. MCP adds only -// vm_copy's note on what `local:` means for this server, and the live desktop view below. +// vm_copy's and vm_sync's notes on what `local:` means for this server, and the live desktop view below. +// vm_sync reads a local checkout, so the same local: policy applies to it (core/sync.mjs). +const LOCAL_SYNC_NOTE = + LOCAL_COPY === false + ? "On this server vm_sync is disabled: it runs on the boxery control plane, not your machine." + : LOCAL_COPY + ? `On this server \`from\` is a checkout on the machine it runs on, confined to ${COPY_ROOT} (relative paths resolve there).` + : null; /** @type {Record Tool>} */ -const MCP_TOOL = { vm_copy: (t) => (LOCAL_COPY_NOTE ? { ...t, description: `${t.description} ${LOCAL_COPY_NOTE}` } : t) }; +const MCP_TOOL = { + vm_copy: (t) => (LOCAL_COPY_NOTE ? { ...t, description: `${t.description} ${LOCAL_COPY_NOTE}` } : t), + vm_sync: (t) => (LOCAL_SYNC_NOTE ? { ...t, description: `${t.description} ${LOCAL_SYNC_NOTE}` } : t), +}; /** @type {McpTool[]} */ const TOOLS = SHARED_TOOLS.map((t) => MCP_TOOL[t.name]?.(t) ?? t).map((t) => ({ name: t.name, @@ -226,7 +236,7 @@ const instructions = (/** @type {Session} */ s) => host: s.host.name, localToolsNote: "Your other tools (shell, files) act on this machine, not a VM; use them only when you mean to work here.", - }) + (LOCAL_COPY_NOTE ? `\nvm_copy: ${LOCAL_COPY_NOTE}` : ""); + }) + (LOCAL_COPY_NOTE ? `\nvm_copy: ${LOCAL_COPY_NOTE}` : "") + (LOCAL_SYNC_NOTE ? `\nvm_sync: ${LOCAL_SYNC_NOTE}` : ""); /** Send the client a request on this call's channel and resolve with its result. Abort sends notifications/cancelled. * @param {Session} session @param {Notify} send @param {string} method @param {object} params @param {AbortSignal} [signal] @returns {Promise} */ diff --git a/test/fixtures/boxery-help-all.txt b/test/fixtures/boxery-help-all.txt index 16ae7de0..54cc52be 100644 --- a/test/fixtures/boxery-help-all.txt +++ b/test/fixtures/boxery-help-all.txt @@ -21,6 +21,7 @@ Work in a VM: ssh NAME[@HOST] [CMD...] Shell into a VM (also: ssh NAME.boxery) run NAME CMD [ARG...] Run a command in ~/project, args as given (like docker exec) cp FROM TO Copy files and directories to, from and between VMs + sync NAME Push a local git checkout into a VM, so its tree matches exactly desktop NAME Open a VM's desktop in your browser (noVNC over ssh) forward NAME PORT [LOCAL] Forward a VM port to localhost attach NAME [AGENT] Attach to an agent in a persistent tmux session (= agent attach) diff --git a/test/fixtures/boxery-help.txt b/test/fixtures/boxery-help.txt index da6f1737..f3091a1c 100644 --- a/test/fixtures/boxery-help.txt +++ b/test/fixtures/boxery-help.txt @@ -21,6 +21,7 @@ Work in a VM: ssh NAME[@HOST] [CMD...] Shell into a VM (also: ssh NAME.boxery) run NAME CMD [ARG...] Run a command in ~/project, args as given (like docker exec) cp FROM TO Copy files and directories to, from and between VMs + sync NAME Push a local git checkout into a VM, so its tree matches exactly desktop NAME Open a VM's desktop in your browser (noVNC over ssh) forward NAME PORT [LOCAL] Forward a VM port to localhost attach NAME [AGENT] Attach to an agent in a persistent tmux session (= agent attach) diff --git a/test/fixtures/copy-ssh.mjs b/test/fixtures/copy-ssh.mjs index d7514470..ecf70e37 100644 --- a/test/fixtures/copy-ssh.mjs +++ b/test/fixtures/copy-ssh.mjs @@ -6,6 +6,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; const HELPER = fileURLToPath(new URL("./copy-helper.mjs", import.meta.url)); const args = process.argv.slice(2); +const argv = [...args]; let conf, vm; while (args.length > 1) { const flag = /** @type {string} */ (args.shift()); @@ -18,7 +19,7 @@ if (!["alpha", "beta", "gamma"].includes(host) || !["vm", "other"].includes(/** const endpoint = `${host}/${vm}`; const root = path.join(/** @type {string} */ (process.env.COPY_ROOT), "vms", host, /** @type {string} */ (vm)); const script = args[0].replaceAll("/home/agent", root).replaceAll("/tmp/.boxery-copy-", `${root}/tmp/.boxery-copy-`); -appendFileSync(/** @type {string} */ (process.env.COPY_LOG), JSON.stringify({ kind: "ssh", endpoint, script, pid: process.pid }) + "\n"); +appendFileSync(/** @type {string} */ (process.env.COPY_LOG), JSON.stringify({ kind: "ssh", endpoint, script, pid: process.pid, argv }) + "\n"); const cleanup = script.includes('for record in "$t"/pgid.') && !script.includes("setsid -w"); if (process.env.COPY_FAIL_CLEANUP === endpoint && cleanup) { process.stderr.write("injected cleanup connection failure\n"); process.exit(255); @@ -35,7 +36,9 @@ if (preparing && process.env.COPY_LATE_PREPARE === endpoint) { } if (preparing && process.env.COPY_SLOW_PREPARE === endpoint) await new Promise((r) => setTimeout(r, 200)); /** @type {NodeJS.ProcessEnv} */ -const env = { ...process.env, COPY_ENDPOINT: endpoint, HOME: root }; +// A real sshd starts the remote command with a fresh environment; git push's ssh inherits git's +// exec-path at the front of PATH, which FAKE_VM_PATH (when set) undoes. +const env = { ...process.env, COPY_ENDPOINT: endpoint, HOME: root, ...(process.env.FAKE_VM_PATH ? { PATH: process.env.FAKE_VM_PATH } : {}) }; if (process.env.COPY_SKIP_EXIT_SNAPSHOT === endpoint) env.BASH_ENV = path.join(/** @type {string} */ (process.env.COPY_ROOT), "no-exit-trap.sh"); const child = spawn("bash", ["-c", script], { env, cwd: root, stdio: "inherit" }); child.on("error", () => { process.exitCode = 255; }); diff --git a/test/fixtures/sync-git.mjs b/test/fixtures/sync-git.mjs new file mode 100644 index 00000000..3cdd04ca --- /dev/null +++ b/test/fixtures/sync-git.mjs @@ -0,0 +1,17 @@ +#!/usr/bin/env node +// git double for test/sync.test.mjs: runs the real git (REAL_GIT), except that with SYNC_SLOW_RECEIVE set, +// receive-pack (the VM end of git push) reads its input slowly and marks that it started, so a test can +// cancel a push mid-transfer. +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { Transform } from "node:stream"; +const args = process.argv.slice(2); +const slow = process.env.SYNC_SLOW_RECEIVE && args.includes("receive-pack"); +const child = spawn(/** @type {string} */ (process.env.REAL_GIT), args, { stdio: [slow ? "pipe" : "inherit", "inherit", "inherit"] }); +if (slow) { + writeFileSync(/** @type {string} */ (process.env.SYNC_SLOW_RECEIVE), String(process.pid)); + const throttle = new Transform({ transform(chunk, _e, cb) { setTimeout(() => cb(null, chunk), 20); } }); + process.stdin.pipe(throttle).pipe(/** @type {import("node:stream").Writable} */ (child.stdin)); +} +for (const sig of /** @type {const} */ (["SIGTERM", "SIGINT", "SIGHUP"])) process.on(sig, () => child.kill(sig)); +child.on("close", (code, signal) => { process.exitCode = code ?? (signal ? 1 : 0); }); diff --git a/test/fixtures/sync-worker.mjs b/test/fixtures/sync-worker.mjs new file mode 100644 index 00000000..cc74e093 --- /dev/null +++ b/test/fixtures/sync-worker.mjs @@ -0,0 +1,18 @@ +// One vm_sync call in its own process (core reads BOXERY_HOME at import), against the fake SSH endpoints +// of test/fixtures/copy-ssh.mjs. argv: VM, then JSON {from, to, local?}. IPC "abort" cancels. +import { withHost } from "../../core/boxery-core.mjs"; +import { syncTool } from "../../core/sync.mjs"; +const [vm, settings = "{}"] = process.argv.slice(2); +const { local, ...params } = JSON.parse(settings); +const ac = new AbortController(); +process.on("message", (m) => { if (m === "abort") ac.abort(); }); +process.once("disconnect", () => ac.abort()); +/** @type {any} */ +let result; +try { + result = { ok: true, ...(await withHost("alpha", () => syncTool(vm, params, { signal: ac.signal, local }))) }; +} catch (e) { + result = { ok: false, error: /** @type {Error} */ (e).message, name: /** @type {Error} */ (e).name }; +} +process.stdout.write(`${JSON.stringify(result)}\n`); +if (process.connected) process.disconnect(); diff --git a/test/lint.sh b/test/lint.sh index 743b75bc..71aface7 100755 --- a/test/lint.sh +++ b/test/lint.sh @@ -2,7 +2,7 @@ # Static checks: shellcheck (warnings are errors), node syntax checks. set -euo pipefail cd "$(dirname "$0")/.." -shellcheck -S warning controller/boxery test/fixtures/bash-profile.sh test/fixtures/ssh-config-golden.sh core/boxery-job-runner.sh host/boxery-host host/backend.sh host/backends/*.sh host/backends/lib/*.sh image/layers/*.sh e2e.sh mcp/test-mcp.sh test/shell/*.sh test/shell/*/*.sh test/bench/*.sh +shellcheck -S warning controller/boxery test/fixtures/bash-profile.sh test/fixtures/ssh-config-golden.sh core/boxery-job-runner.sh core/sync-guest.sh host/boxery-host host/backend.sh host/backends/*.sh host/backends/lib/*.sh image/layers/*.sh e2e.sh mcp/test-mcp.sh test/shell/*.sh test/shell/*/*.sh test/bench/*.sh for f in controller/*.mjs core/*.mjs backends/*.mjs host/backends/lib/*.mjs guests/*.mjs help/*.mjs host/*.mjs mcp/*.mjs proxy/*.mjs test/*.mjs test/fixtures/*.mjs evals/*.mjs evals/lib/*.mjs scripts/*.mjs; do node --check "$f"; done # VM-side TypeScript runs under type stripping: it must parse (types are checked by `npm run typecheck`) # (only where this Node can strip types: 22.18+/23.6+; older Node skips it, like the job tests) diff --git a/test/parity.test.mjs b/test/parity.test.mjs index b4fd6c5e..c91a900a 100644 --- a/test/parity.test.mjs +++ b/test/parity.test.mjs @@ -19,6 +19,7 @@ const CLI_FOR_TOOL = { vm_write: "cp", vm_edit: none("people edit in their editor, over boxery ssh or after boxery cp"), vm_copy: "cp", + vm_sync: "sync", vm_screenshot: DESKTOP, vm_click: DESKTOP, vm_type: DESKTOP, diff --git a/test/sync.test.mjs b/test/sync.test.mjs new file mode 100644 index 00000000..850b38e3 --- /dev/null +++ b/test/sync.test.mjs @@ -0,0 +1,547 @@ +// vm_sync with real git and tar against temporary fake SSH endpoints (test/fixtures/copy-ssh.mjs): each +// "VM" is a directory, and its ssh runs scripts there with /home/agent mapped to it. No real VMs. +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, unlinkSync, utimesSync, writeFileSync } from "node:fs"; +import { randomBytes } from "node:crypto"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, test as nodeTest } from "node:test"; +import { q } from "../core/shell.mjs"; +import { deniedSyncPath } from "../core/sync.mjs"; +const test = (/** @type {string} */ name, /** @type {() => Promise | void} */ fn) => nodeTest(name, { timeout: 60_000 }, fn); + +const FIXTURES = path.resolve(import.meta.dirname, "fixtures"); +/** @type {string} */ let dir; +/** @type {NodeJS.ProcessEnv} */ let env; +/** @type {string} */ let repo; +const vmProject = () => path.join(dir, "vms", "alpha", "vm", "project"); +const GIT_ENV = { GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t", GIT_CONFIG_NOSYSTEM: "1" }; +const git = (/** @type {string} */ cwd, /** @type {string[]} */ ...args) => execFileSync("git", args, { cwd, env, encoding: "utf8" }).trim(); +const write = (/** @type {string} */ root, /** @type {string} */ rel, /** @type {string | Buffer} */ data) => { + mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); + writeFileSync(path.join(root, rel), data); +}; +const read = (/** @type {string} */ root, /** @type {string} */ rel) => readFileSync(path.join(root, rel), "utf8"); + +/** Run vm_sync in a worker; abortWhen(dir) → abort once it returns true. @param {object} [params] @param {{ abortWhen?: () => boolean, vm?: string }} [opts] */ +function sync(params = {}, { abortWhen, vm = "vm" } = {}) { + const child = spawn(process.execPath, [path.join(FIXTURES, "sync-worker.mjs"), vm, JSON.stringify({ from: repo, ...params })], { + env, cwd: dir, stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + let out = "", err = ""; + child.stdout?.on("data", (d) => { out += d; }); + child.stderr?.on("data", (d) => { err += d; }); + let timer; + if (abortWhen) timer = setInterval(() => { if (abortWhen() && child.connected) { child.send("abort"); clearInterval(timer); } }, 5); + return new Promise((resolve, reject) => { + child.on("close", () => { + clearInterval(timer); + try { resolve(JSON.parse(out)); } catch { reject(new Error(`sync worker produced no result: ${err} ${out}`)); } + }); + }); +} +async function ok(params, opts) { + const r = /** @type {any} */ (await sync(params, opts)); + assert.equal(r.ok, true, r.error); + return r; +} +/** Every tracked and untracked (not ignored) file with its content, relative to root. @param {string} root */ +function tree(root) { + const files = git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0").filter(Boolean) + .filter((f) => existsSync(path.join(root, f))); + return Object.fromEntries([...new Set(files)].sort().map((f) => [f, readFileSync(path.join(root, f)).toString("base64")])); +} + +beforeEach(() => { + // Match the canonical --copy-root used by servers (macOS's temp directory is under /var → /private/var). + dir = realpathSync(mkdtempSync(path.join(tmpdir(), "boxery-sync-"))); + mkdirSync(path.join(dir, "bin")); + const home = path.join(dir, "home"); + mkdirSync(path.join(home, "host-registry"), { recursive: true }); + const profile = path.join(home, "profiles", "alpha"); + mkdirSync(profile, { recursive: true }); + writeFileSync(path.join(profile, "config"), "# fake\n"); + writeFileSync(path.join(profile, "ssh_config"), "# fake\n"); + writeFileSync(path.join(home, "host-registry", "alpha"), `${profile}\n`); + writeFileSync(path.join(home, "default-host"), "alpha\n"); + for (const vm of ["vm", "other"]) mkdirSync(path.join(dir, "vms", "alpha", vm, "tmp"), { recursive: true }); + writeFileSync(path.join(dir, "calls"), ""); + writeFileSync(path.join(dir, "bin", "ssh"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "copy-ssh.mjs"))} "$@"\n`, { mode: 0o755 }); + writeFileSync(path.join(dir, "bin", "tar"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "copy-tar.mjs"))} "$@"\n`, { mode: 0o755 }); + // sync deliberately strips caller GIT_* overrides; isolate system config at the executable boundary. + writeFileSync(path.join(dir, "bin", "git"), `#!/usr/bin/env bash\nexport GIT_CONFIG_NOSYSTEM=1\nexec ${q(execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim())} "$@"\n`, { mode: 0o755 }); + writeFileSync(path.join(dir, "bin", "setsid"), "#!/usr/bin/env bash\n[ \"$1\" != -w ] || shift\nexec perl -e 'setpgrp(0,0); exec @ARGV or die' -- \"$@\"\n", { mode: 0o755 }); + env = { + ...process.env, ...GIT_ENV, HOME: home, XDG_CONFIG_HOME: path.join(home, ".config"), PATH: `${path.join(dir, "bin")}:${process.env.PATH}`, BOXERY_HOME: home, + COPY_ROOT: dir, COPY_LOG: path.join(dir, "calls"), REAL_TAR: execFileSync("sh", ["-c", "command -v tar"], { encoding: "utf8" }).trim(), + REAL_GIT: execFileSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).trim(), + }; + repo = path.join(dir, "local", "repo"); + mkdirSync(repo, { recursive: true }); + git(repo, "init", "-q", "-b", "main"); + write(repo, ".gitignore", "node_modules/\ndist/\n"); + write(repo, "README.md", "hello\n"); + write(repo, "src/app.js", "console.log(1);\n"); + write(repo, "src/old name.txt", "rename me\n"); + write(repo, "gone.txt", "delete me\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "first"); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +test("first sync into an empty directory: a detached checkout that matches", async () => { + const r = await ok(); + const head = git(repo, "rev-parse", "HEAD"); + assert.match(r.text, new RegExp(`^synced ${head.slice(0, 7)} \\+ 0 modified, 0 untracked, 0 deleted → vm:/home/agent/project · tree matches$`)); + assert.equal(r.structured.sha, head); + assert.equal(r.structured.tree.match, true); + assert.equal(r.structured.tree.local, r.structured.tree.vm); + assert.equal(r.structured.pushed, true); + assert.ok(r.structured.objects_sent > 0); + assert.ok(readFileSync(path.join(dir, "calls"), "utf8").includes("receive-pack"), "commits went by git push"); + assert.equal(git(vmProject(), "for-each-ref", "refs/boxery/incoming/"), "", "the pushed ref is dropped after apply"); + assert.equal(typeof r.structured.elapsed_ms, "number"); + assert.equal(git(vmProject(), "rev-parse", "HEAD"), head); + assert.throws(() => git(vmProject(), "symbolic-ref", "-q", "HEAD"), "detached"); + assert.deepEqual(tree(vmProject()), tree(repo)); + assert.equal(git(vmProject(), "status", "--porcelain"), "", "a clean checkout in the VM"); + assert.equal(git(vmProject(), "remote"), "", "no remote configured"); +}); + +test("new commits, dirty, untracked, deleted and renamed files, a binary and spaces in names", async () => { + await ok(); + write(repo, "src/app.js", "console.log(2);\n"); + git(repo, "commit", "-qam", "second"); + write(repo, "README.md", "hello, changed\n"); // unstaged + write(repo, "staged.txt", "staged\n"); + git(repo, "add", "staged.txt"); // staged, new + write(repo, "notes with spaces.md", "untracked\n"); + write(repo, "bin/blob.bin", Buffer.from([0, 255, 1, 254, 0, 10, 13, 0])); + unlinkSync(path.join(repo, "gone.txt")); + renameSync(path.join(repo, "src/old name.txt"), path.join(repo, "src/new name.txt")); + const r = await ok(); + assert.equal(r.structured.sha, git(repo, "rev-parse", "HEAD")); + assert.equal(git(vmProject(), "rev-parse", "HEAD"), r.structured.sha); + assert.equal(r.structured.tree.match, true); + assert.deepEqual(r.structured.counts, { modified: 3, untracked: 2, deleted: 1 }, "README, staged.txt, the rename; two untracked; gone.txt"); + assert.deepEqual(tree(vmProject()), tree(repo)); + assert.deepEqual(readFileSync(path.join(vmProject(), "bin/blob.bin")), Buffer.from([0, 255, 1, 254, 0, 10, 13, 0])); + assert.ok(!existsSync(path.join(vmProject(), "gone.txt"))); + assert.ok(!existsSync(path.join(vmProject(), "src/old name.txt"))); + // The VM shows the same working changes against HEAD as the local checkout. + assert.match(git(vmProject(), "status", "--porcelain"), /notes with spaces/); + assert.equal(git(repo, "status", "--porcelain").length > 0, true, "the local checkout is untouched"); + assert.equal(git(repo, "for-each-ref").includes("boxery"), false, "no refs written locally"); + + // Reverting local changes removes the earlier sync's leftovers. + git(repo, "checkout", "-q", "--", "."); + git(repo, "reset", "-q", "--hard"); + git(repo, "clean", "-qfd"); + const r2 = await ok(); + assert.deepEqual(r2.structured.counts, { modified: 0, untracked: 0, deleted: 0 }); + assert.deepEqual(tree(vmProject()), tree(repo)); + assert.ok(!existsSync(path.join(vmProject(), "notes with spaces.md"))); +}); + +test("ignored files in the VM survive a sync", async () => { + await ok(); + write(vmProject(), "node_modules/dep/index.js", "module.exports = 1;\n"); + write(vmProject(), "dist/out.js", "built\n"); + write(repo, "src/app.js", "console.log(3);\n"); + const r = await ok(); + assert.equal(r.structured.tree.match, true); + assert.equal(read(vmProject(), "node_modules/dep/index.js"), "module.exports = 1;\n"); + assert.equal(read(vmProject(), "dist/out.js"), "built\n"); + assert.equal(read(vmProject(), "src/app.js"), "console.log(3);\n"); +}); + +test("a VM-side edit blocks the sync and changes nothing", async () => { + await ok(); + write(vmProject(), "src/app.js", "edited in the VM\n"); + write(vmProject(), "scratch.txt", "new in the VM\n"); + write(repo, "README.md", "local change\n"); + const before = tree(vmProject()); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /refused: vm:\/home\/agent\/project has changes since the last sync that vm_sync didn't make/); + assert.match(r.error, /M src\/app\.js/); + assert.match(r.error, /A scratch\.txt/); + assert.deepEqual(tree(vmProject()), before); + assert.equal(read(vmProject(), "README.md"), "hello\n"); +}); + +test("a dirty VM checkout that was never synced blocks the sync", async () => { + mkdirSync(vmProject(), { recursive: true }); + git(vmProject(), "init", "-q"); + write(vmProject(), "mine.txt", "x\n"); + git(vmProject(), "add", "-A"); + git(vmProject(), "commit", "-qm", "vm"); + write(vmProject(), "mine.txt", "dirty\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /changes against its HEAD/); + assert.equal(read(vmProject(), "mine.txt"), "dirty\n"); +}); + +test("a non-empty directory that isn't a repository is refused", async () => { + write(vmProject(), "stray.txt", "x\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /is not a git repository and is not empty \(stray\.txt/); + assert.ok(!existsSync(path.join(vmProject(), ".git"))); +}); + +test("deny-listed files block the sync, naming the path and vm_grant", async () => { + await ok(); + write(repo, "config/.env.local", "SECRET=1\n"); + write(repo, "README.md", "changed\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /config\/\.env\.local/); + assert.match(r.error, /vm_grant/); + assert.equal(read(vmProject(), "README.md"), "hello\n", "nothing changed in the VM"); + // Once ignored, the rest syncs. + write(repo, ".gitignore", "node_modules/\ndist/\n.env*\n"); + assert.equal((await ok()).structured.tree.match, true); + assert.ok(!existsSync(path.join(vmProject(), "config/.env.local"))); +}); + +test("the deny list", () => { + for (const p of [".env", ".env.local", "a/.env.production", ".envrc", ".npmrc", "x/.pypirc", ".netrc", ".git-credentials", "auth.json", ".docker/config.json", "home/.kube/config", "credentials", "id_ed25519"]) + assert.ok(deniedSyncPath(p), p); + for (const p of [".ENV", ".env-prod", ".env_local", "ID_RSA", "id_rsa_work", "Credentials.json", ".yarnrc.yml", "certs/server.pem", "tls.key", "store.p12", ".terraformrc"]) + assert.ok(deniedSyncPath(p), p); + for (const p of [".env.example", ".env.sample", ".env-template", "environment.ts", "src/auth.js", "npmrc.md", "docs/credentials.md", "id_ed25519.pub", "ID_RSA.PUB", "keys.ts"]) + assert.ok(!deniedSyncPath(p), p); +}); + +test("a second sync with no local changes transfers nothing and still matches", async () => { + write(repo, "wip.txt", "uncommitted\n"); + const first = await ok(); + const calls = readFileSync(path.join(dir, "calls"), "utf8").split("\n").filter(Boolean).length; + const second = await ok(); + assert.equal(second.structured.pushed, false); + assert.equal(second.structured.objects_sent, 0); + assert.equal(second.structured.tree.match, true); + assert.equal(second.structured.lockfile_changed, false); + assert.equal(second.structured.snapshot, first.structured.snapshot, "the same working changes give the same snapshot commit"); + assert.match(second.text, /· tree matches$/); + const pushes = readFileSync(path.join(dir, "calls"), "utf8").split("\n").filter(Boolean).slice(calls).map((l) => JSON.parse(l)).filter((c) => String(c.script).includes("receive-pack")); + assert.deepEqual(pushes, [], "no push the second time"); +}); + +test("a lockfile change is flagged, once", async () => { + write(repo, "package-lock.json", "{\"v\":1}\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "lock"); + assert.equal((await ok()).structured.lockfile_changed, true, "new on the first sync"); + assert.equal((await ok()).structured.lockfile_changed, false); + write(repo, "packages/web/pnpm-lock.yaml", "lockfileVersion: 9\n"); // untracked, nested + const r = await ok(); + assert.equal(r.structured.lockfile_changed, true); + assert.match(r.text, /lockfile changed/); + write(repo, "README.md", "unrelated\n"); + assert.equal((await ok()).structured.lockfile_changed, false); +}); + +test("cancellation mid-push leaves the VM unchanged", async () => { + await ok(); + const before = tree(vmProject()); + const head = git(vmProject(), "rev-parse", "HEAD"); + const refs = git(vmProject(), "for-each-ref"); + write(repo, "big.bin", randomBytes(4 * 1024 * 1024)); + write(repo, "README.md", "changed\n"); + const started = path.join(dir, "receive-started"); + // Only here: a git double whose receive-pack (the VM end of the push) is slow. + writeFileSync(path.join(dir, "bin", "git"), `#!/usr/bin/env bash\nexec ${q(process.execPath)} ${q(path.join(FIXTURES, "sync-git.mjs"))} "$@"\n`, { mode: 0o755 }); + env.SYNC_SLOW_RECEIVE = started; + env.FAKE_VM_PATH = env.PATH; + const r = /** @type {any} */ (await sync({}, { abortWhen: () => existsSync(started) })); + delete env.SYNC_SLOW_RECEIVE; + delete env.FAKE_VM_PATH; + unlinkSync(path.join(dir, "bin", "git")); + assert.ok(existsSync(started), `the push was in flight (${r.error ?? "it finished"})`); + assert.equal(r.ok, false); + assert.equal(r.name, "AbortError"); + assert.deepEqual(tree(vmProject()), before); + assert.equal(git(vmProject(), "rev-parse", "HEAD"), head); + assert.equal(git(vmProject(), "for-each-ref"), refs, "no ref was written"); + // receive-pack quarantines incoming objects and removes them when the push dies. + const objects = path.join(vmProject(), ".git", "objects"); + for (let i = 0; i < 100 && readdirSync(objects).some((n) => n.startsWith("tmp_objdir")); i++) await new Promise((res) => setTimeout(res, 50)); + assert.deepEqual(readdirSync(objects).filter((n) => n.startsWith("tmp_objdir")), [], "no quarantined objects left"); + assert.equal((await ok()).structured.tree.match, true, "a later sync completes"); +}); + +test("a push into a missing directory creates the repository only there", async () => { + const r = await ok({ to: "~/fresh/repo" }); + assert.equal(r.structured.tree.match, true); + const vmHome = path.join(dir, "vms", "alpha", "vm"); + assert.deepEqual(tree(path.join(vmHome, "fresh", "repo")), tree(repo)); +}); + +test("an interrupted swap is undone by the next sync", async () => { + await ok(); + write(repo, "README.md", "next\n"); + // Simulate a crash mid-swap: the pending record is there, the working directory half updated. + const g = path.join(vmProject(), ".git"); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced^{tree}"); + write(vmProject(), "README.md", "half written\n"); + const half = git(vmProject(), "rev-parse", `${git(vmProject(), "stash", "create")}^{tree}`); + writeFileSync(path.join(g, "boxery-sync-pending"), `${synced} ${half}\n`); + writeFileSync(path.join(g, "boxery-sync-index.bak"), readFileSync(path.join(g, "index"))); + const r = await ok(); + assert.equal(r.structured.tree.match, true); + assert.equal(read(vmProject(), "README.md"), "next\n"); + assert.ok(!existsSync(path.join(g, "boxery-sync-pending"))); +}); + +test("a local path that isn't a git checkout gives a clear error", async () => { + const plain = path.join(dir, "plain"); + mkdirSync(plain); + const r = /** @type {any} */ (await sync({ from: plain })); + assert.equal(r.ok, false); + assert.match(r.error, /is not a git checkout .*use vm_copy for other files/); + assert.ok(!existsSync(vmProject()), "the VM was not touched"); +}); + +test("a subdirectory syncs its whole repository; to picks the VM directory", async () => { + const r = await ok({ from: path.join(repo, "src"), to: "elsewhere/repo" }); + assert.equal(r.structured.from, repo); + assert.equal(r.structured.to, "/home/agent/project/elsewhere/repo"); + assert.deepEqual(tree(path.join(vmProject(), "elsewhere/repo")), tree(repo)); +}); + +test("local: refused or confined like vm_copy", async () => { + const refused = /** @type {any} */ (await sync({ local: false })); + assert.equal(refused.ok, false); + assert.match(refused.error, /vm_sync is disabled on this server/); + const outside = /** @type {any} */ (await sync({ local: { root: path.join(dir, "vms") } })); + assert.equal(outside.ok, false); + assert.match(outside.error, /outside the copy root/); + const inside = await ok({ from: "repo", local: { root: path.join(dir, "local") } }); + assert.equal(inside.structured.tree.match, true); +}); + +test("a credential in history the VM lacks blocks the sync, even when it's gone from the tree", async () => { + write(repo, ".env", "SECRET=1\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "oops"); + git(repo, "rm", "-q", ".env"); + git(repo, "commit", "-qm", "remove it"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /\.env is in the history this sync would send/); + assert.ok(!existsSync(vmProject()), "nothing sent"); +}); + +test("a VM edit made after an interrupted swap isn't lost", async () => { + await ok(); + write(repo, "README.md", "next\n"); + const g = path.join(vmProject(), ".git"); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced^{tree}"); + write(vmProject(), "README.md", "half written\n"); + const half = git(vmProject(), "rev-parse", `${git(vmProject(), "stash", "create")}^{tree}`); + writeFileSync(path.join(g, "boxery-sync-pending"), `${synced} ${half}\n`); + write(vmProject(), "src/app.js", "edited in the VM after the crash\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /interrupted earlier sync .* src\/app\.js/s); + assert.equal(read(vmProject(), "src/app.js"), "edited in the VM after the crash\n"); + assert.ok(existsSync(path.join(g, "boxery-sync-pending")), "the record stays for the next attempt"); +}); + +test("an interrupted swap whose refs were written is finished, not undone", async () => { + await ok(); + const g = path.join(vmProject(), ".git"); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced^{tree}"); + writeFileSync(path.join(g, "boxery-sync-pending"), `${"0".repeat(40)} ${synced}\n`); + const r = await ok(); + assert.equal(r.structured.tree.match, true); + assert.ok(!existsSync(path.join(g, "boxery-sync-pending"))); +}); + +test("a path the VM has as an ignored file that local now tracks is refused, not overwritten", async () => { + await ok(); + const synced = git(vmProject(), "rev-parse", "refs/boxery/synced"); + write(vmProject(), "dist/out.js", "the VM's own build\n"); + write(repo, "dist/out.js", "force-added locally\n"); + git(repo, "add", "-f", "dist/out.js"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /tracks paths that exist in the VM as ignored files .*dist\/out\.js/s); + assert.equal(read(vmProject(), "dist/out.js"), "the VM's own build\n"); + assert.equal(git(vmProject(), "rev-parse", "refs/boxery/synced"), synced, "nothing recorded"); + assert.throws(() => git(vmProject(), "rev-parse", "-q", "--verify", "refs/boxery/incoming"), "fetched ref dropped"); +}); + +test("an ignored VM directory replaced by a tracked file is refused", async () => { + await ok(); + write(vmProject(), "dist/keep.js", "built\n"); + write(repo, ".gitignore", "node_modules/\n"); + write(repo, "dist", "now a file\n"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /dist/); + assert.equal(read(vmProject(), "dist/keep.js"), "built\n"); +}); + +test("un-ignoring a path with VM files refuses cleanly, and the next sync isn't stuck", async () => { + await ok(); + write(vmProject(), "dist/out.js", "built\n"); + write(repo, ".gitignore", "node_modules/\n"); + const before = git(vmProject(), "rev-parse", "refs/boxery/synced"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /ignored no longer are: .*dist\/out\.js/s); + assert.equal(git(vmProject(), "rev-parse", "refs/boxery/synced"), before, "nothing recorded"); + assert.equal(read(vmProject(), ".gitignore"), "node_modules/\ndist/\n", "rolled back"); + rmSync(path.join(vmProject(), "dist"), { recursive: true }); + assert.equal((await ok()).structured.tree.match, true); +}); + +test("a VM directory inside another checkout is refused", async () => { + await ok(); + const r = /** @type {any} */ (await sync({ to: "api" })); + assert.equal(r.ok, false); + assert.match(r.error, /is inside the git checkout \S*project/); + assert.ok(!existsSync(path.join(vmProject(), "api"))); +}); + +test("a shallow clone gives a clear error", async () => { + git(repo, "commit", "-q", "--allow-empty", "-m", "second"); + const shallow = path.join(dir, "local", "shallow"); + git(dir, "clone", "-q", "--depth", "1", `file://${repo}`, shallow); + const r = /** @type {any} */ (await sync({ from: shallow })); + assert.equal(r.ok, false); + assert.match(r.error, /shallow clone/); +}); + +test("confined: no repository outside the root through a gitfile, and none of its config runs", async () => { + const root = path.join(dir, "share"); + const outside = path.join(dir, "secret-repo"); + mkdirSync(outside); + git(outside, "init", "-q"); + write(outside, "private.txt", "private\n"); + git(outside, "add", "-A"); + git(outside, "commit", "-qm", "private"); + const marker = path.join(dir, "fsmonitor-ran"); + git(outside, "config", "core.fsmonitor", `touch ${marker}; false`); + mkdirSync(path.join(root, "checkout"), { recursive: true }); + writeFileSync(path.join(root, "checkout", ".git"), `gitdir: ${path.join(outside, ".git")}\n`); + const r = /** @type {any} */ (await sync({ from: "checkout", local: { root } })); + assert.equal(r.ok, false); + assert.match(r.error, /outside the copy root/); + assert.ok(!existsSync(marker), "fsmonitor never ran"); + assert.ok(!existsSync(vmProject())); + + // A repository above the root isn't discovered from inside it either. + git(dir, "init", "-q"); + git(dir, "config", "core.fsmonitor", `touch ${marker}; false`); + mkdirSync(path.join(root, "plain")); + const above = /** @type {any} */ (await sync({ from: "plain", local: { root } })); + assert.equal(above.ok, false); + assert.match(above.error, /not a git checkout/); + assert.ok(!existsSync(marker), "fsmonitor never ran"); +}); + +test("confined: a filter driver in the checkout's config is refused", async () => { + const root = path.join(dir, "local"); + git(repo, "config", "filter.x.clean", "touch /tmp/never"); + const r = /** @type {any} */ (await sync({ from: "repo", local: { root } })); + assert.equal(r.ok, false); + assert.match(r.error, /filter driver/); +}); + +test("an edit in the same second as the index was written, same size, is still seen on both sides", async () => { + // git trusts an entry's stat data unless the file changed in the same second as the index was written; + // a copy of the index with a newer mtime would hide such an edit. Pin both to one second to force it. + const T = new Date("2026-01-01T00:00:00Z"); + const racy = (/** @type {string} */ root, /** @type {string} */ rel, /** @type {string} */ text) => { + git(root, "update-index", "-q", "--refresh"); + utimesSync(path.join(root, rel), T, T); + git(root, "update-index", "-q", "--really-refresh"); + utimesSync(path.join(root, ".git", "index"), T, T); + writeFileSync(path.join(root, rel), text); + utimesSync(path.join(root, rel), T, T); + }; + await ok(); + racy(vmProject(), "README.md", "HELLO\n"); // an agent edit in the VM + const blocked = /** @type {any} */ (await sync()); + assert.equal(blocked.ok, false, "the VM edit was overlooked"); + assert.match(blocked.error, /M README\.md/); + writeFileSync(path.join(vmProject(), "README.md"), "hello\n"); + racy(repo, "README.md", "jello\n"); // a local edit + const r = await ok(); + assert.equal(r.structured.counts.modified, 1, "the local edit was overlooked"); + assert.equal(read(vmProject(), "README.md"), "jello\n"); +}); + +test("a credential added and removed in merge commits blocks the sync", async () => { + git(repo, "checkout", "-q", "-b", "side"); + write(repo, "side.txt", "x\n"); + git(repo, "add", "-A"); + git(repo, "commit", "-qm", "side"); + git(repo, "checkout", "-q", "main"); + git(repo, "merge", "-q", "--no-ff", "--no-commit", "side"); + write(repo, ".env", "SECRET=1\n"); + git(repo, "add", "-f", ".env"); + git(repo, "commit", "-qm", "merge with a secret"); + git(repo, "rm", "-q", ".env"); + git(repo, "commit", "-qm", "drop it"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /\.env is in the history this sync would send/); +}); + +test("the push ignores the user's git config and GIT_* environment, and goes only to the VM", async () => { + await ok(); + const hostile = path.join(dir, "hostile.gitconfig"); + writeFileSync(hostile, '[url "other.boxery:"]\n\tpushInsteadOf = vm.boxery:\n\tinsteadOf = vm.boxery:\n[push]\n\tpushOption = x\n\tnegotiate = true\n'); + Object.assign(env, { GIT_CONFIG_GLOBAL: hostile, GIT_ALLOW_PROTOCOL: "https", GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "url.other.boxery:.pushInsteadOf", GIT_CONFIG_VALUE_0: "vm.boxery:" }); + write(repo, "README.md", "changed\n"); + const mark = readFileSync(path.join(dir, "calls"), "utf8").length; + let r; + try { r = await ok(); } finally { + for (const k of ["GIT_CONFIG_GLOBAL", "GIT_ALLOW_PROTOCOL", "GIT_CONFIG_COUNT", "GIT_CONFIG_KEY_0", "GIT_CONFIG_VALUE_0"]) delete env[k]; + } + assert.equal(r.structured.tree.match, true); + assert.ok(!existsSync(path.join(dir, "vms", "alpha", "other", "project")), "nothing went to another VM"); + const pushes = readFileSync(path.join(dir, "calls"), "utf8").slice(mark).split("\n").filter(Boolean).map((l) => JSON.parse(l)).filter((c) => String(c.script).includes("receive-pack")); + assert.equal(pushes.length, 1); + assert.equal(pushes[0].endpoint, "alpha/vm"); + assert.ok(pushes[0].argv.includes("ControlMaster=no"), "never becomes the shared connection's master"); + assert.ok(pushes[0].argv.includes("BatchMode=yes")); + assert.match(pushes[0].script, /core\.hooksPath=\/dev\/null .*receive\.autogc=false receive-pack/); +}); + +test("a push that fails into a new directory leaves no repository behind", async () => { + writeFileSync(path.join(dir, "bin", "git"), `#!/usr/bin/env bash\nfor a in "$@"; do [ "$a" != receive-pack ] || { echo "injected receive-pack failure" >&2; exit 1; }; done\nexec ${q(env.REAL_GIT ?? "git")} "$@"\n`, { mode: 0o755 }); + env.FAKE_VM_PATH = env.PATH; + let r; + try { r = /** @type {any} */ (await sync({ to: "~/fresh/repo" })); } finally { unlinkSync(path.join(dir, "bin", "git")); delete env.FAKE_VM_PATH; } + assert.equal(r.ok, false); + assert.match(r.error, /git push to VM "vm" failed/); + assert.ok(!existsSync(path.join(dir, "vms", "alpha", "vm", "fresh", "repo")), "init undone"); +}); + +test("a stale incoming ref from an abandoned sync is dropped; a recent one is kept", async () => { + await ok(); + const head = git(vmProject(), "rev-parse", "HEAD"); + git(vmProject(), "update-ref", "refs/boxery/incoming/1000-old", head); + git(vmProject(), "update-ref", `refs/boxery/incoming/${Math.floor(Date.now() / 1000)}-busy`, head); + write(repo, "README.md", "again\n"); + await ok(); + const refs = git(vmProject(), "for-each-ref", "--format=%(refname)", "refs/boxery/incoming/"); + assert.doesNotMatch(refs, /1000-old/); + assert.match(refs, /-busy/); +}); + +test("a partial clone gives a clear error", async () => { + git(repo, "config", "extensions.partialClone", "origin"); + git(repo, "config", "core.repositoryformatversion", "1"); + const r = /** @type {any} */ (await sync()); + assert.equal(r.ok, false); + assert.match(r.error, /partial clone/); +});