fix(mount): flush every exact root

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-relay/sandbox",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "Provider-agnostic sandbox runtimes and orchestration for agent workloads.",
   "license": "Apache-2.0",
   "type": "module",

src/mount-script.test.ts

@@ -1,7 +1,14 @@
 import { describe, it, type TestContext } from "node:test";
 import { strict as assert } from "node:assert";
 import { spawnSync } from "node:child_process";
-import { chmodSync, existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import {
+  chmodSync,
+  existsSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
@@ -59,6 +66,10 @@ if [ -n "\${FAKE_MOUNT_FAIL_LOCAL_DIR:-}" ] && [ "$local_dir" = "$FAKE_MOUNT_FAI
   echo "simulated mount failure: $local_dir" >&2
   exit 23
 fi
+if [ -n "\${FAKE_MOUNT_FAIL_LATER_LOCAL_DIR:-}" ] && [ "$local_dir" = "$FAKE_MOUNT_FAIL_LATER_LOCAL_DIR" ]; then
+  echo "simulated later mount failure: $local_dir" >&2
+  exit 41
+fi
 mkdir -p "$local_dir"
 printf mounted > "$local_dir/.mounted"
 `,
@@ -67,6 +78,10 @@ printf mounted > "$local_dir/.mounted"
   return { binDir, localRoot };
 }
 
+function mountCalls(callsPath: string): string[] {
+  return readFileSync(callsPath, "utf8").trim().split("\n");
+}
+
 function testShellQuote(value: string): string {
   return `'${value.replaceAll("'", "'\\''")}'`;
 }
@@ -250,7 +265,7 @@ describe("exact local-layout contract", () => {
     const { binDir, localRoot } = fakeExactMount(t);
     const firstRoot = join(localRoot, "github/repos/acme/cloud");
     const laterRoot = join(localRoot, "slack/channels/C123");
-    const callsPath = join(localRoot, "cleanup-calls.log");
+    const callsPath = join(binDir, "cleanup-calls.log");
     const shell = buildRelayfileMountCleanupFlushShell({
       ...BASE,
       localDir: localRoot,
@@ -266,18 +281,101 @@ describe("exact local-layout contract", () => {
           PATH: `${binDir}:${process.env.PATH ?? ""}`,
           FAKE_MOUNT_CALLS: callsPath,
           FAKE_MOUNT_FAIL_LOCAL_DIR: firstRoot,
+          FAKE_MOUNT_FAIL_LATER_LOCAL_DIR: laterRoot,
         },
         encoding: "utf8",
       },
     );
 
     assert.equal(result.status, 23, result.stderr);
     assert.equal(existsSync(join(firstRoot, ".mounted")), false);
-    assert.equal(
-      existsSync(join(laterRoot, ".mounted")),
-      true,
-      "a failure in the first exact root must not skip a later teardown flush",
-    );
+    assert.equal(existsSync(join(laterRoot, ".mounted")), false);
+    assert.deepEqual(mountCalls(callsPath), [firstRoot, laterRoot]);
+  });
+
+  it("attempts every ordinary flush root and returns the first failure", (t) => {
+    const { binDir, localRoot } = fakeExactMount(t);
+    const firstRoot = join(localRoot, "github/repos/acme/cloud");
+    const laterRoot = join(localRoot, "slack/channels/C123");
+    const callsPath = join(binDir, "ordinary-flush-calls.log");
+    const shell = buildRelayfileMountFlushShell({
+      ...BASE,
+      localDir: localRoot,
+      paths: ["/github/repos/acme/cloud/**", "/slack/channels/C123/**"],
+    });
+
+    const result = spawnSync("/bin/sh", ["-c", shell], {
+      env: {
+        ...process.env,
+        PATH: `${binDir}:${process.env.PATH ?? ""}`,
+        FAKE_MOUNT_CALLS: callsPath,
+        FAKE_MOUNT_FAIL_LOCAL_DIR: firstRoot,
+        FAKE_MOUNT_FAIL_LATER_LOCAL_DIR: laterRoot,
+      },
+      encoding: "utf8",
+    });
+
+    assert.equal(result.status, 23, result.stderr);
+    assert.equal(existsSync(join(firstRoot, ".mounted")), false);
+    assert.equal(existsSync(join(laterRoot, ".mounted")), false);
+    assert.deepEqual(mountCalls(callsPath), [firstRoot, laterRoot]);
+  });
+
+  it("attempts every initial-sync root and returns the first failure", (t) => {
+    const { binDir, localRoot } = fakeExactMount(t);
+    const firstRoot = join(localRoot, "github/repos/acme/cloud");
+    const laterRoot = join(localRoot, "slack/channels/C123");
+    const callsPath = join(binDir, "initial-sync-calls.log");
+    const shell = buildRelayfileMountInitialSyncShell({
+      ...BASE,
+      localDir: localRoot,
+      paths: ["/github/repos/acme/cloud/**", "/slack/channels/C123/**"],
+    });
+
+    const result = spawnSync("/bin/sh", ["-c", shell], {
+      env: {
+        ...process.env,
+        PATH: `${binDir}:${process.env.PATH ?? ""}`,
+        FAKE_MOUNT_CALLS: callsPath,
+        FAKE_MOUNT_FAIL_LOCAL_DIR: firstRoot,
+        FAKE_MOUNT_FAIL_LATER_LOCAL_DIR: laterRoot,
+      },
+      encoding: "utf8",
+    });
+
+    assert.equal(result.status, 23, result.stderr);
+    assert.equal(existsSync(join(firstRoot, ".mounted")), false);
+    assert.equal(existsSync(join(laterRoot, ".mounted")), false);
+    assert.deepEqual(mountCalls(callsPath), [firstRoot, laterRoot]);
+  });
+
+  it("attempts every timeout-wrapped initial-sync root", (t) => {
+    const { binDir, localRoot } = fakeExactMount(t);
+    const firstRoot = join(localRoot, "github/repos/acme/cloud");
+    const laterRoot = join(localRoot, "slack/channels/C123");
+    const callsPath = join(binDir, "timed-initial-sync-calls.log");
+    const shell = buildRelayfileMountInitialSyncShell({
+      ...BASE,
+      localDir: localRoot,
+      paths: ["/github/repos/acme/cloud/**", "/slack/channels/C123/**"],
+      timeoutSeconds: 2,
+    });
+
+    const result = spawnSync("/bin/sh", ["-c", shell], {
+      env: {
+        ...process.env,
+        PATH: `${binDir}:${process.env.PATH ?? ""}`,
+        FAKE_MOUNT_CALLS: callsPath,
+        FAKE_MOUNT_FAIL_LOCAL_DIR: firstRoot,
+        FAKE_MOUNT_FAIL_LATER_LOCAL_DIR: laterRoot,
+      },
+      encoding: "utf8",
+    });
+
+    assert.equal(result.status, 23, result.stderr);
+    assert.equal(existsSync(join(firstRoot, ".mounted")), false);
+    assert.equal(existsSync(join(laterRoot, ".mounted")), false);
+    assert.deepEqual(mountCalls(callsPath), [firstRoot, laterRoot]);
   });
 
   it("renders late-bound shell templates as separate exact mounts", (t) => {
@@ -332,6 +430,51 @@ describe("exact local-layout contract", () => {
     assert.equal(existsSync(join(localRoot, ".mounted")), false);
   });
 
+  it("attempts every late-bound flush root and returns the first failure", (t) => {
+    const { binDir, localRoot } = fakeExactMount(t);
+    const firstRoot = join(localRoot, "github/repos/acme/cloud");
+    const laterRoot = join(localRoot, "slack/channels/C123");
+    const callsPath = join(binDir, "late-bound-flush-calls.log");
+    const template = buildRelayfileMountShellTemplate({}, {
+      stateDir: join(localRoot, ".state"),
+      websocket: false,
+    });
+    const values = {
+      baseUrl: BASE.baseUrl,
+      workspaceId: BASE.workspaceId,
+      localDir: localRoot,
+      token: BASE.token,
+    };
+    let shell = template.flushShellTemplate;
+    for (const [key, value] of Object.entries(values)) {
+      const placeholder = template.placeholders[key as keyof typeof values];
+      shell = shell.replace(testShellQuote(placeholder), testShellQuote(value));
+    }
+    shell = shell.replace(
+      template.pathArgsPlaceholderArg,
+      buildRelayfileMountPathArgsShell([
+        "/github/repos/acme/cloud/**",
+        "/slack/channels/C123/**",
+      ]),
+    );
+
+    const result = spawnSync("/bin/sh", ["-c", shell], {
+      env: {
+        ...process.env,
+        PATH: `${binDir}:${process.env.PATH ?? ""}`,
+        FAKE_MOUNT_CALLS: callsPath,
+        FAKE_MOUNT_FAIL_LOCAL_DIR: firstRoot,
+        FAKE_MOUNT_FAIL_LATER_LOCAL_DIR: laterRoot,
+      },
+      encoding: "utf8",
+    });
+
+    assert.equal(result.status, 23, result.stderr);
+    assert.equal(existsSync(join(firstRoot, ".mounted")), false);
+    assert.equal(existsSync(join(laterRoot, ".mounted")), false);
+    assert.deepEqual(mountCalls(callsPath), [firstRoot, laterRoot]);
+  });
+
   it("surfaces late-bound daemon argument validation failures", (t) => {
     const { binDir, localRoot } = fakeExactMount(t);
     const template = buildRelayfileMountShellTemplate({}, {
@@ -527,20 +670,24 @@ describe("initial-sync idle watchdog progress files", () => {
       paths: ["/github/agentworkforce/**", "/slack/C0BBTBC1RCM/**"],
     });
     const armed = armedProgressFiles(shell);
+    const normalizedShell = shell.replaceAll("'\\''", "'");
 
     assert.equal(armed.length, 2);
-    assert.equal(shell.match(/relayfile-mount --once --local-layout 'exact'/g)?.length, 2);
+    assert.equal(
+      normalizedShell.match(/relayfile-mount --once --local-layout 'exact'/g)?.length,
+      2,
+    );
     assert.match(
-      shell,
+      normalizedShell,
       /--local-dir '\/home\/user\/workspace\/github\/agentworkforce'.*--remote-path '\/github\/agentworkforce'/,
     );
     assert.match(
-      shell,
+      normalizedShell,
       /--local-dir '\/home\/user\/workspace\/slack\/C0BBTBC1RCM'.*--remote-path '\/slack\/C0BBTBC1RCM'/,
     );
     for (const file of armed) {
       assert.ok(
-        shell.includes(`--state-file '${file}'`),
+        normalizedShell.includes(`--state-file '${file}'`),
         `watchdog watches ${file}, but no --state-file pins the sync to it`,
       );
     }

src/mount-script.ts

@@ -199,7 +199,7 @@ export function buildRelayfileMountFlushShell(opts: RelayfileMountShellOptions):
     `${mountEnvPrefix(opts)}relayfile-mount --once`,
     ...buildMountArgs({ ...opts, ...mount }),
   ].join(" "));
-  return composeMountCommands(commands);
+  return composeIndependentMountCommands(commands);
 }
 
 /**
@@ -233,24 +233,15 @@ export function buildRelayfileMountCleanupFlushShell(
     `${mountEnvPrefix(opts)}relayfile-mount "$1"`,
     ...buildMountArgs({ ...opts, ...mount }),
   ].join(" "));
-  const script = [
-    "relayfile_mount_flush_status=0",
-    ...commands.map((command) => [
-      `${command} || {`,
-      "relayfile_mount_flush_code=$?;",
-      'if [ "$relayfile_mount_flush_status" -eq 0 ]; then relayfile_mount_flush_status=$relayfile_mount_flush_code; fi;',
-      "}",
-    ].join(" ")),
-    'exit "$relayfile_mount_flush_status"',
-  ].join("; ");
+  const script = independentMountCommandsScript(commands);
   return `sh -c ${shellQuote(script)} relayfile-mount-cleanup "$relayfile_mount_flush_mode"`;
 }
 
 export function buildRelayfileMountInitialSyncShell(
   opts: RelayfileMountInitialSyncOptions,
 ): string {
   const commands = buildInitialSyncCommands(opts);
-  const command = commands.join(" && ");
+  const command = composeIndependentMountCommands(commands);
   if (opts.idleTimeoutSeconds && opts.idleTimeoutSeconds > 0) {
     return buildIdleWatchedCommand(
       command,
@@ -262,9 +253,9 @@ export function buildRelayfileMountInitialSyncShell(
     return command;
   }
   const timeout = `${Math.ceil(opts.timeoutSeconds)}s`;
-  const timedCommand = commands
-    .map((entry) => `timeout ${shellQuote(timeout)} ${entry}`)
-    .join(" && ");
+  const timedCommand = composeIndependentMountCommands(
+    commands.map((entry) => `timeout ${shellQuote(timeout)} ${entry}`),
+  );
   return [
     "{",
     "if command -v timeout >/dev/null 2>&1; then",
@@ -720,11 +711,30 @@ export function resolveRelayfileMountExactLayout(
   };
 }
 
-function composeMountCommands(commands: readonly string[]): string {
+/**
+ * Run every independent mount root and return the first failure only after
+ * all roots have had a chance to flush. Teardown must never let a bad first
+ * root discard pending writes from the remaining roots.
+ */
+function independentMountCommandsScript(commands: readonly string[]): string {
+  return [
+    "relayfile_mount_flush_status=0",
+    ...commands.map((command) => [
+      `${command} || {`,
+      "relayfile_mount_flush_code=$?;",
+      'if [ "$relayfile_mount_flush_status" -eq 0 ]; then relayfile_mount_flush_status=$relayfile_mount_flush_code; fi;',
+      "}",
+    ].join(" ")),
+    'exit "$relayfile_mount_flush_status"',
+  ].join("; ");
+}
+
+function composeIndependentMountCommands(commands: readonly string[]): string {
   if (commands.length === 1) {
     return commands[0]!;
   }
-  return `sh -c ${shellQuote(commands.join(" && "))}`;
+  const script = independentMountCommandsScript(commands);
+  return `sh -c ${shellQuote(script)}`;
 }
 
 function buildMountPathArg(path: string): string {
@@ -871,11 +881,16 @@ function buildDynamicMountOnceTemplate(
     'relayfile_mount_local_dir="$relayfile_mount_local_root";',
     `${pathlessOnce};`,
     "else",
+    "relayfile_mount_flush_status=0;",
     'while [ "$#" -gt 0 ]; do',
     'if [ "$#" -lt 2 ] || [ "$1" != "--remote-path" ]; then echo "invalid relayfile mount path args" >&2; exit 2; fi;',
     ...dynamicMountPathSetup(),
-    `${dynamicOnce} || exit $?;`,
+    `${dynamicOnce} || {`,
+    "relayfile_mount_flush_code=$?;",
+    'if [ "$relayfile_mount_flush_status" -eq 0 ]; then relayfile_mount_flush_status=$relayfile_mount_flush_code; fi;',
+    "};",
     "done;",
+    'exit "$relayfile_mount_flush_status";',
     "fi;",
     ")",
   ].join(" ");