Make WorkerShellBackend commands opt-in to reduce bundle size

.gitignore

@@ -6,10 +6,11 @@ PLAN.md
 # tracked code and must not be caught by this rule.
 /artifacts/
 
-# Generated bundle for the worker backend's ShellWorker. Built
-# by packages/computer/src/backends/worker/script/build-bundle.mjs
-# on prepare / pretest / pretypecheck.
-packages/computer/src/backends/worker-shell/generated-bundle.ts
+# Generated shell-module groups for the worker backend's
+# ShellWorker (core plus one file per optional feature). Built by
+# packages/computer/src/backends/worker-shell/script/build-bundle.mjs on
+# prepare / pretest / pretypecheck.
+packages/computer/src/backends/worker-shell/generated/
 
 # SEA binary destinations populated at publish time from
 # artifacts/computerd/ via the build-bin step. The @cloudflare/computer

docs/12_worker_backend.md

@@ -249,22 +249,58 @@ network-bound `git` subcommands do. See
 - `src/index.ts` holds the DO and the HTTP surface (the
   `/c/<name>/file/...` and `/c/<name>/exec` routes the container
   example also exposes).
-- No Dockerfile, no build script. The shell bundle ships with
-  `@cloudflare/computer/backends/worker-shell` as `SHELL_MODULES`
-  (a record of module name → source covering the entry plus
-  every code-split chunk); the backend hands the whole record
-  to the Loader callback itself.
+- No Dockerfile, no build script. The shell ships with
+  `@cloudflare/computer/backends/worker-shell` as feature groups: an
+  always-on core (`SHELL_CORE_MODULES`) plus one optional group per
+  command at `@cloudflare/computer/shell/<feature>`. The backend
+  assembles core with whatever groups you opt into and hands the
+  result to the Loader callback itself.
 
-The DO's backend wiring fits in three lines:
+The DO's backend wiring:
 
 ```ts
+import curlModules from "@cloudflare/computer/shell/curl";
+import sqliteModules from "@cloudflare/computer/shell/sqlite";
+
 new WorkerShellBackend({
   loader: env.LOADER,
   workspace: { binding: "ContainerExample", id: ctx.id.toString() },
   ctx,
+  commands: [curlModules, sqliteModules],
 })
 ```
 
 Run with `npm run dev --workspace @example/computer-worker`.
-The same `curl` recipes from the container example work without
-changes.
+The same `curl` recipes from the container example work once
+`curlModules` is passed to `commands`.
+
+## Optional shell commands
+
+Core carries the always-on command set (`cat`, `ls`, `grep`, `sed`,
+`awk`, `sort`, …). The heavier commands are split into optional
+groups that are opt-in by import: import a group from
+`@cloudflare/computer/shell/<feature>` and pass it to the
+`commands` option, and only then does its code enter your bundle.
+
+```ts
+import curlModules from "@cloudflare/computer/shell/curl";
+import htmlToMarkdownModules from "@cloudflare/computer/shell/html-to-markdown";
+
+// commands: [curlModules, htmlToMarkdownModules]
+```
+
+A group you never import is unreachable in your module graph, so
+the bundler drops it — there is no build-time flag to set and no
+default-on cost to opt out of. The full set of optional groups is
+`curl`, `html-to-markdown`, `python`, `sqlite`, `js-exec`, `yq`,
+`file`, `xan`, and `jq`.
+
+`curl` runs on a `SecureFetch` adapter over the isolate's global
+`fetch` — `undici` is redirected to a throwing stub at build time
+and never ships. Egress stays governed by the Dynamic Worker's
+`globalOutbound` (left `null`, i.e. closed), not by the shell, so
+enabling `curl` does not by itself open the network.
+
+Consumers that build the Loader callback by hand (the `fetcher`
+path) assemble the modules table themselves with
+`assembleShellModules([...groups])` from the same package.

examples/worker-shell/README.md

@@ -101,14 +101,32 @@ POST /c/<name>/exec                    { command | argv, cwd?, encoding? }
 
 ## Run it locally
 
-No Docker, no extra build step. The shell ships as a record of
-pre-bundled modules (`SHELL_MODULES`) inside
-`@cloudflare/computer/backends/worker-shell`; `WorkerShellBackend` spreads
-the whole record into the Loader callback internally so the DO
-constructor stays a three-line backend invocation. The entry
-module parses on cold start; the dynamic chunks (python, js-exec,
-sqlite, curl, html-to-markdown) stay cold until a script reaches
-for them.
+No Docker, no extra build step. The shell ships as pre-bundled
+feature groups inside `@cloudflare/computer/backends/worker-shell`: an
+always-on core plus one optional group per command at
+`@cloudflare/computer/shell/<feature>`. `WorkerShellBackend` assembles
+core with whatever groups you pass to its `commands` option and
+spreads the result into the Loader callback internally. This
+example opts `curl` and `sqlite` in:
+
+```ts
+import curlModules from "@cloudflare/computer/shell/curl";
+import sqliteModules from "@cloudflare/computer/shell/sqlite";
+
+new WorkerShellBackend({
+  loader: env.LOADER,
+  workspace: { binding: "ContainerExample", id: ctx.id.toString() },
+  ctx,
+  commands: [curlModules, sqliteModules],
+});
+```
+
+A group you never import (`html-to-markdown`, `python`, `js-exec`,
+`yq`, `file`, `xan`, `jq`, or either of the two above) is
+unreachable in the bundle and the bundler drops it — opting a
+command in is a single import, and opting out is deleting it. The
+core entry module parses on cold start; each opted-in group's
+chunks stay cold until a script reaches for them.
 
 ```sh
 npm run dev --workspace @example/computer-worker-shell

examples/worker-shell/src/index.ts

@@ -36,6 +36,14 @@ import {
   withWorkspace,
 } from "@cloudflare/computer";
 import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell";
+// Opt-in shell commands. Each import pulls one command group into
+// this Worker's bundle; a group you do not import is unreachable
+// and the bundler drops it. Pass the ones you want to the
+// WorkerShellBackend `commands` option below. Other importable groups:
+// @cloudflare/computer/shell/{html-to-markdown,python,js-exec,yq,
+// file,xan,jq}.
+import curlModules from "@cloudflare/computer/shell/curl";
+import sqliteModules from "@cloudflare/computer/shell/sqlite";
 
 // Re-export so the runtime can wrap WorkspaceServiceProxy into a
 // loopback Fetcher binding. The DO reaches the wrapped class
@@ -60,6 +68,10 @@ export class ContainerExample extends withWorkspace(class extends DurableObject<
         loader: env.LOADER,
         workspace: { binding: "ContainerExample", id: ctx.id.toString() },
         ctx,
+        // Only the groups listed here ship. Core (cat, ls, grep,
+        // sed, …) is always included; drop an import above to
+        // shrink the bundle by that command's cost.
+        commands: [curlModules, sqliteModules],
       }),
     ],
     // Mount the Bucket binding at /workspace/r2. Seed it with

packages/computer/README.md

@@ -100,6 +100,7 @@ quickest way to get `exec` working:
 ```ts
 import { withWorkspace, getWorkspace } from "@cloudflare/computer";
 import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell";
+import curlModules from "@cloudflare/computer/shell/curl";
 import { DurableObject } from "cloudflare:workers";
 
 export class Agent extends withWorkspace(
@@ -111,6 +112,7 @@ export class Agent extends withWorkspace(
         loader: self.env.LOADER,
         workspace: { binding: "Agent", id: self.ctx.id.toString() },
         ctx: self.ctx,
+        commands: [curlModules],
       }),
     ],
   }),
@@ -126,6 +128,16 @@ Add the loader binding and the `experimental` flag to `wrangler.jsonc`:
 }
 ```
 
+The worker shell ships as feature groups: an always-on core plus
+one optional group per command at
+`@cloudflare/computer/shell/<feature>`. Import the groups you want
+and pass them to `WorkerShellBackend`'s `commands` option; a group you
+never import is unreachable in your bundle and the bundler drops
+it. The optional groups are `curl`, `html-to-markdown`, `python`,
+`sqlite`, `js-exec`, `yq`, `file`, `xan`, and `jq`. `curl` runs on
+the isolate's global `fetch` (no `undici` in the bundle); egress
+stays governed by the Dynamic Worker's `globalOutbound`.
+
 Now `exec` runs against the same files your `fs` calls wrote:
 
 ```ts

packages/computer/package.json

@@ -13,6 +13,7 @@
   "publishConfig": {
     "tag": "unreleased"
   },
+  "sideEffects": false,
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
@@ -46,6 +47,46 @@
       "types": "./dist/backends/worker-shell/index.d.ts",
       "import": "./dist/backends/worker-shell/index.js"
     },
+    "./shell/core": {
+      "types": "./dist/backends/worker-shell/shell/core.d.ts",
+      "default": "./dist/backends/worker-shell/shell/core.js"
+    },
+    "./shell/curl": {
+      "types": "./dist/backends/worker-shell/shell/curl.d.ts",
+      "default": "./dist/backends/worker-shell/shell/curl.js"
+    },
+    "./shell/html-to-markdown": {
+      "types": "./dist/backends/worker-shell/shell/html-to-markdown.d.ts",
+      "default": "./dist/backends/worker-shell/shell/html-to-markdown.js"
+    },
+    "./shell/python": {
+      "types": "./dist/backends/worker-shell/shell/python.d.ts",
+      "default": "./dist/backends/worker-shell/shell/python.js"
+    },
+    "./shell/sqlite": {
+      "types": "./dist/backends/worker-shell/shell/sqlite.d.ts",
+      "default": "./dist/backends/worker-shell/shell/sqlite.js"
+    },
+    "./shell/js-exec": {
+      "types": "./dist/backends/worker-shell/shell/js-exec.d.ts",
+      "default": "./dist/backends/worker-shell/shell/js-exec.js"
+    },
+    "./shell/yq": {
+      "types": "./dist/backends/worker-shell/shell/yq.d.ts",
+      "default": "./dist/backends/worker-shell/shell/yq.js"
+    },
+    "./shell/file": {
+      "types": "./dist/backends/worker-shell/shell/file.d.ts",
+      "default": "./dist/backends/worker-shell/shell/file.js"
+    },
+    "./shell/xan": {
+      "types": "./dist/backends/worker-shell/shell/xan.d.ts",
+      "default": "./dist/backends/worker-shell/shell/xan.js"
+    },
+    "./shell/jq": {
+      "types": "./dist/backends/worker-shell/shell/jq.d.ts",
+      "default": "./dist/backends/worker-shell/shell/jq.js"
+    },
     "./observe/cloudflare": {
       "types": "./dist/observe/cloudflare.d.ts",
       "import": "./dist/observe/cloudflare.js"

packages/computer/rolldown.config.ts

@@ -34,6 +34,23 @@ export default defineConfig({
     "backends/container/index": "src/backends/container/index.ts",
     "backends/worker-javascript/index": "src/backends/worker-javascript/index.ts",
     "backends/worker-shell/index": "src/backends/worker-shell/index.ts",
+    // The shell-module groups build-bundle.mjs emits. Each is its
+    // own entry so it lands at the dist path the ./shell/* package
+    // exports point at; shell-modules.ts imports the core group by
+    // subpath (kept external below) and a consumer imports the
+    // optional ones it wants, so the bundler tree-shakes any group
+    // that is never imported.
+    "backends/worker-shell/shell/core": "src/backends/worker-shell/generated/core.ts",
+    "backends/worker-shell/shell/curl": "src/backends/worker-shell/generated/curl.ts",
+    "backends/worker-shell/shell/html-to-markdown":
+      "src/backends/worker-shell/generated/html-to-markdown.ts",
+    "backends/worker-shell/shell/python": "src/backends/worker-shell/generated/python.ts",
+    "backends/worker-shell/shell/sqlite": "src/backends/worker-shell/generated/sqlite.ts",
+    "backends/worker-shell/shell/js-exec": "src/backends/worker-shell/generated/js-exec.ts",
+    "backends/worker-shell/shell/yq": "src/backends/worker-shell/generated/yq.ts",
+    "backends/worker-shell/shell/file": "src/backends/worker-shell/generated/file.ts",
+    "backends/worker-shell/shell/xan": "src/backends/worker-shell/generated/xan.ts",
+    "backends/worker-shell/shell/jq": "src/backends/worker-shell/generated/jq.ts",
     "observe/cloudflare": "src/observe/cloudflare.ts",
   },
   external: [
@@ -44,6 +61,12 @@ export default defineConfig({
     "zod",
     "just-bash",
     /^node:/,
+    // shell-modules.ts imports the generated groups by their
+    // published subpath. Keep the specifiers intact in the emitted
+    // bundle rather than inlining the group here so the consumer's
+    // bundler sees each group as its own module and can drop one it
+    // never imports; each group is built as its own entry above.
+    /^@cloudflare\/computer\/shell\//,
   ],
   resolve: {
     alias: {

packages/computer/src/backends/worker-shell/entrypoint.ts

@@ -18,7 +18,7 @@
 // workspace stub; there's no shared instance state to race.
 
 import { WorkerEntrypoint } from "cloudflare:workers";
-import { Bash, type CustomCommand } from "just-bash";
+import { Bash, type CustomCommand, type SecureFetch } from "just-bash";
 
 import { WorkspaceFsAdapter } from "./adapter.js";
 import { type ArtifactsCommandHost, defineArtifactsCommand } from "./artifacts-command.js";
@@ -40,8 +40,45 @@ export interface ShellWorkerOptions {
   // container backend uses, so scripts that hard-code that path
   // keep working.
   cwd?: string;
+  // Fetch implementation backing `curl`. just-bash registers curl
+  // whenever a fetch is supplied and calls it directly — no undici,
+  // no in-isolate DNS pinning (undici is excluded from the bundle
+  // at build time). Defaults to defaultSecureFetch below, a thin
+  // wrapper over the isolate's global `fetch`, so curl is enabled
+  // by default. Egress is governed by the Dynamic Worker's
+  // globalOutbound, not by this fetch (see worker.ts). Pass a
+  // custom SecureFetch to add an allow-list or credential
+  // injection, or `null` to drop curl entirely.
+  fetch?: SecureFetch | null;
 }
 
+// Default curl fetch: adapt the isolate's global `fetch` to
+// just-bash's SecureFetch contract. No allow-list or private-range
+// checks run here — the Dynamic Worker's globalOutbound is the
+// egress boundary, so requests only leave the isolate once a
+// consumer wires a trusted outbound gateway; policy belongs in
+// that gateway, not the untrusted shell.
+const defaultSecureFetch: SecureFetch = async (url, options) => {
+  const response = await fetch(url, {
+    method: options?.method,
+    headers: options?.headers,
+    body: options?.body,
+    redirect: options?.followRedirects === false ? "manual" : "follow",
+    signal: options?.signal,
+  });
+  const headers: Record<string, string> = Object.create(null);
+  response.headers.forEach((value, key) => {
+    headers[key] = value;
+  });
+  return {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+    body: new Uint8Array(await response.arrayBuffer()),
+    url: response.url || url,
+  };
+};
+
 // Env shape the host Worker is expected to wire through the
 // Loader callback. The shell calls env.HOST.getWorkspace() on
 // every exec; no caching.
@@ -186,6 +223,10 @@ export class ShellWorker<
             ConstructorParameters<typeof Bash>[0]
           >["fs"],
           cwd,
+          fetch:
+            this.shellOptions.fetch === null
+              ? undefined
+              : (this.shellOptions.fetch ?? defaultSecureFetch),
           customCommands,
           // just-bash's in-process DefenseInDepthBox activates by
           // registering ESM loader hooks through node:module's