diff --git a/.changeset/worker-shell-opt-in-commands.md b/.changeset/worker-shell-opt-in-commands.md new file mode 100644 index 00000000..93483072 --- /dev/null +++ b/.changeset/worker-shell-opt-in-commands.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +Heavy worker-shell commands are now opt-in to reduce the final bundle size. See the [documentation](/docs/12_worker_backend.md) for details. diff --git a/.gitignore b/.gitignore index 6c6f75b3..56157203 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,11 @@ PLAN.md # tracked code and must not be caught by this rule. /artifacts/ -# Generated bundle for the worker backend's ShellWorker. Built -# by packages/computer/src/backends/worker/script/build-bundle.mjs -# on prepare / pretest / pretypecheck. -packages/computer/src/backends/worker-shell/generated-bundle.ts +# Generated shell-module groups for the worker backend's +# ShellWorker (core plus one file per optional feature). Built by +# packages/computer/src/backends/worker-shell/script/build-bundle.mjs on +# prepare / pretest / pretypecheck. +packages/computer/src/backends/worker-shell/generated/ # SEA binary destinations populated at publish time from # artifacts/computerd/ via the build-bin step. The @cloudflare/computer diff --git a/docs/12_worker_backend.md b/docs/12_worker_backend.md index a207794a..d434d221 100644 --- a/docs/12_worker_backend.md +++ b/docs/12_worker_backend.md @@ -249,22 +249,58 @@ network-bound `git` subcommands do. See - `src/index.ts` holds the DO and the HTTP surface (the `/c//file/...` and `/c//exec` routes the container example also exposes). -- No Dockerfile, no build script. The shell bundle ships with - `@cloudflare/computer/backends/worker-shell` as `SHELL_MODULES` - (a record of module name → source covering the entry plus - every code-split chunk); the backend hands the whole record - to the Loader callback itself. +- No Dockerfile, no build script. The shell ships with + `@cloudflare/computer/backends/worker-shell` as feature groups: an + always-on core (`SHELL_CORE_MODULES`) plus one optional group per + command at `@cloudflare/computer/shell/`. The backend + assembles core with whatever groups you opt into and hands the + result to the Loader callback itself. -The DO's backend wiring fits in three lines: +The DO's backend wiring: ```ts +import curlModules from "@cloudflare/computer/shell/curl"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; + new WorkerShellBackend({ loader: env.LOADER, workspace: { binding: "ContainerExample", id: ctx.id.toString() }, ctx, + commands: [curlModules, sqliteModules], }) ``` Run with `npm run dev --workspace @example/computer-worker`. -The same `curl` recipes from the container example work without -changes. +The same `curl` recipes from the container example work once +`curlModules` is passed to `commands`. + +## Optional shell commands + +Core carries the always-on command set (`cat`, `ls`, `grep`, `sed`, +`awk`, `sort`, …). The heavier commands are split into optional +groups that are opt-in by import: import a group from +`@cloudflare/computer/shell/` and pass it to the +`commands` option, and only then does its code enter your bundle. + +```ts +import curlModules from "@cloudflare/computer/shell/curl"; +import htmlToMarkdownModules from "@cloudflare/computer/shell/html-to-markdown"; + +// commands: [curlModules, htmlToMarkdownModules] +``` + +A group you never import is unreachable in your module graph, so +the bundler drops it — there is no build-time flag to set and no +default-on cost to opt out of. The full set of optional groups is +`curl`, `html-to-markdown`, `python`, `sqlite`, `js-exec`, `yq`, +`file`, `xan`, and `jq`. + +`curl` runs on a `SecureFetch` adapter over the isolate's global +`fetch` — `undici` is redirected to a throwing stub at build time +and never ships. Egress stays governed by the Dynamic Worker's +`globalOutbound` (left `null`, i.e. closed), not by the shell, so +enabling `curl` does not by itself open the network. + +Consumers that build the Loader callback by hand (the `fetcher` +path) assemble the modules table themselves with +`assembleShellModules([...groups])` from the same package. diff --git a/examples/worker-shell/README.md b/examples/worker-shell/README.md index 76231733..fd7dcb71 100644 --- a/examples/worker-shell/README.md +++ b/examples/worker-shell/README.md @@ -101,14 +101,32 @@ POST /c//exec { command | argv, cwd?, encoding? } ## Run it locally -No Docker, no extra build step. The shell ships as a record of -pre-bundled modules (`SHELL_MODULES`) inside -`@cloudflare/computer/backends/worker-shell`; `WorkerShellBackend` spreads -the whole record into the Loader callback internally so the DO -constructor stays a three-line backend invocation. The entry -module parses on cold start; the dynamic chunks (python, js-exec, -sqlite, curl, html-to-markdown) stay cold until a script reaches -for them. +No Docker, no extra build step. The shell ships as pre-bundled +feature groups inside `@cloudflare/computer/backends/worker-shell`: an +always-on core plus one optional group per command at +`@cloudflare/computer/shell/`. `WorkerShellBackend` assembles +core with whatever groups you pass to its `commands` option and +spreads the result into the Loader callback internally. This +example opts `curl` and `sqlite` in: + +```ts +import curlModules from "@cloudflare/computer/shell/curl"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; + +new WorkerShellBackend({ + loader: env.LOADER, + workspace: { binding: "ContainerExample", id: ctx.id.toString() }, + ctx, + commands: [curlModules, sqliteModules], +}); +``` + +A group you never import (`html-to-markdown`, `python`, `js-exec`, +`yq`, `file`, `xan`, `jq`, or either of the two above) is +unreachable in the bundle and the bundler drops it — opting a +command in is a single import, and opting out is deleting it. The +core entry module parses on cold start; each opted-in group's +chunks stay cold until a script reaches for them. ```sh npm run dev --workspace @example/computer-worker-shell diff --git a/examples/worker-shell/src/index.ts b/examples/worker-shell/src/index.ts index 6099511f..c112714e 100644 --- a/examples/worker-shell/src/index.ts +++ b/examples/worker-shell/src/index.ts @@ -36,6 +36,14 @@ import { withWorkspace, } from "@cloudflare/computer"; import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; +// Opt-in shell commands. Each import pulls one command group into +// this Worker's bundle; a group you do not import is unreachable +// and the bundler drops it. Pass the ones you want to the +// WorkerShellBackend `commands` option below. Other importable groups: +// @cloudflare/computer/shell/{html-to-markdown,python,js-exec,yq, +// file,xan,jq}. +import curl from "@cloudflare/computer/shell/curl"; +import jq from "@cloudflare/computer/shell/jq"; // Re-export so the runtime can wrap WorkspaceServiceProxy into a // loopback Fetcher binding. The DO reaches the wrapped class @@ -60,6 +68,10 @@ export class ContainerExample extends withWorkspace(class extends DurableObject< loader: env.LOADER, workspace: { binding: "ContainerExample", id: ctx.id.toString() }, ctx, + // Only the groups listed here ship. Core (cat, ls, grep, + // sed, …) is always included; drop an import above to + // shrink the bundle by that command's cost. + commands: [curl, jq], }), ], // Mount the Bucket binding at /workspace/r2. Seed it with diff --git a/packages/computer/README.md b/packages/computer/README.md index 1f8a2c47..bffba952 100644 --- a/packages/computer/README.md +++ b/packages/computer/README.md @@ -100,6 +100,7 @@ quickest way to get `exec` working: ```ts import { withWorkspace, getWorkspace } from "@cloudflare/computer"; import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; +import curlModules from "@cloudflare/computer/shell/curl"; import { DurableObject } from "cloudflare:workers"; export class Agent extends withWorkspace( @@ -111,6 +112,7 @@ export class Agent extends withWorkspace( loader: self.env.LOADER, workspace: { binding: "Agent", id: self.ctx.id.toString() }, ctx: self.ctx, + commands: [curlModules], }), ], }), @@ -126,6 +128,16 @@ Add the loader binding and the `experimental` flag to `wrangler.jsonc`: } ``` +The worker shell ships as feature groups: an always-on core plus +one optional group per command at +`@cloudflare/computer/shell/`. Import the groups you want +and pass them to `WorkerShellBackend`'s `commands` option; a group you +never import is unreachable in your bundle and the bundler drops +it. The optional groups are `curl`, `html-to-markdown`, `python`, +`sqlite`, `js-exec`, `yq`, `file`, `xan`, and `jq`. `curl` runs on +the isolate's global `fetch` (no `undici` in the bundle); egress +stays governed by the Dynamic Worker's `globalOutbound`. + Now `exec` runs against the same files your `fs` calls wrote: ```ts diff --git a/packages/computer/package.json b/packages/computer/package.json index 0439a5f9..f915a58f 100644 --- a/packages/computer/package.json +++ b/packages/computer/package.json @@ -13,6 +13,7 @@ "publishConfig": { "tag": "unreleased" }, + "sideEffects": false, "exports": { ".": { "types": "./dist/index.d.ts", @@ -46,6 +47,46 @@ "types": "./dist/backends/worker-shell/index.d.ts", "import": "./dist/backends/worker-shell/index.js" }, + "./shell/core": { + "types": "./dist/backends/worker-shell/shell/core.d.ts", + "default": "./dist/backends/worker-shell/shell/core.js" + }, + "./shell/curl": { + "types": "./dist/backends/worker-shell/shell/curl.d.ts", + "default": "./dist/backends/worker-shell/shell/curl.js" + }, + "./shell/html-to-markdown": { + "types": "./dist/backends/worker-shell/shell/html-to-markdown.d.ts", + "default": "./dist/backends/worker-shell/shell/html-to-markdown.js" + }, + "./shell/python": { + "types": "./dist/backends/worker-shell/shell/python.d.ts", + "default": "./dist/backends/worker-shell/shell/python.js" + }, + "./shell/sqlite": { + "types": "./dist/backends/worker-shell/shell/sqlite.d.ts", + "default": "./dist/backends/worker-shell/shell/sqlite.js" + }, + "./shell/js-exec": { + "types": "./dist/backends/worker-shell/shell/js-exec.d.ts", + "default": "./dist/backends/worker-shell/shell/js-exec.js" + }, + "./shell/yq": { + "types": "./dist/backends/worker-shell/shell/yq.d.ts", + "default": "./dist/backends/worker-shell/shell/yq.js" + }, + "./shell/file": { + "types": "./dist/backends/worker-shell/shell/file.d.ts", + "default": "./dist/backends/worker-shell/shell/file.js" + }, + "./shell/xan": { + "types": "./dist/backends/worker-shell/shell/xan.d.ts", + "default": "./dist/backends/worker-shell/shell/xan.js" + }, + "./shell/jq": { + "types": "./dist/backends/worker-shell/shell/jq.d.ts", + "default": "./dist/backends/worker-shell/shell/jq.js" + }, "./observe/cloudflare": { "types": "./dist/observe/cloudflare.d.ts", "import": "./dist/observe/cloudflare.js" diff --git a/packages/computer/rolldown.config.ts b/packages/computer/rolldown.config.ts index 8143bb8c..7f21309e 100644 --- a/packages/computer/rolldown.config.ts +++ b/packages/computer/rolldown.config.ts @@ -34,6 +34,23 @@ export default defineConfig({ "backends/container/index": "src/backends/container/index.ts", "backends/worker-javascript/index": "src/backends/worker-javascript/index.ts", "backends/worker-shell/index": "src/backends/worker-shell/index.ts", + // The shell-module groups build-bundle.mjs emits. Each is its + // own entry so it lands at the dist path the ./shell/* package + // exports point at; shell-modules.ts imports the core group by + // subpath (kept external below) and a consumer imports the + // optional ones it wants, so the bundler tree-shakes any group + // that is never imported. + "backends/worker-shell/shell/core": "src/backends/worker-shell/generated/core.ts", + "backends/worker-shell/shell/curl": "src/backends/worker-shell/generated/curl.ts", + "backends/worker-shell/shell/html-to-markdown": + "src/backends/worker-shell/generated/html-to-markdown.ts", + "backends/worker-shell/shell/python": "src/backends/worker-shell/generated/python.ts", + "backends/worker-shell/shell/sqlite": "src/backends/worker-shell/generated/sqlite.ts", + "backends/worker-shell/shell/js-exec": "src/backends/worker-shell/generated/js-exec.ts", + "backends/worker-shell/shell/yq": "src/backends/worker-shell/generated/yq.ts", + "backends/worker-shell/shell/file": "src/backends/worker-shell/generated/file.ts", + "backends/worker-shell/shell/xan": "src/backends/worker-shell/generated/xan.ts", + "backends/worker-shell/shell/jq": "src/backends/worker-shell/generated/jq.ts", "observe/cloudflare": "src/observe/cloudflare.ts", }, external: [ @@ -44,6 +61,12 @@ export default defineConfig({ "zod", "just-bash", /^node:/, + // shell-modules.ts imports the generated groups by their + // published subpath. Keep the specifiers intact in the emitted + // bundle rather than inlining the group here so the consumer's + // bundler sees each group as its own module and can drop one it + // never imports; each group is built as its own entry above. + /^@cloudflare\/computer\/shell\//, ], resolve: { alias: { diff --git a/packages/computer/src/backends/worker-shell/entrypoint.ts b/packages/computer/src/backends/worker-shell/entrypoint.ts index 690d6a91..492d2c99 100644 --- a/packages/computer/src/backends/worker-shell/entrypoint.ts +++ b/packages/computer/src/backends/worker-shell/entrypoint.ts @@ -18,7 +18,7 @@ // workspace stub; there's no shared instance state to race. import { WorkerEntrypoint } from "cloudflare:workers"; -import { Bash, type CustomCommand } from "just-bash"; +import { Bash, type CustomCommand, type SecureFetch } from "just-bash"; import { WorkspaceFsAdapter } from "./adapter.js"; import { type ArtifactsCommandHost, defineArtifactsCommand } from "./artifacts-command.js"; @@ -40,8 +40,45 @@ export interface ShellWorkerOptions { // container backend uses, so scripts that hard-code that path // keep working. cwd?: string; + // Fetch implementation backing `curl`. just-bash registers curl + // whenever a fetch is supplied and calls it directly — no undici, + // no in-isolate DNS pinning (undici is excluded from the bundle + // at build time). Defaults to defaultSecureFetch below, a thin + // wrapper over the isolate's global `fetch`, so curl is enabled + // by default. Egress is governed by the Dynamic Worker's + // globalOutbound, not by this fetch (see worker.ts). Pass a + // custom SecureFetch to add an allow-list or credential + // injection, or `null` to drop curl entirely. + fetch?: SecureFetch | null; } +// Default curl fetch: adapt the isolate's global `fetch` to +// just-bash's SecureFetch contract. No allow-list or private-range +// checks run here — the Dynamic Worker's globalOutbound is the +// egress boundary, so requests only leave the isolate once a +// consumer wires a trusted outbound gateway; policy belongs in +// that gateway, not the untrusted shell. +const defaultSecureFetch: SecureFetch = async (url, options) => { + const response = await fetch(url, { + method: options?.method, + headers: options?.headers, + body: options?.body, + redirect: options?.followRedirects === false ? "manual" : "follow", + signal: options?.signal, + }); + const headers: Record = Object.create(null); + response.headers.forEach((value, key) => { + headers[key] = value; + }); + return { + status: response.status, + statusText: response.statusText, + headers, + body: new Uint8Array(await response.arrayBuffer()), + url: response.url || url, + }; +}; + // Env shape the host Worker is expected to wire through the // Loader callback. The shell calls env.HOST.getWorkspace() on // every exec; no caching. @@ -186,6 +223,10 @@ export class ShellWorker< ConstructorParameters[0] >["fs"], cwd, + fetch: + this.shellOptions.fetch === null + ? undefined + : (this.shellOptions.fetch ?? defaultSecureFetch), customCommands, // just-bash's in-process DefenseInDepthBox activates by // registering ESM loader hooks through node:module's diff --git a/packages/computer/src/backends/worker-shell/generated-bundle.test.ts b/packages/computer/src/backends/worker-shell/generated-bundle.test.ts deleted file mode 100644 index c18f2235..00000000 --- a/packages/computer/src/backends/worker-shell/generated-bundle.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -// Tests the shape build-bundle.mjs produces in generated-bundle.ts. -// -// The bundle used to be a single ~3 MB JS string assigned to -// SHELL_BUNDLE. esbuild was inlining every dynamic import() -// just-bash makes (python3, js-exec, sqlite3, html-to-markdown, -// curl, …) into the one string, so workerd's Worker Loader -// parsed all of it on every cold start even though the default -// ShellWorker disables python, javascript, and network. -// -// build-bundle.mjs now runs esbuild with splitting: true and -// emits a record of module name → source. The host Worker -// spreads the whole record into the Loader callback's modules -// table; workerd parses each chunk on first import, so the -// dynamic ones stay cold until a script actually reaches for -// them. These tests are the contract. - -import { describe, expect, it } from "vitest"; - -import { SHELL_MODULES } from "./generated-bundle.js"; - -describe("SHELL_MODULES", () => { - it("exposes shell.js as the main module", () => { - expect(SHELL_MODULES["shell.js"]).toBeDefined(); - expect(typeof SHELL_MODULES["shell.js"].js).toBe("string"); - expect(SHELL_MODULES["shell.js"].js.length).toBeGreaterThan(0); - }); - - it("keeps the main module under 1 MB so cold start parses ~650 KB, not 3 MB", () => { - // Static-reachable set from entrypoint.ts measured at ~651 KB. - // Anything materially above that means esbuild stopped - // splitting and went back to inlining dynamic imports. - const mainBytes = SHELL_MODULES["shell.js"].js.length; - expect(mainBytes).toBeLessThan(1_000_000); - }); - - it("splits dynamic just-bash chunks into separate modules", () => { - // The whole point of (2): the bundle is no longer one blob. - // At least one chunk besides shell.js should be present. - const names = Object.keys(SHELL_MODULES); - expect(names.length).toBeGreaterThan(1); - expect(names).toContain("shell.js"); - }); - - it("emits chunk module names with a .js extension", () => { - // workerd's Worker Loader rejects extensionless module names - // for bare-string modules; chunks must keep their .js suffix. - for (const name of Object.keys(SHELL_MODULES)) { - expect(name.endsWith(".js")).toBe(true); - } - }); - - it("every module entry has a js source string", () => { - for (const [name, mod] of Object.entries(SHELL_MODULES)) { - expect(typeof mod.js, `module ${name}`).toBe("string"); - expect(mod.js.length, `module ${name} non-empty`).toBeGreaterThan(0); - } - }); -}); diff --git a/packages/computer/src/backends/worker-shell/index.ts b/packages/computer/src/backends/worker-shell/index.ts index 8605014b..c4cbf561 100644 --- a/packages/computer/src/backends/worker-shell/index.ts +++ b/packages/computer/src/backends/worker-shell/index.ts @@ -10,24 +10,33 @@ // // import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; // -// The package ships SHELL_MODULES — a record of module name → -// source string covering the pre-built ShellWorker entry plus -// every dynamic chunk just-bash code-splits into — and -// SHELL_RUNTIME_MODULES — the module shims just-bash's static -// native imports need to load under workerd. The backend -// spreads both into the Loader callback's `modules` table -// internally; consumers only need to reach for them when they -// construct the Loader callback by hand (in which case they -// pass a `fetcher` factory to WorkerShellBackend instead of -// `loader` + `workspace` + `ctx`). +// The package ships the shell as feature groups: SHELL_CORE_MODULES +// (the always-on core — the pre-built ShellWorker entry plus every +// dynamic chunk just-bash code-splits into for the base command +// set) and one optional group per command at +// @cloudflare/computer/shell/. A consumer imports the +// optional groups it wants and passes them to WorkerShellBackend's +// `commands` option; groups it never imports drop out of its +// bundle. It also ships SHELL_RUNTIME_MODULES — the module shims +// just-bash's static native imports need to load under workerd. +// The backend assembles core + `commands` and spreads the runtime +// shims into the Loader callback's `modules` table internally. +// assembleShellModules is exposed for consumers that construct the +// Loader callback by hand (in which case they pass a `fetcher` +// factory to WorkerShellBackend instead of `loader` + `workspace` + +// `ctx`). export { type WorkspaceFs, WorkspaceFsAdapter } from "./adapter.js"; export { type ArtifactsCommandHost, defineArtifactsCommand } from "./artifacts-command.js"; export { type AssetsCommandHost, defineAssetsCommand } from "./assets-command.js"; export { type ExecInput, ShellWorker, type ShellWorkerOptions } from "./entrypoint.js"; -export { SHELL_MODULES } from "./generated-bundle.js"; export { defineGitCommand, type GitCommandHost } from "./git-command.js"; export { SHELL_RUNTIME_MODULES } from "./runtime-modules.js"; +export { + assembleShellModules, + SHELL_CORE_MODULES, + type ShellModuleGroup, +} from "./shell-modules.js"; export { WorkerShellBackend, type WorkerShellBackendOptions, diff --git a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs index 406cbba4..ef396c79 100644 --- a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs +++ b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs @@ -1,8 +1,8 @@ -// Bundle entrypoint.ts into a record of module-name → source -// string and write it to generated-bundle.ts as a TypeScript -// module exporting that record (SHELL_MODULES). Consumers -// spread SHELL_MODULES into the Worker Loader callback's -// `modules` field without running a build of their own. +// Bundle entrypoint.ts into a set of module-name → source +// string records and write them under generated/ as TypeScript +// modules, each default-exporting one record. Consumers spread +// the records into the Worker Loader callback's `modules` field +// without running a build of their own. // // Why a record and not a single string? esbuild's bundle: true // inlines dynamic import() targets too, which dragged the full @@ -15,30 +15,77 @@ // it, so the cold-start cost matches the static-reachable set // (~650 KB) and optional features stay free until used. // -// The output file is gitignored. It's regenerated by the +// Why several files and not one? Each optional command +// (html-to-markdown, python, sqlite, js-exec, …) carries chunks +// that are only reachable through that command — html-to-markdown's +// domino (~555 KB) dominates the bundle. Splitting them into their +// own generated module, each published at +// @cloudflare/computer/shell/, lets a consumer select +// commands by import rather than by a build flag: shell-modules.ts +// imports only the core group, so a feature group the consumer +// never imports is unreachable in its module graph and the bundler +// drops the feature's exclusive chunks — and their heavy +// dependencies — from the uploaded Worker. Opting a command in is a +// single import passed to WorkerShellBackend's `commands` option; +// there is no default-on cost and no flag to set. +// +// curl is an optional group like the rest, but its heavy undici +// dependency (~620 KB) is aliased to a throwing stub at bundle time +// (script/undici-stub.mjs) because the Worker backend runs curl on +// the plain-fetch path, never the undici DNS-pinning path — so a +// consumer that imports the curl group gets a working curl without +// undici's bytes. +// +// The generated/ output is gitignored. It's regenerated by the // computer package's prepare / pretest / pretypecheck scripts // so any install from this repo or downstream consumer sees a // fresh bundle matching the source. import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, relative, resolve } from "node:path"; +import { basename, dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; +import { buildModuleGraph, parseCommandRegistry, partitionModules } from "./partition.mjs"; + const here = dirname(fileURLToPath(import.meta.url)); // Script lives at .../backends/worker-shell/script/build-bundle.mjs; // the bundle target is one level up at .../backends/worker-shell/. const root = resolve(here, ".."); -const out = resolve(root, "generated-bundle.ts"); +const outDir = resolve(root, "generated"); -await mkdir(dirname(out), { recursive: true }); +// Optional commands whose chunks are worth splitting out so a +// consumer can drop them. Keyed by the feature name used in the +// package subpath (@cloudflare/computer/shell/); the +// value lists the just-bash command names that pull the feature +// in. A chunk lands in a feature group only when that feature is +// the sole optional owner of it and core can't reach it; anything +// shared with core or with a second feature stays in core so +// dropping one feature never breaks another. +const OPTIONAL_FEATURES = { + curl: ["curl"], + "html-to-markdown": ["html-to-markdown"], + python: ["python3", "python"], + sqlite: ["sqlite3"], + "js-exec": ["js-exec", "node"], + yq: ["yq"], + file: ["file"], + xan: ["xan"], + jq: ["jq"], +}; + +// Start from a clean generated/ so a group that stops being +// emitted (e.g. curl folded into core) doesn't leave a stale file +// behind. The directory is gitignored and fully regenerated here. +await rm(outDir, { recursive: true, force: true }); +await mkdir(outDir, { recursive: true }); // esbuild's code-splitting requires an outdir to compute chunk // paths against. Use a scratch directory so the build output is // transient; we read result.outputFiles and never touch disk // for the .js fragments themselves. -const outdir = await mkdtemp(resolve(tmpdir(), "shell-bundle-")); +const scratch = await mkdtemp(resolve(tmpdir(), "shell-bundle-")); let result; try { @@ -47,7 +94,7 @@ try { bundle: true, write: false, splitting: true, - outdir, + outdir: scratch, format: "esm", target: "es2022", platform: "neutral", @@ -93,6 +140,27 @@ try { // resolves the import at module load time. "seek-bzip", ], + plugins: [ + // curl reaches undici only through just-bash's DNS-pinning + // connection owner, which the Worker backend never activates: + // ShellWorker registers curl on the plain-`fetch` path + // (entrypoint.ts passes a SecureFetch), so just-bash never + // constructs the pinned undici Agent. just-bash ships + // pre-bundled, so it imports undici as a hashed relative + // chunk (./chunks/undici-.js) rather than the bare + // "undici" specifier — redirect both to a throwing stub so + // the real ~620 KB module stays out of the upload while curl + // itself ships in core. + { + name: "exclude-undici", + setup(pluginBuild) { + pluginBuild.onResolve( + { filter: /(^undici$|[/\\]chunks[/\\]undici-[^/\\]*\.js$)/ }, + () => ({ path: resolve(here, "undici-stub.mjs") }), + ); + }, + }, + ], // Treat any stray .node native binding the resolver still // surfaces as an empty module. Belt-and-braces for transitive // deps that pull in native code through a deeper path. @@ -107,32 +175,30 @@ try { define: { "import.meta.url": JSON.stringify("file:///shell.js"), }, + // Emit the module graph as structured data so the partitioner + // reads import edges (and their static/dynamic kind) from + // esbuild rather than scraping the emitted source with regexes. + metafile: true, }); } finally { // The scratch outdir is only used to anchor relative paths. // No files were written (write: false), so removing it is // best-effort cleanup of the empty directory. - await rm(outdir, { recursive: true, force: true }); + await rm(scratch, { recursive: true, force: true }); } if (!result.outputFiles || result.outputFiles.length === 0) { throw new Error("build-bundle: esbuild returned no output"); } -// Collect each emitted file into the modules record keyed by -// its outdir-relative path. esbuild writes the entry as -// shell.js and chunks as chunk-.js per entryNames / -// chunkNames above. Worker Loader's `{ js: "..." }` module -// shape lets the keys carry the .js extension cleanly. -// -// Order is stable: sort by name so the generated file diffs -// predictably across builds when content hashes don't change. +// Collect each emitted file into a modules record keyed by its +// output basename. esbuild writes the entry as shell.js and chunks +// as chunk-.js per entryNames / chunkNames above; the +// basenames are unique and match the keys the module graph and +// partitioner use. const modules = {}; -let totalBytes = 0; for (const file of result.outputFiles) { - const name = relative(outdir, file.path).split(/[\\/]/).join("/"); - modules[name] = { js: file.text }; - totalBytes += file.text.length; + modules[basename(file.path)] = file.text; } if (!modules["shell.js"]) { @@ -141,22 +207,44 @@ if (!modules["shell.js"]) { ); } -const ordered = {}; -for (const name of Object.keys(modules).sort()) { - ordered[name] = modules[name]; -} +// The graph (import edges) comes from esbuild's metafile; command +// identity (which chunks are just-bash commands vs internal +// diagnostics) comes from parsing shell.js's { name, load } +// registry. partitionModules combines the two and throws if any +// optional feature resolves to no command chunk. +const graph = buildModuleGraph(result.metafile.outputs); +const registry = parseCommandRegistry(modules["shell.js"]); +const partition = partitionModules({ graph, registry, optionalFeatures: OPTIONAL_FEATURES }); -const header = "// Generated by script/build-bundle.mjs — do not edit.\n"; -const body = `export const SHELL_MODULES: Readonly> = Object.freeze(${JSON.stringify( - ordered, - null, - 2, -)});\n`; -const source = `${header}\n${body}`; +// Emit one generated file per group. shell-modules.ts imports +// each by its @cloudflare/computer/shell/ subpath and +// spreads them back together. +const groupNames = ["core", ...Object.keys(OPTIONAL_FEATURES)]; +let totalBytes = 0; +for (const group of groupNames) { + const names = (partition[group] ?? []).sort(); + const record = {}; + for (const name of names) { + record[name] = { js: modules[name] }; + totalBytes += modules[name].length; + } + const header = + `// Generated by script/build-bundle.mjs — do not edit.\n` + + `// Shell modules exclusive to the "${group}" feature group.\n`; + const body = `export default Object.freeze(${JSON.stringify( + record, + null, + 2, + )}) as Readonly>;\n`; + await writeFile(resolve(outDir, `${group}.ts`), `${header}\n${body}`); +} -await writeFile(out, source); -const chunkCount = Object.keys(ordered).length; -const mainBytes = ordered["shell.js"].js.length; +const coreCount = (partition.core ?? []).length; +const mainBytes = modules["shell.js"].length; +const featureSummary = Object.keys(OPTIONAL_FEATURES) + .map((f) => `${f} ${(partition[f] ?? []).length}`) + .join(", "); console.log( - `Wrote ${out} (${chunkCount} modules, shell.js ${mainBytes} bytes, total ${totalBytes} bytes)`, + `Wrote ${outDir} (core ${coreCount} modules, shell.js ${mainBytes} bytes, ` + + `features: ${featureSummary}, total ${totalBytes} bytes)`, ); diff --git a/packages/computer/src/backends/worker-shell/script/partition.mjs b/packages/computer/src/backends/worker-shell/script/partition.mjs new file mode 100644 index 00000000..0a8b1d45 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/partition.mjs @@ -0,0 +1,179 @@ +// Pure partitioning logic for build-bundle.mjs, split out so it can +// be unit-tested against synthetic module graphs without running a +// real esbuild bundle. +// +// The problem: esbuild emits shell.js plus a flat set of +// chunk-.js code-split fragments. We assign each fragment to +// exactly one group — "core" (always shipped) or one optional +// command feature — so a consumer that never imports a feature's +// group drops the feature's exclusive chunks from its bundle. +// +// Two independent facts drive the assignment, and they come from +// two different sources on purpose: +// +// 1. The module graph — which chunk imports which, and whether +// the edge is a static `import` or a lazy `import()`. This +// comes from esbuild's metafile (imports[].kind), not from +// scraping the emitted source. The metafile is esbuild's own +// structured account of the graph, so it survives codegen and +// minification changes that would break a text scrape. +// +// 2. Command identity — which chunks are just-bash *commands* +// (curl, python3, …) versus internal diagnostics that also +// lazy-load chunks (e.g. flag-coverage, which fans out to +// every command). Only just-bash's `{ name, load }` registry +// in shell.js carries this: a diagnostic's dynamic edges must +// not drag every command's heavy dependency into core, so we +// follow dynamic edges only out of genuine commands. The +// metafile can't distinguish the two — both look like +// code-split entry points — so the registry parse stays. + +import { basename } from "node:path"; + +// Build the module graph from esbuild's metafile `outputs`. Keys +// are output basenames (shell.js, chunk-.js) to match the +// modules record build-bundle.mjs assembles from result.outputFiles. +// Static and dynamic edges are kept apart so the partitioner can +// follow them selectively; external imports (node:*, workerd +// built-ins) are dropped since they resolve at runtime, not from +// the modules table. +export function buildModuleGraph(metafileOutputs) { + const modules = new Set(); + const staticEdges = new Map(); + const dynamicEdges = new Map(); + const entryPointOf = new Map(); + for (const [output, meta] of Object.entries(metafileOutputs)) { + const name = basename(output); + modules.add(name); + const staticTargets = new Set(); + const dynamicTargets = new Set(); + for (const edge of meta.imports ?? []) { + if (edge.external) continue; + const target = basename(edge.path); + if (edge.kind === "dynamic-import") dynamicTargets.add(target); + else if (edge.kind === "import-statement") staticTargets.add(target); + } + staticEdges.set(name, staticTargets); + dynamicEdges.set(name, dynamicTargets); + if (meta.entryPoint) entryPointOf.set(name, basename(meta.entryPoint)); + } + return { modules, staticEdges, dynamicEdges, entryPointOf }; +} + +// Parse just-bash's lazy command registry out of shell.js. Each +// entry looks like +// { name: "curl", load: async () => (await import("./chunk-…js")).curlCommand } +// Returns a command-name -> chunk-basename map. Diagnostics that +// lazy-load chunks through a different shape (flag-coverage's +// `{ …FlagCoverage } = await import(…)`) are deliberately not +// matched: they are not commands, and following their fan-out would +// pull every command's dependency into core. +export function parseCommandRegistry(shellSource) { + const registry = {}; + const re = + /\{\s*name:\s*"([^"]+)",\s*load:\s*async\s*\(\)\s*=>\s*\(await import\("(\.\/chunk-[^"]+)"\)\)/g; + for (const match of shellSource.matchAll(re)) { + registry[match[1]] = basename(match[2]); + } + return registry; +} + +// Resolve each optional feature to the chunk(s) its command entries +// load. A feature may list several command names (aliases such as +// python/python3 or node/js-exec) that resolve to the same chunk; +// duplicates collapse. Throws when a feature resolves to nothing: +// that means the registry parse came back empty or the command +// names drifted, and silently continuing would fold the feature's +// heavy chunk into core — the exact regression the split exists to +// prevent. Also throws if a resolved chunk is missing from the +// module graph, which signals the registry and the emitted output +// have drifted apart. +export function resolveFeatureRoots(registry, optionalFeatures, modules) { + const roots = new Map(); + for (const [feature, commands] of Object.entries(optionalFeatures)) { + const chunks = new Set(); + for (const command of commands) { + const chunk = registry[command]; + if (chunk !== undefined) chunks.add(chunk); + } + if (chunks.size === 0) { + throw new Error( + `partition: optional feature "${feature}" resolved no command chunk ` + + `from the shell registry (commands: ${commands.join(", ")}). The ` + + `registry parse likely broke or just-bash's command names changed; ` + + `refusing to fold the feature's chunks into core silently.`, + ); + } + for (const chunk of chunks) { + if (modules !== undefined && !modules.has(chunk)) { + throw new Error( + `partition: feature "${feature}" command chunk "${chunk}" is not in ` + + `the emitted module set. The shell registry and the bundle output ` + + `have drifted apart.`, + ); + } + } + roots.set(feature, [...chunks]); + } + return roots; +} + +// Assign every emitted module to exactly one group: "core" or one +// optional feature. A module belongs to a feature only when that +// feature is its sole reacher and core can't reach it; everything +// else — shared chunks, chunks a non-optional command reaches, the +// shell.js entry — stays in core. That invariant is what makes +// dropping one feature safe: no core code and no other feature can +// depend on a feature's exclusive chunks. +export function partitionModules({ graph, registry, optionalFeatures }) { + const { modules, staticEdges, dynamicEdges } = graph; + + const closure = (starts, followDynamic) => { + const seen = new Set(); + const stack = [...starts]; + while (stack.length > 0) { + const current = stack.pop(); + if (seen.has(current) || !modules.has(current)) continue; + seen.add(current); + for (const next of staticEdges.get(current) ?? []) stack.push(next); + if (followDynamic) for (const next of dynamicEdges.get(current) ?? []) stack.push(next); + } + return seen; + }; + + const featureRoots = resolveFeatureRoots(registry, optionalFeatures, modules); + const optionalCommands = new Set(Object.values(optionalFeatures).flat()); + + // Core reach: everything statically pulled by shell.js (the + // always-parsed entry) plus the full closure of every command + // that isn't optional. Dynamic edges out of shell.js are the + // per-command import() fan-out — following them would drag every + // optional chunk into core, so core uses shell.js's static edges + // only, then adds the closure of each kept command explicitly. + const coreReach = closure(["shell.js"], /* followDynamic */ false); + for (const [command, chunk] of Object.entries(registry)) { + if (!optionalCommands.has(command)) { + for (const name of closure([chunk], /* followDynamic */ true)) coreReach.add(name); + } + } + + const featureReach = new Map(); + for (const [feature, roots] of featureRoots) { + featureReach.set(feature, closure(roots, /* followDynamic */ true)); + } + + const partition = { core: [] }; + for (const feature of Object.keys(optionalFeatures)) partition[feature] = []; + for (const name of modules) { + const optionalOwners = []; + for (const feature of Object.keys(optionalFeatures)) { + if (featureReach.get(feature).has(name)) optionalOwners.push(feature); + } + if (!coreReach.has(name) && optionalOwners.length === 1) { + partition[optionalOwners[0]].push(name); + } else { + partition.core.push(name); + } + } + return partition; +} diff --git a/packages/computer/src/backends/worker-shell/script/partition.test.ts b/packages/computer/src/backends/worker-shell/script/partition.test.ts new file mode 100644 index 00000000..7c90de00 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/partition.test.ts @@ -0,0 +1,206 @@ +// Unit tests for the bundle partitioner. These drive synthetic +// module graphs through the pure functions so the assignment rules +// are pinned without running a real esbuild bundle — the real bundle +// is exercised end-to-end by shell-modules.test.ts against the +// generated output. + +import { describe, expect, it } from "vitest"; + +import { + buildModuleGraph, + parseCommandRegistry, + partitionModules, + resolveFeatureRoots, +} from "./partition.mjs"; + +describe("buildModuleGraph", () => { + it("keys modules by output basename and splits static from dynamic edges", () => { + const graph = buildModuleGraph({ + "out/shell.js": { + imports: [ + { path: "out/chunk-core.js", kind: "import-statement" }, + { path: "out/chunk-curl.js", kind: "dynamic-import" }, + ], + }, + "out/chunk-core.js": { imports: [] }, + "out/chunk-curl.js": { imports: [], entryPoint: "vendor/curl-ABCD1234.js" }, + }); + expect([...graph.modules].sort()).toEqual(["chunk-core.js", "chunk-curl.js", "shell.js"]); + expect([...graph.staticEdges.get("shell.js")]).toEqual(["chunk-core.js"]); + expect([...graph.dynamicEdges.get("shell.js")]).toEqual(["chunk-curl.js"]); + expect(graph.entryPointOf.get("chunk-curl.js")).toBe("curl-ABCD1234.js"); + }); + + it("drops external imports, which resolve at runtime not from the modules table", () => { + const graph = buildModuleGraph({ + "out/shell.js": { + imports: [ + { path: "node:fs", kind: "import-statement", external: true }, + { path: "out/chunk-core.js", kind: "import-statement" }, + ], + }, + "out/chunk-core.js": { imports: [] }, + }); + expect([...graph.staticEdges.get("shell.js")]).toEqual(["chunk-core.js"]); + }); +}); + +describe("parseCommandRegistry", () => { + it("maps command names to the chunk each lazy loader imports", () => { + const source = ` + const commands = [ + { name: "curl", load: async () => (await import("./chunk-CURL1234.js")).curlCommand }, + { name: "jq", load: async () => (await import("./chunk-JQAB5678.js")).jqCommand }, + ]; + `; + expect(parseCommandRegistry(source)).toEqual({ + curl: "chunk-CURL1234.js", + jq: "chunk-JQAB5678.js", + }); + }); + + it("ignores diagnostics that lazy-load through a different shape", () => { + // flag-coverage loads a chunk but is not a { name, load } command; + // matching it would drag every command's dependency into core. + const source = ` + const { instrumentFlagCoverage } = await import("./chunks/flag-coverage-THYQHOT3.js"); + const commands = [ + { name: "curl", load: async () => (await import("./chunk-CURL1234.js")).curlCommand }, + ]; + `; + expect(parseCommandRegistry(source)).toEqual({ curl: "chunk-CURL1234.js" }); + }); +}); + +describe("resolveFeatureRoots", () => { + const modules = new Set(["shell.js", "chunk-curl.js", "chunk-python.js"]); + + it("collapses command aliases that resolve to the same chunk", () => { + const registry = { python3: "chunk-python.js", python: "chunk-python.js" }; + const roots = resolveFeatureRoots(registry, { python: ["python3", "python"] }, modules); + expect(roots.get("python")).toEqual(["chunk-python.js"]); + }); + + it("throws when a feature resolves to no chunk", () => { + // An empty or drifted registry would otherwise fold the feature's + // heavy chunk into core silently — the regression the split + // exists to prevent. + expect(() => resolveFeatureRoots({}, { curl: ["curl"] }, modules)).toThrow( + /optional feature "curl" resolved no command chunk/, + ); + }); + + it("throws when a resolved chunk is absent from the module graph", () => { + const registry = { curl: "chunk-missing.js" }; + expect(() => resolveFeatureRoots(registry, { curl: ["curl"] }, modules)).toThrow( + /chunk "chunk-missing.js" is not in the emitted module set/, + ); + }); +}); + +describe("partitionModules", () => { + // A synthetic graph shaped like the real one: shell.js statically + // pulls its core, and lazy-loads commands. `diag` is a diagnostic + // (not a registry command) that fans out to the optional curl + // command; `req` is a required (non-optional) command. `shared` is + // reachable from the required command, so it must stay in core even + // though curl reaches it too. + const outputs = { + "out/shell.js": { + imports: [ + { path: "out/chunk-corelib.js", kind: "import-statement" }, + { path: "out/chunk-req.js", kind: "dynamic-import" }, + { path: "out/chunk-curl.js", kind: "dynamic-import" }, + { path: "out/chunk-diag.js", kind: "dynamic-import" }, + ], + }, + "out/chunk-corelib.js": { imports: [] }, + "out/chunk-req.js": { + imports: [{ path: "out/chunk-shared.js", kind: "import-statement" }], + entryPoint: "vendor/cat-REQ00000.js", + }, + "out/chunk-curl.js": { + imports: [ + { path: "out/chunk-shared.js", kind: "import-statement" }, + { path: "out/chunk-curlonly.js", kind: "import-statement" }, + ], + entryPoint: "vendor/curl-CURL0000.js", + }, + "out/chunk-curlonly.js": { imports: [] }, + "out/chunk-shared.js": { imports: [] }, + "out/chunk-diag.js": { + imports: [{ path: "out/chunk-curl.js", kind: "dynamic-import" }], + entryPoint: "vendor/flag-coverage-DIAG0000.js", + }, + }; + const registry = { + cat: "chunk-req.js", + curl: "chunk-curl.js", + }; + const optionalFeatures = { curl: ["curl"] }; + + it("assigns a feature's exclusive chunk to that feature", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + expect(part.curl).toContain("chunk-curlonly.js"); + expect(part.curl).toContain("chunk-curl.js"); + }); + + it("keeps a chunk a non-optional command reaches in core", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + // shared.js is reached by the required `cat` command, so it must + // ship in core even though curl reaches it too. + expect(part.core).toContain("chunk-shared.js"); + expect(part.curl).not.toContain("chunk-shared.js"); + }); + + it("does not follow a diagnostic's dynamic fan-out into core", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + // diag lazy-loads curl, but diag is not a command; its fan-out + // must not pull curl's exclusive chunk into core. + expect(part.core).not.toContain("chunk-curlonly.js"); + // The diagnostic chunk itself has no single optional owner, so it + // lands in core. + expect(part.core).toContain("chunk-diag.js"); + }); + + it("keeps shell.js and statically-reachable core in core", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + expect(part.core).toContain("shell.js"); + expect(part.core).toContain("chunk-corelib.js"); + }); + + it("keeps a chunk two features share in core", () => { + // fa and fb are both optional; a chunk both reach has more than + // one optional owner, so it belongs in core, not in either group. + const twoFeatureOutputs = { + "out/shell.js": { + imports: [ + { path: "out/chunk-fa.js", kind: "dynamic-import" }, + { path: "out/chunk-fb.js", kind: "dynamic-import" }, + ], + }, + "out/chunk-fa.js": { + imports: [{ path: "out/chunk-both.js", kind: "import-statement" }], + entryPoint: "vendor/xan-FA000000.js", + }, + "out/chunk-fb.js": { + imports: [{ path: "out/chunk-both.js", kind: "import-statement" }], + entryPoint: "vendor/jq-FB000000.js", + }, + "out/chunk-both.js": { imports: [] }, + }; + const graph = buildModuleGraph(twoFeatureOutputs); + const part = partitionModules({ + graph, + registry: { xan: "chunk-fa.js", jq: "chunk-fb.js" }, + optionalFeatures: { xan: ["xan"], jq: ["jq"] }, + }); + expect(part.core).toContain("chunk-both.js"); + expect(part.xan).toEqual(["chunk-fa.js"]); + expect(part.jq).toEqual(["chunk-fb.js"]); + }); +}); diff --git a/packages/computer/src/backends/worker-shell/script/undici-stub.mjs b/packages/computer/src/backends/worker-shell/script/undici-stub.mjs new file mode 100644 index 00000000..04578218 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/undici-stub.mjs @@ -0,0 +1,29 @@ +// Build-time replacement for `undici`, aliased in by +// build-bundle.mjs so the real ~620 KB network stack never enters +// the Worker shell bundle. +// +// just-bash only reaches undici through its DNS-pinning connection +// owner (network/dns-pin.ts), which runs solely when a curl/wget +// request is made with `denyPrivateRanges` enabled. The Worker +// backend registers curl on the plain-fetch path with +// `denyPrivateRanges` off (egress is governed by the Dynamic +// Worker's globalOutbound, not by in-isolate DNS pinning), so this +// code is never executed. The throwing members exist only to keep +// the dynamic `import("undici")` resolvable; reaching them means +// pinning was switched on without shipping the real dependency. + +const excluded = () => { + throw new Error( + "undici is excluded from the Worker shell bundle; curl runs on the fetch path with denyPrivateRanges disabled", + ); +}; + +export class Agent { + constructor() { + excluded(); + } +} + +export const fetch = excluded; + +export default { Agent, fetch }; diff --git a/packages/computer/src/backends/worker-shell/shell-modules.test.ts b/packages/computer/src/backends/worker-shell/shell-modules.test.ts new file mode 100644 index 00000000..445c3683 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/shell-modules.test.ts @@ -0,0 +1,154 @@ +// Tests the groups build-bundle.mjs emits under generated/ and the +// import-based assembly shell-modules.ts performs on top of them. +// +// build-bundle.mjs runs esbuild with splitting: true and partitions +// the emitted modules into a core group (every always-on command +// plus the ShellWorker entry) and one group per optional command +// (curl included), each published on its own +// @cloudflare/computer/shell/* subpath. shell-modules.ts imports +// only the core group; a consumer opts a command in by importing +// its group and passing it to assembleShellModules (or to +// WorkerBackend's `commands` option). A group the consumer never +// imports is unreachable in their bundle and drops out. These tests +// are the contract. + +import curlModules from "@cloudflare/computer/shell/curl"; +import fileModules from "@cloudflare/computer/shell/file"; +import htmlToMarkdownModules from "@cloudflare/computer/shell/html-to-markdown"; +import jqModules from "@cloudflare/computer/shell/jq"; +import jsExecModules from "@cloudflare/computer/shell/js-exec"; +import pythonModules from "@cloudflare/computer/shell/python"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; +import xanModules from "@cloudflare/computer/shell/xan"; +import yqModules from "@cloudflare/computer/shell/yq"; +import { describe, expect, it } from "vitest"; +import { assembleShellModules, SHELL_CORE_MODULES } from "./shell-modules.js"; + +const OPTIONAL_GROUPS = { + curl: curlModules, + "html-to-markdown": htmlToMarkdownModules, + python: pythonModules, + sqlite: sqliteModules, + "js-exec": jsExecModules, + yq: yqModules, + file: fileModules, + xan: xanModules, + jq: jqModules, +}; + +describe("SHELL_CORE_MODULES", () => { + it("exposes shell.js as the main module", () => { + expect(SHELL_CORE_MODULES["shell.js"]).toBeDefined(); + expect(typeof SHELL_CORE_MODULES["shell.js"].js).toBe("string"); + expect(SHELL_CORE_MODULES["shell.js"].js.length).toBeGreaterThan(0); + }); + + it("keeps the main module under 1 MB so cold start parses ~650 KB, not 3 MB", () => { + // Static-reachable set from entrypoint.ts measured at ~651 KB. + // Anything materially above that means esbuild stopped + // splitting and went back to inlining dynamic imports. + const mainBytes = SHELL_CORE_MODULES["shell.js"].js.length; + expect(mainBytes).toBeLessThan(1_000_000); + }); + + it("splits dynamic just-bash chunks into separate modules", () => { + // The whole point of splitting: the bundle is no longer one + // blob. At least one chunk besides shell.js should be present. + const names = Object.keys(SHELL_CORE_MODULES); + expect(names.length).toBeGreaterThan(1); + expect(names).toContain("shell.js"); + }); + + it("emits chunk module names with a .js extension", () => { + // workerd's Worker Loader rejects extensionless module names + // for bare-string modules; chunks must keep their .js suffix. + for (const name of Object.keys(SHELL_CORE_MODULES)) { + expect(name.endsWith(".js")).toBe(true); + } + }); + + it("every module entry has a js source string", () => { + for (const [name, mod] of Object.entries(SHELL_CORE_MODULES)) { + expect(typeof mod.js, `module ${name}`).toBe("string"); + expect(mod.js.length, `module ${name} non-empty`).toBeGreaterThan(0); + } + }); + + it("carries no optional command's chunks", () => { + // Every optional command lives in its own group; core holds + // none of them. Importing a group is the only way to ship it. + for (const [feature, groupModules] of Object.entries(OPTIONAL_GROUPS)) { + const names = Object.keys(groupModules); + expect(names.length, `${feature} group non-empty on disk`).toBeGreaterThan(0); + for (const name of names) { + expect( + SHELL_CORE_MODULES[name], + `${feature} chunk ${name} absent from core`, + ).toBeUndefined(); + } + } + }); + + it("excludes the real undici dependency, leaving only the stub", () => { + // undici is redirected to a throwing stub at bundle time, so + // even when curl is included it runs on the fetch path and the + // ~620 KB real dependency never ships. The stub marker lives in + // core (the secure-fetch seam), so it is present by default. + const marker = "undici is excluded from the Worker shell bundle"; + const stubPresent = Object.values(SHELL_CORE_MODULES).some((mod) => mod.js.includes(marker)); + expect(stubPresent).toBe(true); + }); +}); + +describe("assembleShellModules", () => { + it("returns core only when no groups are passed", () => { + const assembled = assembleShellModules(); + expect(Object.keys(assembled).sort()).toEqual(Object.keys(SHELL_CORE_MODULES).sort()); + }); + + it("folds an imported group in on top of core", () => { + const assembled = assembleShellModules([curlModules]); + // Core is still there. + expect(assembled["shell.js"]).toBeDefined(); + // curl's chunks are now present. + for (const name of Object.keys(curlModules)) { + expect(assembled[name], `curl chunk ${name} present`).toBeDefined(); + } + // A group that was not passed stays out. + for (const name of Object.keys(sqliteModules)) { + expect(assembled[name], `sqlite chunk ${name} absent`).toBeUndefined(); + } + }); + + it("folds multiple imported groups in", () => { + const assembled = assembleShellModules([curlModules, sqliteModules]); + for (const name of [...Object.keys(curlModules), ...Object.keys(sqliteModules)]) { + expect(assembled[name], `chunk ${name} present`).toBeDefined(); + } + }); +}); + +describe("shell feature groups", () => { + it("keeps curl in its own group with its arg-check message", () => { + const curlOnDisk = Object.values(curlModules).some((mod) => + mod.js.includes("curl: no URL specified"), + ); + expect(curlOnDisk).toBe(true); + }); + + it("keeps feature groups disjoint from each other", () => { + // A chunk owned by one feature must not also appear in another; + // a shared chunk belongs in core. + const entries = Object.entries(OPTIONAL_GROUPS); + for (let i = 0; i < entries.length; i++) { + for (let j = i + 1; j < entries.length; j++) { + const [aName, aMods] = entries[i]; + const [bName, bMods] = entries[j]; + const aKeys = new Set(Object.keys(aMods)); + for (const name of Object.keys(bMods)) { + expect(aKeys.has(name), `${name} in both ${aName} and ${bName}`).toBe(false); + } + } + } + }); +}); diff --git a/packages/computer/src/backends/worker-shell/shell-modules.ts b/packages/computer/src/backends/worker-shell/shell-modules.ts new file mode 100644 index 00000000..4a2ea850 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/shell-modules.ts @@ -0,0 +1,40 @@ +// Assembles the Worker Loader modules table the shell runs from. +// +// build-bundle.mjs runs esbuild with splitting: true and +// partitions the emitted modules into a core group (every +// always-on command plus the ShellWorker entry) and one group per +// optional command, each published on its own +// @cloudflare/computer/shell/ subpath. +// +// Selection is import-based, not flag-based: this module imports +// only the core group. A consumer opts a command in by importing +// its group module and passing it to WorkerBackend's `commands` +// option (or into assembleShellModules directly). Because nothing +// here references the optional groups, a group the consumer never +// imports is unreachable in their module graph and the bundler +// drops it — no build-time define, no explicit opt-out needed. + +import coreModules from "@cloudflare/computer/shell/core"; + +// One generated feature group: module name -> source string. The +// core group and every @cloudflare/computer/shell/ import +// share this shape. +export type ShellModuleGroup = Readonly>; + +// The always-on core group. Ships in every Worker shell; carries +// the ShellWorker entry (shell.js), the base command set, and the +// secure-fetch seam curl runs on when its group is included. +export const SHELL_CORE_MODULES: ShellModuleGroup = Object.freeze({ ...coreModules }); + +// Merge the core group with the optional groups the consumer +// imported and passed. Later groups win on key collisions, but the +// build keeps groups disjoint so order never matters in practice. +export function assembleShellModules( + groups: readonly ShellModuleGroup[] = [], +): Readonly> { + const modules: Record = { ...coreModules }; + for (const group of groups) { + Object.assign(modules, group); + } + return Object.freeze(modules); +} diff --git a/packages/computer/src/backends/worker-shell/worker-shell.ts b/packages/computer/src/backends/worker-shell/worker-shell.ts index e413ed42..6890ce5f 100644 --- a/packages/computer/src/backends/worker-shell/worker-shell.ts +++ b/packages/computer/src/backends/worker-shell/worker-shell.ts @@ -5,7 +5,8 @@ // caller hands the backend a Loader binding plus a {binding, id} // reference to the host DO, and the backend takes care of the // rest. It builds the Worker Loader callback's modules table -// (SHELL_MODULES plus the runtime stubs), wires a +// (core plus any opted-in command groups, plus the runtime +// stubs), wires a // WorkspaceServiceProxy loopback into the loaded Worker's env so // the shell can call env.HOST.getWorkspace() back into the host // DO, mints the Dynamic Worker stub through env.LOADER.get(...), @@ -26,8 +27,8 @@ import type { ExecEvent, ShellRPC, SyncRPC, WorkspaceRPC } from "@cloudflare/com import type { BackendHandle, WorkspaceBackend } from "../../backend.js"; import type { WorkspaceServiceProxyProps } from "../../proxy.js"; -import { SHELL_MODULES } from "./generated-bundle.js"; import { SHELL_RUNTIME_MODULES } from "./runtime-modules.js"; +import { assembleShellModules, type ShellModuleGroup } from "./shell-modules.js"; // The shape the loaded ShellWorker exposes. The host-side // implementation lives in ./entrypoint.ts; the backend consumes @@ -136,6 +137,17 @@ export interface WorkerShellBackendOptions { // workers on different loaders or with different shell // configurations). id?: string; + + // Optional shell command groups to include beyond the always-on + // core. Import the groups you want from + // @cloudflare/computer/shell/ and pass them here; the + // backend folds them into the Loader modules table on top of + // core. A group you never import is unreachable in your bundle + // and the bundler drops it, so this is how you opt a command in + // without shipping the rest. Ignored on the `fetcher` path, + // where the caller assembles the modules table itself (use + // assembleShellModules there). + commands?: readonly ShellModuleGroup[]; } const DEFAULT_COMPAT_DATE = "2026-06-17"; @@ -227,7 +239,7 @@ export class WorkerShellBackend implements WorkspaceBackend { compatibilityFlags, mainModule: "shell.js", modules: { - ...SHELL_MODULES, + ...assembleShellModules(this.#options.commands), ...SHELL_RUNTIME_MODULES, }, env: { diff --git a/packages/computer/test-helpers/shell-module-aliases.ts b/packages/computer/test-helpers/shell-module-aliases.ts new file mode 100644 index 00000000..fda19e62 --- /dev/null +++ b/packages/computer/test-helpers/shell-module-aliases.ts @@ -0,0 +1,28 @@ +// Resolve the published @cloudflare/computer/shell/* subpaths to +// their source files for the test runners. shell-modules.ts +// imports the core group by subpath and a consumer imports the +// optional ones it wants; those subpaths resolve through the +// package's dist exports, which a src-based test run doesn't +// build. Point them at the generated src files instead. + +import { resolve } from "node:path"; + +const src = resolve(import.meta.dirname, "..", "src", "backends", "worker-shell"); + +const groups = [ + "core", + "curl", + "html-to-markdown", + "python", + "sqlite", + "js-exec", + "yq", + "file", + "xan", + "jq", +] as const; + +export const shellModuleAliases = groups.map((group) => ({ + find: `@cloudflare/computer/shell/${group}`, + replacement: resolve(src, "generated", `${group}.ts`), +})); diff --git a/packages/computer/tests/worker-backend-worker.ts b/packages/computer/tests/worker-backend-worker.ts index 1e210136..a116d541 100644 --- a/packages/computer/tests/worker-backend-worker.ts +++ b/packages/computer/tests/worker-backend-worker.ts @@ -18,6 +18,7 @@ // SELF.fetch instead of holding a DO reference itself. import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; +import curlModules from "@cloudflare/computer/shell/curl"; import { WorkerShellBackend } from "../src/backends/worker-shell/index.js"; import type { DurableObjectStorageLike, WorkspaceStub } from "../src/index.js"; import { Workspace } from "../src/index.js"; @@ -42,6 +43,9 @@ export class HostDO extends DurableObject { loader: env.LOADER, workspace: { binding: "HOST", id: ctx.id.toString() }, ctx, + // Opt curl in by importing its group and passing it; the + // fetch-path curl integration test exercises the wiring. + commands: [curlModules], }), ], }); diff --git a/packages/computer/tests/worker-backend.test.ts b/packages/computer/tests/worker-backend.test.ts index 6030d352..6bc42510 100644 --- a/packages/computer/tests/worker-backend.test.ts +++ b/packages/computer/tests/worker-backend.test.ts @@ -120,6 +120,22 @@ describe("WorkerShellBackend end-to-end", () => { expect(result.stdout).toMatch(/done/); }); + it("registers curl on the fetch path when its group is imported", async () => { + // The harness opts curl in by passing the imported curl group + // to WorkerBackend's `commands`. ShellWorker wires curl to a + // SecureFetch over the isolate's global fetch (no undici). A + // registered curl with no URL fails its own arg check ("curl: + // no URL specified"); an unregistered command would instead be + // reported as not found. This proves the fetch-path curl is + // wired without depending on egress, which globalOutbound keeps + // closed. + const id = freshId(); + const result = await exec(id, "curl 2>&1; echo done"); + expect(result.stdout).toMatch(/done/); + expect(result.stdout).toMatch(/curl: no URL specified/); + expect(result.stdout).not.toMatch(/command not found/); + }); + it("isolates state between separate workspace ids", async () => { // Two host-DO names → two distinct workspaces, two distinct // Dynamic Worker isolates (the loader caches by diff --git a/packages/computer/tsconfig.json b/packages/computer/tsconfig.json index 0d9b0289..39d2c4e4 100644 --- a/packages/computer/tsconfig.json +++ b/packages/computer/tsconfig.json @@ -10,7 +10,21 @@ "esModuleInterop": true, "resolveJsonModule": true, "isolatedModules": true, - "types": ["@cloudflare/workers-types", "vitest/globals", "node"] + "types": ["@cloudflare/workers-types", "vitest/globals", "node"], + "paths": { + "@cloudflare/computer/shell/core": ["./src/backends/worker-shell/generated/core.ts"], + "@cloudflare/computer/shell/curl": ["./src/backends/worker-shell/generated/curl.ts"], + "@cloudflare/computer/shell/html-to-markdown": [ + "./src/backends/worker-shell/generated/html-to-markdown.ts" + ], + "@cloudflare/computer/shell/python": ["./src/backends/worker-shell/generated/python.ts"], + "@cloudflare/computer/shell/sqlite": ["./src/backends/worker-shell/generated/sqlite.ts"], + "@cloudflare/computer/shell/js-exec": ["./src/backends/worker-shell/generated/js-exec.ts"], + "@cloudflare/computer/shell/yq": ["./src/backends/worker-shell/generated/yq.ts"], + "@cloudflare/computer/shell/file": ["./src/backends/worker-shell/generated/file.ts"], + "@cloudflare/computer/shell/xan": ["./src/backends/worker-shell/generated/xan.ts"], + "@cloudflare/computer/shell/jq": ["./src/backends/worker-shell/generated/jq.ts"] + } }, "include": ["src/**/*.ts", "*.ts"] } diff --git a/packages/computer/vitest.config.ts b/packages/computer/vitest.config.ts index d8927922..54742b25 100644 --- a/packages/computer/vitest.config.ts +++ b/packages/computer/vitest.config.ts @@ -2,6 +2,8 @@ import { resolve } from "node:path"; import { defineConfig } from "vitest/config"; +import { shellModuleAliases } from "./test-helpers/shell-module-aliases.js"; + export default defineConfig({ resolve: { alias: [ @@ -27,6 +29,12 @@ export default defineConfig({ find: "pako", replacement: resolve(__dirname, "src/git/pako-zlib-shim.ts"), }, + // shell-modules.ts imports the generated groups by their + // published @cloudflare/computer/shell/* subpath. Those + // subpaths resolve through the package's dist exports, which + // don't exist under the src test runner — point them at the + // generated src files instead. + ...shellModuleAliases, ], }, test: { diff --git a/packages/computer/vitest.config.worker-backend.ts b/packages/computer/vitest.config.worker-backend.ts index ce1a53ab..3721d2c3 100644 --- a/packages/computer/vitest.config.worker-backend.ts +++ b/packages/computer/vitest.config.worker-backend.ts @@ -8,12 +8,22 @@ import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; import { defineConfig } from "vitest/config"; +import { shellModuleAliases } from "./test-helpers/shell-module-aliases.js"; + export default defineConfig({ plugins: [ cloudflareTest({ wrangler: { configPath: "./tests/wrangler.worker-backend.jsonc" }, }), ], + resolve: { + // The backend imports the core shell group, and the harness + // imports the curl group it opts in, both by their + // @cloudflare/computer/shell/* subpath. Those resolve through + // dist exports that this src test run doesn't build; map them + // to the generated src files. + alias: shellModuleAliases, + }, test: { globals: true, include: ["tests/worker-backend.test.ts"],