diff --git a/.github/DEPLOYMENT.md b/.github/DEPLOYMENT.md index 2eff66e1..615e4159 100644 --- a/.github/DEPLOYMENT.md +++ b/.github/DEPLOYMENT.md @@ -44,6 +44,25 @@ The Rust workflow maps both names and runs Rust release scripts from Rust PRs that change crate code must add a changelog fragment in `rust/changelog.d/`. +## Feature Documentation + +The feature catalog in `js/examples/features/catalog.mjs` drives executable +examples for both language packages and the generated documentation in +`docs/`. Pull requests run every catalog entry with Node.js, Bun and Rust, then +verify that the committed guide is current. + +Generate and validate the guide locally from the repository root: + +```bash +node scripts/generate-docs.mjs +node scripts/check-parity.mjs +node scripts/generate-docs.mjs --check +``` + +After changes reach `main`, `.github/workflows/docs.yml` publishes +`docs/site/` to GitHub Pages. Configure the repository's Pages source as +**GitHub Actions** before the first deployment. + ## Local Release Checks JavaScript: diff --git a/.github/scripts/check-language-parity.sh b/.github/scripts/check-language-parity.sh index 88f1e0a9..dab228c5 100755 --- a/.github/scripts/check-language-parity.sh +++ b/.github/scripts/check-language-parity.sh @@ -50,6 +50,7 @@ while IFS= read -r f; do js/src/*) js_source_changed=true ;; rust/src/*) rust_source_changed=true ;; js/benchmarks/* | js/tests/benchmark-*) js_benchmarks_changed=true ;; + rust/benchmarks/Cargo.lock) ;; rust/benchmarks/*) rust_benchmarks_changed=true ;; esac done <, >> and < redirect command input and output with shell-compatible behavior. +- [Command sequences](features/sequences.md) — &&, ||, ; and parentheses execute with the expected shell semantics. +- [Safe interpolation](features/interpolation.md) — Interpolated values are escaped as arguments; each language also exposes an explicit raw form. +- [Shell settings](features/shell-settings.md) — Shell settings model errexit, pipefail, verbose, xtrace and nounset behavior. + +### Utilities + +- [ANSI and control character helpers](features/ansi-utils.md) — Helpers can strip colours and control characters from captured output. + +## Libraries compared + +| Library | Version | Runs in | +| ------------------------------------------------------------------- | --------------- | ------------------ | +| [command-stream](https://github.com/link-foundation/command-stream) | this repository | Node.js, Bun | +| [Bun.$](https://bun.com/docs/runtime/shell) | 1.4 | Bun | +| [zx](https://github.com/google/zx) | 8 | Node.js, Bun, Deno | +| [execa](https://github.com/sindresorhus/execa) | 9.6 | Node.js, Bun, Deno | +| [ShellJS](https://github.com/shelljs/shelljs) | 0.10 | Node.js, Bun | +| [node:child_process](https://nodejs.org/api/child_process.html) | this repository | Node.js, Bun, Deno | diff --git a/docs/features/ansi-utils.md b/docs/features/ansi-utils.md new file mode 100644 index 00000000..528192f0 --- /dev/null +++ b/docs/features/ansi-utils.md @@ -0,0 +1,130 @@ +# ANSI and control character helpers + +Helpers can strip colours and control characters from captured output. + +**Category:** Utilities + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `AnsiUtils`, `configureAnsi`, `getAnsiConfig`, `processOutput` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/ansi-utils.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/ansi-utils.mjs) + +```js +// Helpers for dealing with ANSI escape sequences and control characters in +// captured output. +import { + AnsiUtils, + processOutput, + configureAnsi, + getAnsiConfig, +} from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const ESC = String.fromCharCode(27); +const BELL = String.fromCharCode(7); + +await example( + { id: 'ansi-utils', title: 'ANSI and control character helpers' }, + async ({ record }) => { + const coloured = `${ESC}[31mred${ESC}[0m and ${ESC}[32mgreen${ESC}[0m`; + record('stripAnsi removes the colours', AnsiUtils.stripAnsi(coloured)); + record( + 'stripControlChars keeps text readable', + AnsiUtils.stripControlChars(`beep${BELL}boop`) + ); + record( + 'stripAll does both', + AnsiUtils.stripAll(`${ESC}[31mred${ESC}[0m${BELL}`) + ); + record( + 'cleanForProcessing handles buffers', + AnsiUtils.cleanForProcessing(Buffer.from(coloured)).toString() + ); + + // The same helpers can be applied to every captured chunk through the global + // configuration. + const original = getAnsiConfig(); + record('default config', original); + configureAnsi({ preserveAnsi: false }); + record('processOutput with preserveAnsi disabled', processOutput(coloured)); + configureAnsi(original); + record('config restored', getAnsiConfig()); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# ansi-utils — ANSI and control character helpers +stripAnsi removes the colours: "red and green" +stripControlChars keeps text readable: "beepboop" +stripAll does both: "[31mred[0m" +cleanForProcessing handles buffers: "[31mred[0m and [32mgreen[0m" +default config: {"preserveAnsi":true,"preserveControlChars":true} +processOutput with preserveAnsi disabled: "red and green" +config restored: {"preserveAnsi":true,"preserveControlChars":true} +``` + +## Rust + +**API:** `AnsiUtils`, `AnsiConfig` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn ansi_utils() -> ExampleResult { + Ok(vec![observation( + "stripped output", + AnsiUtils::strip_all("\u{1b}[31mred\u{1b}[0m"), + )]) +} +``` + +### Output + +``` +# ansi-utils — Rust +stripped output: "red" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — no helper; strip the codes yourself. + +### [zx](https://github.com/google/zx) + +```js +chalk is re-exported for adding colour, but there is no helper for removing it +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +await execa({ stripFinalNewline: true })`echo hi`; // trailing newline only, not ANSI +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +Not supported — no helper; strip the codes yourself. + +### [node:child_process](https://nodejs.org/api/child_process.html) + +Not supported — no helper; strip the codes yourself. + +--- + +[← All features](../README.md) diff --git a/docs/features/async-iteration.md b/docs/features/async-iteration.md new file mode 100644 index 00000000..c90a048f --- /dev/null +++ b/docs/features/async-iteration.md @@ -0,0 +1,148 @@ +# Async iteration over output + +A command is an async iterable of chunks, so output can be handled as it arrives. + +**Category:** Streaming + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#[Symbol.asyncIterator]`, `ProcessRunner#stream` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/async-iteration.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/async-iteration.mjs) + +```js +// A command is an async iterable of output chunks, so output can be processed +// while the command is still running. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'async-iteration', title: 'Async iteration over output' }, + async ({ record }) => { + const lines = []; + for await (const chunk of $q`seq 1 5`.stream()) { + if (chunk.type === 'exit') { + continue; + } + lines.push({ type: chunk.type, data: chunk.data.toString() }); + } + record('chunk types', [...new Set(lines.map((l) => l.type))]); + record('collected output', lines.map((l) => l.data).join('')); + + // stdout and stderr are tagged, so both can be consumed from one loop. + const tagged = []; + for await (const chunk of $q`sh -c 'echo to-stdout; echo to-stderr >&2'`.stream()) { + if (chunk.type === 'exit') { + continue; + } + tagged.push([chunk.type, chunk.data.toString().trim()]); + } + record('tagged chunks', tagged.sort()); + + // Leaving the loop early terminates the command. + let seen = 0; + for await (const _chunk of $q`seq 1 1000`.stream()) { + seen++; + break; + } + record('iteration can stop early', seen === 1); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# async-iteration — Async iteration over output +chunk types: ["stdout"] +collected output: "1\n2\n3\n4\n5\n" +tagged chunks: [["stderr","to-stderr"],["stdout","to-stdout"]] +iteration can stop early: true +``` + +## Rust + +**API:** `StreamingRunner`, `OutputStream::next` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn async_iteration() -> ExampleResult { + let mut stream = StreamingRunner::new("printf 'one\\ntwo\\n'").stream(); + let mut stdout = Vec::new(); + let mut exit_code = None; + while let Some(chunk) = stream.next().await { + match chunk { + OutputChunk::Stdout(data) => stdout.extend(data), + OutputChunk::Stderr(_) => {} + OutputChunk::Exit(code) => exit_code = Some(code), + } + } + Ok(vec![ + observation("collected chunks", String::from_utf8(stdout)?), + observation("exit code", exit_code), + ]) +} +``` + +### Output + +``` +# async-iteration — Rust +collected chunks: "one\ntwo\n" +exit code: 0 +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +for await (const line of $`printf 'a\nb\n'`.lines()) { + /* line by line only */ +} +``` + +### [zx](https://github.com/google/zx) + +```js +for await (const line of $`printf 'a\nb\n'`) { + /* lines */ +} +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +for await (const line of execa`printf 'a\nb\n'`) { + /* lines */ +} +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +Not supported — output is only delivered as a whole string, or through the raw child process in async mode. + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +for await (const chunk of spawn('printf', ['a\nb\n']).stdout) { + /* Buffers */ +} +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/await-result.md b/docs/features/await-result.md new file mode 100644 index 00000000..9b1099be --- /dev/null +++ b/docs/features/await-result.md @@ -0,0 +1,122 @@ +# Await a command + +Awaiting a command returns an object with stdout, stderr and the exit code. + +**Category:** Running commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/await-result.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/await-result.mjs) + +```js +// Awaiting a command returns a result object with stdout, stderr and the exit code. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'await-result', title: 'Await a command' }, + async ({ record }) => { + const result = await $q`echo "hello world"`; + record('stdout', result.stdout); + record('stderr', result.stderr); + record('code', result.code); + + const system = await $q`sh -c 'printf out; printf err >&2'`; + record('stdout of a system binary', system.stdout); + record('stderr of a system binary', system.stderr); + + record('interpolated value', (await $q`echo ${'a value'}`).stdout); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# await-result — Await a command +stdout: "hello world\n" +stderr: "" +code: 0 +stdout of a system binary: "out" +stderr of a system binary: "err" +interpolated value: "a value\n" +``` + +## Rust + +**API:** `run`, `CommandResult` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn await_result() -> ExampleResult { + let result = quiet("echo hello").await?; + Ok(vec![ + observation("stdout", result.stdout), + observation("stderr", result.stderr), + observation("exit code", result.code), + ]) +} +``` + +### Output + +``` +# await-result — Rust +stdout: "hello\n" +stderr: "" +exit code: 0 +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +const { stdout, stderr, exitCode } = await $`echo hi`.quiet(); +// stdout and stderr are Buffers, not strings +``` + +### [zx](https://github.com/google/zx) + +```js +const { stdout, stderr, exitCode } = await $`echo hi`; +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const { stdout, stderr, exitCode } = await execa`echo hi`; +// no shell is involved, so `echo hi` is the binary `echo` with one argument +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +const result = shell.exec('echo hi', { silent: true }); +// result.stdout, result.stderr, result.code +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +const { stdout, stderr } = await promisify(execFile)('echo', ['hi']); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/buffers-strings.md b/docs/features/buffers-strings.md new file mode 100644 index 00000000..a04d3a44 --- /dev/null +++ b/docs/features/buffers-strings.md @@ -0,0 +1,123 @@ +# Buffer and string interfaces + +Output is available as a string and as raw bytes, without running the command twice. + +**Category:** Reading output + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#text`, `ProcessRunner#buffers` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/buffers-strings.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/buffers-strings.mjs) + +```js +// .buffers and .strings expose the output as Buffers or as decoded strings. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'buffers-strings', title: 'Buffer and string interfaces' }, + async ({ record }) => { + const asBuffer = await $q`echo buffered`.buffers.stdout; + record('buffers.stdout is a Buffer', Buffer.isBuffer(asBuffer)); + record('buffers.stdout content', asBuffer.toString()); + + const asString = await $q`echo stringified`.strings.stdout; + record('strings.stdout', asString); + + const stderrBuffer = await $q`sh -c 'echo problem >&2'`.buffers.stderr; + record('buffers.stderr content', stderrBuffer.toString()); + + // Binary-safe: bytes survive the round trip unchanged. + const bytes = await $q`printf 'a\\tb'`.buffers.stdout; + record('raw bytes', Array.from(bytes)); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# buffers-strings — Buffer and string interfaces +buffers.stdout is a Buffer: true +buffers.stdout content: "buffered\n" +strings.stdout: "stringified\n" +buffers.stderr content: "problem\n" +raw bytes: [97,9,98] +``` + +## Rust + +**API:** `CommandResult::stdout`, `OutputChunk` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn buffers_strings() -> ExampleResult { + let result = quiet("printf bytes").await?; + Ok(vec![ + observation("string", &result.stdout), + observation("bytes", result.stdout.as_bytes()), + ]) +} +``` + +### Output + +``` +# buffers-strings — Rust +string: "bytes" +bytes: [98,121,116,101,115] +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +const result = await $`echo hi`.quiet(); +result.stdout; // Buffer +await $`echo hi`.text(); // string, but runs the command again +``` + +### [zx](https://github.com/google/zx) + +```js +const p = await $`echo hi`; +p.stdout; // string +Buffer.from(p.stdout); // bytes by conversion +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const { stdout } = await execa({ encoding: 'buffer' })`echo hi`; // choose one up front +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +Not supported — output is decoded to a string; raw bytes are not available. + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +const { stdout } = await promisify(execFile)('echo', ['hi'], { + encoding: 'buffer', +}); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/builtin-catalog.md b/docs/features/builtin-catalog.md new file mode 100644 index 00000000..4d8406ee --- /dev/null +++ b/docs/features/builtin-catalog.md @@ -0,0 +1,119 @@ +# The built-in command catalog + +Common commands are implemented in-process in both languages for portable behavior. + +**Category:** Built-in commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `listCommands`, `enableVirtualCommands`, `disableVirtualCommands` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/builtin-catalog.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/builtin-catalog.mjs) + +```js +// command-stream ships built-in implementations of common shell commands, so +// scripts behave the same even where those binaries are missing. +import { + $, + listCommands, + enableVirtualCommands, + disableVirtualCommands, +} from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'builtin-catalog', title: 'The built-in command catalog' }, + async ({ record }) => { + record('available built-ins', listCommands().sort()); + record('number of built-ins', listCommands().length); + + // Built-ins can be switched off, which falls back to the real binaries. + record('with built-ins', (await $q`echo built-in`).stdout); + disableVirtualCommands(); + record('with built-ins disabled', (await $q`echo real binary`).stdout); + enableVirtualCommands(); + record('built-ins enabled again', listCommands().length > 0); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# builtin-catalog — The built-in command catalog +available built-ins: ["basename","cat","cd","cp","dirname","echo","env","exit","false","ls","mkdir","mv","pwd","rm","seq","sleep","tee","test","touch","true","which","yes"] +number of built-ins: 22 +with built-ins: "built-in\n" +with built-ins disabled: "real binary\n" +built-ins enabled again: true +``` + +## Rust + +**API:** `VirtualCommandRegistry::with_builtins` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn builtin_catalog() -> ExampleResult { + let registry = VirtualCommandRegistry::with_builtins(); + let mut commands = registry.list(); + commands.sort_unstable(); + Ok(vec![ + observation("available built-ins", &commands), + observation("number of built-ins", commands.len()), + ]) +} +``` + +### Output + +``` +# builtin-catalog — Rust +available built-ins: ["basename","cat","cd","cp","dirname","echo","env","exit","false","ls","mkdir","mv","pwd","rm","seq","sleep","tee","test","touch","true","which","yes"] +number of built-ins: 22 +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +// a fixed set of built-ins (cd, echo, ls, rm, ...) that cannot be listed or turned off +``` + +### [zx](https://github.com/google/zx) + +Not supported — every command is handed to the system shell; the fs and glob helpers are separate APIs, not commands. + +### [execa](https://github.com/sindresorhus/execa) + +Not supported — every command is a real binary. + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.ls(); +shell.cat(); +shell.mkdir(); // built-ins, but as functions rather than commands +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +Not supported — every command is a real binary. + +--- + +[← All features](../README.md) diff --git a/docs/features/builtin-environment.md b/docs/features/builtin-environment.md new file mode 100644 index 00000000..270adbca --- /dev/null +++ b/docs/features/builtin-environment.md @@ -0,0 +1,140 @@ +# Environment built-ins + +cd, pwd, env, which and exit affect the command they run in, not the host process. + +**Category:** Built-in commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/builtin-environment.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/builtin-environment.mjs) + +```js +// Environment built-ins: pwd, cd, env, which, sleep, exit. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import path from 'path'; + +await example( + { id: 'builtin-environment', title: 'Environment built-ins' }, + async ({ record }) => { + const dir = makeTempDir('env'); + const $q = $({ mirror: false }); + + record( + 'pwd inside a chosen directory', + (await $({ mirror: false, cwd: dir })`pwd`).stdout + ); + + // cd changes the working directory of the process, and is remembered by the + // following commands. + const before = (await $q`pwd`).stdout.trim(); + await $q`cd ${dir}`; + record('pwd after cd', (await $q`pwd`).stdout); + await $q`cd ${before}`; + record('back in the original directory', (await $q`pwd`).stdout); + + const withEnv = await $({ mirror: false, env: { DEMO: 'value' } })`env`; + record('env lists the variables', withEnv.stdout); + + record('which finds a binary', (await $q`which sh`).code); + + const started = Date.now(); + await $q`sleep 0.1`; + record('sleep waited', Date.now() - started >= 90); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# builtin-environment — Environment built-ins +pwd inside a chosen directory: "\n" +pwd after cd: "\n" +back in the original directory: "\n" +env lists the variables: "DEMO=value\n" +which finds a binary: 0 +sleep waited: true +``` + +## Rust + +**API:** `pwd`, `cd`, `env` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn builtin_environment() -> ExampleResult { + let mut env = HashMap::new(); + env.insert("COMMAND_STREAM_DEMO".to_string(), "visible".to_string()); + let result = exec( + "env", + RunOptions { + mirror: false, + env: Some(env), + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![observation( + "configured environment visible", + result.stdout.contains("COMMAND_STREAM_DEMO=visible"), + )]) +} +``` + +### Output + +``` +# builtin-environment — Rust +configured environment visible: true +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`cd /tmp && pwd`.text(); // cd is scoped to the command +``` + +### [zx](https://github.com/google/zx) + +```js +cd('/tmp'); // changes the directory for every later command +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +execa({ cwd: '/tmp' })`pwd`; // an option, not a command +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.cd('/tmp'); +shell.pwd(); // changes the process working directory +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +execFile('pwd', [], { cwd: '/tmp' }); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/builtin-filesystem.md b/docs/features/builtin-filesystem.md new file mode 100644 index 00000000..64bdf894 --- /dev/null +++ b/docs/features/builtin-filesystem.md @@ -0,0 +1,142 @@ +# File system built-ins + +ls, cat, mkdir, touch, cp, mv, rm and test run in-process. + +**Category:** Built-in commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/builtin-filesystem.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/builtin-filesystem.mjs) + +```js +// File system built-ins: mkdir, touch, ls, cp, mv, rm. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import fs from 'fs'; +import path from 'path'; + +await example( + { id: 'builtin-filesystem', title: 'File system built-ins' }, + async ({ record }) => { + const dir = makeTempDir('fs'); + const $q = $({ mirror: false, cwd: dir }); + + await $q`mkdir -p project/src`; + record( + 'mkdir -p created the tree', + fs.existsSync(path.join(dir, 'project/src')) + ); + + await $q`touch project/src/index.mjs`; + record( + 'touch created the file', + fs.existsSync(path.join(dir, 'project/src/index.mjs')) + ); + + record('ls', (await $q`ls project/src`).stdout); + + await $q`cp project/src/index.mjs project/src/copy.mjs`; + record('after cp', (await $q`ls project/src`).stdout); + + await $q`mv project/src/copy.mjs project/src/renamed.mjs`; + record('after mv', (await $q`ls project/src`).stdout); + + await $q`rm project/src/renamed.mjs`; + record('after rm', (await $q`ls project/src`).stdout); + + await $q`rm -rf project`; + record( + 'the tree still exists after rm -rf', + fs.existsSync(path.join(dir, 'project')) + ); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# builtin-filesystem — File system built-ins +mkdir -p created the tree: true +touch created the file: true +ls: "index.mjs\n" +after cp: "copy.mjs\nindex.mjs\n" +after mv: "index.mjs\nrenamed.mjs\n" +after rm: "index.mjs\n" +the tree still exists after rm -rf: false +``` + +## Rust + +**API:** `mkdir`, `touch`, `ls`, `rm` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn builtin_filesystem() -> ExampleResult { + let directory = tempfile::tempdir()?; + let options = RunOptions { + mirror: false, + cwd: Some(directory.path().to_path_buf()), + ..RunOptions::default() + }; + exec("mkdir demo", options.clone()).await?; + exec("touch demo/file.txt", options.clone()).await?; + let listed = exec("ls demo", options.clone()).await?; + exec("rm -r demo", options).await?; + Ok(vec![observation("created and listed", listed.stdout)]) +} +``` + +### Output + +``` +# builtin-filesystem — Rust +created and listed: "file.txt\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`mkdir -p dir`; +await $`ls dir`.text(); // built-in, same idea +``` + +### [zx](https://github.com/google/zx) + +```js +await fs.mkdirp('dir'); // zx re-exports fs-extra instead of implementing commands +``` + +### [execa](https://github.com/sindresorhus/execa) + +Not supported — use node:fs. + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.mkdir('-p', 'dir'); +shell.ls('dir'); +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +Not supported — use node:fs. + +--- + +[← All features](../README.md) diff --git a/docs/features/builtin-text.md b/docs/features/builtin-text.md new file mode 100644 index 00000000..2a92e9fc --- /dev/null +++ b/docs/features/builtin-text.md @@ -0,0 +1,130 @@ +# Text and value built-ins + +echo, seq, yes, basename, dirname, true and false run in-process. + +**Category:** Built-in commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/builtin-text.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/builtin-text.mjs) + +```js +// Text and value built-ins: echo, cat, seq, basename, dirname, true, false, test. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import fs from 'fs'; +import path from 'path'; + +await example( + { id: 'builtin-text', title: 'Text and value built-ins' }, + async ({ record }) => { + const dir = makeTempDir('text'); + const file = path.join(dir, 'greeting.txt'); + fs.writeFileSync(file, 'hello from a file\n'); + const $q = $({ mirror: false }); + + record('echo', (await $q`echo hello`).stdout); + record('echo -n', (await $q`echo -n no newline`).stdout); + record('cat', (await $q`cat ${file}`).stdout); + record('seq', (await $q`seq 1 4`).stdout); + record('basename', (await $q`basename /usr/local/lib/file.txt`).stdout); + record('dirname', (await $q`dirname /usr/local/lib/file.txt`).stdout); + record('true', (await $q`true`).code); + record('false', (await $q`false`).code); + record('test on an existing file', (await $q`test -f ${file}`).code); + record( + 'test on a missing file', + (await $q`test -f ${path.join(dir, 'missing')}`).code + ); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# builtin-text — Text and value built-ins +echo: "hello\n" +echo -n: "no newline" +cat: "hello from a file\n" +seq: "1\n2\n3\n4\n" +basename: "file.txt\n" +dirname: "/usr/local/lib\n" +true: 0 +false: 1 +test on an existing file: 0 +test on a missing file: 1 +``` + +## Rust + +**API:** `echo`, `seq`, `basename`, `dirname`, `test`, `which` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn builtin_text() -> ExampleResult { + let sequence = quiet("seq 1 3").await?; + let basename = quiet("basename /tmp/example.txt").await?; + Ok(vec![ + observation("sequence", sequence.stdout), + observation("basename", basename.stdout), + ]) +} +``` + +### Output + +``` +# builtin-text — Rust +sequence: "1\n2\n3\n" +basename: "example.txt\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`echo hi`.text(); // echo is a built-in; seq and yes are not +``` + +### [zx](https://github.com/google/zx) + +```js +await $`echo hi`; // the system binaries +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +await execa('echo', ['hi']); // the system binaries +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.echo('hi'); // echo only +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +execFile('echo', ['hi']); // the system binaries +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/cancellation.md b/docs/features/cancellation.md new file mode 100644 index 00000000..d1ba803b --- /dev/null +++ b/docs/features/cancellation.md @@ -0,0 +1,147 @@ +# Killing and cancelling commands + +A running command can be killed, and cancelling one leaves the rest of the script running. + +**Category:** Running commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#kill`, `forceCleanupAll` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/cancellation.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/cancellation.mjs) + +```js +// Running commands can be killed, and virtual commands are told about it +// through abortSignal / isCancelled(). +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'cancellation', title: 'Killing and cancelling commands' }, + async ({ record }) => { + const runner = $q`sleep 30`; + runner.start(); + setTimeout(() => runner.kill(), 100); + const killed = await runner; + record('exit code after kill()', killed.code); + + // The handler reports back as soon as it notices the cancellation, so the + // example does not depend on timing. + let noticed; + const noticedCancellation = new Promise((resolve) => { + noticed = resolve; + }); + + register('cancellable', async ({ abortSignal, isCancelled }) => { + for (let i = 0; i < 200; i++) { + if (abortSignal?.aborted || isCancelled()) { + noticed({ + aborted: abortSignal?.aborted === true, + cancelled: isCancelled(), + }); + break; + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } + return { stdout: '', code: 0 }; + }); + + const virtualRunner = $q`cancellable`; + virtualRunner.start(); + setTimeout(() => virtualRunner.kill(), 50); + await virtualRunner; + record('what the virtual command observed', await noticedCancellation); + unregister('cancellable'); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# cancellation — Killing and cancelling commands +exit code after kill(): 143 +what the virtual command observed: {"aborted":true,"cancelled":true} +``` + +## Rust + +**API:** `ProcessRunner::kill`, `OutputStream::kill` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn cancellation() -> ExampleResult { + let mut stream = StreamingRunner::new("sleep 30").stream(); + let started = stream.wait_for_pid().await.is_some(); + stream.kill(); + let mut exit_code = 0; + while let Some(chunk) = stream.next().await { + if let OutputChunk::Exit(code) = chunk { + exit_code = code; + } + } + Ok(vec![ + observation("process started", started), + observation("cancelled exit is non-zero", exit_code != 0), + ]) +} +``` + +### Output + +``` +# cancellation — Rust +process started: true +cancelled exit is non-zero: true +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — a ShellPromise has no kill method; the command runs to completion. + +### [zx](https://github.com/google/zx) + +```js +const p = $({ nothrow: true })`sleep 5`; +p.kill(); +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const p = execa({ reject: false })`sleep 5`; +p.kill(); +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +const child = shell.exec('sleep 5', { async: true }); +child.kill(); +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +const child = spawn('sleep', ['5']); +child.kill(); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/events.md b/docs/features/events.md new file mode 100644 index 00000000..b0b031f7 --- /dev/null +++ b/docs/features/events.md @@ -0,0 +1,144 @@ +# Event-driven output + +Event APIs report output and lifecycle signals as work progresses. + +**Category:** Streaming + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#on`, `ProcessRunner#off` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/events.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/events.mjs) + +```js +// Commands are EventEmitters: 'stdout', 'stderr', 'data' and 'end'. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'events', title: 'EventEmitter interface' }, + async ({ record }) => { + const events = []; + + await new Promise((resolve, reject) => { + $q`sh -c 'echo out; echo err >&2'` + .on('stdout', (data) => events.push(['stdout', data.toString().trim()])) + .on('stderr', (data) => events.push(['stderr', data.toString().trim()])) + .on('end', (result) => { + events.push(['end', result.code]); + resolve(); + }) + .on('error', reject) + .start(); + }); + + record( + 'events (sorted: stdout/stderr order is up to the OS)', + events.sort() + ); + + // The 'data' event receives both streams with a type tag. + const tagged = []; + await new Promise((resolve) => { + $q`echo tagged` + .on('data', (chunk) => + tagged.push([chunk.type, chunk.data.toString().trim()]) + ) + .on('end', () => resolve()) + .start(); + }); + record('data events', tagged); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# events — EventEmitter interface +events (sorted: stdout/stderr order is up to the OS): [["end",0],["stderr","err"],["stdout","out"]] +data events: [["stdout","tagged"]] +``` + +## Rust + +**API:** `StreamEmitter`, `EventType`, `EventData` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn events() -> ExampleResult { + let emitter = StreamEmitter::new(); + let count = Arc::new(AtomicUsize::new(0)); + let listener_count = Arc::clone(&count); + emitter + .on(EventType::Stdout, move |_| { + listener_count.fetch_add(1, Ordering::SeqCst); + }) + .await; + emitter + .emit(EventType::Stdout, EventData::String("hello".to_string())) + .await; + Ok(vec![observation( + "stdout events", + count.load(Ordering::SeqCst), + )]) +} +``` + +### Output + +``` +# events — Rust +stdout events: 1 +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — a ShellPromise is not an EventEmitter and exposes no streams. + +### [zx](https://github.com/google/zx) + +```js +$`echo hi`.stdout.on('data', (chunk) => { + /* Node stream events */ +}); +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +execa`echo hi`.stdout.on('data', (chunk) => { + /* Node stream events */ +}); +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.exec('echo hi', { async: true }).stdout.on('data', (chunk) => {}); +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +spawn('echo', ['hi']).stdout.on('data', (chunk) => {}); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/exit-codes.md b/docs/features/exit-codes.md new file mode 100644 index 00000000..a4d566d1 --- /dev/null +++ b/docs/features/exit-codes.md @@ -0,0 +1,126 @@ +# Exit codes and errors + +A non-zero exit code is reported on the result instead of thrown, unless errexit is set. + +**Category:** Running commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `shell.errexit` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/exit-codes.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/exit-codes.mjs) + +```js +// Exit codes are reported on the result; errors are thrown only when asked for. +import { $, shell } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'exit-codes', title: 'Exit codes and errors' }, + async ({ record }) => { + record('successful command', (await $q`sh -c 'exit 0'`).code); + record('failing command', (await $q`sh -c 'exit 42'`).code); + record( + 'stderr of a failing command', + (await $q`sh -c 'echo nope >&2; exit 1'`).stderr + ); + + // With errexit (set -e) a non-zero exit code becomes an exception. + shell.errexit(true); + try { + await $q`sh -c 'exit 42'`; + record('errexit', 'no error thrown'); + } catch (error) { + record('errexit throws', { code: error.code, hasResult: !!error.result }); + } finally { + shell.errexit(false); + } + + record('after disabling errexit', (await $q`sh -c 'exit 42'`).code); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# exit-codes — Exit codes and errors +successful command: 0 +failing command: 42 +stderr of a failing command: "nope\n" +errexit throws: {"code":42,"hasResult":true} +after disabling errexit: 42 +``` + +## Rust + +**API:** `CommandResult::code`, `CommandResult::error_for_status` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn exit_codes() -> ExampleResult { + let result = quiet("false").await?; + let checked = result.clone().error_for_status().unwrap_err(); + Ok(vec![ + observation("result code", result.code), + observation("checked error code", checked.code()), + ]) +} +``` + +### Output + +``` +# exit-codes — Rust +result code: 1 +checked error code: 1 +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +const { exitCode } = await $`exit 3`.nothrow(); // throws without .nothrow() +``` + +### [zx](https://github.com/google/zx) + +```js +const { exitCode } = await $({ nothrow: true })`exit 3`; // throws without nothrow +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const { exitCode } = await execa({ reject: false })`sh -c 'exit 3'`; // throws without reject: false +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +const code = shell.exec('exit 3', { silent: true }).code; // never throws +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +// execFile rejects on a non-zero exit; the code is on error.code +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/function-api.md b/docs/features/function-api.md new file mode 100644 index 00000000..14d60ef2 --- /dev/null +++ b/docs/features/function-api.md @@ -0,0 +1,121 @@ +# Function and builder APIs + +Commands can also be built from plain strings instead of template literals. + +**Category:** Running commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `sh`, `exec`, `run`, `create`, `shell` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/function-api.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/function-api.mjs) + +```js +// Besides the template tag there are plain functions: sh, exec, run and create. +import { $, sh, exec, run, create } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +await example( + { id: 'function-api', title: 'sh(), exec(), run() and create()' }, + async ({ record }) => { + record('sh(command)', (await sh('echo from-sh', { mirror: false })).stdout); + record( + 'exec(file, args)', + (await exec('echo', ['from-exec'], { mirror: false })).stdout + ); + record('run(command)', (await run('echo from-run')).stdout); + + // create() returns a $ with preset options. + const $quiet = create({ mirror: false, capture: true }); + record('create(options)', (await $quiet`echo from-create`).stdout); + + // $ itself can be called with options for the same effect. + record('$(options)', (await $({ mirror: false })`echo from-dollar`).stdout); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# function-api — sh(), exec(), run() and create() +sh(command): "from-sh\n" +exec(file, args): "from-exec\n" +run(command): "from-run\n" +create(options): "from-create\n" +$(options): "from-dollar\n" +``` + +## Rust + +**API:** `run`, `exec`, `create` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn function_api() -> ExampleResult { + let simple = run("echo run").await?; + let configured = exec("echo exec", quiet_options()).await?; + let mut runner = create("echo create", quiet_options()); + let created = runner.run().await?; + Ok(vec![observation( + "run, exec and create", + [ + simple.stdout.trim(), + configured.stdout.trim(), + created.stdout.trim(), + ], + )]) +} +``` + +### Output + +``` +# function-api — Rust +run, exec and create: ["run","exec","create"] +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — Bun.$ only accepts a tagged template; a string has to be turned back into one by hand. + +### [zx](https://github.com/google/zx) + +```js +await $({ input: '' })`sh -c ${'echo hi'}`; // or build a template array manually +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +await execa('echo', ['hi']); // the classic function form +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.exec('echo hi'); // strings are the only form +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +execFile('echo', ['hi']); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/interpolation.md b/docs/features/interpolation.md new file mode 100644 index 00000000..d4bf269f --- /dev/null +++ b/docs/features/interpolation.md @@ -0,0 +1,121 @@ +# Safe interpolation + +Interpolated values are escaped as arguments; each language also exposes an explicit raw form. + +**Category:** Shell syntax + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `quote`, `raw` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/interpolation.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/interpolation.mjs) + +```js +// Interpolated values are quoted automatically, so user input cannot turn into +// extra shell syntax. +import { $, quote, raw } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'interpolation', title: 'Safe interpolation' }, + async ({ record }) => { + const name = "it's a name"; + record('quotes are handled', (await $q`echo ${name}`).stdout); + + const dangerous = 'hello; rm -rf /tmp/nothing'; + record( + 'injection stays one argument', + (await $q`echo ${dangerous}`).stdout + ); + + const args = ['one', 'two three']; + record( + 'an array becomes separate arguments', + (await $q`echo ${args}`).stdout + ); + + record('quote() shows what interpolation does', quote("it's a name")); + + // raw() opts out of quoting when you really mean shell syntax. + record('raw() keeps shell syntax', (await $q`echo ${raw('a b')}`).stdout); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# interpolation — Safe interpolation +quotes are handled: "it's a name\n" +injection stays one argument: "hello; rm -rf /nothing\n" +an array becomes separate arguments: "one two three\n" +quote() shows what interpolation does: "'it'\\''s a name'" +raw() keeps shell syntax: "a b\n" +``` + +## Rust + +**API:** `cmd!`, `quote` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn interpolation() -> ExampleResult { + let value = "hello from Rust"; + let result = cmd!("echo {}", value).await?; + Ok(vec![observation("macro interpolation", result.stdout)]) +} +``` + +### Output + +``` +# interpolation — Rust +macro interpolation: "hello from Rust\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`echo ${value}`; // escaped; $.escape(value) shows the result +``` + +### [zx](https://github.com/google/zx) + +```js +await $`echo ${value}`; // escaped; quote(value) shows the result +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +await execa`echo ${value}`; // passed as an argument, no shell to escape for +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +Not supported — shell.exec takes a string, so escaping is the caller’s job. + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +execFile('echo', [value]); // arguments are never parsed as shell syntax +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/mirror-capture.md b/docs/features/mirror-capture.md new file mode 100644 index 00000000..8e909a0e --- /dev/null +++ b/docs/features/mirror-capture.md @@ -0,0 +1,126 @@ +# Mirroring and capturing output + +Output can be shown, captured, both or neither, chosen independently. + +**Category:** Reading output + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `create` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/mirror-capture.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/mirror-capture.mjs) + +```js +// mirror controls whether output is shown, capture whether it is kept. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +await example( + { id: 'mirror-capture', title: 'Mirroring and capturing output' }, + async ({ record }) => { + // The default: output is shown and captured. + const both = await $`echo shown and captured`; + record('default mirror', true); + record('default capture', both.stdout); + + const quiet = await $({ mirror: false })`echo only captured`; + record('mirror: false still captures', quiet.stdout); + + const dropped = await $({ mirror: false, capture: false })`echo neither`; + record('capture: false returns no stdout', dropped.stdout); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +shown and captured +# mirror-capture — Mirroring and capturing output +default mirror: true +default capture: "shown and captured\n" +mirror: false still captures: "only captured\n" +capture: false returns no stdout: undefined +``` + +## Rust + +**API:** `RunOptions::mirror`, `RunOptions::capture` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn mirror_capture() -> ExampleResult { + let captured = quiet("echo captured").await?; + let uncaptured = exec( + "true", + RunOptions { + mirror: false, + capture: false, + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![ + observation("captured output", captured.stdout), + observation("capture can be disabled", uncaptured.stdout.is_empty()), + ]) +} +``` + +### Output + +``` +# mirror-capture — Rust +captured output: "captured\n" +capture can be disabled: true +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`echo hi`; // shown and captured +await $`echo hi`.quiet(); // captured only +``` + +### [zx](https://github.com/google/zx) + +```js +$.verbose = true; // shown and captured +await $({ quiet: true })`echo hi`; +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +await execa({ stdout: ['pipe', 'inherit'] })`echo hi`; // both, by listing destinations +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.exec('echo hi'); // shown and captured +shell.exec('echo hi', { silent: true }); // captured only +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +spawn('echo', ['hi'], { stdio: 'inherit' }); // shown, but then not captured +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/options.md b/docs/features/options.md new file mode 100644 index 00000000..ed3a459e --- /dev/null +++ b/docs/features/options.md @@ -0,0 +1,142 @@ +# Options: capture, cwd, env, stdin + +Execution options control capture, cwd, environment and stdin for a command or reusable runner. + +**Category:** Running commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `create` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/options.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/options.mjs) + +```js +// $({ ... }) configures capture, mirroring, cwd, env and stdin. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import path from 'path'; +import fs from 'fs'; + +await example( + { id: 'options', title: 'Options: capture, cwd, env, stdin' }, + async ({ record }) => { + const dir = makeTempDir('options'); + fs.writeFileSync(path.join(dir, 'marker.txt'), 'here\n'); + + record( + 'captured output', + (await $({ mirror: false, capture: true })`echo captured`).stdout + ); + record( + 'capture disabled', + (await $({ mirror: false, capture: false })`echo dropped`).stdout + ); + + const inDir = await $({ mirror: false, cwd: dir })`ls`; + record('cwd option', inDir.stdout); + + const withEnv = await $({ + mirror: false, + env: { ...process.env, DEMO_VALUE: 'from-env' }, + })`printenv DEMO_VALUE`; + record('env option', withEnv.stdout); + + const withStdin = await $({ mirror: false, stdin: 'piped in\n' })`cat`; + record('stdin option', withStdin.stdout); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# options — Options: capture, cwd, env, stdin +captured output: "captured\n" +capture disabled: undefined +cwd option: "marker.txt\n" +env option: "from-env\n" +stdin option: "piped in\n" +``` + +## Rust + +**API:** `exec`, `RunOptions` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn options() -> ExampleResult { + let directory = tempfile::tempdir()?; + let mut env = HashMap::new(); + env.insert( + "COMMAND_STREAM_DEMO".to_string(), + "from-options".to_string(), + ); + let result = exec( + "cat", + RunOptions { + mirror: false, + cwd: Some(directory.path().to_path_buf()), + env: Some(env), + stdin: StdinOption::Content("from-stdin\n".to_string()), + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![observation("stdin and cwd options", result.stdout)]) +} +``` + +### Output + +``` +# options — Rust +stdin and cwd options: "from-stdin\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`pwd`.cwd('/tmp').env({ KEY: 'value' }).quiet(); +``` + +### [zx](https://github.com/google/zx) + +```js +const $$ = $({ cwd: '/tmp', env: { KEY: 'value' } }); +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const run = execa({ cwd: '/tmp', env: { KEY: 'value' } }); +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.cd('/tmp'); +shell.env.KEY = 'value'; // process-wide, not per command +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +execFile('pwd', [], { cwd: '/tmp', env: { KEY: 'value' } }); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/pipelines.md b/docs/features/pipelines.md new file mode 100644 index 00000000..9203b77c --- /dev/null +++ b/docs/features/pipelines.md @@ -0,0 +1,139 @@ +# Pipelines + +Commands can be composed into pipelines whose output feeds the next stage. + +**Category:** Shell syntax + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#pipe` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/pipelines.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/pipelines.mjs) + +```js +// Pipelines mix built-ins, your own commands and real binaries freely. +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example({ id: 'pipelines', title: 'Pipelines' }, async ({ record }) => { + register('upper', async ({ stdin }) => ({ + stdout: String(stdin ?? '').toUpperCase(), + code: 0, + })); + + record('built-in into built-in', (await $q`seq 1 3 | cat`).stdout); + record('built-in into your command', (await $q`echo hello | upper`).stdout); + record( + 'your command into a real binary', + (await $q`echo hello | upper | tr A-Z a-z`).stdout + ); + record( + 'real binary into your command', + (await $q`printf 'abc' | upper`).stdout + ); + + // The exit code of a pipeline is the exit code of its last stage. + record( + 'exit code of the last stage', + (await $q`echo x | sh -c 'exit 7'`).code + ); + record( + 'an earlier failure does not change it', + (await $q`sh -c 'exit 3' | cat`).code + ); + + // The .pipe() method builds the same pipeline from separate commands. + const piped = await $({ mirror: false })`echo method`.pipe( + $({ mirror: false })`upper` + ); + record('.pipe() method', piped.stdout); + + unregister('upper'); +}); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# pipelines — Pipelines +built-in into built-in: "1\n2\n3\n" +built-in into your command: "HELLO\n" +your command into a real binary: "hello\n" +real binary into your command: "ABC" +exit code of the last stage: 7 +an earlier failure does not change it: 0 +.pipe() method: "METHOD\n" +``` + +## Rust + +**API:** `Pipeline`, `PipelineExt` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn pipelines() -> ExampleResult { + let result = Pipeline::new() + .add("printf 'hello\\nworld\\n'") + .add("grep world") + .mirror_output(false) + .run() + .await?; + Ok(vec![observation("pipeline output", result.stdout)]) +} +``` + +### Output + +``` +# pipelines — Rust +pipeline output: "world\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`echo hi | tr a-z A-Z`.text(); +``` + +### [zx](https://github.com/google/zx) + +```js +await $`echo hi`.pipe($`tr a-z A-Z`); +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +await execa`echo hi`.pipe`tr a-z A-Z`; +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.echo('hi').exec('tr a-z A-Z'); +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +// connect the streams by hand: a.stdout.pipe(b.stdin) +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/redirection.md b/docs/features/redirection.md new file mode 100644 index 00000000..04911787 --- /dev/null +++ b/docs/features/redirection.md @@ -0,0 +1,140 @@ +# Redirecting output and input + +> , >> and < redirect command input and output with shell-compatible behavior. + +**Category:** Shell syntax + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/redirection.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/redirection.mjs) + +```js +// Output and input redirection work with built-ins and with your own commands, +// without handing the command line to a real shell. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import fs from 'fs'; +import path from 'path'; + +await example( + { id: 'redirection', title: 'Redirecting output and input' }, + async ({ record }) => { + const dir = makeTempDir('redirect'); + const file = path.join(dir, 'out.txt'); + const $q = $({ mirror: false }); + + const written = await $q`echo first > ${file}`; + record('the command itself prints nothing', written.stdout); + record('the file holds the output', fs.readFileSync(file, 'utf8')); + + await $q`echo second >> ${file}`; + record('>> appends', fs.readFileSync(file, 'utf8')); + + const numbers = path.join(dir, 'numbers.txt'); + await $q`seq 1 3 | cat > ${numbers}`; + record('a pipeline can redirect too', fs.readFileSync(numbers, 'utf8')); + + record('< feeds a command from a file', (await $q`cat < ${file}`).stdout); + record( + 'a quoted > stays a literal argument', + (await $q`echo "a > b"`).stdout + ); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# redirection — Redirecting output and input +the command itself prints nothing: "" +the file holds the output: "first\n" +>> appends: "first\nsecond\n" +a pipeline can redirect too: "1\n2\n3\n" +< feeds a command from a file: "first\nsecond\n" +a quoted > stays a literal argument: "a > b\n" +``` + +## Rust + +**API:** `exec` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn redirection() -> ExampleResult { + let directory = tempfile::tempdir()?; + let file = directory.path().join("output.txt"); + let result = exec( + "echo redirected > output.txt", + RunOptions { + mirror: false, + cwd: Some(directory.path().to_path_buf()), + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![ + observation("exit code", result.code), + observation("file contents", std::fs::read_to_string(file)?), + ]) +} +``` + +### Output + +``` +# redirection — Rust +exit code: 0 +file contents: "redirected\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`echo hi > out.txt`; +``` + +### [zx](https://github.com/google/zx) + +```js +await $`echo hi > out.txt`; // handled by the system shell +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +await execa({ stdout: { file: 'out.txt' } })`echo hi`; +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.echo('hi').to('out.txt'); +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +spawn('echo', ['hi'], { + stdio: ['ignore', fs.openSync('out.txt', 'w'), 'inherit'], +}); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/result-text.md b/docs/features/result-text.md new file mode 100644 index 00000000..49a91c79 --- /dev/null +++ b/docs/features/result-text.md @@ -0,0 +1,110 @@ +# Read the output with text() + +Captured stdout is available as text through each language’s result API. + +**Category:** Reading output + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#text` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/result-text.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/result-text.mjs) + +```js +// Every result exposes an async text() method, like Bun's built-in $. +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'result-text', title: 'Read the output with text()' }, + async ({ record }) => { + record('system command', await (await $q`sh -c 'echo system'`).text()); + record('built-in command', await (await $q`echo built-in`).text()); + record('synchronous command', await $q`echo sync`.sync().text()); + record('pipeline', await (await $q`echo piped | cat`).text()); + + register('text-demo', async () => ({ stdout: 'virtual\n', code: 0 })); + record('virtual command', await (await $q`text-demo`).text()); + unregister('text-demo'); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# result-text — Read the output with text() +system command: "system\n" +built-in command: "built-in\n" +synchronous command: "sync\n" +pipeline: "piped\n" +virtual command: "virtual\n" +``` + +## Rust + +**API:** `CommandResult::stdout` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn result_text() -> ExampleResult { + let result = quiet("echo hello").await?; + Ok(vec![observation("text output", result.stdout)]) +} +``` + +### Output + +``` +# result-text — Rust +text output: "hello\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +const text = await $`echo hi`.text(); +``` + +### [zx](https://github.com/google/zx) + +```js +const text = (await $`echo hi`).toString(); +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const text = (await execa`echo hi`).stdout; +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +const text = shell.exec('echo hi', { silent: true }).stdout; +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +const text = (await promisify(execFile)('echo', ['hi'])).stdout; +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/sequences.md b/docs/features/sequences.md new file mode 100644 index 00000000..3ba255a5 --- /dev/null +++ b/docs/features/sequences.md @@ -0,0 +1,112 @@ +# Command sequences + +&&, ||, ; and parentheses execute with the expected shell semantics. + +**Category:** Shell syntax + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/sequences.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/sequences.mjs) + +```js +// Operators between commands: && runs on success, || runs on failure, +// ; runs unconditionally and ( ) groups commands into a subshell. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'sequences', title: 'Command sequences' }, + async ({ record }) => { + record('&& after a success', (await $q`true && echo ran`).stdout); + record('&& after a failure', (await $q`false && echo ran`).stdout); + record('|| after a failure', (await $q`false || echo fallback`).stdout); + record('|| after a success', (await $q`true || echo fallback`).stdout); + record('; runs both', (await $q`echo one ; echo two`).stdout); + record('( ) groups commands', (await $q`(echo a ; echo b)`).stdout); + + const chain = await $q`false && echo skipped`; + record('exit code of a short-circuited chain', chain.code); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# sequences — Command sequences +&& after a success: "ran\n" +&& after a failure: "" +|| after a failure: "fallback\n" +|| after a success: "" +; runs both: "one\ntwo\n" +( ) groups commands: "a\nb\n" +exit code of a short-circuited chain: 1 +``` + +## Rust + +**API:** `exec` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn sequences() -> ExampleResult { + let result = quiet("false || echo fallback; echo next").await?; + Ok(vec![observation("sequence output", result.stdout)]) +} +``` + +### Output + +``` +# sequences — Rust +sequence output: "fallback\nnext\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`mkdir -p dir && cd dir && pwd`.text(); +``` + +### [zx](https://github.com/google/zx) + +```js +await $`mkdir -p dir && cd dir && pwd`; // the system shell runs it +``` + +### [execa](https://github.com/sindresorhus/execa) + +Not supported — no shell operators unless the shell option is turned on, which gives up escaping. + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.exec('mkdir -p dir && cd dir && pwd'); // the system shell runs it +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +execFile('sh', ['-c', 'mkdir -p dir && cd dir && pwd']); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/shell-settings.md b/docs/features/shell-settings.md new file mode 100644 index 00000000..cc113766 --- /dev/null +++ b/docs/features/shell-settings.md @@ -0,0 +1,137 @@ +# Shell settings + +Shell settings model errexit, pipefail, verbose, xtrace and nounset behavior. + +**Category:** Shell syntax + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `shell`, `set`, `unset` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/shell-settings.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/shell-settings.mjs) + +```js +// Shell settings mirror `set -e`, `set -x`, `set -v` and `set -o pipefail`. +import { $, shell, set, unset } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'shell-settings', title: 'Shell settings' }, + async ({ record }) => { + record('defaults', shell.settings()); + + set('e'); + record('set("e") enables errexit', shell.settings().errexit); + try { + await $q`sh -c 'exit 5'`; + record('failing command with errexit', 'did not throw'); + } catch (error) { + record('failing command with errexit', `threw with code ${error.code}`); + } + unset('e'); + + shell.pipefail(true); + record( + 'pipefail makes an early failure win', + (await $q`sh -c 'exit 3' | cat`).code + ); + shell.pipefail(false); + record( + 'without pipefail the last stage wins', + (await $q`sh -c 'exit 3' | cat`).code + ); + + set('x'); + record('xtrace on', shell.settings().xtrace); + unset('x'); + record('settings restored', shell.settings()); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# shell-settings — Shell settings +defaults: {"errexit":false,"verbose":false,"xtrace":false,"pipefail":false,"nounset":false} +set("e") enables errexit: true +failing command with errexit: "threw with code 5" +pipefail makes an early failure win: 3 +without pipefail the last stage wins: 0 +xtrace on: true +settings restored: {"errexit":false,"verbose":false,"xtrace":false,"pipefail":false,"nounset":false} +``` + +## Rust + +**API:** `ShellSettings`, `set_shell_option`, `unset_shell_option` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn shell_settings() -> ExampleResult { + set_shell_option("pipefail").await; + let with_pipefail = Pipeline::new().add("false").add("true").run().await?; + unset_shell_option("pipefail").await; + let without_pipefail = Pipeline::new().add("false").add("true").run().await?; + Ok(vec![ + observation("with pipefail", with_pipefail.code), + observation("without pipefail", without_pipefail.code), + ]) +} +``` + +### Output + +``` +# shell-settings — Rust +with pipefail: 1 +without pipefail: 0 +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +$.throws(true); // errexit only +``` + +### [zx](https://github.com/google/zx) + +```js +$.verbose = true; // verbose only; the rest belong to the system shell +``` + +### [execa](https://github.com/sindresorhus/execa) + +Not supported — no shell settings; the equivalents are per-command options. + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.config.fatal = true; +shell.config.verbose = true; // errexit and verbose +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +execFile('sh', ['-c', 'set -eo pipefail; ...']); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/stdin-streaming.md b/docs/features/stdin-streaming.md new file mode 100644 index 00000000..7a0508fc --- /dev/null +++ b/docs/features/stdin-streaming.md @@ -0,0 +1,129 @@ +# Writing to stdin while a command runs + +Input can be supplied up front or written to a running command. + +**Category:** Streaming + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#stdin` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/stdin-streaming.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/stdin-streaming.mjs) + +```js +// .streams.stdin gives write access to a running command. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'stdin-streaming', title: 'Writing to stdin while a command runs' }, + async ({ record }) => { + const runner = $q`cat`; + const stdin = await runner.streams.stdin; + stdin.write('first line\n'); + stdin.write('second line\n'); + stdin.end(); + record('what cat echoed back', (await runner).stdout); + + // A whole string can also be handed over up front. + record( + 'stdin option', + (await $({ mirror: false, stdin: 'up front\n' })`cat`).stdout + ); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# stdin-streaming — Writing to stdin while a command runs +what cat echoed back: "first line\nsecond line\n" +stdin option: "up front\n" +``` + +## Rust + +**API:** `ProcessRunner::write_stdin`, `ProcessRunner::close_stdin` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn stdin_streaming() -> ExampleResult { + let mut runner = ProcessRunner::new( + "cat", + RunOptions { + mirror: false, + stdin: StdinOption::Pipe, + ..RunOptions::default() + }, + ); + runner.start().await?; + runner.write_stdin("first line\n").await?; + runner.write_stdin("second line\n").await?; + runner.close_stdin().await?; + let result = runner.run().await?; + Ok(vec![observation("what cat echoed back", result.stdout)]) +} +``` + +### Output + +``` +# stdin-streaming — Rust +what cat echoed back: "first line\nsecond line\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +```js +await $`cat < ${new Response('x')}`.quiet(); // a value, not a live stream +``` + +### [zx](https://github.com/google/zx) + +```js +const p = $`cat`; +p.stdin.write('x'); +p.stdin.end(); +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const p = execa`cat`; +p.stdin.write('x'); +p.stdin.end(); +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +shell.ShellString('x').exec('cat'); // value only +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +const p = spawn('cat'); +p.stdin.write('x'); +p.stdin.end(); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/sync-execution.md b/docs/features/sync-execution.md new file mode 100644 index 00000000..00d8b9c8 --- /dev/null +++ b/docs/features/sync-execution.md @@ -0,0 +1,121 @@ +# Synchronous execution + +The same command can be run without awaiting, blocking until it finishes. + +**Category:** Running commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `$`, `ProcessRunner#sync` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/sync-execution.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/sync-execution.mjs) + +```js +// .sync() runs a command synchronously and returns the finished result. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'sync-execution', title: 'Synchronous execution' }, + async ({ record }) => { + const result = $q`echo synchronous`.sync(); + record('stdout', result.stdout); + record('code', result.code); + record( + 'result is available without await', + typeof result.stdout === 'string' + ); + + const failed = $q`sh -c 'exit 3'`.sync(); + record('exit code of a failing command', failed.code); + + record( + 'order of execution', + (() => { + const order = []; + order.push('before'); + $q`echo ignored`.sync(); + order.push('after'); + return order; + })() + ); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# sync-execution — Synchronous execution +stdout: "synchronous\n" +code: 0 +result is available without await: true +exit code of a failing command: 3 +order of execution: ["before","after"] +``` + +## Rust + +**API:** `run_sync` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn sync_execution() -> ExampleResult { + let result = tokio::task::spawn_blocking(|| run_sync("echo synchronous")).await??; + Ok(vec![observation("stdout", result.stdout)]) +} +``` + +### Output + +``` +# sync-execution — Rust +stdout: "synchronous\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — Bun.$ is always asynchronous; Bun.spawnSync is the synchronous escape hatch, and it takes an argument array rather than a command line. + +### [zx](https://github.com/google/zx) + +```js +const { stdout } = $.sync`echo hi`; +``` + +### [execa](https://github.com/sindresorhus/execa) + +```js +const { stdout } = execaSync`echo hi`; +``` + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +const stdout = shell.exec('echo hi', { silent: true }).stdout; // synchronous by default +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +```js +const stdout = execFileSync('echo', ['hi'], { encoding: 'utf8' }); +``` + +--- + +[← All features](../README.md) diff --git a/docs/features/virtual-commands.md b/docs/features/virtual-commands.md new file mode 100644 index 00000000..d5cad04c --- /dev/null +++ b/docs/features/virtual-commands.md @@ -0,0 +1,126 @@ +# Registering your own commands + +A handler can be registered by name and invoked through a registry or command runner. + +**Category:** Your own commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `register`, `unregister`, `listCommands` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/virtual-commands.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/virtual-commands.mjs) + +```js +// Any JavaScript function can be registered as a command and then used from a +// command line like a real binary. +import { $, register, unregister, listCommands } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'virtual-commands', title: 'Registering your own commands' }, + async ({ record }) => { + register('greet', async ({ args }) => ({ + stdout: `Hello, ${args.join(' ') || 'world'}!\n`, + code: 0, + })); + + record('the command is registered', listCommands().includes('greet')); + record('without arguments', (await $q`greet`).stdout); + record('with arguments', (await $q`greet Node and Bun`).stdout); + + // A handler decides its own exit code and may write to stderr. + register('fail-with', async ({ args }) => ({ + stderr: `failing on purpose\n`, + code: Number(args[0] ?? 1), + })); + const failed = await $q`fail-with 42`; + record('custom exit code', failed.code); + record('custom stderr', failed.stderr); + + unregister('greet'); + unregister('fail-with'); + record('unregistered again', listCommands().includes('greet')); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# virtual-commands — Registering your own commands +the command is registered: true +without arguments: "Hello, world!\n" +with arguments: "Hello, Node and Bun!\n" +custom exit code: 42 +custom stderr: "failing on purpose\n" +unregistered again: false +``` + +## Rust + +**API:** `VirtualCommandRegistry::register`, `VirtualCommandRegistry::unregister` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn virtual_commands() -> ExampleResult { + let mut registry = VirtualCommandRegistry::new(); + registry.register("greet", greet_handler); + let handler = registry.get("greet").expect("registered handler"); + let result = handler(CommandContext::new(vec!["Rust".to_string()])).await; + let removed = registry.unregister("greet"); + Ok(vec![ + observation("custom command output", result.stdout), + observation("unregistered again", removed), + ]) +} +``` + +### Output + +``` +# virtual-commands — Rust +custom command output: "Hello, Rust!\n" +unregistered again: true +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — the built-in set is fixed; a name cannot be bound to a JavaScript function. + +### [zx](https://github.com/google/zx) + +Not supported — a command name always resolves to a binary in PATH. + +### [execa](https://github.com/sindresorhus/execa) + +Not supported — a command name always resolves to a binary in PATH. + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +require('shelljs/plugin').register('greet', (options, name) => `hi ${name}\n`); +shell.greet('bob'); // a method, not a command usable inside a pipeline string +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +Not supported — a command name always resolves to a binary in PATH. + +--- + +[← All features](../README.md) diff --git a/docs/features/virtual-context.md b/docs/features/virtual-context.md new file mode 100644 index 00000000..08c747d8 --- /dev/null +++ b/docs/features/virtual-context.md @@ -0,0 +1,121 @@ +# The handler context + +A handler receives args, stdin, cwd, env and a cancellation signal. + +**Category:** Your own commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `register` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/virtual-context.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/virtual-context.mjs) + +```js +// A command handler receives a context object describing how it was invoked. +import { $, register, unregister } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; + +await example( + { id: 'virtual-context', title: 'The handler context' }, + async ({ record }) => { + const dir = makeTempDir('context'); + + register('describe', async ({ args, stdin, cwd, env, options }) => ({ + stdout: + JSON.stringify({ + args, + stdin, + cwdIsTheOneWeAskedFor: cwd === dir, + envValue: env.DEMO, + mirror: options.mirror, + }) + '\n', + code: 0, + })); + + const result = await $({ + mirror: false, + cwd: dir, + env: { DEMO: 'from-options' }, + })`echo piped | describe one two`; + record('context seen by the handler', JSON.parse(result.stdout)); + + unregister('describe'); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# virtual-context — The handler context +context seen by the handler: {"args":["one","two"],"stdin":"piped\n","cwdIsTheOneWeAskedFor":true,"envValue":"from-options","mirror":false} +``` + +## Rust + +**API:** `CommandContext` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn virtual_context() -> ExampleResult { + let mut context = CommandContext::new(vec!["one".to_string(), "two".to_string()]); + context.stdin = Some("piped\n".to_string()); + context.cwd = Some(std::env::temp_dir()); + context.env = Some(HashMap::from([("DEMO".to_string(), "value".to_string())])); + Ok(vec![observation( + "handler context", + json!({ + "args": context.args, + "stdin": context.stdin, + "has_cwd": context.cwd.is_some(), + "env_value": context.env.and_then(|env| env.get("DEMO").cloned()), + }), + )]) +} +``` + +### Output + +``` +# virtual-context — Rust +handler context: {"args":["one","two"],"env_value":"value","has_cwd":true,"stdin":"piped\n"} +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — no handler API. + +### [zx](https://github.com/google/zx) + +Not supported — no handler API. + +### [execa](https://github.com/sindresorhus/execa) + +Not supported — no handler API. + +### [ShellJS](https://github.com/shelljs/shelljs) + +```js +require('shelljs/plugin').readFromPipe(); // stdin only; no cwd, env or cancellation +``` + +### [node:child_process](https://nodejs.org/api/child_process.html) + +Not supported — no handler API. + +--- + +[← All features](../README.md) diff --git a/docs/features/virtual-streaming.md b/docs/features/virtual-streaming.md new file mode 100644 index 00000000..1089fdbb --- /dev/null +++ b/docs/features/virtual-streaming.md @@ -0,0 +1,129 @@ +# Streaming commands + +A streaming handler publishes output incrementally like a real process. + +**Category:** Your own commands + +**Languages:** JavaScript, Rust + +## JavaScript + +**API:** `register` + +**Verified in:** Node.js, Bun + +### Example + +[`js/examples/features/virtual-streaming.mjs`](https://github.com/link-foundation/command-stream/blob/main/js/examples/features/virtual-streaming.mjs) + +```js +// A handler written as an async generator streams its output chunk by chunk, +// so consumers see data before the command has finished. +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +await example( + { id: 'virtual-streaming', title: 'Streaming commands' }, + async ({ record }) => { + register('countdown', async function* ({ args }) { + for (let i = Number(args[0] ?? 3); i > 0; i--) { + yield `${i}\n`; + } + yield 'liftoff\n'; + }); + + const chunks = []; + for await (const chunk of $({ mirror: false })`countdown 3`.stream()) { + if (chunk.type === 'exit') { + continue; + } + chunks.push(chunk.data.toString()); + } + record('chunks received one by one', chunks); + record( + 'same command awaited as a whole', + (await $({ mirror: false })`countdown 2`).stdout + ); + + // Streaming commands compose with the rest of a pipeline. + record( + 'piped into a built-in', + (await $({ mirror: false })`countdown 2 | cat`).stdout + ); + + unregister('countdown'); + } +); +``` + +### Output + +Identical in Node.js and Bun: + +``` +# virtual-streaming — Streaming commands +chunks received one by one: ["3\n","2\n","1\n","liftoff\n"] +same command awaited as a whole: "2\n1\nliftoff\n" +piped into a built-in: "2\n1\nliftoff\n" +``` + +## Rust + +**API:** `CommandContext::output_tx`, `StreamChunk` + +### Example + +[`rust/examples/language_features.rs`](https://github.com/link-foundation/command-stream/blob/main/rust/examples/language_features.rs) + +```rust +async fn virtual_streaming() -> ExampleResult { + let (sender, mut receiver) = tokio::sync::mpsc::channel(4); + let mut context = CommandContext::new(Vec::new()); + context.output_tx = Some(sender); + let result = streaming_handler(context).await; + let mut chunks = Vec::new(); + while let Ok(chunk) = receiver.try_recv() { + if let command_stream::StreamChunk::Stdout(text) = chunk { + chunks.push(text); + } + } + Ok(vec![ + observation("chunks", chunks), + observation("collected output", result.stdout), + ]) +} +``` + +### Output + +``` +# virtual-streaming — Rust +chunks: ["one\n","two\n"] +collected output: "one\ntwo\n" +``` + +## The same thing in other libraries + +### [Bun.$](https://bun.com/docs/runtime/shell) + +Not supported — no handler API. + +### [zx](https://github.com/google/zx) + +Not supported — no handler API. + +### [execa](https://github.com/sindresorhus/execa) + +Not supported — no handler API. + +### [ShellJS](https://github.com/shelljs/shelljs) + +Not supported — a plugin returns its output as one value when it is done. + +### [node:child_process](https://nodejs.org/api/child_process.html) + +Not supported — no handler API. + +--- + +[← All features](../README.md) diff --git a/docs/screenshots/feature-guide.png b/docs/screenshots/feature-guide.png new file mode 100644 index 00000000..6f7a8d92 Binary files /dev/null and b/docs/screenshots/feature-guide.png differ diff --git a/docs/site/index.html b/docs/site/index.html new file mode 100644 index 00000000..1fd59f83 --- /dev/null +++ b/docs/site/index.html @@ -0,0 +1,1120 @@ + + + + + + command-stream — feature comparison + + + +
+

command-stream — feature comparison

+

+ Every feature in JavaScript and Rust, plus equivalent code in other + shell libraries. +

+
+
+ +
+
+
+ Generated from executable examples in + js/examples/features/ and + rust/examples/language_features.rs. +
+ + + + diff --git a/experiments/alt-libs-probe.mjs b/experiments/alt-libs-probe.mjs new file mode 100644 index 00000000..c45b99a4 --- /dev/null +++ b/experiments/alt-libs-probe.mjs @@ -0,0 +1,88 @@ +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; + +const requireFromJs = createRequire( + new URL('../js/package.json', import.meta.url) +); +const importFromJs = (name) => + import(pathToFileURL(requireFromJs.resolve(name)).href); +// shelljs includes an older transitive Execa tree. Load these sequentially to +// avoid Node's ESM/CJS loader observing path-key while another import owns it. +const zxModule = await importFromJs('zx'); +const execaModule = await importFromJs('execa'); +const shelljsModule = await importFromJs('shelljs'); +const { $: zx } = zxModule; +const { execa, execaSync, $: execa$ } = execaModule; +const shelljs = shelljsModule.default; + +const out = []; +const t = async (label, fn) => { + try { + out.push([label, 'ok', await fn()]); + } catch (e) { + out.push([label, 'ERR', e.message.split('\n')[0]]); + } +}; + +zx.verbose = false; +await t('zx stdout', async () => (await zx`echo hi`).stdout); +await t('zx sync', () => zx.sync`echo hi`.stdout); +await t( + 'zx nothrow exitCode', + async () => (await zx({ nothrow: true })`exit 3`).exitCode +); +await t('zx pipe', async () => (await zx`echo hi`.pipe(zx`tr a-z A-Z`)).stdout); +await t('zx iterate', async () => { + const lines = []; + for await (const l of zx`printf 'a\nb\n'`) { + lines.push(l); + } + return lines; +}); +await t('zx stdin', async () => (await zx({ input: 'x' })`cat`).stdout); +await t('zx kill', async () => { + const p = zx({ nothrow: true })`sleep 5`; + setTimeout(() => p.kill(), 50); + return (await p).exitCode; +}); + +await t('execa stdout', async () => (await execa`echo hi`).stdout); +await t('execa sync', () => execaSync`echo hi`.stdout); +await t( + 'execa reject false', + async () => (await execa({ reject: false })`sh -c 'exit 3'`).exitCode +); +await t( + 'execa pipe', + async () => (await execa`echo hi`.pipe`tr a-z A-Z`).stdout +); +await t('execa iterate', async () => { + const lines = []; + for await (const l of execa`printf 'a\nb\n'`) { + lines.push(l); + } + return lines; +}); +await t('execa input', async () => (await execa({ input: 'x' })`cat`).stdout); +await t('execa $ template', async () => (await execa$`echo hi`).stdout); + +shelljs.config.silent = true; +await t('shelljs exec', () => { + const r = shelljs.exec('echo hi'); + return [r.stdout, r.code]; +}); +await t( + 'shelljs async', + () => + new Promise((r) => + shelljs.exec('echo hi', { async: true }, (code, stdout) => + r([code, stdout]) + ) + ) +); +await t('shelljs ls', () => shelljs.ls('/tmp').length >= 0); +await t('shelljs pipe', () => shelljs.echo('hi').exec('tr a-z A-Z').stdout); + +for (const [l, s, v] of out) { + console.log(s.padEnd(4), l.padEnd(24), JSON.stringify(v)); +} diff --git a/experiments/api-probe.mjs b/experiments/api-probe.mjs new file mode 100644 index 00000000..4b18bcab --- /dev/null +++ b/experiments/api-probe.mjs @@ -0,0 +1,137 @@ +// Probe of command-stream API behaviours used by the comparison examples. +// Run with: node experiments/api-probe.mjs and bun experiments/api-probe.mjs +import { + $, + sh, + exec, + run, + create, + quote, + raw, + register, + unregister, + listCommands, + shell, + set, + unset, + AnsiUtils, + getAnsiConfig, +} from '../js/src/$.mjs'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const out = (k, v) => console.log(`[${runtime}] ${k}:`, JSON.stringify(v)); + +const $q = $({ mirror: false, capture: true }); + +out('basic', (await $q`echo hi`).stdout); +out('exitcode', (await $q`sh -c 'exit 3'`).code); +out('sync', $({ mirror: false })`echo sync`.sync().stdout); +out('text', await (await $q`echo text`).text()); +out('pipe-shell', (await $q`echo hello | tr a-z A-Z`).stdout); + +register('upper', async ({ stdin }) => ({ + stdout: String(stdin || '').toUpperCase(), + code: 0, +})); +out('virtual', (await $q`echo abc | upper`).stdout); +out( + 'pipe-method', + (await $({ mirror: false })`echo pm`.pipe($({ mirror: false })`upper`)).stdout +); +unregister('upper'); + +register('gen', async function* ({ args }) { + for (let i = 1; i <= Number(args[0] || 2); i++) { + yield `n${i}\n`; + } +}); +out('virtual-stream', (await $q`gen 3`).stdout); +unregister('gen'); + +out('builtins-count', listCommands().length); +out('quote', quote("it's a test")); +out('raw', raw('*')); +out( + 'opts-env', + ( + await $({ + mirror: false, + env: { ...process.env, PROBE: 'yes' }, + })`printenv PROBE` + ).stdout +); +out('opts-cwd', (await $({ mirror: false, cwd: '/tmp' })`pwd`).stdout); +out( + 'opts-stdin', + (await $({ mirror: false, stdin: 'from-stdin\n' })`cat`).stdout +); + +const buf = await $({ mirror: false })`echo buf`.buffers.stdout; +out('buffers', [Buffer.isBuffer(buf), buf.length]); +const str = await $({ mirror: false })`echo str`.strings.stdout; +out('strings', str); + +const chunks = []; +for await (const chunk of $({ mirror: false })`seq 1 3`.stream()) { + if (chunk.type === 'exit') { + continue; + } + chunks.push([chunk.type, chunk.data.toString()]); +} +out('stream', chunks); + +const ev = []; +await new Promise((resolve) => { + $({ mirror: false })`sh -c 'echo o; echo e >&2'` + .on('stdout', (d) => ev.push(['stdout', d.toString().trim()])) + .on('stderr', (d) => ev.push(['stderr', d.toString().trim()])) + .on('end', (r) => { + ev.push(['end', r.code]); + resolve(); + }) + .start(); +}); +out('events', ev); + +const g = $({ mirror: false })`cat`; +const stdinStream = await g.streams.stdin; +stdinStream.write('line1\n'); +stdinStream.end(); +out('streams-stdin', (await g).stdout); + +shell.errexit(true); +try { + await $q`sh -c 'exit 7'`; + out('errexit', 'no-throw'); +} catch (e) { + out('errexit', ['threw', e.code]); +} +shell.errexit(false); + +set('x'); +const xOn = shell.settings().xtrace; +unset('x'); +out('set-unset', `${xOn}/${shell.settings().xtrace}`); + +out( + 'ansi', + AnsiUtils.stripAnsi( + String.fromCharCode(27) + '[31mred' + String.fromCharCode(27) + '[0m' + ) +); +out('ansi-config', getAnsiConfig()); +out('sh-fn', (await sh('echo shfn', { mirror: false, capture: true })).stdout); +out('run-fn', (await run('echo runfn')).stdout); +out( + 'exec-fn', + (await exec('echo', ['execfn'], { mirror: false, capture: true })).stdout +); + +const $c = create({ mirror: false, capture: true }); +out('create-fn', (await $c`echo createfn`).stdout); + +const k = $({ mirror: false })`sleep 5`; +k.start(); +setTimeout(() => k.kill(), 200); +const kr = await k; +out('kill', kr.code); diff --git a/experiments/bun-shell-probe.mjs b/experiments/bun-shell-probe.mjs new file mode 100644 index 00000000..511c53f5 --- /dev/null +++ b/experiments/bun-shell-probe.mjs @@ -0,0 +1,44 @@ +const $ = Bun.$; +const out = []; +const t = async (label, fn) => { + try { + out.push([label, 'ok', await fn()]); + } catch (e) { + out.push([label, 'ERR', String(e.message).split('\n')[0]]); + } +}; + +await t('text', async () => await $`echo hi`.text()); +await t('quiet stdout', async () => + (await $`echo hi`.quiet()).stdout.toString() +); +await t( + 'nothrow code', + async () => (await $`exit 3`.nothrow().quiet()).exitCode +); +await t('json', async () => await $`echo '{"a":1}'`.json()); +await t('lines', async () => { + const l = []; + for await (const line of $`printf 'a\nb\n'`.lines()) { + l.push(line); + } + return l; +}); +await t('cwd', async () => + (await $`pwd`.cwd('/tmp').quiet()).stdout.toString().trim() +); +await t('env', async () => + (await $`printenv X`.env({ X: 'y' }).quiet()).stdout.toString() +); +await t('stdin', async () => + (await $`cat < ${new Response('x')}`.quiet()).stdout.toString() +); +await t('escape', () => $.escape("it's")); +await t('pipe builtin', async () => + (await $`echo hi | tr a-z A-Z`.quiet()).stdout.toString() +); +await t('sync', () => String($`echo hi`.sync?.())); +await t('register custom cmd', () => typeof $.Shell); +for (const [l, s, v] of out) { + console.log(s.padEnd(4), l.padEnd(20), JSON.stringify(v)); +} diff --git a/experiments/echo-redirect-probe.mjs b/experiments/echo-redirect-probe.mjs new file mode 100644 index 00000000..3000cdb3 --- /dev/null +++ b/experiments/echo-redirect-probe.mjs @@ -0,0 +1,17 @@ +// Probes `echo ... > file` redirection with built-in commands. +import { $ } from '../js/src/$.mjs'; +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const $q = $({ mirror: false, capture: true }); +const dir = `/tmp/redirect-probe-${runtime}`; +await $q`rm -rf ${dir}`; +await $q`mkdir -p ${dir}`; +const f = `${dir}/out.txt`; +const w = await $q`echo "test content" > ${f}`; +console.log( + `[${runtime}] write code=${w.code} stdout=${JSON.stringify(w.stdout)} stderr=${JSON.stringify(w.stderr)}` +); +const r = await $q`cat ${f}`; +console.log( + `[${runtime}] read code=${r.code} stdout=${JSON.stringify(r.stdout)} stderr=${JSON.stringify(r.stderr)}` +); +await $q`rm -rf ${dir}`; diff --git a/experiments/env-builtin-probe.mjs b/experiments/env-builtin-probe.mjs new file mode 100644 index 00000000..07c5500c --- /dev/null +++ b/experiments/env-builtin-probe.mjs @@ -0,0 +1,23 @@ +// Probes the environment built-ins one by one, printing before/after each step, +// so a hanging step is obvious. +import { $ } from '../js/src/$.mjs'; + +const $q = $({ mirror: false }); +const step = async (label, fn) => { + process.stdout.write(`-> ${label} ... `); + try { + console.log(JSON.stringify(await fn())); + } catch (e) { + console.log(`ERROR ${e.message}`); + } +}; + +await step('pwd', async () => (await $q`pwd`).stdout); +await step('cd /tmp', async () => (await $q`cd /tmp`).code); +await step('pwd after cd', async () => (await $q`pwd`).stdout); +await step( + 'env with custom env', + async () => (await $({ mirror: false, env: { DEMO: 'value' } })`env`).stdout +); +await step('which sh', async () => (await $q`which sh`).code); +await step('sleep 0.1', async () => (await $q`sleep 0.1`).code); diff --git a/experiments/ls-order-probe.mjs b/experiments/ls-order-probe.mjs new file mode 100644 index 00000000..f96bc6a3 --- /dev/null +++ b/experiments/ls-order-probe.mjs @@ -0,0 +1,18 @@ +// Reproduces the `ls` built-in returning entries in directory order instead of +// sorted order. Real `ls` sorts by name, and readdir order differs between +// Node.js and Bun, so the same script prints different output per runtime. +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { $ } from '../js/src/$.mjs'; + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ls-order-')); +for (const name of ['zebra.txt', 'alpha.txt', 'middle.txt']) { + fs.writeFileSync(path.join(dir, name), ''); +} +console.log('readdir order:', JSON.stringify(fs.readdirSync(dir))); +console.log( + 'ls built-in :', + JSON.stringify((await $({ mirror: false })`ls ${dir}`).stdout) +); +fs.rmSync(dir, { recursive: true, force: true }); diff --git a/experiments/parse-redirect-probe.mjs b/experiments/parse-redirect-probe.mjs new file mode 100644 index 00000000..35361c61 --- /dev/null +++ b/experiments/parse-redirect-probe.mjs @@ -0,0 +1,14 @@ +// What does the enhanced shell parser produce for simple commands with redirects? +import { parseShellCommand } from '../js/src/shell-parser.mjs'; + +for (const cmd of [ + 'echo hello > /tmp/a.txt', + 'echo hello >> /tmp/a.txt', + 'echo "a > b"', + "echo 'a > b' > /tmp/a.txt", + 'cat < /tmp/a.txt', + 'echo hi 2> /tmp/err.txt', + 'echo a | cat > /tmp/a.txt', +]) { + console.log(cmd, '=>', JSON.stringify(parseShellCommand(cmd))); +} diff --git a/experiments/pipefail-parity.mjs b/experiments/pipefail-parity.mjs new file mode 100644 index 00000000..a9609737 --- /dev/null +++ b/experiments/pipefail-parity.mjs @@ -0,0 +1,39 @@ +// Compares `set -o pipefail` behaviour between runtimes and against a real shell. +import { $, shell, register, unregister } from '../js/src/$.mjs'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const $q = $({ mirror: false, capture: true }); + +register('cat-virtual', async ({ stdin }) => ({ + stdout: String(stdin ?? ''), + code: 0, +})); + +const probe = async (label, fn) => { + try { + const r = await fn(); + console.log( + `[${runtime}] ${label.padEnd(34)} -> code=${r.code} stdout=${JSON.stringify(r.stdout)}` + ); + } catch (e) { + console.log( + `[${runtime}] ${label.padEnd(34)} -> THREW ${JSON.stringify(e.message)} code=${e.code}` + ); + } +}; + +shell.pipefail(true); +await probe('system | system', () => $q`sh -c 'exit 3' | cat`); +await probe('system | built-in', () => $q`sh -c 'echo x; exit 3' | cat`); +await probe('system | virtual', () => $q`sh -c 'echo x; exit 3' | cat-virtual`); +await probe('built-in | system', () => $q`echo x | sh -c 'exit 4'`); +shell.pipefail(false); +await probe('no pipefail: system | system', () => $q`sh -c 'exit 3' | cat`); + +const real = + await $q`sh -c 'set -o pipefail; sh -c "exit 3" | cat; echo code=$?'`; +console.log( + `[${runtime}] real shell with pipefail -> ${JSON.stringify(real.stdout)}` +); + +unregister('cat-virtual'); diff --git a/experiments/pipeline-exitcode-parity.mjs b/experiments/pipeline-exitcode-parity.mjs new file mode 100644 index 00000000..6ebd09fa --- /dev/null +++ b/experiments/pipeline-exitcode-parity.mjs @@ -0,0 +1,25 @@ +// Parity probe: exit code propagation out of pipelines. +import { $, register, unregister } from '../js/src/$.mjs'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const $q = $({ mirror: false, capture: true }); + +register('fail7', async () => ({ stdout: '', stderr: 'boom\n', code: 7 })); + +const cases = { + 'virtual last fails': () => $q`echo a | fail7`, + 'virtual only fails': () => $q`fail7`, + 'system last fails': () => $q`echo a | sh -c 'exit 7'`, + 'builtin cat missing file': () => $q`echo a | cat /no/such/file`, + 'virtual first fails': () => $q`fail7 | cat`, + 'system first fails': () => $q`sh -c 'exit 7' | cat`, +}; + +for (const [label, run] of Object.entries(cases)) { + const r = await run(); + console.log( + `[${runtime}] ${label.padEnd(26)} code=${r.code} stdout=${JSON.stringify(r.stdout)} stderr=${JSON.stringify(r.stderr.trim())}` + ); +} + +unregister('fail7'); diff --git a/experiments/pipeline-input-sentinel.mjs b/experiments/pipeline-input-sentinel.mjs new file mode 100644 index 00000000..44eea703 --- /dev/null +++ b/experiments/pipeline-input-sentinel.mjs @@ -0,0 +1,20 @@ +// The default `stdin: 'inherit'` must not be fed into a pipeline as data. +import { $, register, unregister } from '../js/src/$.mjs'; +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const $q = $({ mirror: false, capture: true }); +register('count-bytes', async ({ stdin }) => ({ + stdout: `bytes=${String(stdin ?? '').length}\n`, + code: 0, +})); +console.log( + `[${runtime}] echo hi | count-bytes ->`, + JSON.stringify((await $q`echo hi | count-bytes`).stdout) +); +console.log( + `[${runtime}] stdin option pipeline ->`, + JSON.stringify( + (await $({ mirror: false, capture: true, stdin: 'abc' })`cat | count-bytes`) + .stdout + ) +); +unregister('count-bytes'); diff --git a/experiments/pipeline-redirect-probe.mjs b/experiments/pipeline-redirect-probe.mjs new file mode 100644 index 00000000..d8c9ded6 --- /dev/null +++ b/experiments/pipeline-redirect-probe.mjs @@ -0,0 +1,29 @@ +// README documents `seq 1 5 | cat > numbers.txt`. Does it actually redirect? +import { $ } from '../js/src/$.mjs'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const dir = fs.mkdtempSync(path.join(os.tmpdir(), `pipe-redirect-${runtime}-`)); +const $q = $({ mirror: false, capture: true }); + +async function probe(label, run, file) { + const r = await run(); + console.log( + `[${runtime}] ${label.padEnd(30)} code=${r.code} stdout=${JSON.stringify(r.stdout)} stderr=${JSON.stringify(r.stderr.trim())} file=${fs.existsSync(file) ? JSON.stringify(fs.readFileSync(file, 'utf8')) : 'MISSING'}` + ); +} + +const f1 = path.join(dir, 'a.txt'); +await probe('seq 1 3 | cat > f', () => $q`seq 1 3 | cat > ${f1}`, f1); +const f2 = path.join(dir, 'b.txt'); +await probe('sh -c seq | cat > f', () => $q`sh -c 'seq 1 3' | cat > ${f2}`, f2); +const f3 = path.join(dir, 'c.txt'); +fs.writeFileSync(f3, 'from-file\n'); +const r = await $q`cat < ${f3}`; +console.log( + `[${runtime}] ${'cat < f'.padEnd(30)} code=${r.code} stdout=${JSON.stringify(r.stdout)} stderr=${JSON.stringify(r.stderr.trim())}` +); + +fs.rmSync(dir, { recursive: true, force: true }); diff --git a/experiments/pipeline-stdin-parity.mjs b/experiments/pipeline-stdin-parity.mjs new file mode 100644 index 00000000..d5af2e11 --- /dev/null +++ b/experiments/pipeline-stdin-parity.mjs @@ -0,0 +1,40 @@ +// Minimal reproduction: piping into a virtual command. +// Bun yields "ABC\n"; Node yields "INHERIT" (the literal default stdin option). +import { $, register, unregister } from '../js/src/$.mjs'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const $q = $({ mirror: false, capture: true }); + +register('upper', async ({ stdin }) => ({ + stdout: String(stdin ?? '').toUpperCase(), + code: 0, +})); + +register('show-stdin', async ({ stdin }) => ({ + stdout: `stdin=${JSON.stringify(stdin)}\n`, + code: 0, +})); + +console.log( + `[${runtime}] echo abc | upper ->`, + JSON.stringify((await $q`echo abc | upper`).stdout) +); +console.log( + `[${runtime}] echo abc | show-stdin ->`, + JSON.stringify((await $q`echo abc | show-stdin`).stdout) +); +console.log( + `[${runtime}] seq 1 3 | show-stdin ->`, + JSON.stringify((await $q`seq 1 3 | show-stdin`).stdout) +); +console.log( + `[${runtime}] sh -c echo | show-stdin ->`, + JSON.stringify((await $q`sh -c 'echo sys' | show-stdin`).stdout) +); +console.log( + `[${runtime}] upper (no pipe) ->`, + JSON.stringify((await $q`upper`).stdout) +); + +unregister('upper'); +unregister('show-stdin'); diff --git a/experiments/quote-parity.mjs b/experiments/quote-parity.mjs new file mode 100644 index 00000000..8d3bcdcb --- /dev/null +++ b/experiments/quote-parity.mjs @@ -0,0 +1,42 @@ +// Compares how an interpolated value with a single quote reaches a command. +// A real shell prints the value unchanged; the built-in path used to leak the +// quoting that command-stream added. +import { + $, + quote, + enableVirtualCommands, + disableVirtualCommands, +} from '../js/src/$.mjs'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const $q = $({ mirror: false, capture: true }); +const name = "it's a name"; +const withSpaces = 'two spaces'; + +console.log( + `[${runtime}] quote() ->`, + JSON.stringify(quote(name)) +); +enableVirtualCommands(); +console.log( + `[${runtime}] built-in echo ->`, + JSON.stringify((await $q`echo ${name}`).stdout) +); +console.log( + `[${runtime}] built-in echo spaces ->`, + JSON.stringify((await $q`echo ${withSpaces}`).stdout) +); +console.log( + `[${runtime}] built-in cat arg ->`, + JSON.stringify((await $q`echo ${name} | cat`).stdout) +); +disableVirtualCommands(); +console.log( + `[${runtime}] system echo ->`, + JSON.stringify((await $q`echo ${name}`).stdout) +); +console.log( + `[${runtime}] system echo spaces ->`, + JSON.stringify((await $q`echo ${withSpaces}`).stdout) +); +enableVirtualCommands(); diff --git a/experiments/redirect-path-probe.mjs b/experiments/redirect-path-probe.mjs new file mode 100644 index 00000000..99cc488c --- /dev/null +++ b/experiments/redirect-path-probe.mjs @@ -0,0 +1,34 @@ +// Root-cause probe for output redirection with built-in/virtual commands. +// Hypothesis: redirection is only honoured when the *enhanced* shell parser runs, +// which happens only when the command contains &&, ||, ; or ( ... ). +// Without one of those, _parseCommand() treats ">" as a literal argument. +import { $ } from '../js/src/$.mjs'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const dir = fs.mkdtempSync(path.join(os.tmpdir(), `redirect-${runtime}-`)); +const $q = $({ mirror: false, capture: true }); + +async function probe(label, run, file) { + const r = await run(); + const exists = fs.existsSync(file); + console.log( + `[${runtime}] ${label.padEnd(28)} code=${r.code} stdout=${JSON.stringify(r.stdout)} file=${exists ? JSON.stringify(fs.readFileSync(file, 'utf8')) : 'MISSING'}` + ); +} + +const f1 = path.join(dir, 'plain.txt'); +await probe('echo x > f', () => $q`echo hello > ${f1}`, f1); + +const f2 = path.join(dir, 'sequence.txt'); +await probe('echo x > f ; true', () => $q`echo hello > ${f2} ; true`, f2); + +const f3 = path.join(dir, 'system.txt'); +await probe('sh -c echo x > f', () => $q`sh -c 'echo hello' > ${f3}`, f3); + +const f4 = path.join(dir, 'append.txt'); +await probe('echo x >> f', () => $q`echo hello >> ${f4}`, f4); + +fs.rmSync(dir, { recursive: true, force: true }); diff --git a/experiments/sleep-exit-probe.mjs b/experiments/sleep-exit-probe.mjs new file mode 100644 index 00000000..60ff5559 --- /dev/null +++ b/experiments/sleep-exit-probe.mjs @@ -0,0 +1,12 @@ +// Reproduces the hang caused by the `sleep` built-in: the interval it starts to +// poll for cancellation is never cleared when the sleep finishes normally, so +// the event loop stays alive and the host script never exits. +// Expected: "done" is printed and the process exits immediately. +import { $ } from '../js/src/$.mjs'; + +const started = Date.now(); +await $({ mirror: false })`sleep 0.1`; +console.log( + `done after ${Date.now() - started >= 90 ? 'the full delay' : 'too little time'}` +); +console.log('if the process does not exit now, a timer was leaked'); diff --git a/experiments/special-path-probe.mjs b/experiments/special-path-probe.mjs new file mode 100644 index 00000000..1b8863b9 --- /dev/null +++ b/experiments/special-path-probe.mjs @@ -0,0 +1,27 @@ +// Reproduces the `cd` into a path containing quotes and `$1`, which the +// built-in path has to unquote exactly like a shell would. +import { $ } from '../js/src/$.mjs'; +import { mkdtempSync, rmSync, existsSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const $q = $({ mirror: false, capture: true }); +const base = mkdtempSync(join(tmpdir(), 'special-chars-')); +const specialDir = join(base, "test-'dir'-$1"); + +try { + const mk = await $q`mkdir -p ${specialDir}`; + console.log(`[${runtime}] mkdir code`, mk.code, JSON.stringify(mk.stderr)); + console.log(`[${runtime}] exists `, existsSync(specialDir)); + const init = await $q`cd ${specialDir} && git init`; + console.log(`[${runtime}] git init `, init.code, JSON.stringify(init.stderr)); + const status = await $q`cd ${specialDir} && git status`; + console.log( + `[${runtime}] git statu`, + status.code, + JSON.stringify(status.stderr) + ); +} finally { + rmSync(base, { recursive: true, force: true }); +} diff --git a/experiments/text-method-probe.mjs b/experiments/text-method-probe.mjs new file mode 100644 index 00000000..0e1c17c3 --- /dev/null +++ b/experiments/text-method-probe.mjs @@ -0,0 +1,23 @@ +// Probes which execution paths expose the documented `.text()` method on results. +import { $, register, unregister } from '../js/src/$.mjs'; + +const $q = $({ mirror: false, capture: true }); +const report = (label, value) => + console.log(`${label.padEnd(34)} text(): ${typeof value.text}`); + +report('system command (async)', await $q`sh -c 'echo system'`); +report('built-in command (async)', await $q`echo builtin`); +report('built-in command (sync)', $({ mirror: false })`echo builtin`.sync()); +report( + 'system command (sync)', + $({ mirror: false })`sh -c 'echo system'`.sync() +); +report('pipeline (async)', await $q`echo a | cat`); +report( + '.pipe() method', + await $({ mirror: false })`echo a`.pipe($({ mirror: false })`cat`) +); + +register('probe-virtual', async () => ({ stdout: 'virtual\n', code: 0 })); +report('virtual command (async)', await $q`probe-virtual`); +unregister('probe-virtual'); diff --git a/experiments/virtual-cancel-probe.mjs b/experiments/virtual-cancel-probe.mjs new file mode 100644 index 00000000..949cf1b6 --- /dev/null +++ b/experiments/virtual-cancel-probe.mjs @@ -0,0 +1,39 @@ +// Does kill() reach a running virtual command handler? +import { $, register, unregister } from '../js/src/$.mjs'; + +const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; +const events = []; + +register('cancellable', async ({ abortSignal, isCancelled }) => { + events.push([ + 'handler start', + { hasSignal: !!abortSignal, aborted: abortSignal?.aborted }, + ]); + abortSignal?.addEventListener?.('abort', () => + events.push(['abort event', true]) + ); + for (let i = 0; i < 20; i++) { + if (abortSignal?.aborted) { + events.push(['saw aborted at', i]); + break; + } + if (isCancelled?.()) { + events.push(['saw isCancelled at', i]); + break; + } + await new Promise((r) => setTimeout(r, 10)); + } + events.push(['handler end', null]); + return { stdout: '', code: 0 }; +}); + +const runner = $({ mirror: false })`cancellable`; +runner.start(); +setTimeout(() => { + events.push(['kill called', null]); + runner.kill(); +}, 50); +const result = await runner; +events.push(['result code', result.code]); +console.log(`[${runtime}]`, JSON.stringify(events)); +unregister('cancellable'); diff --git a/js/.changeset/bright-streams-agree.md b/js/.changeset/bright-streams-agree.md new file mode 100644 index 00000000..666f1810 --- /dev/null +++ b/js/.changeset/bright-streams-agree.md @@ -0,0 +1,5 @@ +--- +'command-stream': patch +--- + +Keep built-in and streaming pipeline results consistent across Node.js and Bun, and publish executable cross-language feature documentation. diff --git a/js/bun.lock b/js/bun.lock index 6a08fca6..60010c47 100644 --- a/js/bun.lock +++ b/js/bun.lock @@ -12,6 +12,7 @@ }, "devDependencies": { "@changesets/cli": "^2.31.1", + "@eslint/js": "^9.39.5", "cross-spawn": "7.0.6", "dejavu-fonts-ttf": "^2.37.3", "esbuild": "0.28.2", diff --git a/js/examples/features/_harness.mjs b/js/examples/features/_harness.mjs new file mode 100644 index 00000000..e783b6fd --- /dev/null +++ b/js/examples/features/_harness.mjs @@ -0,0 +1,121 @@ +// Shared harness for the feature examples. +// +// Every example in this directory describes one feature of command-stream and +// records what that feature actually produced. Running an example prints a +// readable report; running it with COMMAND_STREAM_PARITY=1 additionally prints a +// JSON block that `scripts/check-parity.mjs` compares between runtimes. +// +// Recorded values are redacted, so the report of an example is identical in +// every runtime, on every machine and in every checkout. +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +export const runtime = typeof globalThis.Bun !== 'undefined' ? 'bun' : 'node'; + +export const runtimeLabel = runtime === 'bun' ? 'Bun' : 'Node.js'; + +export const PARITY_START = '<<'); +redact(os.tmpdir(), ''); + +// Creates a throwaway directory that is redacted and removed automatically. +export function makeTempDir(name = 'example') { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `command-stream-${name}-`)); + tempDirs.push(dir); + redact(dir, `<${name}-dir>`); + return dir; +} + +function cleanup() { + while (tempDirs.length) { + try { + fs.rmSync(tempDirs.pop(), { recursive: true, force: true }); + } catch { + // best effort + } + } +} + +function sanitize(value) { + if (typeof value === 'string') { + let out = value; + // Longest needle first, so a temp directory is replaced as a whole instead + // of having its `os.tmpdir()` prefix swapped out from under it. + for (const [needle, placeholder] of [...redactions].sort( + (a, b) => b[0].length - a[0].length + )) { + out = out.split(needle).join(placeholder); + } + return out; + } + if (Array.isArray(value)) { + return value.map(sanitize); + } + if (value && typeof value === 'object') { + const out = {}; + for (const [key, item] of Object.entries(value)) { + out[key] = sanitize(item); + } + return out; + } + return value; +} + +function format(value) { + if (typeof value === 'string') { + return JSON.stringify(value); + } + return JSON.stringify(value, null, 0); +} + +// Runs one example. `body` receives a `record(label, value)` callback; each +// recorded value becomes one line of the report and one entry of the JSON block. +export async function example(meta, body) { + const observations = []; + const record = (label, value) => { + observations.push({ label, value: sanitize(value) }); + }; + + let failure = null; + try { + await body({ record }); + } catch (error) { + failure = sanitize(error?.message ?? String(error)); + } finally { + cleanup(); + } + + console.log(`# ${meta.id} — ${meta.title}`); + for (const { label, value } of observations) { + console.log(`${label}: ${format(value)}`); + } + if (failure) { + console.log(`error: ${format(failure)}`); + } + + if (process.env.COMMAND_STREAM_PARITY === '1') { + console.log(PARITY_START); + console.log( + JSON.stringify({ id: meta.id, runtime, observations, failure }) + ); + console.log(PARITY_END); + } + + if (failure) { + process.exitCode = 1; + } +} diff --git a/js/examples/features/ansi-utils.mjs b/js/examples/features/ansi-utils.mjs new file mode 100644 index 00000000..95c20f79 --- /dev/null +++ b/js/examples/features/ansi-utils.mjs @@ -0,0 +1,41 @@ +// Helpers for dealing with ANSI escape sequences and control characters in +// captured output. +import { + AnsiUtils, + processOutput, + configureAnsi, + getAnsiConfig, +} from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const ESC = String.fromCharCode(27); +const BELL = String.fromCharCode(7); + +await example( + { id: 'ansi-utils', title: 'ANSI and control character helpers' }, + async ({ record }) => { + const coloured = `${ESC}[31mred${ESC}[0m and ${ESC}[32mgreen${ESC}[0m`; + record('stripAnsi removes the colours', AnsiUtils.stripAnsi(coloured)); + record( + 'stripControlChars keeps text readable', + AnsiUtils.stripControlChars(`beep${BELL}boop`) + ); + record( + 'stripAll does both', + AnsiUtils.stripAll(`${ESC}[31mred${ESC}[0m${BELL}`) + ); + record( + 'cleanForProcessing handles buffers', + AnsiUtils.cleanForProcessing(Buffer.from(coloured)).toString() + ); + + // The same helpers can be applied to every captured chunk through the global + // configuration. + const original = getAnsiConfig(); + record('default config', original); + configureAnsi({ preserveAnsi: false }); + record('processOutput with preserveAnsi disabled', processOutput(coloured)); + configureAnsi(original); + record('config restored', getAnsiConfig()); + } +); diff --git a/js/examples/features/async-iteration.mjs b/js/examples/features/async-iteration.mjs new file mode 100644 index 00000000..fbf323f1 --- /dev/null +++ b/js/examples/features/async-iteration.mjs @@ -0,0 +1,39 @@ +// A command is an async iterable of output chunks, so output can be processed +// while the command is still running. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'async-iteration', title: 'Async iteration over output' }, + async ({ record }) => { + const lines = []; + for await (const chunk of $q`seq 1 5`.stream()) { + if (chunk.type === 'exit') { + continue; + } + lines.push({ type: chunk.type, data: chunk.data.toString() }); + } + record('chunk types', [...new Set(lines.map((l) => l.type))]); + record('collected output', lines.map((l) => l.data).join('')); + + // stdout and stderr are tagged, so both can be consumed from one loop. + const tagged = []; + for await (const chunk of $q`sh -c 'echo to-stdout; echo to-stderr >&2'`.stream()) { + if (chunk.type === 'exit') { + continue; + } + tagged.push([chunk.type, chunk.data.toString().trim()]); + } + record('tagged chunks', tagged.sort()); + + // Leaving the loop early terminates the command. + let seen = 0; + for await (const _chunk of $q`seq 1 1000`.stream()) { + seen++; + break; + } + record('iteration can stop early', seen === 1); + } +); diff --git a/js/examples/features/await-result.mjs b/js/examples/features/await-result.mjs new file mode 100644 index 00000000..1175b5f7 --- /dev/null +++ b/js/examples/features/await-result.mjs @@ -0,0 +1,21 @@ +// Awaiting a command returns a result object with stdout, stderr and the exit code. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'await-result', title: 'Await a command' }, + async ({ record }) => { + const result = await $q`echo "hello world"`; + record('stdout', result.stdout); + record('stderr', result.stderr); + record('code', result.code); + + const system = await $q`sh -c 'printf out; printf err >&2'`; + record('stdout of a system binary', system.stdout); + record('stderr of a system binary', system.stderr); + + record('interpolated value', (await $q`echo ${'a value'}`).stdout); + } +); diff --git a/js/examples/features/buffers-strings.mjs b/js/examples/features/buffers-strings.mjs new file mode 100644 index 00000000..297cbbb8 --- /dev/null +++ b/js/examples/features/buffers-strings.mjs @@ -0,0 +1,24 @@ +// .buffers and .strings expose the output as Buffers or as decoded strings. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'buffers-strings', title: 'Buffer and string interfaces' }, + async ({ record }) => { + const asBuffer = await $q`echo buffered`.buffers.stdout; + record('buffers.stdout is a Buffer', Buffer.isBuffer(asBuffer)); + record('buffers.stdout content', asBuffer.toString()); + + const asString = await $q`echo stringified`.strings.stdout; + record('strings.stdout', asString); + + const stderrBuffer = await $q`sh -c 'echo problem >&2'`.buffers.stderr; + record('buffers.stderr content', stderrBuffer.toString()); + + // Binary-safe: bytes survive the round trip unchanged. + const bytes = await $q`printf 'a\\tb'`.buffers.stdout; + record('raw bytes', Array.from(bytes)); + } +); diff --git a/js/examples/features/builtin-catalog.mjs b/js/examples/features/builtin-catalog.mjs new file mode 100644 index 00000000..40516a29 --- /dev/null +++ b/js/examples/features/builtin-catalog.mjs @@ -0,0 +1,26 @@ +// command-stream ships built-in implementations of common shell commands, so +// scripts behave the same even where those binaries are missing. +import { + $, + listCommands, + enableVirtualCommands, + disableVirtualCommands, +} from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'builtin-catalog', title: 'The built-in command catalog' }, + async ({ record }) => { + record('available built-ins', listCommands().sort()); + record('number of built-ins', listCommands().length); + + // Built-ins can be switched off, which falls back to the real binaries. + record('with built-ins', (await $q`echo built-in`).stdout); + disableVirtualCommands(); + record('with built-ins disabled', (await $q`echo real binary`).stdout); + enableVirtualCommands(); + record('built-ins enabled again', listCommands().length > 0); + } +); diff --git a/js/examples/features/builtin-environment.mjs b/js/examples/features/builtin-environment.mjs new file mode 100644 index 00000000..81176d79 --- /dev/null +++ b/js/examples/features/builtin-environment.mjs @@ -0,0 +1,34 @@ +// Environment built-ins: pwd, cd, env, which, sleep, exit. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import path from 'path'; + +await example( + { id: 'builtin-environment', title: 'Environment built-ins' }, + async ({ record }) => { + const dir = makeTempDir('env'); + const $q = $({ mirror: false }); + + record( + 'pwd inside a chosen directory', + (await $({ mirror: false, cwd: dir })`pwd`).stdout + ); + + // cd changes the working directory of the process, and is remembered by the + // following commands. + const before = (await $q`pwd`).stdout.trim(); + await $q`cd ${dir}`; + record('pwd after cd', (await $q`pwd`).stdout); + await $q`cd ${before}`; + record('back in the original directory', (await $q`pwd`).stdout); + + const withEnv = await $({ mirror: false, env: { DEMO: 'value' } })`env`; + record('env lists the variables', withEnv.stdout); + + record('which finds a binary', (await $q`which sh`).code); + + const started = Date.now(); + await $q`sleep 0.1`; + record('sleep waited', Date.now() - started >= 90); + } +); diff --git a/js/examples/features/builtin-filesystem.mjs b/js/examples/features/builtin-filesystem.mjs new file mode 100644 index 00000000..d1837ded --- /dev/null +++ b/js/examples/features/builtin-filesystem.mjs @@ -0,0 +1,42 @@ +// File system built-ins: mkdir, touch, ls, cp, mv, rm. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import fs from 'fs'; +import path from 'path'; + +await example( + { id: 'builtin-filesystem', title: 'File system built-ins' }, + async ({ record }) => { + const dir = makeTempDir('fs'); + const $q = $({ mirror: false, cwd: dir }); + + await $q`mkdir -p project/src`; + record( + 'mkdir -p created the tree', + fs.existsSync(path.join(dir, 'project/src')) + ); + + await $q`touch project/src/index.mjs`; + record( + 'touch created the file', + fs.existsSync(path.join(dir, 'project/src/index.mjs')) + ); + + record('ls', (await $q`ls project/src`).stdout); + + await $q`cp project/src/index.mjs project/src/copy.mjs`; + record('after cp', (await $q`ls project/src`).stdout); + + await $q`mv project/src/copy.mjs project/src/renamed.mjs`; + record('after mv', (await $q`ls project/src`).stdout); + + await $q`rm project/src/renamed.mjs`; + record('after rm', (await $q`ls project/src`).stdout); + + await $q`rm -rf project`; + record( + 'the tree still exists after rm -rf', + fs.existsSync(path.join(dir, 'project')) + ); + } +); diff --git a/js/examples/features/builtin-text.mjs b/js/examples/features/builtin-text.mjs new file mode 100644 index 00000000..fc4e8965 --- /dev/null +++ b/js/examples/features/builtin-text.mjs @@ -0,0 +1,29 @@ +// Text and value built-ins: echo, cat, seq, basename, dirname, true, false, test. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import fs from 'fs'; +import path from 'path'; + +await example( + { id: 'builtin-text', title: 'Text and value built-ins' }, + async ({ record }) => { + const dir = makeTempDir('text'); + const file = path.join(dir, 'greeting.txt'); + fs.writeFileSync(file, 'hello from a file\n'); + const $q = $({ mirror: false }); + + record('echo', (await $q`echo hello`).stdout); + record('echo -n', (await $q`echo -n no newline`).stdout); + record('cat', (await $q`cat ${file}`).stdout); + record('seq', (await $q`seq 1 4`).stdout); + record('basename', (await $q`basename /usr/local/lib/file.txt`).stdout); + record('dirname', (await $q`dirname /usr/local/lib/file.txt`).stdout); + record('true', (await $q`true`).code); + record('false', (await $q`false`).code); + record('test on an existing file', (await $q`test -f ${file}`).code); + record( + 'test on a missing file', + (await $q`test -f ${path.join(dir, 'missing')}`).code + ); + } +); diff --git a/js/examples/features/cancellation.mjs b/js/examples/features/cancellation.mjs new file mode 100644 index 00000000..9f1a5bf9 --- /dev/null +++ b/js/examples/features/cancellation.mjs @@ -0,0 +1,45 @@ +// Running commands can be killed, and virtual commands are told about it +// through abortSignal / isCancelled(). +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'cancellation', title: 'Killing and cancelling commands' }, + async ({ record }) => { + const runner = $q`sleep 30`; + runner.start(); + setTimeout(() => runner.kill(), 100); + const killed = await runner; + record('exit code after kill()', killed.code); + + // The handler reports back as soon as it notices the cancellation, so the + // example does not depend on timing. + let noticed; + const noticedCancellation = new Promise((resolve) => { + noticed = resolve; + }); + + register('cancellable', async ({ abortSignal, isCancelled }) => { + for (let i = 0; i < 200; i++) { + if (abortSignal?.aborted || isCancelled()) { + noticed({ + aborted: abortSignal?.aborted === true, + cancelled: isCancelled(), + }); + break; + } + await new Promise((resolve) => setTimeout(resolve, 5)); + } + return { stdout: '', code: 0 }; + }); + + const virtualRunner = $q`cancellable`; + virtualRunner.start(); + setTimeout(() => virtualRunner.kill(), 50); + await virtualRunner; + record('what the virtual command observed', await noticedCancellation); + unregister('cancellable'); + } +); diff --git a/js/examples/features/catalog.mjs b/js/examples/features/catalog.mjs new file mode 100644 index 00000000..4ea1f905 --- /dev/null +++ b/js/examples/features/catalog.mjs @@ -0,0 +1,605 @@ +// The feature catalog: one entry per feature of command-stream. +// +// Each entry names the example that demonstrates the feature and shows how the +// same thing is written with the other shell libraries, so the generated +// documentation is a side-by-side comparison rather than a list of links. +// +// An alternative is either a code snippet or `{ unsupported: 'reason' }`. The +// reasons are deliberately specific: "no equivalent" is not useful to a reader +// deciding between libraries. +// +// Representative alternatives were executed against the listed versions; +// see experiments/alt-libs-probe.mjs and experiments/bun-shell-probe.mjs. + +export const libraries = [ + { + id: 'command-stream', + name: 'command-stream', + url: 'https://github.com/link-foundation/command-stream', + runtimes: ['Node.js', 'Bun'], + }, + { + id: 'bun-shell', + name: 'Bun.$', + version: '1.4', + url: 'https://bun.com/docs/runtime/shell', + runtimes: ['Bun'], + }, + { + id: 'zx', + name: 'zx', + version: '8', + url: 'https://github.com/google/zx', + runtimes: ['Node.js', 'Bun', 'Deno'], + }, + { + id: 'execa', + name: 'execa', + version: '9.6', + url: 'https://github.com/sindresorhus/execa', + runtimes: ['Node.js', 'Bun', 'Deno'], + }, + { + id: 'shelljs', + name: 'ShellJS', + version: '0.10', + url: 'https://github.com/shelljs/shelljs', + runtimes: ['Node.js', 'Bun'], + }, + { + id: 'child_process', + name: 'node:child_process', + url: 'https://nodejs.org/api/child_process.html', + runtimes: ['Node.js', 'Bun', 'Deno'], + }, +]; + +export const categories = [ + 'Running commands', + 'Reading output', + 'Streaming', + 'Built-in commands', + 'Your own commands', + 'Shell syntax', + 'Utilities', +]; + +export const features = [ + { + id: 'await-result', + title: 'Await a command', + category: 'Running commands', + summary: + 'Awaiting a command returns an object with stdout, stderr and the exit code.', + file: 'js/examples/features/await-result.mjs', + api: ['$'], + alternatives: { + 'bun-shell': + 'const { stdout, stderr, exitCode } = await $`echo hi`.quiet();\n// stdout and stderr are Buffers, not strings', + zx: 'const { stdout, stderr, exitCode } = await $`echo hi`;', + execa: + 'const { stdout, stderr, exitCode } = await execa`echo hi`;\n// no shell is involved, so `echo hi` is the binary `echo` with one argument', + shelljs: + "const result = shell.exec('echo hi', { silent: true });\n// result.stdout, result.stderr, result.code", + child_process: + "const { stdout, stderr } = await promisify(execFile)('echo', ['hi']);", + }, + }, + { + id: 'result-text', + title: 'Read the output with text()', + category: 'Reading output', + summary: + 'Captured stdout is available as text through each language’s result API.', + file: 'js/examples/features/result-text.mjs', + api: ['$', 'ProcessRunner#text'], + alternatives: { + 'bun-shell': 'const text = await $`echo hi`.text();', + zx: 'const text = (await $`echo hi`).toString();', + execa: 'const text = (await execa`echo hi`).stdout;', + shelljs: "const text = shell.exec('echo hi', { silent: true }).stdout;", + child_process: + "const text = (await promisify(execFile)('echo', ['hi'])).stdout;", + }, + }, + { + id: 'sync-execution', + title: 'Synchronous execution', + category: 'Running commands', + summary: + 'The same command can be run without awaiting, blocking until it finishes.', + file: 'js/examples/features/sync-execution.mjs', + api: ['$', 'ProcessRunner#sync'], + alternatives: { + 'bun-shell': { + unsupported: + 'Bun.$ is always asynchronous; Bun.spawnSync is the synchronous escape hatch, and it takes an argument array rather than a command line', + }, + zx: 'const { stdout } = $.sync`echo hi`;', + execa: 'const { stdout } = execaSync`echo hi`;', + shelljs: + "const stdout = shell.exec('echo hi', { silent: true }).stdout; // synchronous by default", + child_process: + "const stdout = execFileSync('echo', ['hi'], { encoding: 'utf8' });", + }, + }, + { + id: 'exit-codes', + title: 'Exit codes and errors', + category: 'Running commands', + summary: + 'A non-zero exit code is reported on the result instead of thrown, unless errexit is set.', + file: 'js/examples/features/exit-codes.mjs', + api: ['$', 'shell.errexit'], + alternatives: { + 'bun-shell': + 'const { exitCode } = await $`exit 3`.nothrow(); // throws without .nothrow()', + zx: 'const { exitCode } = await $({ nothrow: true })`exit 3`; // throws without nothrow', + execa: + "const { exitCode } = await execa({ reject: false })`sh -c 'exit 3'`; // throws without reject: false", + shelljs: + "const code = shell.exec('exit 3', { silent: true }).code; // never throws", + child_process: + '// execFile rejects on a non-zero exit; the code is on error.code', + }, + }, + { + id: 'options', + title: 'Options: capture, cwd, env, stdin', + category: 'Running commands', + summary: + 'Execution options control capture, cwd, environment and stdin for a command or reusable runner.', + file: 'js/examples/features/options.mjs', + api: ['$', 'create'], + alternatives: { + 'bun-shell': "await $`pwd`.cwd('/tmp').env({ KEY: 'value' }).quiet();", + zx: "const $$ = $({ cwd: '/tmp', env: { KEY: 'value' } });", + execa: "const run = execa({ cwd: '/tmp', env: { KEY: 'value' } });", + shelljs: + "shell.cd('/tmp'); shell.env.KEY = 'value'; // process-wide, not per command", + child_process: + "execFile('pwd', [], { cwd: '/tmp', env: { KEY: 'value' } });", + }, + }, + { + id: 'function-api', + title: 'Function and builder APIs', + category: 'Running commands', + summary: + 'Commands can also be built from plain strings instead of template literals.', + file: 'js/examples/features/function-api.mjs', + api: ['sh', 'exec', 'run', 'create', 'shell'], + alternatives: { + 'bun-shell': { + unsupported: + 'Bun.$ only accepts a tagged template; a string has to be turned back into one by hand', + }, + zx: "await $({ input: '' })`sh -c ${'echo hi'}`; // or build a template array manually", + execa: "await execa('echo', ['hi']); // the classic function form", + shelljs: "shell.exec('echo hi'); // strings are the only form", + child_process: "execFile('echo', ['hi']);", + }, + }, + { + id: 'cancellation', + title: 'Killing and cancelling commands', + category: 'Running commands', + summary: + 'A running command can be killed, and cancelling one leaves the rest of the script running.', + file: 'js/examples/features/cancellation.mjs', + api: ['$', 'ProcessRunner#kill', 'forceCleanupAll'], + alternatives: { + 'bun-shell': { + unsupported: + 'a ShellPromise has no kill method; the command runs to completion', + }, + zx: 'const p = $({ nothrow: true })`sleep 5`; p.kill();', + execa: 'const p = execa({ reject: false })`sleep 5`; p.kill();', + shelljs: + "const child = shell.exec('sleep 5', { async: true }); child.kill();", + child_process: "const child = spawn('sleep', ['5']); child.kill();", + }, + }, + { + id: 'async-iteration', + title: 'Async iteration over output', + category: 'Streaming', + summary: + 'A command is an async iterable of chunks, so output can be handled as it arrives.', + file: 'js/examples/features/async-iteration.mjs', + api: ['$', 'ProcessRunner#[Symbol.asyncIterator]', 'ProcessRunner#stream'], + alternatives: { + 'bun-shell': + "for await (const line of $`printf 'a\\nb\\n'`.lines()) { /* line by line only */ }", + zx: "for await (const line of $`printf 'a\\nb\\n'`) { /* lines */ }", + execa: + "for await (const line of execa`printf 'a\\nb\\n'`) { /* lines */ }", + shelljs: { + unsupported: + 'output is only delivered as a whole string, or through the raw child process in async mode', + }, + child_process: + "for await (const chunk of spawn('printf', ['a\\nb\\n']).stdout) { /* Buffers */ }", + }, + }, + { + id: 'events', + title: 'Event-driven output', + category: 'Streaming', + summary: + 'Event APIs report output and lifecycle signals as work progresses.', + file: 'js/examples/features/events.mjs', + api: ['$', 'ProcessRunner#on', 'ProcessRunner#off'], + alternatives: { + 'bun-shell': { + unsupported: + 'a ShellPromise is not an EventEmitter and exposes no streams', + }, + zx: "$`echo hi`.stdout.on('data', chunk => { /* Node stream events */ });", + execa: + "execa`echo hi`.stdout.on('data', chunk => { /* Node stream events */ });", + shelljs: + "shell.exec('echo hi', { async: true }).stdout.on('data', chunk => {});", + child_process: "spawn('echo', ['hi']).stdout.on('data', chunk => {});", + }, + }, + { + id: 'stdin-streaming', + title: 'Writing to stdin while a command runs', + category: 'Streaming', + summary: 'Input can be supplied up front or written to a running command.', + file: 'js/examples/features/stdin-streaming.mjs', + api: ['$', 'ProcessRunner#stdin'], + alternatives: { + 'bun-shell': + "await $`cat < ${new Response('x')}`.quiet(); // a value, not a live stream", + zx: "const p = $`cat`; p.stdin.write('x'); p.stdin.end();", + execa: "const p = execa`cat`; p.stdin.write('x'); p.stdin.end();", + shelljs: "shell.ShellString('x').exec('cat'); // value only", + child_process: + "const p = spawn('cat'); p.stdin.write('x'); p.stdin.end();", + }, + }, + { + id: 'buffers-strings', + title: 'Buffer and string interfaces', + category: 'Reading output', + summary: + 'Output is available as a string and as raw bytes, without running the command twice.', + file: 'js/examples/features/buffers-strings.mjs', + api: ['$', 'ProcessRunner#text', 'ProcessRunner#buffers'], + alternatives: { + 'bun-shell': + 'const result = await $`echo hi`.quiet(); result.stdout; // Buffer\nawait $`echo hi`.text(); // string, but runs the command again', + zx: 'const p = await $`echo hi`; p.stdout; // string\nBuffer.from(p.stdout); // bytes by conversion', + execa: + "const { stdout } = await execa({ encoding: 'buffer' })`echo hi`; // choose one up front", + shelljs: { + unsupported: + 'output is decoded to a string; raw bytes are not available', + }, + child_process: + "const { stdout } = await promisify(execFile)('echo', ['hi'], { encoding: 'buffer' });", + }, + }, + { + id: 'mirror-capture', + title: 'Mirroring and capturing output', + category: 'Reading output', + summary: + 'Output can be shown, captured, both or neither, chosen independently.', + file: 'js/examples/features/mirror-capture.mjs', + api: ['$', 'create'], + alternatives: { + 'bun-shell': + 'await $`echo hi`; // shown and captured\nawait $`echo hi`.quiet(); // captured only', + zx: '$.verbose = true; // shown and captured\nawait $({ quiet: true })`echo hi`;', + execa: + "await execa({ stdout: ['pipe', 'inherit'] })`echo hi`; // both, by listing destinations", + shelljs: + "shell.exec('echo hi'); // shown and captured\nshell.exec('echo hi', { silent: true }); // captured only", + child_process: + "spawn('echo', ['hi'], { stdio: 'inherit' }); // shown, but then not captured", + }, + }, + { + id: 'builtin-catalog', + title: 'The built-in command catalog', + category: 'Built-in commands', + summary: + 'Common commands are implemented in-process in both languages for portable behavior.', + file: 'js/examples/features/builtin-catalog.mjs', + api: ['listCommands', 'enableVirtualCommands', 'disableVirtualCommands'], + alternatives: { + 'bun-shell': + '// a fixed set of built-ins (cd, echo, ls, rm, ...) that cannot be listed or turned off', + zx: { + unsupported: + 'every command is handed to the system shell; the fs and glob helpers are separate APIs, not commands', + }, + execa: { unsupported: 'every command is a real binary' }, + shelljs: + 'shell.ls(); shell.cat(); shell.mkdir(); // built-ins, but as functions rather than commands', + child_process: { unsupported: 'every command is a real binary' }, + }, + }, + { + id: 'builtin-filesystem', + title: 'File system built-ins', + category: 'Built-in commands', + summary: 'ls, cat, mkdir, touch, cp, mv, rm and test run in-process.', + file: 'js/examples/features/builtin-filesystem.mjs', + api: ['$'], + alternatives: { + 'bun-shell': + 'await $`mkdir -p dir`; await $`ls dir`.text(); // built-in, same idea', + zx: "await fs.mkdirp('dir'); // zx re-exports fs-extra instead of implementing commands", + execa: { unsupported: 'use node:fs' }, + shelljs: "shell.mkdir('-p', 'dir'); shell.ls('dir');", + child_process: { unsupported: 'use node:fs' }, + }, + }, + { + id: 'builtin-text', + title: 'Text and value built-ins', + category: 'Built-in commands', + summary: + 'echo, seq, yes, basename, dirname, true and false run in-process.', + file: 'js/examples/features/builtin-text.mjs', + api: ['$'], + alternatives: { + 'bun-shell': + 'await $`echo hi`.text(); // echo is a built-in; seq and yes are not', + zx: 'await $`echo hi`; // the system binaries', + execa: "await execa('echo', ['hi']); // the system binaries", + shelljs: "shell.echo('hi'); // echo only", + child_process: "execFile('echo', ['hi']); // the system binaries", + }, + }, + { + id: 'builtin-environment', + title: 'Environment built-ins', + category: 'Built-in commands', + summary: + 'cd, pwd, env, which and exit affect the command they run in, not the host process.', + file: 'js/examples/features/builtin-environment.mjs', + api: ['$'], + alternatives: { + 'bun-shell': + 'await $`cd /tmp && pwd`.text(); // cd is scoped to the command', + zx: "cd('/tmp'); // changes the directory for every later command", + execa: "execa({ cwd: '/tmp' })`pwd`; // an option, not a command", + shelljs: + "shell.cd('/tmp'); shell.pwd(); // changes the process working directory", + child_process: "execFile('pwd', [], { cwd: '/tmp' });", + }, + }, + { + id: 'virtual-commands', + title: 'Registering your own commands', + category: 'Your own commands', + summary: + 'A handler can be registered by name and invoked through a registry or command runner.', + file: 'js/examples/features/virtual-commands.mjs', + api: ['register', 'unregister', 'listCommands'], + alternatives: { + 'bun-shell': { + unsupported: + 'the built-in set is fixed; a name cannot be bound to a JavaScript function', + }, + zx: { unsupported: 'a command name always resolves to a binary in PATH' }, + execa: { + unsupported: 'a command name always resolves to a binary in PATH', + }, + shelljs: + "require('shelljs/plugin').register('greet', (options, name) => `hi ${name}\\n`);\nshell.greet('bob'); // a method, not a command usable inside a pipeline string", + child_process: { + unsupported: 'a command name always resolves to a binary in PATH', + }, + }, + }, + { + id: 'virtual-context', + title: 'The handler context', + category: 'Your own commands', + summary: + 'A handler receives args, stdin, cwd, env and a cancellation signal.', + file: 'js/examples/features/virtual-context.mjs', + api: ['register'], + alternatives: { + 'bun-shell': { unsupported: 'no handler API' }, + zx: { unsupported: 'no handler API' }, + execa: { unsupported: 'no handler API' }, + shelljs: + "require('shelljs/plugin').readFromPipe(); // stdin only; no cwd, env or cancellation", + child_process: { unsupported: 'no handler API' }, + }, + }, + { + id: 'virtual-streaming', + title: 'Streaming commands', + category: 'Your own commands', + summary: + 'A streaming handler publishes output incrementally like a real process.', + file: 'js/examples/features/virtual-streaming.mjs', + api: ['register'], + alternatives: { + 'bun-shell': { unsupported: 'no handler API' }, + zx: { unsupported: 'no handler API' }, + execa: { unsupported: 'no handler API' }, + shelljs: { + unsupported: 'a plugin returns its output as one value when it is done', + }, + child_process: { unsupported: 'no handler API' }, + }, + }, + { + id: 'pipelines', + title: 'Pipelines', + category: 'Shell syntax', + summary: + 'Commands can be composed into pipelines whose output feeds the next stage.', + file: 'js/examples/features/pipelines.mjs', + api: ['$', 'ProcessRunner#pipe'], + alternatives: { + 'bun-shell': 'await $`echo hi | tr a-z A-Z`.text();', + zx: 'await $`echo hi`.pipe($`tr a-z A-Z`);', + execa: 'await execa`echo hi`.pipe`tr a-z A-Z`;', + shelljs: "shell.echo('hi').exec('tr a-z A-Z');", + child_process: '// connect the streams by hand: a.stdout.pipe(b.stdin)', + }, + }, + { + id: 'redirection', + title: 'Redirecting output and input', + category: 'Shell syntax', + summary: + '>, >> and < redirect command input and output with shell-compatible behavior.', + file: 'js/examples/features/redirection.mjs', + api: ['$'], + alternatives: { + 'bun-shell': 'await $`echo hi > out.txt`;', + zx: 'await $`echo hi > out.txt`; // handled by the system shell', + execa: "await execa({ stdout: { file: 'out.txt' } })`echo hi`;", + shelljs: "shell.echo('hi').to('out.txt');", + child_process: + "spawn('echo', ['hi'], { stdio: ['ignore', fs.openSync('out.txt', 'w'), 'inherit'] });", + }, + }, + { + id: 'sequences', + title: 'Command sequences', + category: 'Shell syntax', + summary: + '&&, ||, ; and parentheses execute with the expected shell semantics.', + file: 'js/examples/features/sequences.mjs', + api: ['$'], + alternatives: { + 'bun-shell': 'await $`mkdir -p dir && cd dir && pwd`.text();', + zx: 'await $`mkdir -p dir && cd dir && pwd`; // the system shell runs it', + execa: { + unsupported: + 'no shell operators unless the shell option is turned on, which gives up escaping', + }, + shelljs: + "shell.exec('mkdir -p dir && cd dir && pwd'); // the system shell runs it", + child_process: "execFile('sh', ['-c', 'mkdir -p dir && cd dir && pwd']);", + }, + }, + { + id: 'interpolation', + title: 'Safe interpolation', + category: 'Shell syntax', + summary: + 'Interpolated values are escaped as arguments; each language also exposes an explicit raw form.', + file: 'js/examples/features/interpolation.mjs', + api: ['$', 'quote', 'raw'], + alternatives: { + 'bun-shell': + 'await $`echo ${value}`; // escaped; $.escape(value) shows the result', + zx: 'await $`echo ${value}`; // escaped; quote(value) shows the result', + execa: + 'await execa`echo ${value}`; // passed as an argument, no shell to escape for', + shelljs: { + unsupported: + 'shell.exec takes a string, so escaping is the caller’s job', + }, + child_process: + "execFile('echo', [value]); // arguments are never parsed as shell syntax", + }, + }, + { + id: 'shell-settings', + title: 'Shell settings', + category: 'Shell syntax', + summary: + 'Shell settings model errexit, pipefail, verbose, xtrace and nounset behavior.', + file: 'js/examples/features/shell-settings.mjs', + api: ['shell', 'set', 'unset'], + alternatives: { + 'bun-shell': '$.throws(true); // errexit only', + zx: '$.verbose = true; // verbose only; the rest belong to the system shell', + execa: { + unsupported: + 'no shell settings; the equivalents are per-command options', + }, + shelljs: + 'shell.config.fatal = true; shell.config.verbose = true; // errexit and verbose', + child_process: "execFile('sh', ['-c', 'set -eo pipefail; ...']);", + }, + }, + { + id: 'ansi-utils', + title: 'ANSI and control character helpers', + category: 'Utilities', + summary: + 'Helpers can strip colours and control characters from captured output.', + file: 'js/examples/features/ansi-utils.mjs', + api: ['AnsiUtils', 'configureAnsi', 'getAnsiConfig', 'processOutput'], + alternatives: { + 'bun-shell': { unsupported: 'no helper; strip the codes yourself' }, + zx: 'chalk is re-exported for adding colour, but there is no helper for removing it', + execa: + 'await execa({ stripFinalNewline: true })`echo hi`; // trailing newline only, not ANSI', + shelljs: { unsupported: 'no helper; strip the codes yourself' }, + child_process: { unsupported: 'no helper; strip the codes yourself' }, + }, + }, +]; + +export const featuresById = new Map( + features.map((feature) => [feature.id, feature]) +); + +export const languages = [ + { + id: 'javascript', + name: 'JavaScript', + source: 'js/examples/features/', + }, + { + id: 'rust', + name: 'Rust', + source: 'rust/examples/language_features.rs', + }, +]; + +export const rustApiByFeature = new Map( + Object.entries({ + 'await-result': ['run', 'CommandResult'], + 'result-text': ['CommandResult::stdout'], + 'sync-execution': ['run_sync'], + 'exit-codes': ['CommandResult::code', 'CommandResult::error_for_status'], + options: ['exec', 'RunOptions'], + 'function-api': ['run', 'exec', 'create'], + cancellation: ['ProcessRunner::kill', 'OutputStream::kill'], + 'async-iteration': ['StreamingRunner', 'OutputStream::next'], + events: ['StreamEmitter', 'EventType', 'EventData'], + 'stdin-streaming': [ + 'ProcessRunner::write_stdin', + 'ProcessRunner::close_stdin', + ], + 'buffers-strings': ['CommandResult::stdout', 'OutputChunk'], + 'mirror-capture': ['RunOptions::mirror', 'RunOptions::capture'], + 'builtin-catalog': ['VirtualCommandRegistry::with_builtins'], + 'builtin-filesystem': ['mkdir', 'touch', 'ls', 'rm'], + 'builtin-text': ['echo', 'seq', 'basename', 'dirname', 'test', 'which'], + 'builtin-environment': ['pwd', 'cd', 'env'], + 'virtual-commands': [ + 'VirtualCommandRegistry::register', + 'VirtualCommandRegistry::unregister', + ], + 'virtual-context': ['CommandContext'], + 'virtual-streaming': ['CommandContext::output_tx', 'StreamChunk'], + pipelines: ['Pipeline', 'PipelineExt'], + redirection: ['exec'], + sequences: ['exec'], + interpolation: ['cmd!', 'quote'], + 'shell-settings': [ + 'ShellSettings', + 'set_shell_option', + 'unset_shell_option', + ], + 'ansi-utils': ['AnsiUtils', 'AnsiConfig'], + }) +); diff --git a/js/examples/features/events.mjs b/js/examples/features/events.mjs new file mode 100644 index 00000000..280d2b62 --- /dev/null +++ b/js/examples/features/events.mjs @@ -0,0 +1,41 @@ +// Commands are EventEmitters: 'stdout', 'stderr', 'data' and 'end'. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'events', title: 'EventEmitter interface' }, + async ({ record }) => { + const events = []; + + await new Promise((resolve, reject) => { + $q`sh -c 'echo out; echo err >&2'` + .on('stdout', (data) => events.push(['stdout', data.toString().trim()])) + .on('stderr', (data) => events.push(['stderr', data.toString().trim()])) + .on('end', (result) => { + events.push(['end', result.code]); + resolve(); + }) + .on('error', reject) + .start(); + }); + + record( + 'events (sorted: stdout/stderr order is up to the OS)', + events.sort() + ); + + // The 'data' event receives both streams with a type tag. + const tagged = []; + await new Promise((resolve) => { + $q`echo tagged` + .on('data', (chunk) => + tagged.push([chunk.type, chunk.data.toString().trim()]) + ) + .on('end', () => resolve()) + .start(); + }); + record('data events', tagged); + } +); diff --git a/js/examples/features/exit-codes.mjs b/js/examples/features/exit-codes.mjs new file mode 100644 index 00000000..b69a9839 --- /dev/null +++ b/js/examples/features/exit-codes.mjs @@ -0,0 +1,30 @@ +// Exit codes are reported on the result; errors are thrown only when asked for. +import { $, shell } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'exit-codes', title: 'Exit codes and errors' }, + async ({ record }) => { + record('successful command', (await $q`sh -c 'exit 0'`).code); + record('failing command', (await $q`sh -c 'exit 42'`).code); + record( + 'stderr of a failing command', + (await $q`sh -c 'echo nope >&2; exit 1'`).stderr + ); + + // With errexit (set -e) a non-zero exit code becomes an exception. + shell.errexit(true); + try { + await $q`sh -c 'exit 42'`; + record('errexit', 'no error thrown'); + } catch (error) { + record('errexit throws', { code: error.code, hasResult: !!error.result }); + } finally { + shell.errexit(false); + } + + record('after disabling errexit', (await $q`sh -c 'exit 42'`).code); + } +); diff --git a/js/examples/features/function-api.mjs b/js/examples/features/function-api.mjs new file mode 100644 index 00000000..7b238f39 --- /dev/null +++ b/js/examples/features/function-api.mjs @@ -0,0 +1,22 @@ +// Besides the template tag there are plain functions: sh, exec, run and create. +import { $, sh, exec, run, create } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +await example( + { id: 'function-api', title: 'sh(), exec(), run() and create()' }, + async ({ record }) => { + record('sh(command)', (await sh('echo from-sh', { mirror: false })).stdout); + record( + 'exec(file, args)', + (await exec('echo', ['from-exec'], { mirror: false })).stdout + ); + record('run(command)', (await run('echo from-run')).stdout); + + // create() returns a $ with preset options. + const $quiet = create({ mirror: false, capture: true }); + record('create(options)', (await $quiet`echo from-create`).stdout); + + // $ itself can be called with options for the same effect. + record('$(options)', (await $({ mirror: false })`echo from-dollar`).stdout); + } +); diff --git a/js/examples/features/interpolation.mjs b/js/examples/features/interpolation.mjs new file mode 100644 index 00000000..cf5658a1 --- /dev/null +++ b/js/examples/features/interpolation.mjs @@ -0,0 +1,31 @@ +// Interpolated values are quoted automatically, so user input cannot turn into +// extra shell syntax. +import { $, quote, raw } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'interpolation', title: 'Safe interpolation' }, + async ({ record }) => { + const name = "it's a name"; + record('quotes are handled', (await $q`echo ${name}`).stdout); + + const dangerous = 'hello; rm -rf /tmp/nothing'; + record( + 'injection stays one argument', + (await $q`echo ${dangerous}`).stdout + ); + + const args = ['one', 'two three']; + record( + 'an array becomes separate arguments', + (await $q`echo ${args}`).stdout + ); + + record('quote() shows what interpolation does', quote("it's a name")); + + // raw() opts out of quoting when you really mean shell syntax. + record('raw() keeps shell syntax', (await $q`echo ${raw('a b')}`).stdout); + } +); diff --git a/js/examples/features/mirror-capture.mjs b/js/examples/features/mirror-capture.mjs new file mode 100644 index 00000000..29d50115 --- /dev/null +++ b/js/examples/features/mirror-capture.mjs @@ -0,0 +1,19 @@ +// mirror controls whether output is shown, capture whether it is kept. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +await example( + { id: 'mirror-capture', title: 'Mirroring and capturing output' }, + async ({ record }) => { + // The default: output is shown and captured. + const both = await $`echo shown and captured`; + record('default mirror', true); + record('default capture', both.stdout); + + const quiet = await $({ mirror: false })`echo only captured`; + record('mirror: false still captures', quiet.stdout); + + const dropped = await $({ mirror: false, capture: false })`echo neither`; + record('capture: false returns no stdout', dropped.stdout); + } +); diff --git a/js/examples/features/options.mjs b/js/examples/features/options.mjs new file mode 100644 index 00000000..67a5ce26 --- /dev/null +++ b/js/examples/features/options.mjs @@ -0,0 +1,34 @@ +// $({ ... }) configures capture, mirroring, cwd, env and stdin. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import path from 'path'; +import fs from 'fs'; + +await example( + { id: 'options', title: 'Options: capture, cwd, env, stdin' }, + async ({ record }) => { + const dir = makeTempDir('options'); + fs.writeFileSync(path.join(dir, 'marker.txt'), 'here\n'); + + record( + 'captured output', + (await $({ mirror: false, capture: true })`echo captured`).stdout + ); + record( + 'capture disabled', + (await $({ mirror: false, capture: false })`echo dropped`).stdout + ); + + const inDir = await $({ mirror: false, cwd: dir })`ls`; + record('cwd option', inDir.stdout); + + const withEnv = await $({ + mirror: false, + env: { ...process.env, DEMO_VALUE: 'from-env' }, + })`printenv DEMO_VALUE`; + record('env option', withEnv.stdout); + + const withStdin = await $({ mirror: false, stdin: 'piped in\n' })`cat`; + record('stdin option', withStdin.stdout); + } +); diff --git a/js/examples/features/pipelines.mjs b/js/examples/features/pipelines.mjs new file mode 100644 index 00000000..1da0813b --- /dev/null +++ b/js/examples/features/pipelines.mjs @@ -0,0 +1,41 @@ +// Pipelines mix built-ins, your own commands and real binaries freely. +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example({ id: 'pipelines', title: 'Pipelines' }, async ({ record }) => { + register('upper', async ({ stdin }) => ({ + stdout: String(stdin ?? '').toUpperCase(), + code: 0, + })); + + record('built-in into built-in', (await $q`seq 1 3 | cat`).stdout); + record('built-in into your command', (await $q`echo hello | upper`).stdout); + record( + 'your command into a real binary', + (await $q`echo hello | upper | tr A-Z a-z`).stdout + ); + record( + 'real binary into your command', + (await $q`printf 'abc' | upper`).stdout + ); + + // The exit code of a pipeline is the exit code of its last stage. + record( + 'exit code of the last stage', + (await $q`echo x | sh -c 'exit 7'`).code + ); + record( + 'an earlier failure does not change it', + (await $q`sh -c 'exit 3' | cat`).code + ); + + // The .pipe() method builds the same pipeline from separate commands. + const piped = await $({ mirror: false })`echo method`.pipe( + $({ mirror: false })`upper` + ); + record('.pipe() method', piped.stdout); + + unregister('upper'); +}); diff --git a/js/examples/features/redirection.mjs b/js/examples/features/redirection.mjs new file mode 100644 index 00000000..f27c10f6 --- /dev/null +++ b/js/examples/features/redirection.mjs @@ -0,0 +1,32 @@ +// Output and input redirection work with built-ins and with your own commands, +// without handing the command line to a real shell. +import { $ } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; +import fs from 'fs'; +import path from 'path'; + +await example( + { id: 'redirection', title: 'Redirecting output and input' }, + async ({ record }) => { + const dir = makeTempDir('redirect'); + const file = path.join(dir, 'out.txt'); + const $q = $({ mirror: false }); + + const written = await $q`echo first > ${file}`; + record('the command itself prints nothing', written.stdout); + record('the file holds the output', fs.readFileSync(file, 'utf8')); + + await $q`echo second >> ${file}`; + record('>> appends', fs.readFileSync(file, 'utf8')); + + const numbers = path.join(dir, 'numbers.txt'); + await $q`seq 1 3 | cat > ${numbers}`; + record('a pipeline can redirect too', fs.readFileSync(numbers, 'utf8')); + + record('< feeds a command from a file', (await $q`cat < ${file}`).stdout); + record( + 'a quoted > stays a literal argument', + (await $q`echo "a > b"`).stdout + ); + } +); diff --git a/js/examples/features/result-text.mjs b/js/examples/features/result-text.mjs new file mode 100644 index 00000000..e664f1c7 --- /dev/null +++ b/js/examples/features/result-text.mjs @@ -0,0 +1,19 @@ +// Every result exposes an async text() method, like Bun's built-in $. +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'result-text', title: 'Read the output with text()' }, + async ({ record }) => { + record('system command', await (await $q`sh -c 'echo system'`).text()); + record('built-in command', await (await $q`echo built-in`).text()); + record('synchronous command', await $q`echo sync`.sync().text()); + record('pipeline', await (await $q`echo piped | cat`).text()); + + register('text-demo', async () => ({ stdout: 'virtual\n', code: 0 })); + record('virtual command', await (await $q`text-demo`).text()); + unregister('text-demo'); + } +); diff --git a/js/examples/features/sequences.mjs b/js/examples/features/sequences.mjs new file mode 100644 index 00000000..d385a6b8 --- /dev/null +++ b/js/examples/features/sequences.mjs @@ -0,0 +1,21 @@ +// Operators between commands: && runs on success, || runs on failure, +// ; runs unconditionally and ( ) groups commands into a subshell. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'sequences', title: 'Command sequences' }, + async ({ record }) => { + record('&& after a success', (await $q`true && echo ran`).stdout); + record('&& after a failure', (await $q`false && echo ran`).stdout); + record('|| after a failure', (await $q`false || echo fallback`).stdout); + record('|| after a success', (await $q`true || echo fallback`).stdout); + record('; runs both', (await $q`echo one ; echo two`).stdout); + record('( ) groups commands', (await $q`(echo a ; echo b)`).stdout); + + const chain = await $q`false && echo skipped`; + record('exit code of a short-circuited chain', chain.code); + } +); diff --git a/js/examples/features/shell-settings.mjs b/js/examples/features/shell-settings.mjs new file mode 100644 index 00000000..5cfb2ecc --- /dev/null +++ b/js/examples/features/shell-settings.mjs @@ -0,0 +1,38 @@ +// Shell settings mirror `set -e`, `set -x`, `set -v` and `set -o pipefail`. +import { $, shell, set, unset } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'shell-settings', title: 'Shell settings' }, + async ({ record }) => { + record('defaults', shell.settings()); + + set('e'); + record('set("e") enables errexit', shell.settings().errexit); + try { + await $q`sh -c 'exit 5'`; + record('failing command with errexit', 'did not throw'); + } catch (error) { + record('failing command with errexit', `threw with code ${error.code}`); + } + unset('e'); + + shell.pipefail(true); + record( + 'pipefail makes an early failure win', + (await $q`sh -c 'exit 3' | cat`).code + ); + shell.pipefail(false); + record( + 'without pipefail the last stage wins', + (await $q`sh -c 'exit 3' | cat`).code + ); + + set('x'); + record('xtrace on', shell.settings().xtrace); + unset('x'); + record('settings restored', shell.settings()); + } +); diff --git a/js/examples/features/stdin-streaming.mjs b/js/examples/features/stdin-streaming.mjs new file mode 100644 index 00000000..18f18833 --- /dev/null +++ b/js/examples/features/stdin-streaming.mjs @@ -0,0 +1,23 @@ +// .streams.stdin gives write access to a running command. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'stdin-streaming', title: 'Writing to stdin while a command runs' }, + async ({ record }) => { + const runner = $q`cat`; + const stdin = await runner.streams.stdin; + stdin.write('first line\n'); + stdin.write('second line\n'); + stdin.end(); + record('what cat echoed back', (await runner).stdout); + + // A whole string can also be handed over up front. + record( + 'stdin option', + (await $({ mirror: false, stdin: 'up front\n' })`cat`).stdout + ); + } +); diff --git a/js/examples/features/sync-execution.mjs b/js/examples/features/sync-execution.mjs new file mode 100644 index 00000000..ca54aa39 --- /dev/null +++ b/js/examples/features/sync-execution.mjs @@ -0,0 +1,32 @@ +// .sync() runs a command synchronously and returns the finished result. +import { $ } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'sync-execution', title: 'Synchronous execution' }, + async ({ record }) => { + const result = $q`echo synchronous`.sync(); + record('stdout', result.stdout); + record('code', result.code); + record( + 'result is available without await', + typeof result.stdout === 'string' + ); + + const failed = $q`sh -c 'exit 3'`.sync(); + record('exit code of a failing command', failed.code); + + record( + 'order of execution', + (() => { + const order = []; + order.push('before'); + $q`echo ignored`.sync(); + order.push('after'); + return order; + })() + ); + } +); diff --git a/js/examples/features/virtual-commands.mjs b/js/examples/features/virtual-commands.mjs new file mode 100644 index 00000000..900ccf55 --- /dev/null +++ b/js/examples/features/virtual-commands.mjs @@ -0,0 +1,33 @@ +// Any JavaScript function can be registered as a command and then used from a +// command line like a real binary. +import { $, register, unregister, listCommands } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +const $q = $({ mirror: false }); + +await example( + { id: 'virtual-commands', title: 'Registering your own commands' }, + async ({ record }) => { + register('greet', async ({ args }) => ({ + stdout: `Hello, ${args.join(' ') || 'world'}!\n`, + code: 0, + })); + + record('the command is registered', listCommands().includes('greet')); + record('without arguments', (await $q`greet`).stdout); + record('with arguments', (await $q`greet Node and Bun`).stdout); + + // A handler decides its own exit code and may write to stderr. + register('fail-with', async ({ args }) => ({ + stderr: `failing on purpose\n`, + code: Number(args[0] ?? 1), + })); + const failed = await $q`fail-with 42`; + record('custom exit code', failed.code); + record('custom stderr', failed.stderr); + + unregister('greet'); + unregister('fail-with'); + record('unregistered again', listCommands().includes('greet')); + } +); diff --git a/js/examples/features/virtual-context.mjs b/js/examples/features/virtual-context.mjs new file mode 100644 index 00000000..da7e723d --- /dev/null +++ b/js/examples/features/virtual-context.mjs @@ -0,0 +1,31 @@ +// A command handler receives a context object describing how it was invoked. +import { $, register, unregister } from '../../src/$.mjs'; +import { example, makeTempDir } from './_harness.mjs'; + +await example( + { id: 'virtual-context', title: 'The handler context' }, + async ({ record }) => { + const dir = makeTempDir('context'); + + register('describe', async ({ args, stdin, cwd, env, options }) => ({ + stdout: + JSON.stringify({ + args, + stdin, + cwdIsTheOneWeAskedFor: cwd === dir, + envValue: env.DEMO, + mirror: options.mirror, + }) + '\n', + code: 0, + })); + + const result = await $({ + mirror: false, + cwd: dir, + env: { DEMO: 'from-options' }, + })`echo piped | describe one two`; + record('context seen by the handler', JSON.parse(result.stdout)); + + unregister('describe'); + } +); diff --git a/js/examples/features/virtual-streaming.mjs b/js/examples/features/virtual-streaming.mjs new file mode 100644 index 00000000..37ae8acd --- /dev/null +++ b/js/examples/features/virtual-streaming.mjs @@ -0,0 +1,37 @@ +// A handler written as an async generator streams its output chunk by chunk, +// so consumers see data before the command has finished. +import { $, register, unregister } from '../../src/$.mjs'; +import { example } from './_harness.mjs'; + +await example( + { id: 'virtual-streaming', title: 'Streaming commands' }, + async ({ record }) => { + register('countdown', async function* ({ args }) { + for (let i = Number(args[0] ?? 3); i > 0; i--) { + yield `${i}\n`; + } + yield 'liftoff\n'; + }); + + const chunks = []; + for await (const chunk of $({ mirror: false })`countdown 3`.stream()) { + if (chunk.type === 'exit') { + continue; + } + chunks.push(chunk.data.toString()); + } + record('chunks received one by one', chunks); + record( + 'same command awaited as a whole', + (await $({ mirror: false })`countdown 2`).stdout + ); + + // Streaming commands compose with the rest of a pipeline. + record( + 'piped into a built-in', + (await $({ mirror: false })`countdown 2 | cat`).stdout + ); + + unregister('countdown'); + } +); diff --git a/js/package.json b/js/package.json index 56ca64ec..e5ded114 100644 --- a/js/package.json +++ b/js/package.json @@ -45,6 +45,9 @@ "format": "cd .. && js/node_modules/.bin/prettier --write .", "format:check": "cd .. && js/node_modules/.bin/prettier --check .", "check:duplication": "jscpd src scripts", + "check:parity": "cd .. && node scripts/check-parity.mjs", + "docs:generate": "cd .. && node scripts/generate-docs.mjs", + "docs:check": "cd .. && node scripts/generate-docs.mjs --check", "check": "bun run lint && bun run format:check && bun run check:duplication", "build:terminal-font": "node scripts/build-terminal-font.mjs", "prepare": "cd .. && js/node_modules/.bin/husky || true", @@ -79,6 +82,7 @@ ], "devDependencies": { "@changesets/cli": "^2.31.1", + "@eslint/js": "^9.39.5", "cross-spawn": "7.0.6", "dejavu-fonts-ttf": "^2.37.3", "esbuild": "0.28.2", diff --git a/js/src/$.process-runner-base.mjs b/js/src/$.process-runner-base.mjs index 3cb3b494..81be900c 100644 --- a/js/src/$.process-runner-base.mjs +++ b/js/src/$.process-runner-base.mjs @@ -11,6 +11,7 @@ import { } from './$.state.mjs'; import { StreamEmitter } from './$.stream-emitter.mjs'; import { processOutput } from './$.ansi.mjs'; +import { ensureResultText } from './$.result.mjs'; const isBun = typeof globalThis.Bun !== 'undefined'; @@ -442,6 +443,7 @@ class ProcessRunner extends StreamEmitter { if (result && result.exitCode === undefined && result.code !== undefined) { result.exitCode = result.code; } + ensureResultText(result); trace( 'ProcessRunner', diff --git a/js/src/$.process-runner-pipeline.mjs b/js/src/$.process-runner-pipeline.mjs index 22709aef..d9899f0b 100644 --- a/js/src/$.process-runner-pipeline.mjs +++ b/js/src/$.process-runner-pipeline.mjs @@ -178,20 +178,21 @@ function getStdinString(options) { } /** - * Handle pipefail check + * Compute the status reported by a pipeline. Without pipefail this is the last + * stage; with pipefail it is the rightmost failing stage, matching Bash. + * pipefail changes the status only. errexit decides whether that status throws. + * * @param {number[]} exitCodes - Exit codes from pipeline * @param {object} shellSettings - Shell settings + * @returns {number} Pipeline exit status */ -function checkPipefail(exitCodes, shellSettings) { - if (shellSettings.pipefail) { - const failedIndex = exitCodes.findIndex((code) => code !== 0); - if (failedIndex !== -1) { - throw createCommandError( - `Pipeline command at index ${failedIndex} failed with exit code ${exitCodes[failedIndex]}`, - { code: exitCodes[failedIndex] } - ); - } +function pipelineExitCode(exitCodes, shellSettings) { + const codes = exitCodes.map((code) => code ?? 0); + const last = codes.at(-1) ?? 0; + if (!shellSettings.pipefail) { + return last; } + return codes.findLast((code) => code !== 0) ?? last; } /** @@ -623,7 +624,7 @@ async function handleVirtualPipelineCommand( isLastCommand, deps ) { - const { virtualCommands, globalShellSettings } = deps; + const { virtualCommands, globalShellSettings, exitCodes } = deps; const handler = virtualCommands.get(command.cmd); const argValues = getArgValues(command.args); logShellTrace(globalShellSettings, command.cmd, argValues); @@ -635,31 +636,29 @@ async function handleVirtualPipelineCommand( currentInput, { ...runner.options, + options: runner.options, cwd: effectiveCwd(runner), env: effectiveEnv(runner) ?? process.env, } ); applyVirtualProcessContext(runner, result); + exitCodes.push(result.code); if (isLastCommand) { emitFinalOutput(runner, result); return { finalResult: createFinalPipelineResult( runner, - result, + { + ...result, + code: pipelineExitCode(exitCodes, globalShellSettings), + }, result.stdout, globalShellSettings ), }; } - if (globalShellSettings.errexit && result.code !== 0) { - throw createCommandError( - `Pipeline command failed with exit code ${result.code}`, - { code: result.code, result } - ); - } - return { input: result.stdout }; } @@ -679,7 +678,7 @@ async function handleShellPipelineCommand( isLastCommand, deps ) { - const { globalShellSettings } = deps; + const { globalShellSettings, exitCodes } = deps; const commandStr = buildCommandParts(command).join(' '); logShellTrace(globalShellSettings, commandStr, []); @@ -695,13 +694,7 @@ async function handleShellPipelineCommand( stdout: proc.stdout || '', stderr: proc.stderr || '', }; - - if (globalShellSettings.pipefail && result.code !== 0) { - throw createCommandError( - `Pipeline command '${commandStr}' failed with exit code ${result.code}`, - { code: result.code } - ); - } + exitCodes.push(result.code); if (isLastCommand) { let allStderr = ''; @@ -712,7 +705,7 @@ async function handleShellPipelineCommand( allStderr += result.stderr; } const finalResult = createResult({ - code: result.code, + code: pipelineExitCode(exitCodes, globalShellSettings), stdout: result.stdout, stderr: allStderr, stdin: getStdinString(runner.options), @@ -813,10 +806,8 @@ export function attachPipelineMethods(ProcessRunner, deps) { } const exitCodes = await Promise.all(processes.map((p) => p.exited)); - checkPipefail(exitCodes, globalShellSettings); - const result = createResult({ - code: exitCodes[exitCodes.length - 1] || 0, + code: pipelineExitCode(exitCodes, globalShellSettings), stdout: finalOutput, stderr: collector.stderr, stdin: getStdinString(this.options), @@ -904,10 +895,8 @@ export function attachPipelineMethods(ProcessRunner, deps) { } const exitCodes = await Promise.all(processes.map((p) => p.exited)); - checkPipefail(exitCodes, globalShellSettings); - const result = createResult({ - code: exitCodes[exitCodes.length - 1] || 0, + code: pipelineExitCode(exitCodes, globalShellSettings), stdout: finalOutput, stderr: collector.stderr, stdin: getStdinString(this.options), @@ -929,6 +918,7 @@ export function attachPipelineMethods(ProcessRunner, deps) { let currentInputStream = createInitialInputStream(this.options); let finalOutput = ''; const collector = { stderr: '' }; + const stageCodes = []; for (let i = 0; i < commands.length; i++) { const command = commands[i]; @@ -945,43 +935,55 @@ export function attachPipelineMethods(ProcessRunner, deps) { if (handler.constructor.name === 'AsyncGeneratorFunction') { const chunks = []; const self = this; + let generatorDone; currentInputStream = new ReadableStream({ - async start(controller) { + start(controller) { const { stdin: _, ...opts } = self.options; - for await (const chunk of handler({ - args: argValues, - stdin: inputData, - ...opts, - cwd: effectiveCwd(self), - env: effectiveEnv(self) ?? process.env, - })) { - const data = Buffer.from(chunk); - controller.enqueue(data); - if (isLastCommand) { - chunks.push(data); - if (self.options.mirror) { - safeWrite(process.stdout, data); + generatorDone = (async () => { + for await (const chunk of handler({ + args: argValues, + stdin: inputData, + ...opts, + options: self.options, + cwd: effectiveCwd(self), + env: effectiveEnv(self) ?? process.env, + })) { + const data = Buffer.from(chunk); + controller.enqueue(data); + if (isLastCommand) { + chunks.push(data); + if (self.options.mirror) { + safeWrite(process.stdout, data); + } + self.emit('stdout', data); + self.emit('data', { type: 'stdout', data }); } - self.emit('stdout', data); - self.emit('data', { type: 'stdout', data }); } - } - controller.close(); - if (isLastCommand) { - finalOutput = Buffer.concat(chunks).toString('utf8'); - } + controller.close(); + if (isLastCommand) { + finalOutput = Buffer.concat(chunks).toString('utf8'); + } + })(); + return generatorDone; }, }); + // Track completion without awaiting it here. The next process must + // start now so it can consume chunks while the generator is still + // producing them (and so producer/consumer handshakes cannot + // deadlock). + stageCodes.push(generatorDone.then(() => 0)); } else { const { stdin: _, ...opts } = this.options; const result = await handler({ args: argValues, stdin: inputData, ...opts, + options: this.options, cwd: effectiveCwd(this), env: effectiveEnv(this) ?? process.env, }); applyVirtualProcessContext(this, result); + stageCodes.push(result.code ?? 0); const outputData = result.stdout || ''; if (isLastCommand) { finalOutput = outputData; @@ -1006,6 +1008,7 @@ export function attachPipelineMethods(ProcessRunner, deps) { pipeStreamToProcess(currentInputStream, proc); currentInputStream = proc.stdout; + stageCodes.push(proc.exited); collectStderrAsync(this, proc, isLastCommand, collector); if (isLastCommand) { @@ -1015,14 +1018,17 @@ export function attachPipelineMethods(ProcessRunner, deps) { } } + const exitCodes = await Promise.all(stageCodes); + const result = createResult({ - code: 0, + code: pipelineExitCode(exitCodes, globalShellSettings), stdout: finalOutput, stderr: collector.stderr, stdin: getStdinString(this.options), }); this.finish(result); + throwErrexitError(result, globalShellSettings); return result; }; @@ -1034,7 +1040,11 @@ export function attachPipelineMethods(ProcessRunner, deps) { const currentOutput = ''; let currentInput = getStdinString(this.options); - const pipelineDeps = { virtualCommands, globalShellSettings }; + const pipelineDeps = { + virtualCommands, + globalShellSettings, + exitCodes: [], + }; for (let i = 0; i < commands.length; i++) { const command = commands[i]; diff --git a/js/src/$.result.mjs b/js/src/$.result.mjs index 24f4d762..5c1b2829 100644 --- a/js/src/$.result.mjs +++ b/js/src/$.result.mjs @@ -24,6 +24,33 @@ export function createResult({ code, stdout = '', stderr = '', stdin = '' }) { }; } +/** + * Add the Bun.$-compatible text() helper to results created by execution paths + * that return a plain object (notably virtual and built-in commands). + * + * @param {object} result - Result object to normalize + * @returns {object} The same result object + */ +export function ensureResultText(result) { + if ( + !result || + typeof result !== 'object' || + typeof result.text === 'function' + ) { + return result; + } + + Object.defineProperty(result, 'text', { + value() { + return Promise.resolve(result.stdout ?? ''); + }, + writable: true, + configurable: true, + enumerable: false, + }); + return result; +} + /** * Create an Error describing a command that exited with a failing status. * diff --git a/js/src/commands/$.ls.mjs b/js/src/commands/$.ls.mjs index 22475f83..c7a3baf7 100644 --- a/js/src/commands/$.ls.mjs +++ b/js/src/commands/$.ls.mjs @@ -39,7 +39,10 @@ export default async function ls({ args, stdin: _stdin, cwd }) { const stats = fs.statSync(resolvedPath); if (stats.isDirectory()) { - let entries = fs.readdirSync(resolvedPath); + // readdir returns entries in directory order, which differs between + // file systems and between Node.js and Bun. Real `ls` sorts by name, + // so sort here to keep the output stable everywhere. + let entries = fs.readdirSync(resolvedPath).sort(); if (!showAll) { entries = entries.filter((e) => !e.startsWith('.')); diff --git a/js/tests/cross-runtime-parity.test.mjs b/js/tests/cross-runtime-parity.test.mjs new file mode 100644 index 00000000..12509b00 --- /dev/null +++ b/js/tests/cross-runtime-parity.test.mjs @@ -0,0 +1,453 @@ +// Regression tests for behaviours that used to differ between Node.js and Bun, +// or that silently diverged from the documented API. +// +// Every expectation here is runtime-independent on purpose: the whole point of +// these tests is that `bun test` and the Node parity runner +// (`node scripts/check-parity.mjs`) must observe the very same values. +import { describe, test, expect, afterEach } from 'bun:test'; +import './test-helper.mjs'; +import { + $, + register, + unregister, + shell, + enableVirtualCommands, +} from '../src/$.mjs'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { spawn } from 'child_process'; + +const $q = $({ mirror: false, capture: true }); + +const tempDirs = []; +function tempDir() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cs-parity-')); + tempDirs.push(dir); + return dir; +} + +afterEach(() => { + while (tempDirs.length) { + fs.rmSync(tempDirs.pop(), { recursive: true, force: true }); + } +}); + +describe('result.text() is available on every execution path', () => { + test('system command (async)', async () => { + const result = await $q`sh -c 'echo system'`; + expect(typeof result.text).toBe('function'); + expect(await result.text()).toBe('system\n'); + }); + + test('built-in command (async)', async () => { + const result = await $q`echo builtin`; + expect(typeof result.text).toBe('function'); + expect(await result.text()).toBe('builtin\n'); + }); + + test('built-in command (sync)', async () => { + const result = $({ mirror: false })`echo builtin`.sync(); + expect(typeof result.text).toBe('function'); + expect(await result.text()).toBe('builtin\n'); + }); + + test('pipeline', async () => { + const result = await $q`echo a | cat`; + expect(typeof result.text).toBe('function'); + expect(await result.text()).toBe('a\n'); + }); + + test('.pipe() method', async () => { + const result = await $({ mirror: false })`echo a`.pipe( + $({ mirror: false })`cat` + ); + expect(typeof result.text).toBe('function'); + expect(await result.text()).toBe('a\n'); + }); + + test('virtual command', async () => { + register('parity-text', async () => ({ stdout: 'virtual\n', code: 0 })); + try { + const result = await $q`parity-text`; + expect(typeof result.text).toBe('function'); + expect(await result.text()).toBe('virtual\n'); + } finally { + unregister('parity-text'); + } + }); +}); + +describe('virtual command stdin', () => { + test('a standalone virtual command receives empty stdin, never the "inherit" sentinel', async () => { + register('parity-stdin', async ({ stdin }) => ({ + stdout: JSON.stringify(stdin), + code: 0, + })); + try { + const result = await $q`parity-stdin`; + expect(result.stdout).toBe('""'); + } finally { + unregister('parity-stdin'); + } + }); + + test('a virtual command receives the previous built-in command output', async () => { + register('parity-upper', async ({ stdin }) => ({ + stdout: String(stdin).toUpperCase(), + code: 0, + })); + try { + expect((await $q`echo abc | parity-upper`).stdout).toBe('ABC\n'); + } finally { + unregister('parity-upper'); + } + }); + + test('a virtual command receives the previous system command output', async () => { + register('parity-upper', async ({ stdin }) => ({ + stdout: String(stdin).toUpperCase(), + code: 0, + })); + try { + expect((await $q`sh -c 'echo sys' | parity-upper`).stdout).toBe('SYS\n'); + } finally { + unregister('parity-upper'); + } + }); + + test('explicit stdin is forwarded to a virtual command', async () => { + register('parity-upper', async ({ stdin }) => ({ + stdout: String(stdin).toUpperCase(), + code: 0, + })); + try { + const result = await $({ + mirror: false, + capture: true, + stdin: 'given\n', + })`parity-upper`; + expect(result.stdout).toBe('GIVEN\n'); + } finally { + unregister('parity-upper'); + } + }); + + test('the handler context exposes the documented fields', async () => { + let seen; + register('parity-ctx', async (ctx) => { + seen = ctx; + return { stdout: '', code: 0 }; + }); + try { + await $({ + mirror: false, + capture: true, + cwd: os.tmpdir(), + })`parity-ctx one two`; + expect(seen.args).toEqual(['one', 'two']); + expect(seen.stdin).toBe(''); + expect(seen.cwd).toBe(os.tmpdir()); + expect(typeof seen.isCancelled).toBe('function'); + expect(seen.options).toBeDefined(); + expect(seen.env).toBeDefined(); + } finally { + unregister('parity-ctx'); + } + }); +}); + +describe('virtual command streaming', () => { + test('starts a downstream process before an async generator finishes', async () => { + const dir = tempDir(); + const consumer = path.join(dir, 'consumer.cjs'); + const marker = path.join(dir, 'consumer-started'); + fs.writeFileSync( + consumer, + [ + "const fs = require('fs');", + 'let started = false;', + "process.stdin.on('data', (chunk) => {", + ' if (!started) {', + ' started = true;', + ' fs.writeFileSync(process.argv[2], "");', + ' }', + ' process.stdout.write(chunk);', + '});', + ].join('\n') + ); + + register('parity-handshake', async function* () { + yield 'first\n'; + const deadline = Date.now() + 2000; + while (!fs.existsSync(marker)) { + if (Date.now() >= deadline) { + throw new Error('downstream process did not consume the first chunk'); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + yield 'second\n'; + }); + + try { + const result = + await $q`parity-handshake | ${process.execPath} ${consumer} ${marker}`; + expect(result.stdout).toBe('first\nsecond\n'); + expect(fs.existsSync(marker)).toBe(true); + } finally { + unregister('parity-handshake'); + } + }, 10000); +}); + +describe('pipeline exit codes', () => { + test('the exit code of the last virtual command is propagated', async () => { + register('parity-fail', async () => ({ + stdout: '', + stderr: 'boom\n', + code: 7, + })); + try { + const result = await $q`echo a | parity-fail`; + expect(result.code).toBe(7); + expect(result.stderr).toContain('boom'); + } finally { + unregister('parity-fail'); + } + }); + + test('the exit code of the last system command is propagated', async () => { + const result = await $q`echo a | sh -c 'exit 7'`; + expect(result.code).toBe(7); + }); + + test('a failing built-in command in the last position is propagated', async () => { + const result = await $q`echo a | cat /definitely/not/here`; + expect(result.code).not.toBe(0); + }); + + test('a failure in an earlier stage does not mask the final exit code', async () => { + register('parity-fail', async () => ({ + stdout: '', + stderr: 'boom\n', + code: 7, + })); + try { + const result = await $q`parity-fail | cat`; + expect(result.code).toBe(0); + expect(result.stderr).toContain('boom'); + } finally { + unregister('parity-fail'); + } + }); +}); + +describe('output redirection with built-in and virtual commands', () => { + test('`command > file` writes the file instead of passing ">" as an argument', async () => { + const file = path.join(tempDir(), 'out.txt'); + const result = await $q`echo hello > ${file}`; + expect(result.code).toBe(0); + expect(result.stdout).toBe(''); + expect(fs.readFileSync(file, 'utf8')).toBe('hello\n'); + }); + + test('`command >> file` appends', async () => { + const file = path.join(tempDir(), 'out.txt'); + await $q`echo one > ${file}`; + await $q`echo two >> ${file}`; + expect(fs.readFileSync(file, 'utf8')).toBe('one\ntwo\n'); + }); + + test('redirection at the end of a pipeline writes the file', async () => { + const file = path.join(tempDir(), 'numbers.txt'); + const result = await $q`seq 1 3 | cat > ${file}`; + expect(result.code).toBe(0); + expect(result.stdout).toBe(''); + expect(fs.readFileSync(file, 'utf8')).toBe('1\n2\n3\n'); + }); + + test('a quoted ">" stays a literal argument', async () => { + const result = await $q`echo "a > b"`; + expect(result.stdout).toBe('a > b\n'); + }); + + test('input redirection feeds a built-in command', async () => { + const file = path.join(tempDir(), 'in.txt'); + fs.writeFileSync(file, 'from-file\n'); + const result = await $q`cat < ${file}`; + expect(result.code).toBe(0); + expect(result.stdout).toBe('from-file\n'); + }); +}); + +describe('built-in commands behave like their POSIX counterparts', () => { + test('ls sorts entries by name, like real ls', async () => { + // Other test files switch the built-ins off, so be explicit about needing + // the built-in `ls` rather than the system one. + enableVirtualCommands(); + const dir = tempDir(); + // Written in an order that is neither sorted nor reverse sorted, so a + // readdir that happens to be ordered cannot make this pass by accident. + for (const name of ['zebra.txt', 'alpha.txt', 'middle.txt']) { + fs.writeFileSync(path.join(dir, name), ''); + } + const result = await $q`ls ${dir}`; + expect(result.stdout).toBe('alpha.txt\nmiddle.txt\nzebra.txt\n'); + }); + + test('ls -a sorts the dot entries in too', async () => { + enableVirtualCommands(); + const dir = tempDir(); + for (const name of ['visible.txt', '.hidden']) { + fs.writeFileSync(path.join(dir, name), ''); + } + const result = await $q`ls -a ${dir}`; + expect(result.stdout).toBe('.hidden\nvisible.txt\n'); + }); + + test('sleep does not keep the process alive after it finishes', async () => { + // The built-in used to start an interval to poll for cancellation and never + // clear it on the success path, so any script using `sleep` hung forever. + const dir = tempDir(); + const script = path.join(dir, 'sleep-exit.mjs'); + const entry = path.resolve( + import.meta.dirname ?? path.dirname(new URL(import.meta.url).pathname), + '../src/$.mjs' + ); + fs.writeFileSync( + script, + [ + `import { $ } from ${JSON.stringify(entry)};`, + 'await $({ mirror: false })`sleep 0.05`;', + "console.log('finished');", + ].join('\n') + ); + + const exited = await new Promise((resolve) => { + const child = spawn(process.execPath, [script], { stdio: 'ignore' }); + const timer = setTimeout(() => { + child.kill('SIGKILL'); + resolve('timed out'); + }, 10000); + child.on('exit', (code) => { + clearTimeout(timer); + resolve(`exited with ${code}`); + }); + }); + expect(exited).toBe('exited with 0'); + }, 20000); +}); + +describe('pipefail reports an exit code instead of throwing', () => { + afterEach(() => { + shell.pipefail(false); + shell.errexit(false); + }); + + test('without pipefail the last stage decides', async () => { + const result = await $q`sh -c 'echo x; exit 3' | cat`; + expect(result.code).toBe(0); + expect(result.stdout).toBe('x\n'); + }); + + test('with pipefail the rightmost failing stage decides', async () => { + shell.pipefail(true); + const result = await $q`sh -c 'echo x; exit 3' | cat`; + expect(result.code).toBe(3); + // bash keeps the output of a pipeline that pipefail marked as failed. + expect(result.stdout).toBe('x\n'); + }); + + test('with pipefail a failing built-in stage decides', async () => { + shell.pipefail(true); + enableVirtualCommands(); + const result = await $q`false | cat`; + expect(result.code).toBe(1); + }); + + test('with pipefail the rightmost failure wins over an earlier one', async () => { + shell.pipefail(true); + const result = await $q`sh -c 'exit 3' | sh -c 'exit 4' | cat`; + expect(result.code).toBe(4); + }); + + test('pipefail alone does not throw, errexit does', async () => { + shell.pipefail(true); + shell.errexit(true); + let thrown = null; + try { + await $q`sh -c 'exit 3' | cat`; + } catch (error) { + thrown = error; + } + expect(thrown).not.toBe(null); + expect(thrown.code).toBe(3); + }); +}); + +describe('quoting survives the trip to a command', () => { + // The built-in path parses the command line itself instead of handing it to a + // shell, so it has to understand the same quoting the shell would. Each case + // below asserts that a built-in and the system command agree. + const cases = [ + ["it's a name", 'an apostrophe inside the value'], + ['two spaces', 'repeated spaces'], + ['say "hi"', 'double quotes inside the value'], + ['back\\slash', 'a backslash'], + ['a|b', 'a pipe character'], + ['$HOME', 'something that looks like a variable'], + ]; + + for (const [value, description] of cases) { + test(`echo passes through ${description}`, async () => { + enableVirtualCommands(); + const builtin = await $q`echo ${value}`; + expect(builtin.stdout).toBe(`${value}\n`); + }); + } + + test('an interpolated apostrophe does not split the pipeline', async () => { + enableVirtualCommands(); + const result = await $q`echo ${"it's a name"} | cat`; + expect(result.stdout).toBe("it's a name\n"); + }); + + test('a pipe inside a quoted argument is not a pipeline separator', async () => { + enableVirtualCommands(); + const result = await $q`echo "a | b"`; + expect(result.stdout).toBe('a | b\n'); + }); + + test('adjacent quoted and unquoted pieces form one argument', async () => { + enableVirtualCommands(); + const result = await $q`echo pre"in quotes"post`; + expect(result.stdout).toBe('prein quotespost\n'); + }); + + test('a system command still sees the shell expansion it was given', async () => { + // `printf` has no built-in, so this goes to a real shell. Rebuilding the + // command line must keep `$HOME` unexpanded for the shell to expand. + const result = await $q`printf '%s' $HOME`; + expect(result.stdout.length).toBeGreaterThan(0); + expect(result.stdout).not.toBe('$HOME'); + }); + + test('a system command keeps a quoted expansion literal', async () => { + const result = await $q`printf '%s' '$HOME'`; + expect(result.stdout).toBe('$HOME'); + }); + + test('the enhanced parser unquotes a path the same way', async () => { + // A command line containing `&&` takes the enhanced parser instead of the + // simple one. Both have to agree, or a directory created by one is + // unreachable by the other. + enableVirtualCommands(); + const dir = path.join(tempDir(), "odd-'name'-$1"); + await $q`mkdir -p ${dir}`; + expect(fs.existsSync(dir)).toBe(true); + const marker = 'parser-reached-directory.txt'; + const result = await $q`cd ${dir} && touch ${marker}`; + expect(result.code).toBe(0); + expect(fs.existsSync(path.join(dir, marker))).toBe(true); + }); +}); diff --git a/js/tests/docs-validation.test.mjs b/js/tests/docs-validation.test.mjs index e67a0d19..a1133a2e 100644 --- a/js/tests/docs-validation.test.mjs +++ b/js/tests/docs-validation.test.mjs @@ -111,6 +111,16 @@ describe('documentation validation', () => { expect(executable).toEqual([]); }); + test('the generated website never reparses feature data as HTML', () => { + // Catalog entries and captured command output are text. Assigning rendered + // strings to an HTML sink would turn any markup in that data into active + // DOM content (the DOM-XSS pattern reported by CodeQL). + for (const file of ['scripts/generate-docs.mjs', 'docs/site/index.html']) { + const text = readFileSync(join(repoRoot, file), 'utf8'); + expect(text).not.toMatch(/\.(?:inner|outer)HTML\s*=|insertAdjacentHTML/); + } + }); + // A reader following a cross-reference lands on a heading. These are the // headings other documents and the workflows point at. test.each([ diff --git a/js/tests/language-parity.test.mjs b/js/tests/language-parity.test.mjs index a5ef4d47..6e310ee0 100644 --- a/js/tests/language-parity.test.mjs +++ b/js/tests/language-parity.test.mjs @@ -81,6 +81,13 @@ describe.skipIf(process.platform === 'win32')('language parity guard', () => { expect(result.stdout).toContain('Language parity check passed.'); }); + test('a generated Rust benchmark lockfile does not require a JavaScript edit', () => { + const result = parityResult(['rust/benchmarks/Cargo.lock']); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('Language parity check passed.'); + }); + test('a benchmark edit cannot stand in for a source implementation', () => { const result = parityResult(['js/src/.keep', 'rust/benchmarks/.keep']); diff --git a/js/tests/repository-layout.test.mjs b/js/tests/repository-layout.test.mjs index 1e3620b9..5b5a13b6 100644 --- a/js/tests/repository-layout.test.mjs +++ b/js/tests/repository-layout.test.mjs @@ -75,8 +75,10 @@ describe('repository language layout', () => { ); }); - test('does not keep language release scripts at the repository root', () => { - expect(existsFromRepo('scripts')).toBe(false); + test('keeps shared tooling at root and language release scripts in their packages', () => { + expect(existsFromRepo('scripts')).toBe(true); + expect(existsFromRepo('scripts/check-parity.mjs')).toBe(true); + expect(existsFromRepo('scripts/generate-docs.mjs')).toBe(true); expect(existsFromRepo('scripts/publish-to-npm.mjs')).toBe(false); expect(existsFromRepo('scripts/publish-to-crates.mjs')).toBe(false); expect(existsFromRepo('scripts/sync-rust-version.mjs')).toBe(false); diff --git a/js/tests/virtual.test.mjs b/js/tests/virtual.test.mjs index 0f931f1d..646a24e5 100644 --- a/js/tests/virtual.test.mjs +++ b/js/tests/virtual.test.mjs @@ -10,6 +10,7 @@ import { listCommands, enableVirtualCommands, } from '../src/$.mjs'; +import builtinLs from '../src/commands/$.ls.mjs'; // Helper function to setup shell settings function setupShellSettings() { @@ -184,6 +185,10 @@ describe('Virtual Commands System', () => { const systemResult = await $`ls`; expect(systemResult.stdout).not.toBe('virtual ls output\n'); expect(systemResult.code).toBe(0); // System ls should work + + // The registry is process-wide, so put the built-in back. Leaving it + // unregistered would silently hand `ls` to the system in every later test. + register('ls', builtinLs); }); test('should fall back to system commands when virtual not found', async () => { diff --git a/js/tests/workflow-hygiene.test.mjs b/js/tests/workflow-hygiene.test.mjs index 4fb4b924..440a4491 100644 --- a/js/tests/workflow-hygiene.test.mjs +++ b/js/tests/workflow-hygiene.test.mjs @@ -23,17 +23,19 @@ const workflows = workflowFiles.map((name) => { }); /** - * Jobs that mutate the repository: push a commit or a tag to main, publish a - * package, or open a release pull request. These are the ones that must never - * be cancelled halfway. + * Jobs that mutate repository state: push a commit or tag, publish a package, + * open a release pull request, or deploy GitHub Pages. These are the ones that + * must never be cancelled halfway. * - * `contents: write` is the test, not `pull-requests: write`. A job can hold the - * latter alone and still change nothing that outlives the run -- the security - * workflow's dependency-review only uses it to leave a review comment -- and - * putting such a job in the shared non-cancellable group would serialise every - * pull request behind main's releases for no benefit. + * `contents: write` and `pages: write` identify persistent writes. A job can + * hold `pull-requests: write` alone and still change no package, tag or site -- + * the security workflow only uses it to leave a review comment -- so putting + * that job in the shared group would serialise every pull request needlessly. */ -const isWriterJob = (job) => (job.permissions ?? {})['contents'] === 'write'; +const isWriterJob = (job) => { + const permissions = job.permissions ?? {}; + return permissions.contents === 'write' || permissions.pages === 'write'; +}; const WRITER_GROUP = 'main-writer-${{ github.repository }}-main'; @@ -333,6 +335,28 @@ describe('workflow linting is itself wired into CI', () => { }); }); +describe('benchmark baselines stay reproducible across package releases', () => { + const benchmarks = workflows.find((w) => w.name === 'benchmarks.yml'); + + test('the Rust baseline refreshes only its local package before running locked', () => { + const baseline = benchmarks.doc.jobs.rust.steps.find( + (step) => step.name === 'Benchmark the pull request base' + ); + const refresh = + 'cargo update --offline --manifest-path benchmarks/Cargo.toml -p command-stream'; + const runLocked = + 'cargo run --release --locked --manifest-path benchmarks/Cargo.toml'; + + // The base branch can legitimately contain the previous package version in + // this nested lockfile. Refreshing just the local path dependency keeps all + // third-party versions pinned, after which --locked protects the benchmark. + expect(baseline.run).toContain(refresh); + expect(baseline.run.indexOf(refresh)).toBeLessThan( + baseline.run.indexOf(runLocked) + ); + }); +}); + describe('every shipped ecosystem is audited', () => { const security = workflows.find((w) => w.name === 'security.yml'); const runs = Object.values(security.doc.jobs) diff --git a/rust/benchmarks/Cargo.lock b/rust/benchmarks/Cargo.lock index 596e5222..36045a44 100644 --- a/rust/benchmarks/Cargo.lock +++ b/rust/benchmarks/Cargo.lock @@ -218,7 +218,7 @@ dependencies = [ [[package]] name = "command-stream" -version = "0.18.6" +version = "0.23.0" dependencies = [ "async-trait", "chrono", diff --git a/rust/changelog.d/20260916_181500_language_feature_parity.md b/rust/changelog.d/20260916_181500_language_feature_parity.md new file mode 100644 index 00000000..a1fba57f --- /dev/null +++ b/rust/changelog.d/20260916_181500_language_feature_parity.md @@ -0,0 +1,14 @@ +--- +bump: minor +--- + +### Added + +- Added live stdin writes with `ProcessRunner::write_stdin` and `ProcessRunner::close_stdin`. +- Added executable Rust counterparts for every feature in the generated language-parity guide. + +### Fixed + +- Pipelines now use the last stage's status by default and the rightmost failure with `pipefail`. +- Shell sequence operators are executed with shell-compatible behavior. +- `VirtualCommandRegistry::with_builtins` now returns the complete built-in catalog. diff --git a/rust/examples/language_features.rs b/rust/examples/language_features.rs new file mode 100644 index 00000000..e2e3545e --- /dev/null +++ b/rust/examples/language_features.rs @@ -0,0 +1,484 @@ +//! Executable Rust examples for the generated cross-language feature guide. +//! +//! Each `feature:*` region is extracted into the matching documentation page. +//! The binary executes one region at a time so CI verifies every example: +//! `cargo run --example language_features -- await-result`. + +use command_stream::commands::{CommandContext, VirtualCommandRegistry}; +use command_stream::{ + cmd, create, exec, run, run_sync, set_shell_option, unset_shell_option, AnsiUtils, + CommandResult, EventData, EventType, OutputChunk, Pipeline, ProcessRunner, RunOptions, + StdinOption, StreamEmitter, StreamingRunner, +}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::error::Error; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +const PARITY_START: &str = "<<, Box>; + +fn observation(label: &'static str, value: impl Serialize) -> Observation { + Observation { + label, + value: serde_json::to_value(value).expect("example observations are serializable"), + } +} + +fn quiet_options() -> RunOptions { + RunOptions { + mirror: false, + ..RunOptions::default() + } +} + +async fn quiet(command: &str) -> command_stream::Result { + exec(command, quiet_options()).await +} + +// feature:await-result +async fn await_result() -> ExampleResult { + let result = quiet("echo hello").await?; + Ok(vec![ + observation("stdout", result.stdout), + observation("stderr", result.stderr), + observation("exit code", result.code), + ]) +} +// endfeature:await-result + +// feature:result-text +async fn result_text() -> ExampleResult { + let result = quiet("echo hello").await?; + Ok(vec![observation("text output", result.stdout)]) +} +// endfeature:result-text + +// feature:sync-execution +async fn sync_execution() -> ExampleResult { + let result = tokio::task::spawn_blocking(|| run_sync("echo synchronous")).await??; + Ok(vec![observation("stdout", result.stdout)]) +} +// endfeature:sync-execution + +// feature:exit-codes +async fn exit_codes() -> ExampleResult { + let result = quiet("false").await?; + let checked = result.clone().error_for_status().unwrap_err(); + Ok(vec![ + observation("result code", result.code), + observation("checked error code", checked.code()), + ]) +} +// endfeature:exit-codes + +// feature:options +async fn options() -> ExampleResult { + let directory = tempfile::tempdir()?; + let mut env = HashMap::new(); + env.insert( + "COMMAND_STREAM_DEMO".to_string(), + "from-options".to_string(), + ); + let result = exec( + "cat", + RunOptions { + mirror: false, + cwd: Some(directory.path().to_path_buf()), + env: Some(env), + stdin: StdinOption::Content("from-stdin\n".to_string()), + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![observation("stdin and cwd options", result.stdout)]) +} +// endfeature:options + +// feature:function-api +async fn function_api() -> ExampleResult { + let simple = run("echo run").await?; + let configured = exec("echo exec", quiet_options()).await?; + let mut runner = create("echo create", quiet_options()); + let created = runner.run().await?; + Ok(vec![observation( + "run, exec and create", + [ + simple.stdout.trim(), + configured.stdout.trim(), + created.stdout.trim(), + ], + )]) +} +// endfeature:function-api + +// feature:cancellation +async fn cancellation() -> ExampleResult { + let mut stream = StreamingRunner::new("sleep 30").stream(); + let started = stream.wait_for_pid().await.is_some(); + stream.kill(); + let mut exit_code = 0; + while let Some(chunk) = stream.next().await { + if let OutputChunk::Exit(code) = chunk { + exit_code = code; + } + } + Ok(vec![ + observation("process started", started), + observation("cancelled exit is non-zero", exit_code != 0), + ]) +} +// endfeature:cancellation + +// feature:async-iteration +async fn async_iteration() -> ExampleResult { + let mut stream = StreamingRunner::new("printf 'one\\ntwo\\n'").stream(); + let mut stdout = Vec::new(); + let mut exit_code = None; + while let Some(chunk) = stream.next().await { + match chunk { + OutputChunk::Stdout(data) => stdout.extend(data), + OutputChunk::Stderr(_) => {} + OutputChunk::Exit(code) => exit_code = Some(code), + } + } + Ok(vec![ + observation("collected chunks", String::from_utf8(stdout)?), + observation("exit code", exit_code), + ]) +} +// endfeature:async-iteration + +// feature:events +async fn events() -> ExampleResult { + let emitter = StreamEmitter::new(); + let count = Arc::new(AtomicUsize::new(0)); + let listener_count = Arc::clone(&count); + emitter + .on(EventType::Stdout, move |_| { + listener_count.fetch_add(1, Ordering::SeqCst); + }) + .await; + emitter + .emit(EventType::Stdout, EventData::String("hello".to_string())) + .await; + Ok(vec![observation( + "stdout events", + count.load(Ordering::SeqCst), + )]) +} +// endfeature:events + +// feature:stdin-streaming +async fn stdin_streaming() -> ExampleResult { + let mut runner = ProcessRunner::new( + "cat", + RunOptions { + mirror: false, + stdin: StdinOption::Pipe, + ..RunOptions::default() + }, + ); + runner.start().await?; + runner.write_stdin("first line\n").await?; + runner.write_stdin("second line\n").await?; + runner.close_stdin().await?; + let result = runner.run().await?; + Ok(vec![observation("what cat echoed back", result.stdout)]) +} +// endfeature:stdin-streaming + +// feature:buffers-strings +async fn buffers_strings() -> ExampleResult { + let result = quiet("printf bytes").await?; + Ok(vec![ + observation("string", &result.stdout), + observation("bytes", result.stdout.as_bytes()), + ]) +} +// endfeature:buffers-strings + +// feature:mirror-capture +async fn mirror_capture() -> ExampleResult { + let captured = quiet("echo captured").await?; + let uncaptured = exec( + "true", + RunOptions { + mirror: false, + capture: false, + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![ + observation("captured output", captured.stdout), + observation("capture can be disabled", uncaptured.stdout.is_empty()), + ]) +} +// endfeature:mirror-capture + +// feature:builtin-catalog +async fn builtin_catalog() -> ExampleResult { + let registry = VirtualCommandRegistry::with_builtins(); + let mut commands = registry.list(); + commands.sort_unstable(); + Ok(vec![ + observation("available built-ins", &commands), + observation("number of built-ins", commands.len()), + ]) +} +// endfeature:builtin-catalog + +// feature:builtin-filesystem +async fn builtin_filesystem() -> ExampleResult { + let directory = tempfile::tempdir()?; + let options = RunOptions { + mirror: false, + cwd: Some(directory.path().to_path_buf()), + ..RunOptions::default() + }; + exec("mkdir demo", options.clone()).await?; + exec("touch demo/file.txt", options.clone()).await?; + let listed = exec("ls demo", options.clone()).await?; + exec("rm -r demo", options).await?; + Ok(vec![observation("created and listed", listed.stdout)]) +} +// endfeature:builtin-filesystem + +// feature:builtin-text +async fn builtin_text() -> ExampleResult { + let sequence = quiet("seq 1 3").await?; + let basename = quiet("basename /tmp/example.txt").await?; + Ok(vec![ + observation("sequence", sequence.stdout), + observation("basename", basename.stdout), + ]) +} +// endfeature:builtin-text + +// feature:builtin-environment +async fn builtin_environment() -> ExampleResult { + let mut env = HashMap::new(); + env.insert("COMMAND_STREAM_DEMO".to_string(), "visible".to_string()); + let result = exec( + "env", + RunOptions { + mirror: false, + env: Some(env), + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![observation( + "configured environment visible", + result.stdout.contains("COMMAND_STREAM_DEMO=visible"), + )]) +} +// endfeature:builtin-environment + +fn greet_handler( + context: CommandContext, +) -> Pin + Send>> { + Box::pin(async move { CommandResult::success(format!("Hello, {}!\n", context.args.join(" "))) }) +} + +// feature:virtual-commands +async fn virtual_commands() -> ExampleResult { + let mut registry = VirtualCommandRegistry::new(); + registry.register("greet", greet_handler); + let handler = registry.get("greet").expect("registered handler"); + let result = handler(CommandContext::new(vec!["Rust".to_string()])).await; + let removed = registry.unregister("greet"); + Ok(vec![ + observation("custom command output", result.stdout), + observation("unregistered again", removed), + ]) +} +// endfeature:virtual-commands + +// feature:virtual-context +async fn virtual_context() -> ExampleResult { + let mut context = CommandContext::new(vec!["one".to_string(), "two".to_string()]); + context.stdin = Some("piped\n".to_string()); + context.cwd = Some(std::env::temp_dir()); + context.env = Some(HashMap::from([("DEMO".to_string(), "value".to_string())])); + Ok(vec![observation( + "handler context", + json!({ + "args": context.args, + "stdin": context.stdin, + "has_cwd": context.cwd.is_some(), + "env_value": context.env.and_then(|env| env.get("DEMO").cloned()), + }), + )]) +} +// endfeature:virtual-context + +fn streaming_handler( + context: CommandContext, +) -> Pin + Send>> { + Box::pin(async move { + if let Some(output) = context.output_tx { + let _ = output + .send(command_stream::StreamChunk::Stdout("one\n".to_string())) + .await; + let _ = output + .send(command_stream::StreamChunk::Stdout("two\n".to_string())) + .await; + } + CommandResult::success("one\ntwo\n") + }) +} + +// feature:virtual-streaming +async fn virtual_streaming() -> ExampleResult { + let (sender, mut receiver) = tokio::sync::mpsc::channel(4); + let mut context = CommandContext::new(Vec::new()); + context.output_tx = Some(sender); + let result = streaming_handler(context).await; + let mut chunks = Vec::new(); + while let Ok(chunk) = receiver.try_recv() { + if let command_stream::StreamChunk::Stdout(text) = chunk { + chunks.push(text); + } + } + Ok(vec![ + observation("chunks", chunks), + observation("collected output", result.stdout), + ]) +} +// endfeature:virtual-streaming + +// feature:pipelines +async fn pipelines() -> ExampleResult { + let result = Pipeline::new() + .add("printf 'hello\\nworld\\n'") + .add("grep world") + .mirror_output(false) + .run() + .await?; + Ok(vec![observation("pipeline output", result.stdout)]) +} +// endfeature:pipelines + +// feature:redirection +async fn redirection() -> ExampleResult { + let directory = tempfile::tempdir()?; + let file = directory.path().join("output.txt"); + let result = exec( + "echo redirected > output.txt", + RunOptions { + mirror: false, + cwd: Some(directory.path().to_path_buf()), + ..RunOptions::default() + }, + ) + .await?; + Ok(vec![ + observation("exit code", result.code), + observation("file contents", std::fs::read_to_string(file)?), + ]) +} +// endfeature:redirection + +// feature:sequences +async fn sequences() -> ExampleResult { + let result = quiet("false || echo fallback; echo next").await?; + Ok(vec![observation("sequence output", result.stdout)]) +} +// endfeature:sequences + +// feature:interpolation +async fn interpolation() -> ExampleResult { + let value = "hello from Rust"; + let result = cmd!("echo {}", value).await?; + Ok(vec![observation("macro interpolation", result.stdout)]) +} +// endfeature:interpolation + +// feature:shell-settings +async fn shell_settings() -> ExampleResult { + set_shell_option("pipefail").await; + let with_pipefail = Pipeline::new().add("false").add("true").run().await?; + unset_shell_option("pipefail").await; + let without_pipefail = Pipeline::new().add("false").add("true").run().await?; + Ok(vec![ + observation("with pipefail", with_pipefail.code), + observation("without pipefail", without_pipefail.code), + ]) +} +// endfeature:shell-settings + +// feature:ansi-utils +async fn ansi_utils() -> ExampleResult { + Ok(vec![observation( + "stripped output", + AnsiUtils::strip_all("\u{1b}[31mred\u{1b}[0m"), + )]) +} +// endfeature:ansi-utils + +async fn execute(id: &str) -> ExampleResult { + match id { + "await-result" => await_result().await, + "result-text" => result_text().await, + "sync-execution" => sync_execution().await, + "exit-codes" => exit_codes().await, + "options" => options().await, + "function-api" => function_api().await, + "cancellation" => cancellation().await, + "async-iteration" => async_iteration().await, + "events" => events().await, + "stdin-streaming" => stdin_streaming().await, + "buffers-strings" => buffers_strings().await, + "mirror-capture" => mirror_capture().await, + "builtin-catalog" => builtin_catalog().await, + "builtin-filesystem" => builtin_filesystem().await, + "builtin-text" => builtin_text().await, + "builtin-environment" => builtin_environment().await, + "virtual-commands" => virtual_commands().await, + "virtual-context" => virtual_context().await, + "virtual-streaming" => virtual_streaming().await, + "pipelines" => pipelines().await, + "redirection" => redirection().await, + "sequences" => sequences().await, + "interpolation" => interpolation().await, + "shell-settings" => shell_settings().await, + "ansi-utils" => ansi_utils().await, + _ => Err(format!("unknown feature: {id}").into()), + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let id = std::env::args().nth(1).ok_or("pass a feature id")?; + let observations = execute(&id).await?; + + println!("# {id} — Rust"); + for item in &observations { + println!("{}: {}", item.label, item.value); + } + println!("{PARITY_START}"); + println!( + "{}", + serde_json::to_string(&json!({ + "id": id, + "language": "rust", + "observations": observations, + "failure": Value::Null, + }))? + ); + println!("{PARITY_END}"); + Ok(()) +} diff --git a/rust/src/commands/mod.rs b/rust/src/commands/mod.rs index a7f4636a..4bfa53f6 100644 --- a/rust/src/commands/mod.rs +++ b/rust/src/commands/mod.rs @@ -175,9 +175,34 @@ impl VirtualCommandRegistry { /// Register all built-in commands pub fn register_builtins(&mut self) { - // Note: These are placeholder registrations - actual async handlers - // would need proper wrapper functions - // The actual commands are available as standalone functions + macro_rules! register { + ($name:literal, $function:path) => { + self.register($name, |ctx| Box::pin($function(ctx))); + }; + } + + register!("echo", echo); + register!("pwd", pwd); + register!("cd", cd); + register!("true", r#true); + register!("false", r#false); + register!("sleep", sleep); + register!("cat", cat); + register!("ls", ls); + register!("mkdir", mkdir); + register!("rm", rm); + register!("touch", touch); + register!("cp", cp); + register!("mv", mv); + register!("basename", basename); + register!("dirname", dirname); + register!("env", env); + register!("exit", exit); + register!("which", which); + register!("yes", yes); + register!("seq", seq); + register!("tee", tee); + register!("test", test); } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 6843b148..da59b29c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -83,7 +83,7 @@ use std::collections::HashMap; use std::path::PathBuf; use std::process::Stdio; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; -use tokio::process::{Child, Command}; +use tokio::process::Child; use tokio::sync::mpsc; pub use commands::{CommandContext, StreamChunk}; @@ -412,7 +412,10 @@ impl ProcessRunner { // arguments, so `echo hello > out.txt` printed the redirection instead // of writing the file, and `git push ... 2>&1` reported success while // nothing was pushed (#46). - let first_word = if has_shell_escapes(&self.command) || needs_real_shell(&self.command) { + let first_word = if matches!(self.options.stdin, StdinOption::Pipe) + || has_shell_escapes(&self.command) + || needs_real_shell(&self.command) + { "" } else { self.command.split_whitespace().next().unwrap_or("") @@ -430,13 +433,7 @@ impl ProcessRunner { }; // Execute via real shell if needed - let shell = find_available_shell(); - - let mut cmd = Command::new(&shell.cmd); - for arg in &shell.args { - cmd.arg(arg); - } - utils::append_shell_command(&mut cmd, &self.command, self.options.env.as_ref()); + let mut cmd = utils::shell_command(&self.command, self.options.env.as_ref()); // Configure stdin match &self.options.stdin { @@ -504,6 +501,35 @@ impl ProcessRunner { Ok(()) } + /// Write bytes to the stdin pipe of a running command. + /// + /// Configure the runner with [`StdinOption::Pipe`], call [`start`](Self::start), + /// write as many chunks as needed, and finish with [`close_stdin`](Self::close_stdin). + pub async fn write_stdin(&mut self, data: impl AsRef<[u8]>) -> Result<()> { + self.start().await?; + let stdin = self + .child + .as_mut() + .and_then(|child| child.stdin.as_mut()) + .ok_or_else(|| { + Error::Io(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "command stdin is not available; use StdinOption::Pipe", + )) + })?; + stdin.write_all(data.as_ref()).await?; + Ok(()) + } + + /// Close a running command's stdin pipe so it can observe end-of-input. + pub async fn close_stdin(&mut self) -> Result<()> { + self.start().await?; + if let Some(mut stdin) = self.child.as_mut().and_then(|child| child.stdin.take()) { + stdin.shutdown().await?; + } + Ok(()) + } + /// Run the process to completion pub async fn run(&mut self) -> Result { self.start().await?; @@ -771,62 +797,6 @@ impl ProcessRunner { } } -/// Shell configuration -#[derive(Debug, Clone)] -struct ShellConfig { - cmd: String, - args: Vec, -} - -/// Find an available shell -fn find_available_shell() -> ShellConfig { - let is_windows = cfg!(windows); - - if is_windows { - // Windows shells - let shells = [ - ("cmd.exe", vec!["/c"]), - ("powershell.exe", vec!["-Command"]), - ]; - - for (cmd, args) in shells { - if which::which(cmd).is_ok() { - return ShellConfig { - cmd: cmd.to_string(), - args: args.into_iter().map(String::from).collect(), - }; - } - } - - ShellConfig { - cmd: "cmd.exe".to_string(), - args: vec!["/c".to_string()], - } - } else { - // Unix shells - let shells = [ - ("/bin/sh", vec!["-c"]), - ("/usr/bin/sh", vec!["-c"]), - ("/bin/bash", vec!["-c"]), - ("sh", vec!["-c"]), - ]; - - for (cmd, args) in shells { - if std::path::Path::new(cmd).exists() || which::which(cmd).is_ok() { - return ShellConfig { - cmd: cmd.to_string(), - args: args.into_iter().map(String::from).collect(), - }; - } - } - - ShellConfig { - cmd: "/bin/sh".to_string(), - args: vec!["-c".to_string()], - } - } -} - /// Execute a command and return the result /// /// This is the main entry point for simple command execution. diff --git a/rust/src/pipeline.rs b/rust/src/pipeline.rs index dcf1b141..ea811fd0 100644 --- a/rust/src/pipeline.rs +++ b/rust/src/pipeline.rs @@ -28,7 +28,6 @@ use std::collections::HashMap; use std::path::PathBuf; use std::process::Stdio; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::process::Command; use crate::trace::trace_lazy; use crate::{CommandResult, Result, RunOptions, StdinOption}; @@ -38,6 +37,20 @@ struct VirtualCommandResult { cd_context: Option, } +fn pipeline_exit_code(exit_codes: &[i32], pipefail: bool) -> i32 { + let last = exit_codes.last().copied().unwrap_or(0); + if pipefail { + exit_codes + .iter() + .rev() + .copied() + .find(|code| *code != 0) + .unwrap_or(last) + } else { + last + } +} + /// A pipeline of commands to be executed sequentially /// /// Each command's stdout is piped to the next command's stdin. @@ -139,6 +152,8 @@ impl Pipeline { code: 0, }; let mut accumulated_stderr = String::new(); + let mut exit_codes = Vec::with_capacity(self.commands.len()); + let pipefail = crate::get_shell_settings().await.pipefail; for (i, cmd_str) in self.commands.iter().enumerate() { let is_last = i == self.commands.len() - 1; @@ -166,23 +181,23 @@ impl Pipeline { .await { let VirtualCommandResult { result, cd_context } = result; - if result.code != 0 { - return Ok(CommandResult { - stdout: result.stdout, - stderr: accumulated_stderr + &result.stderr, - code: result.code, - }); - } + exit_codes.push(result.code); current_stdin = Some(result.stdout.clone()); accumulated_stderr.push_str(&result.stderr); - if let Some(context) = cd_context { - let env = effective_env.get_or_insert_with(|| std::env::vars().collect()); - env.insert( - "OLDPWD".to_string(), - context.oldpwd.to_string_lossy().to_string(), - ); - env.insert("PWD".to_string(), context.cwd.to_string_lossy().to_string()); - effective_cwd = Some(context.cwd); + if result.code == 0 { + if let Some(context) = cd_context { + let env = + effective_env.get_or_insert_with(|| std::env::vars().collect()); + env.insert( + "OLDPWD".to_string(), + context.oldpwd.to_string_lossy().to_string(), + ); + env.insert( + "PWD".to_string(), + context.cwd.to_string_lossy().to_string(), + ); + effective_cwd = Some(context.cwd); + } } last_result = result; continue; @@ -190,12 +205,7 @@ impl Pipeline { } // Execute via shell - let shell = find_available_shell(); - let mut cmd = Command::new(&shell.cmd); - for arg in &shell.args { - cmd.arg(arg); - } - crate::utils::append_shell_command(&mut cmd, cmd_str, effective_env.as_ref()); + let mut cmd = crate::utils::shell_command(cmd_str, effective_env.as_ref()); // Configure stdio cmd.stdin(Stdio::piped()); @@ -256,14 +266,7 @@ impl Pipeline { let code = status.code().unwrap_or(-1); accumulated_stderr.push_str(&stderr_content); - - if code != 0 { - return Ok(CommandResult { - stdout: stdout_content, - stderr: accumulated_stderr, - code, - }); - } + exit_codes.push(code); // Set up stdin for next command current_stdin = Some(stdout_content.clone()); @@ -277,7 +280,7 @@ impl Pipeline { Ok(CommandResult { stdout: last_result.stdout, stderr: accumulated_stderr, - code: last_result.code, + code: pipeline_exit_code(&exit_codes, pipefail), }) } @@ -333,45 +336,6 @@ impl Pipeline { } } -/// Shell configuration -#[derive(Debug, Clone)] -struct ShellConfig { - cmd: String, - args: Vec, -} - -/// Find an available shell -fn find_available_shell() -> ShellConfig { - let is_windows = cfg!(windows); - - if is_windows { - ShellConfig { - cmd: "cmd.exe".to_string(), - args: vec!["/c".to_string()], - } - } else { - let shells = [ - ("/bin/sh", "-c"), - ("/usr/bin/sh", "-c"), - ("/bin/bash", "-c"), - ]; - - for (cmd, arg) in shells { - if std::path::Path::new(cmd).exists() { - return ShellConfig { - cmd: cmd.to_string(), - args: vec![arg.to_string()], - }; - } - } - - ShellConfig { - cmd: "/bin/sh".to_string(), - args: vec!["-c".to_string()], - } - } -} - /// Extension trait to add `.pipe()` method to ProcessRunner pub trait PipelineExt { /// Pipe the output of this command to another command @@ -404,19 +368,13 @@ impl PipelineBuilder { pub async fn run(mut self) -> Result { // First, run the initial command let first_result = self.first.run().await?; - - if first_result.code != 0 { - return Ok(first_result); - } + let pipefail = crate::get_shell_settings().await.pipefail; + let mut exit_codes = vec![first_result.code]; // Then run the rest as a pipeline - let mut current_stdin = Some(first_result.stdout); - let mut accumulated_stderr = first_result.stderr; - let mut last_result = CommandResult { - stdout: String::new(), - stderr: String::new(), - code: 0, - }; + let mut current_stdin = Some(first_result.stdout.clone()); + let mut accumulated_stderr = first_result.stderr.clone(); + let mut last_result = first_result; for cmd_str in &self.additional { let mut runner = crate::ProcessRunner::new( @@ -431,14 +389,7 @@ impl PipelineBuilder { let result = runner.run().await?; accumulated_stderr.push_str(&result.stderr); - - if result.code != 0 { - return Ok(CommandResult { - stdout: result.stdout, - stderr: accumulated_stderr, - code: result.code, - }); - } + exit_codes.push(result.code); current_stdin = Some(result.stdout.clone()); last_result = result; @@ -447,7 +398,22 @@ impl PipelineBuilder { Ok(CommandResult { stdout: last_result.stdout, stderr: accumulated_stderr, - code: last_result.code, + code: pipeline_exit_code(&exit_codes, pipefail), }) } } + +#[cfg(test)] +mod tests { + use super::pipeline_exit_code; + + #[test] + fn pipeline_status_uses_last_stage_by_default() { + assert_eq!(pipeline_exit_code(&[3, 0], false), 0); + } + + #[test] + fn pipefail_uses_rightmost_failing_stage() { + assert_eq!(pipeline_exit_code(&[2, 7, 0], true), 7); + } +} diff --git a/rust/src/shell_parser.rs b/rust/src/shell_parser.rs index dc813b2a..7e56a06e 100644 --- a/rust/src/shell_parser.rs +++ b/rust/src/shell_parser.rs @@ -533,6 +533,11 @@ pub fn needs_real_shell(command: &str) -> bool { '*', // Glob patterns '?', // Glob patterns '[', // Glob patterns + '|', // Pipelines and boolean OR + '&', // Boolean AND and backgrounding + ';', // Command sequences + '(', // Subshells + ')', // Subshells '>', // Output redirection, in every form (>, >>, 2>, &>, >&) '<', // Input redirection, in every form (<, <<, <<<) ]; @@ -625,8 +630,8 @@ mod tests { assert!(needs_real_shell("echo $(date)")); assert!(needs_real_shell("ls *.txt")); assert!(needs_real_shell("echo ${HOME}")); + assert!(needs_real_shell("ls | grep foo")); assert!(!needs_real_shell("echo hello")); - assert!(!needs_real_shell("ls | grep foo")); } #[test] diff --git a/rust/src/stream.rs b/rust/src/stream.rs index b96e81ee..f5d05c50 100644 --- a/rust/src/stream.rs +++ b/rust/src/stream.rs @@ -420,13 +420,7 @@ async fn run_streaming_process( }); let mut cmd = match command { - StreamingCommand::Shell(command) => { - let shell = find_available_shell(); - let mut cmd = Command::new(&shell.cmd); - cmd.args(&shell.args); - crate::utils::append_shell_command(&mut cmd, &command, env.as_ref()); - cmd - } + StreamingCommand::Shell(command) => crate::utils::shell_command(&command, env.as_ref()), StreamingCommand::Argv { program, args } => { let mut cmd = Command::new(program); cmd.args(args); @@ -627,45 +621,6 @@ fn status_to_code(status: std::process::ExitStatus) -> i32 { -1 } -/// Shell configuration -#[derive(Debug, Clone)] -struct ShellConfig { - cmd: String, - args: Vec, -} - -/// Find an available shell -fn find_available_shell() -> ShellConfig { - let is_windows = cfg!(windows); - - if is_windows { - ShellConfig { - cmd: "cmd.exe".to_string(), - args: vec!["/c".to_string()], - } - } else { - let shells = [ - ("/bin/sh", "-c"), - ("/usr/bin/sh", "-c"), - ("/bin/bash", "-c"), - ]; - - for (cmd, arg) in shells { - if std::path::Path::new(cmd).exists() { - return ShellConfig { - cmd: cmd.to_string(), - args: vec![arg.to_string()], - }; - } - } - - ShellConfig { - cmd: "/bin/sh".to_string(), - args: vec!["-c".to_string()], - } - } -} - /// Async iterator trait for output streams #[async_trait::async_trait] pub trait AsyncIterator { diff --git a/rust/src/utils.rs b/rust/src/utils.rs index dc4f4149..529428bd 100644 --- a/rust/src/utils.rs +++ b/rust/src/utils.rs @@ -60,28 +60,95 @@ pub(crate) fn with_exported_process_context( command.to_string() } -/// Append a command string using the platform shell's argument convention. -pub(crate) fn append_shell_command( - process: &mut tokio::process::Command, - command: &str, - env: Option<&HashMap>, -) { - let command = with_exported_process_context(command, env); +#[derive(Debug, Clone)] +struct ShellConfig { + cmd: String, + args: Vec, + raw_command_arg: bool, +} +fn find_available_shell() -> ShellConfig { #[cfg(windows)] - { + let shells: &[(&str, &[&str], bool)] = &[ + (r"C:\Program Files\Git\bin\bash.exe", &["-c"], false), + (r"C:\Program Files\Git\usr\bin\bash.exe", &["-c"], false), + (r"C:\Program Files (x86)\Git\bin\bash.exe", &["-c"], false), + ("bash.exe", &["-c"], false), + ("wsl.exe", &["bash", "-c"], false), + ("powershell.exe", &["-Command"], false), + ("pwsh.exe", &["-Command"], false), + ("cmd.exe", &["/c"], true), + ]; + + #[cfg(not(windows))] + let shells: &[(&str, &[&str], bool)] = &[ + ("/bin/sh", &["-c"], false), + ("/usr/bin/sh", &["-c"], false), + ("/bin/bash", &["-c"], false), + ("sh", &["-c"], false), + ]; + + for (cmd, args, raw_command_arg) in shells { + if Path::new(cmd).exists() || which::which(cmd).is_ok() { + return ShellConfig { + cmd: (*cmd).to_string(), + args: args.iter().map(|arg| (*arg).to_string()).collect(), + raw_command_arg: *raw_command_arg, + }; + } + } + + #[cfg(windows)] + return ShellConfig { + cmd: "cmd.exe".to_string(), + args: vec!["/c".to_string()], + raw_command_arg: true, + }; + + #[cfg(not(windows))] + ShellConfig { + cmd: "/bin/sh".to_string(), + args: vec!["-c".to_string()], + raw_command_arg: false, + } +} + +#[cfg(windows)] +fn append_command_arg(process: &mut tokio::process::Command, command: &str, raw_command_arg: bool) { + if raw_command_arg { // `cmd.exe /c` does not use the C runtime's argument decoder. Passing // the command through `arg` would therefore expose Rust's backslash // escapes as literal characters. The extra outer quotes are required // to preserve a quoted executable path at the start of the command. use std::os::windows::process::CommandExt; process.as_std_mut().raw_arg(format!("\"{command}\"")); + } else { + process.arg(command); } +} - #[cfg(not(windows))] +#[cfg(not(windows))] +fn append_command_arg( + process: &mut tokio::process::Command, + command: &str, + _raw_command_arg: bool, +) { process.arg(command); } +/// Build a command using the best platform shell and its argument convention. +pub(crate) fn shell_command( + command: &str, + env: Option<&HashMap>, +) -> tokio::process::Command { + let shell = find_available_shell(); + let mut process = tokio::process::Command::new(&shell.cmd); + process.args(&shell.args); + let command = with_exported_process_context(command, env); + append_command_arg(&mut process, &command, shell.raw_command_arg); + process +} + /// Result type for virtual command operations #[derive(Debug, Clone)] pub struct CommandResult { diff --git a/rust/tests/pipeline.rs b/rust/tests/pipeline.rs index 7e5cf92d..8ee8bb54 100644 --- a/rust/tests/pipeline.rs +++ b/rust/tests/pipeline.rs @@ -63,17 +63,19 @@ async fn test_pipeline_empty() { } #[tokio::test] -async fn test_pipeline_failure_propagation() { +async fn test_pipeline_status_comes_from_last_stage_without_pipefail() { let result = Pipeline::new() .add("echo hello") .add("false") // This command always fails - .add("echo should not reach here") + .add("echo reached last stage") .run() .await .unwrap(); - // Pipeline should fail because 'false' returns non-zero - assert!(!result.is_success()); + // POSIX pipelines report the final stage unless pipefail is enabled. A + // failed stage must therefore not stop the rest of the pipeline. + assert!(result.is_success()); + assert!(result.stdout.contains("reached last stage")); } #[tokio::test] diff --git a/rust/tests/process_runner.rs b/rust/tests/process_runner.rs index aca3c2c3..9cddfa82 100644 --- a/rust/tests/process_runner.rs +++ b/rust/tests/process_runner.rs @@ -35,10 +35,10 @@ async fn test_command_with_arguments() { #[tokio::test] async fn test_real_shell_preserves_missing_final_newlines() { - #[cfg(unix)] + // Windows deliberately prefers Git Bash too, keeping the command language + // consistent with the JavaScript implementation and the documented + // cross-language examples. let command = "printf stdout; printf stderr >&2"; - #[cfg(windows)] - let command = "&2 set /p x=stderr&exit /b 0"; let result = exec( command, diff --git a/rust/tests/redirection_silent_failure.rs b/rust/tests/redirection_silent_failure.rs index 7ca93994..72d667e5 100644 --- a/rust/tests/redirection_silent_failure.rs +++ b/rust/tests/redirection_silent_failure.rs @@ -199,7 +199,7 @@ fn needs_real_shell_recognises_redirection() { assert!(needs_real_shell("cat < in.txt")); assert!(needs_real_shell("git push origin main 2>&1")); assert!(needs_real_shell("cat < {marker}")).cwd(temp_dir.path()); let result = runner.collect().await.unwrap(); assert!(result.is_success()); - let stdout = result.stdout.trim().replace('\\', "/"); - let expected = temp_dir.path().to_string_lossy().replace('\\', "/"); - assert!( - stdout.contains(&expected), - "expected stdout {stdout:?} to contain cwd {expected:?}" + assert_eq!( + std::fs::read_to_string(temp_dir.path().join(marker)) + .unwrap() + .trim(), + "reached" ); } diff --git a/scripts/check-parity.mjs b/scripts/check-parity.mjs new file mode 100644 index 00000000..274aa711 --- /dev/null +++ b/scripts/check-parity.mjs @@ -0,0 +1,48 @@ +#!/usr/bin/env node +// Runs every feature example in JavaScript and Rust. JavaScript observations +// must agree in Node and Bun; every Rust counterpart must compile and run. +// +// Each example prints a JSON block under COMMAND_STREAM_PARITY=1 listing what it +// observed. Comparing those blocks is what "the feature behaves the same +// everywhere" means in this repository, and it is checked in CI. +// +// node scripts/check-parity.mjs compare every installed runtime +// node scripts/check-parity.mjs --json print the report as JSON +import { runExamples } from './run-examples.mjs'; + +const asJson = process.argv.includes('--json'); +const report = await runExamples(); + +if (asJson) { + console.log(JSON.stringify(report, null, 2)); +} else { + console.log( + `JavaScript runtimes: ${report.runtimes.map((r) => `${r.label} ${r.version}`).join(', ')}` + ); + console.log( + `Languages: ${report.languages.map((language) => `${language.name} ${language.version}`).join('; ')}` + ); + console.log(''); + for (const feature of report.features) { + const mark = feature.parity ? '✓' : '✗'; + console.log(`${mark} ${feature.id}`); + if (!feature.parity) { + for (const difference of feature.differences) { + console.log(` ${difference}`); + } + } + } + console.log(''); +} + +const broken = report.features.filter((feature) => !feature.parity); +if (broken.length > 0) { + console.error( + `${broken.length} feature(s) failed language/runtime parity: ${broken.map((f) => f.id).join(', ')}` + ); + process.exit(1); +} + +console.log( + `All ${report.features.length} features are executable in JavaScript and Rust; JavaScript observations match in ${report.runtimes.length} runtime(s).` +); diff --git a/scripts/generate-docs.mjs b/scripts/generate-docs.mjs new file mode 100644 index 00000000..3c580d54 --- /dev/null +++ b/scripts/generate-docs.mjs @@ -0,0 +1,579 @@ +#!/usr/bin/env node +// Generates the feature documentation from the catalog and from what the +// examples actually printed. +// +// Nothing here is written by hand: the code shown is the example file, and the +// output shown is the output that example produced in each installed runtime. +// Documentation therefore cannot drift from the library - if it did, the parity +// check would have failed first. +// +// node scripts/generate-docs.mjs write docs/ +// node scripts/generate-docs.mjs --check fail if docs/ is out of date +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import prettier from '../js/node_modules/prettier/index.mjs'; +import { runExamples } from './run-examples.mjs'; +import { + features, + libraries, + categories, + languages, + rustApiByFeature, +} from '../js/examples/features/catalog.mjs'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const docsDir = path.join(root, 'docs'); +const featuresDir = path.join(docsDir, 'features'); +const siteDir = path.join(docsDir, 'site'); + +const checkOnly = process.argv.includes('--check'); + +const REPO = 'https://github.com/link-foundation/command-stream'; + +const generated = new Map(); +function emit(relativePath, contents) { + generated.set(relativePath, contents); +} + +function alternativeText(alternative) { + if (!alternative) { + return null; + } + if (typeof alternative === 'string') { + return { supported: true, code: alternative }; + } + return { supported: false, reason: alternative.unsupported }; +} + +// ---------------------------------------------------------------- feature page + +// The sequential pushes mirror the document's section order and keep the +// generated Markdown easy to compare with the rendered page. +// eslint-disable-next-line max-statements +function featurePage(feature, run, runtimes) { + const lines = []; + lines.push(`# ${feature.title}`); + lines.push(''); + lines.push(feature.summary); + lines.push(''); + lines.push(`**Category:** ${feature.category}`); + lines.push(''); + lines.push( + `**Languages:** ${languages.map((language) => language.name).join(', ')}` + ); + lines.push(''); + lines.push('## JavaScript'); + lines.push(''); + lines.push(`**API:** ${feature.api.map((name) => `\`${name}\``).join(', ')}`); + lines.push(''); + lines.push( + `**Verified in:** ${runtimes.map((runtime) => runtime.label).join(', ')}` + ); + lines.push(''); + lines.push('### Example'); + lines.push(''); + lines.push(`[\`${feature.file}\`](${REPO}/blob/main/${feature.file})`); + lines.push(''); + lines.push('```js'); + lines.push(run.source.trimEnd()); + lines.push('```'); + lines.push(''); + lines.push('### Output'); + lines.push(''); + + const reports = runtimes.map((runtime) => run.runs[runtime.id]?.report ?? ''); + const identical = reports.every((report) => report === reports[0]); + + if (identical) { + lines.push( + `Identical in ${runtimes.map((runtime) => runtime.label).join(' and ')}:` + ); + lines.push(''); + lines.push('```'); + lines.push(reports[0].trimEnd()); + lines.push('```'); + } else { + for (const [index, runtime] of runtimes.entries()) { + lines.push(`### ${runtime.label}`); + lines.push(''); + lines.push('```'); + lines.push(reports[index].trimEnd()); + lines.push('```'); + lines.push(''); + } + } + lines.push(''); + lines.push('## Rust'); + lines.push(''); + lines.push( + `**API:** ${rustApiByFeature + .get(feature.id) + .map((name) => `\`${name}\``) + .join(', ')}` + ); + lines.push(''); + lines.push('### Example'); + lines.push(''); + lines.push( + `[\`rust/examples/language_features.rs\`](${REPO}/blob/main/rust/examples/language_features.rs)` + ); + lines.push(''); + lines.push('```rust'); + lines.push(run.rust.source.trimEnd()); + lines.push('```'); + lines.push(''); + lines.push('### Output'); + lines.push(''); + lines.push('```'); + lines.push(run.rust.report.trimEnd()); + lines.push('```'); + lines.push(''); + lines.push('## The same thing in other libraries'); + lines.push(''); + + for (const library of libraries.filter( + (library) => library.id !== 'command-stream' + )) { + const alternative = alternativeText(feature.alternatives[library.id]); + lines.push(`### [${library.name}](${library.url})`); + lines.push(''); + if (!alternative) { + lines.push('_Not compared._'); + } else if (alternative.supported) { + lines.push('```js'); + lines.push(alternative.code); + lines.push('```'); + } else { + lines.push(`Not supported — ${alternative.reason}.`); + } + lines.push(''); + } + + lines.push('---'); + lines.push(''); + lines.push('[← All features](../README.md)'); + lines.push(''); + return lines.join('\n'); +} + +// ----------------------------------------------------------------- index page + +function indexPage(report) { + const { runtimes } = report; + const lines = []; + lines.push('# Feature documentation'); + lines.push(''); + lines.push( + 'Every feature of command-stream, with executable JavaScript and Rust examples,' + ); + lines.push( + 'captured output, and the same thing written with other shell libraries.' + ); + lines.push(''); + lines.push( + 'This file is generated by `node scripts/generate-docs.mjs`. Edit the examples in' + ); + lines.push( + '`js/examples/features/` or the catalog in `js/examples/features/catalog.mjs` instead.' + ); + lines.push(''); + lines.push('## Language and runtime parity'); + lines.push(''); + lines.push( + `All ${report.features.length} examples were executed in JavaScript and Rust. JavaScript was checked in ${runtimes.map((runtime) => runtime.label).join(' and ')}.` + ); + lines.push(''); + lines.push( + `| Feature | ${runtimes.map((runtime) => `JavaScript (${runtime.label})`).join(' | ')} | Rust |` + ); + lines.push(`| --- | ${runtimes.map(() => '---').join(' | ')} | --- |`); + for (const run of report.features) { + const feature = features.find((entry) => entry.id === run.id); + const cells = runtimes.map((runtime) => + run.runs[runtime.id]?.failed ? '✗' : '✓' + ); + lines.push( + `| [${feature.title}](features/${feature.id}.md) | ${cells.join(' | ')} | ${run.rust.failed ? '✗' : '✓'} |` + ); + } + lines.push(''); + lines.push('## Library comparison'); + lines.push(''); + lines.push( + '✓ supported, — not supported. Follow a feature for the code in each library.' + ); + lines.push(''); + const others = libraries.filter((library) => library.id !== 'command-stream'); + lines.push( + `| Feature | command-stream | ${others.map((library) => library.name).join(' | ')} |` + ); + lines.push(`| --- | --- | ${others.map(() => '---').join(' | ')} |`); + for (const feature of features) { + const cells = others.map((library) => { + const alternative = alternativeText(feature.alternatives[library.id]); + return alternative?.supported ? '✓' : '—'; + }); + lines.push( + `| [${feature.title}](features/${feature.id}.md) | ✓ | ${cells.join(' | ')} |` + ); + } + lines.push(''); + lines.push('## Features by category'); + lines.push(''); + for (const category of categories) { + const inCategory = features.filter( + (feature) => feature.category === category + ); + if (inCategory.length === 0) { + continue; + } + lines.push(`### ${category}`); + lines.push(''); + for (const feature of inCategory) { + lines.push( + `- [${feature.title}](features/${feature.id}.md) — ${feature.summary}` + ); + } + lines.push(''); + } + lines.push('## Libraries compared'); + lines.push(''); + lines.push('| Library | Version | Runs in |'); + lines.push('| --- | --- | --- |'); + for (const library of libraries) { + lines.push( + `| [${library.name}](${library.url}) | ${library.version ?? 'this repository'} | ${library.runtimes.join(', ')} |` + ); + } + lines.push(''); + return lines.join('\n'); +} + +// -------------------------------------------------------------------- website + +// Keeping the site in one template makes the single-file Pages artifact +// portable and avoids a second asset-generation pipeline. +// eslint-disable-next-line max-lines-per-function +function website(report) { + const data = { + runtimes: report.runtimes.map((runtime) => ({ + id: runtime.id, + label: runtime.label, + })), + languages, + libraries, + categories, + features: features.map((feature) => { + const run = report.features.find((entry) => entry.id === feature.id); + const reports = report.runtimes.map( + (runtime) => run.runs[runtime.id]?.report ?? '' + ); + return { + ...feature, + source: run.source.trimEnd(), + rustApi: rustApiByFeature.get(feature.id), + rustSource: run.rust.source.trimEnd(), + rustOutput: run.rust.report.trimEnd(), + identicalOutput: reports.every((text) => text === reports[0]), + output: Object.fromEntries( + report.runtimes.map((runtime, index) => [ + runtime.id, + reports[index].trimEnd(), + ]) + ), + }; + }), + }; + + return ` + + + + +command-stream — feature comparison + + + +
+

command-stream — feature comparison

+

Every feature in JavaScript and Rust, plus equivalent code in other shell libraries.

+
+
+ +
+
+
Generated from executable examples in js/examples/features/ and rust/examples/language_features.rs.
+ + + + +`; +} + +// ------------------------------------------------------------------ generate + +const report = await runExamples(); + +const broken = report.features.filter((feature) => !feature.parity); +if (broken.length > 0) { + console.error( + `Refusing to document behaviour that differs between runtimes: ${broken.map((feature) => feature.id).join(', ')}` + ); + console.error('Run `node scripts/check-parity.mjs` to see the differences.'); + process.exit(1); +} + +for (const feature of features) { + const run = report.features.find((entry) => entry.id === feature.id); + emit( + path.join('docs', 'features', `${feature.id}.md`), + featurePage(feature, run, report.runtimes) + ); +} +emit(path.join('docs', 'README.md'), indexPage(report)); +emit(path.join('docs', 'site', 'index.html'), website(report)); + +const prettierConfig = + (await prettier.resolveConfig(path.join(root, 'README.md'))) ?? {}; +for (const [relativePath, contents] of generated) { + generated.set( + relativePath, + await prettier.format(contents, { + ...prettierConfig, + filepath: path.join(root, relativePath), + }) + ); +} + +if (checkOnly) { + const stale = []; + for (const [relativePath, contents] of generated) { + const absolute = path.join(root, relativePath); + if ( + !fs.existsSync(absolute) || + fs.readFileSync(absolute, 'utf8') !== contents + ) { + stale.push(relativePath); + } + } + if (stale.length > 0) { + console.error('Generated documentation is out of date:'); + for (const file of stale) { + console.error(` ${file}`); + } + console.error( + '\nRun `node scripts/generate-docs.mjs` and commit the result.' + ); + process.exit(1); + } + console.log(`Documentation is up to date (${generated.size} files).`); +} else { + fs.mkdirSync(featuresDir, { recursive: true }); + fs.mkdirSync(siteDir, { recursive: true }); + for (const [relativePath, contents] of generated) { + fs.writeFileSync(path.join(root, relativePath), contents); + } + console.log(`Wrote ${generated.size} files to docs/.`); +} diff --git a/scripts/run-examples.mjs b/scripts/run-examples.mjs new file mode 100644 index 00000000..d6f5a307 --- /dev/null +++ b/scripts/run-examples.mjs @@ -0,0 +1,252 @@ +// Runs the feature examples and collects what each runtime observed. +// +// Used by scripts/check-parity.mjs to compare runtimes and by +// scripts/generate-docs.mjs to put real, captured output into the documentation. +import { execFile } from 'child_process'; +import { promisify } from 'util'; +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { availableRuntimes } from './runtimes.mjs'; +import { + features, + languages as languageCatalog, + rustApiByFeature, +} from '../js/examples/features/catalog.mjs'; + +const execFileAsync = promisify(execFile); + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const PARITY_START = '<< !rustApiByFeature.has(feature.id) + ); + if (missingRustApi.length > 0) { + throw new Error( + `Rust API catalog is missing: ${missingRustApi.map((feature) => feature.id).join(', ')}` + ); + } + + const rust = await prepareRustExamples(); + const results = []; + + for (const feature of features) { + const file = path.join(root, feature.file); + if (!fs.existsSync(file)) { + throw new Error( + `Catalog entry "${feature.id}" points at a missing file: ${feature.file}` + ); + } + + const runs = {}; + for (const runtime of runtimes) { + runs[runtime.id] = await runOne(runtime, file); + } + + const differences = []; + const [first, ...rest] = runtimes; + const reference = runs[first.id]; + + if (reference.failed) { + differences.push( + `${first.label} failed: ${(reference.stderr ?? '').trim().split('\n').pop()}` + ); + } + + for (const runtime of rest) { + const run = runs[runtime.id]; + if (run.failed && !reference.failed) { + differences.push( + `${runtime.label} failed while ${first.label} succeeded` + ); + continue; + } + if (!run.parity || !reference.parity) { + differences.push(`${runtime.label} produced no parity block`); + continue; + } + for (const difference of compare( + reference.parity.observations, + run.parity.observations + )) { + differences.push(`${runtime.label}: ${difference}`); + } + if ( + JSON.stringify(run.parity.failure) !== + JSON.stringify(reference.parity.failure) + ) { + differences.push( + `${runtime.label}: error ${JSON.stringify(run.parity.failure)} instead of ${JSON.stringify(reference.parity.failure)}` + ); + } + } + + const rustRun = await runRustExample(rust, feature); + if (rustRun.failed) { + differences.push( + `Rust failed: ${(rustRun.stderr ?? '').trim().split('\n').pop()}` + ); + } else if (!rustRun.parity) { + differences.push('Rust produced no feature result block'); + } else if (rustRun.parity.id !== feature.id) { + differences.push( + `Rust reported feature ${rustRun.parity.id} instead of ${feature.id}` + ); + } + + results.push({ + id: feature.id, + title: feature.title, + parity: differences.length === 0, + differences, + runs, + source: fs.readFileSync(file, 'utf8'), + rust: rustRun, + }); + } + + return { + runtimes, + languages: languageCatalog.map((language) => + language.id === 'rust' + ? { ...language, version: rust.version } + : { + ...language, + version: runtimes + .map((runtime) => `${runtime.label} ${runtime.version}`) + .join(', '), + } + ), + features: results, + }; +} diff --git a/scripts/runtimes.mjs b/scripts/runtimes.mjs new file mode 100644 index 00000000..bb1fe688 --- /dev/null +++ b/scripts/runtimes.mjs @@ -0,0 +1,47 @@ +// Which runtimes the examples can be executed with. +// +// The parity check and the documentation generator both need the same answer to +// "which runtimes are available here", so they ask this module. +import { execFileSync } from 'child_process'; + +const CANDIDATES = [ + { + id: 'node', + label: 'Node.js', + command: process.execPath.includes('bun') ? 'node' : process.execPath, + versionArgs: ['--version'], + }, + { id: 'bun', label: 'Bun', command: 'bun', versionArgs: ['--version'] }, +]; + +function probe(candidate) { + try { + const version = execFileSync(candidate.command, candidate.versionArgs, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + return { + ...candidate, + runArgs: [], + version: version.trim().split('\n')[0].replace(/^v/, ''), + }; + } catch { + return null; + } +} + +// Returns the runtimes that are actually installed, in a stable order. +export function availableRuntimes() { + return CANDIDATES.map(probe).filter(Boolean); +} + +export function requireRuntimes(ids) { + const available = availableRuntimes(); + const missing = ids.filter( + (id) => !available.some((runtime) => runtime.id === id) + ); + if (missing.length > 0) { + throw new Error(`Required runtime(s) not installed: ${missing.join(', ')}`); + } + return available.filter((runtime) => ids.includes(runtime.id)); +}