From 2d85d4ca05175b3d56a7917f28930a32b8b51e7d Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 18:48:19 -0400 Subject: [PATCH 01/14] feat: implement specs/testing-spec.md core: - expandInvocation extension hook on the Component Api - extensions claim raw invocations before built-in expansion; default answers unhandled - ComponentInvocation carries SourcePosition (original-file offset, line, column - frontmatter included, body start computed as a verbatim suffix) - runDocument output becomes a replay-safe stream (retained history, one queue per subscriber); error path closes with accumulated output @executablemd/testing (new workspace package): - Test Api: testing/inTest/verbose values, record/results/boundary ops - boundaries: subtree activation, delegating collectors, per-boundary zero-test rule; nested boundaries delegate outward - : child Effection scope + test-owned EvalScope lease, stable isolated binding env, raise interception (ErrorSegments fail the test), 20s timebox, setup/body/teardown phase classification, text-only returns - 14 @std/assert assertion components: live expression evaluation over projected+current env, expected-children (Capture parity) XOR expected prop, assert-then-format diagnostics with guarded fallback - executeDocument: single scoped wrapper for run and test modes; lazy output stream, run-level collectors, TestFailureError after the report cli: - xmd test subcommand; run and test share one driver over executeDocument - exit 1 on test failure or document abort; --verbose renders assertion diagnostics during regular execution release: publish workflow regenerated, lockstep spec updated, testing in all workspace manifests and lockfiles; @jsr scope registry for npm/bun --- .github/workflows/publish-packages.yml | 11 +- .npmrc | 1 + bun.lock | 29 +- bunfig.toml | 3 + cli/deno.json | 3 +- cli/package.json | 1 + cli/src/cli.ts | 74 ++++- core/mod.ts | 17 ++ core/src/component-api.ts | 18 ++ core/src/expand.ts | 21 ++ core/src/replay-stream.ts | 70 +++++ core/src/run-document.ts | 37 ++- core/src/scanner.ts | 80 +++++- core/src/types.ts | 44 +++ core/tests/expand-invocation.test.ts | 188 +++++++++++++ core/tests/replay-stream.test.ts | 99 +++++++ core/tests/source-position.test.ts | 121 +++++++++ deno.json | 10 +- deno.lock | 32 ++- package.json | 5 +- pnpm-lock.yaml | 36 +++ pnpm-workspace.yaml | 1 + specs/release-process-spec.md | 10 +- specs/testing-spec.md | 148 ++++++++++ testing/deno.json | 11 + testing/mod.ts | 15 ++ testing/package.json | 17 ++ testing/src/assertions.ts | 356 +++++++++++++++++++++++++ testing/src/execute.ts | 139 ++++++++++ testing/src/handlers.ts | 310 +++++++++++++++++++++ testing/src/test-api.ts | 72 +++++ testing/src/vocabulary.ts | 52 ++++ testing/tests/assertions.test.ts | 180 +++++++++++++ testing/tests/cli.test.ts | 89 +++++++ testing/tests/execute.test.ts | 187 +++++++++++++ testing/tests/fixtures/failing.md | 4 + testing/tests/fixtures/passing.md | 8 + testing/tests/helpers.ts | 85 ++++++ testing/tests/testing-mode.test.ts | 304 +++++++++++++++++++++ 39 files changed, 2845 insertions(+), 43 deletions(-) mode change 100644 => 100755 cli/src/cli.ts create mode 100644 core/src/replay-stream.ts create mode 100644 core/tests/expand-invocation.test.ts create mode 100644 core/tests/replay-stream.test.ts create mode 100644 core/tests/source-position.test.ts create mode 100644 specs/testing-spec.md create mode 100644 testing/deno.json create mode 100644 testing/mod.ts create mode 100644 testing/package.json create mode 100644 testing/src/assertions.ts create mode 100644 testing/src/execute.ts create mode 100644 testing/src/handlers.ts create mode 100644 testing/src/test-api.ts create mode 100644 testing/src/vocabulary.ts create mode 100644 testing/tests/assertions.test.ts create mode 100644 testing/tests/cli.test.ts create mode 100644 testing/tests/execute.test.ts create mode 100644 testing/tests/fixtures/failing.md create mode 100644 testing/tests/fixtures/passing.md create mode 100644 testing/tests/helpers.ts create mode 100644 testing/tests/testing-mode.test.ts diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index d8057c0..5a42247 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -30,7 +30,7 @@ jobs: - name: Validate the manifests declare this version run: | VERSION="${{ steps.resolve.outputs.value }}" - for f in durable-streams/deno.json runtime/deno.json core/deno.json cli/deno.json packages/code-review-agent/deno.json; do + for f in durable-streams/deno.json runtime/deno.json core/deno.json testing/deno.json cli/deno.json packages/code-review-agent/deno.json; do declared="$(jq -r .version "$f")" if [ "$declared" != "$VERSION" ]; then echo "::error::$f declares $declared, not $VERSION — the tag does not match the manifests" @@ -82,8 +82,15 @@ jobs: package: core version: ${{ needs.version.outputs.value }} + testing: + needs: [version, core] + uses: ./.github/workflows/publish-one.yml + with: + package: testing + version: ${{ needs.version.outputs.value }} + cli: - needs: [version, core, durable-streams] + needs: [version, core, durable-streams, testing] uses: ./.github/workflows/publish-one.yml with: package: cli diff --git a/.npmrc b/.npmrc index 319e41e..190f758 100644 --- a/.npmrc +++ b/.npmrc @@ -1 +1,2 @@ strict-peer-dependencies=false +@jsr:registry=https://npm.jsr.io diff --git a/bun.lock b/bun.lock index 8c43fd3..39c3244 100644 --- a/bun.lock +++ b/bun.lock @@ -40,7 +40,7 @@ }, "cli": { "name": "@executablemd/cli", - "version": "0.2.0", + "version": "0.3.1", "bin": { "xmd": "./src/cli.ts", }, @@ -48,6 +48,7 @@ "@effectionx/stream-helpers": "0.8.3", "@executablemd/core": "workspace:*", "@executablemd/durable-streams": "workspace:*", + "@executablemd/testing": "workspace:*", "configliere": "^0.2.3", "effection": "4.1.0-alpha.7", "zod": "^4.3.6", @@ -55,7 +56,7 @@ }, "core": { "name": "@executablemd/core", - "version": "0.2.0", + "version": "0.3.1", "dependencies": { "@effectionx/context-api": "0.6.0", "@effectionx/converge": "0.1.4", @@ -83,7 +84,7 @@ }, "durable-streams": { "name": "@executablemd/durable-streams", - "version": "0.2.0", + "version": "0.3.1", "dependencies": { "@durable-streams/client": "^0.2.2", "effection": "4.1.0-alpha.7", @@ -91,11 +92,11 @@ }, "packages/code-review-agent": { "name": "@executablemd/code-review-agent", - "version": "0.2.0", + "version": "0.3.1", }, "runtime": { "name": "@executablemd/runtime", - "version": "0.2.0", + "version": "0.3.1", "dependencies": { "@effectionx/context-api": "0.6.0", "@effectionx/fetch": "0.2.0", @@ -112,6 +113,18 @@ "expect": "^30.0.0", }, }, + "testing": { + "name": "@executablemd/testing", + "version": "0.3.1", + "dependencies": { + "@effectionx/context-api": "0.6.0", + "@effectionx/scope-eval": "0.1.3", + "@effectionx/timebox": "0.4.3", + "@executablemd/core": "workspace:*", + "@std/assert": "npm:@jsr/std__assert@^1.0.17", + "effection": "4.1.0-alpha.7", + }, + }, }, "packages": { "@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "7.28.5", "js-tokens": "4.0.0", "picocolors": "1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="], @@ -210,6 +223,8 @@ "@executablemd/runtime": ["@executablemd/runtime@workspace:runtime"], + "@executablemd/testing": ["@executablemd/testing@workspace:testing"], + "@jest/diff-sequences": ["@jest/diff-sequences@30.3.0", "", {}, "sha512-cG51MVnLq1ecVUaQ3fr6YuuAOitHK1S4WUJHnsPFE/quQr33ADUx1FfrTCpMCRxvy0Yr9BThKpDjSlcTi91tMA=="], "@jest/expect-utils": ["@jest/expect-utils@30.3.0", "", { "dependencies": { "@jest/get-type": "30.1.0" } }, "sha512-j0+W5iQQ8hBh7tHZkTQv3q2Fh/M7Je72cIsYqC4OaktgtO7v1So9UTjp6uPBHIaB6beoF/RRsCgMJKvti0wADA=="], @@ -224,6 +239,8 @@ "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + "@jsr/std__internal": ["@jsr/std__internal@1.0.14", "https://npm.jsr.io/~/11/@jsr/std__internal/1.0.14.tgz", {}, "sha512-JT8b/t40WcR9q0GDwRZUooY7aXeXnFal8iidE/5TckdAFtvpVAsaJ71/Xgf1SfoChs41s6CZkuCYM8toaNgdvA=="], + "@microsoft/fetch-event-source": ["@microsoft/fetch-event-source@2.0.1", "", {}, "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA=="], "@oxfmt/binding-android-arm-eabi": ["@oxfmt/binding-android-arm-eabi@0.41.0", "", { "os": "android", "cpu": "arm" }, "sha512-REfrqeMKGkfMP+m/ScX4f5jJBSmVNYcpoDF8vP8f8eYPDuPGZmzp56NIUsYmx3h7f6NzC6cE3gqh8GDWrJHCKw=="], @@ -308,6 +325,8 @@ "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "@std/assert": ["@jsr/std__assert@1.0.19", "https://npm.jsr.io/~/11/@jsr/std__assert/1.0.19.tgz", { "dependencies": { "@jsr/std__internal": "^1.0.12" } }, "sha512-pEj6RPkGbqlgRmyKwATp4cUs6+ijxtdrv3bq8v1d2I2CEcMEyPaO8cVKro61wGRDH4cNg8Zx6haztvK/9m7gkA=="], + "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "2.1.0" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], "@types/istanbul-lib-coverage": ["@types/istanbul-lib-coverage@2.0.6", "", {}, "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w=="], diff --git a/bunfig.toml b/bunfig.toml index 8743232..b6b4282 100644 --- a/bunfig.toml +++ b/bunfig.toml @@ -1,2 +1,5 @@ [test] tsconfig-override = "tsconfig.node.json" + +[install.scopes] +"@jsr" = "https://npm.jsr.io" diff --git a/cli/deno.json b/cli/deno.json index d5ea873..3b1edd8 100644 --- a/cli/deno.json +++ b/cli/deno.json @@ -4,6 +4,7 @@ "exports": "./src/cli.ts", "imports": { "@executablemd/core": "../core/mod.ts", - "zod": "npm:zod@^4.3.6" + "zod": "npm:zod@^4.3.6", + "@executablemd/testing": "../testing/mod.ts" } } diff --git a/cli/package.json b/cli/package.json index 7713dac..1a2e66e 100644 --- a/cli/package.json +++ b/cli/package.json @@ -11,6 +11,7 @@ "@effectionx/stream-helpers": "0.8.3", "@executablemd/core": "workspace:*", "@executablemd/durable-streams": "workspace:*", + "@executablemd/testing": "workspace:*", "configliere": "^0.2.3", "effection": "4.1.0-alpha.7", "zod": "^4.3.6" diff --git a/cli/src/cli.ts b/cli/src/cli.ts old mode 100644 new mode 100755 index 8126c93..056b16c --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -25,7 +25,8 @@ import { inspect } from "node:util"; import process from "node:process"; import { program, object, field, cli, commands, type Mods } from "configliere"; import { z } from "zod"; -import { runDocument, useNormalizedOutput, useTerminalOutput } from "@executablemd/core"; +import { useNormalizedOutput, useTerminalOutput } from "@executablemd/core"; +import { executeDocument, TestFailureError } from "@executablemd/testing"; import { FileStream } from "./file-stream.ts"; import denoJson from "../deno.json" with { type: "json" }; @@ -66,10 +67,35 @@ const runConfig = object({ }, }); +const testConfig = object({ + docPath: { + description: "markdown document to test", + ...field(z.string(), cli.argument()), + }, + componentDir: { + description: "component search directory", + ...field(z.array(z.string()), defaults(["components", "."]), field.array()), + }, + verbose: { + description: "log journal entries to stderr", + aliases: ["-V"], + ...field(z.boolean(), defaults(false)), + }, + journal: { + description: "write a diagnostic JSONL trace (path must not exist)", + aliases: ["-j"], + ...field(z.string().optional()), + }, + raw: { + description: "output raw markdown without normalization or terminal formatting", + ...field(z.boolean(), defaults(false)), + }, +}); + const xmd = program({ name: "xmd", version: denoJson.version, - config: commands({ run: runConfig }, { default: "run" }), + config: commands({ run: runConfig, test: testConfig }, { default: "run" }), }); // --------------------------------------------------------------------------- @@ -142,13 +168,16 @@ function* createJournalFile(filePath: string): Operation { yield* until(handle.close()); } -function* run(config: { - docPath: string; - componentDir: string[]; - verbose: boolean; - journal: string | undefined; - raw: boolean; -}): Operation { +function* run( + config: { + docPath: string; + componentDir: string[]; + verbose: boolean; + journal: string | undefined; + raw: boolean; + }, + mode: { testing: boolean }, +): Operation { const { docPath, componentDir, verbose, journal, raw } = config; // Every CLI invocation starts from an empty stream. --journal writes @@ -200,12 +229,15 @@ function* run(config: { yield* useTerminalOutput(); } - // Run the document — returns a DocumentExecution. - // yield* execution waits for completion. execution.output streams chunks. - const execution = yield* runDocument({ + // Run the document through the testing wrapper — the vocabulary is + // registered for both commands (assertions work in regular documents), + // while testing MODE activates only for `xmd test`. + const execution = yield* executeDocument({ docPath, stream, componentDirs: componentDir, + testing: mode.testing, + verbose, }); // Consume the output stream with forEach. @@ -227,6 +259,19 @@ function* run(config: { signal.close(); yield* writer; } + + // Observe the execution's outcome AFTER the report finished streaming: + // test failures, assertion aborts, and any document abort exit nonzero. + try { + yield* execution; + } catch (error) { + if (error instanceof TestFailureError) { + console.error(`\ntests failed: ${error.message}`); + } else { + console.error(error instanceof Error ? error.message : String(error)); + } + yield* exit(1); + } } // --------------------------------------------------------------------------- @@ -254,7 +299,10 @@ await main(function* (args) { } switch (parsed.value.name) { case "run": - yield* run(parsed.value.config); + yield* run(parsed.value.config, { testing: false }); + break; + case "test": + yield* run(parsed.value.config, { testing: true }); break; } } diff --git a/core/mod.ts b/core/mod.ts index bbe4c5e..2529239 100644 --- a/core/mod.ts +++ b/core/mod.ts @@ -24,6 +24,9 @@ export type { ResolveResult, SampleContext, Json, + SourcePosition, + InvocationContext, + InvocationHandling, } from "./src/types.ts"; export type { Workflow } from "@executablemd/durable-streams"; @@ -53,11 +56,25 @@ export { raise, env, evalScope, + expandInvocation, codeBlock, persistent, content, } from "./src/component-api.ts"; +// --------------------------------------------------------------------------- +// Rendering +// --------------------------------------------------------------------------- + +export { renderSegments } from "./src/render.ts"; + +// --------------------------------------------------------------------------- +// Replay-safe stream +// --------------------------------------------------------------------------- + +export { createReplayStream } from "./src/replay-stream.ts"; +export type { ReplayStream } from "./src/replay-stream.ts"; + // --------------------------------------------------------------------------- // Eval system (generator eval blocks) // --------------------------------------------------------------------------- diff --git a/core/src/component-api.ts b/core/src/component-api.ts index 321263c..6fc2e07 100644 --- a/core/src/component-api.ts +++ b/core/src/component-api.ts @@ -20,9 +20,12 @@ import type { CodeBlockContext, CodeBlockResult, ComponentDefinition, + ComponentInvocation, ErrorSegment, EvalEnv, FunctionComponentDefinition, + InvocationContext, + InvocationHandling, Modifier, } from "./types.ts"; @@ -38,6 +41,16 @@ export interface ComponentApi { raise(error: ErrorSegment): Operation; env: EvalEnv | undefined; evalScope: EvalScope | undefined; + /** + * Offer a raw component invocation to extensions before built-in expansion. + * Extensions install middleware that returns `{ segments }` for the names + * they claim and delegates to `next` for everything else. The default + * answers `undefined` — unhandled — so expansion proceeds normally. + */ + expandInvocation( + invocation: ComponentInvocation, + ctx: InvocationContext, + ): Operation; codeBlock(): Operation; /** Whether the current block runs with persistent resource lifetime. */ persistent: boolean; @@ -67,6 +80,10 @@ export const Component = createApi("Component", { env: undefined, evalScope: undefined, // deno-lint-ignore require-yield + *expandInvocation(): Operation { + return undefined; + }, + // deno-lint-ignore require-yield *codeBlock(): Operation { throw new Error( "Component.codeBlock() has no provider: no code block is executing in this scope.", @@ -87,6 +104,7 @@ export const { raise, env, evalScope, + expandInvocation, codeBlock, persistent, content, diff --git a/core/src/expand.ts b/core/src/expand.ts index a98558a..16fa10e 100644 --- a/core/src/expand.ts +++ b/core/src/expand.ts @@ -33,6 +33,7 @@ import { applyModifiers, env, evalScope, + expandInvocation, importComponent, raise, } from "./component-api.ts"; @@ -122,6 +123,26 @@ export function* expandSegments( } case "component": { + // Extension hook: an installed vocabulary may claim this invocation + // before built-in expansion. Returned error segments follow the + // ambient raise policy, like any component-produced error. + const handling = yield* expandInvocation(segment, { + meta: parentMeta, + props: parentProps, + projectedEnv: segment.projectedEnv, + expand: (segments) => expandSegments(segments, parentMeta, parentProps, hideSet, counter), + }); + if (handling) { + for (const handled of handling.segments) { + if (handled.type === "error") { + result.push(yield* raise(handled)); + } else { + result.push(handled); + } + } + break; + } + if (segment.name === "Output") { // Definition-owned is consumed by buildBody before it // reaches here. Reaching this branch means a misplaced or diff --git a/core/src/replay-stream.ts b/core/src/replay-stream.ts new file mode 100644 index 0000000..4fb8455 --- /dev/null +++ b/core/src/replay-stream.ts @@ -0,0 +1,70 @@ +/** + * Replay-safe stream — the document output transport (spec §9). + * + * An unbuffered channel drops values sent before a consumer subscribes. The + * replay stream retains its full event history so late subscribers receive + * every chunk and the close value, and multiple subscribers each get the + * complete sequence exactly once. + * + * Race-free by construction: delivery appends to the history and forwards to + * every registered subscriber queue in one synchronous step, and a new + * subscription replays the history into its own queue and registers for live + * events before yielding — no event can fall between replay and registration. + */ + +import { createQueue, ensure } from "effection"; +import type { Operation, Queue, Stream } from "effection"; + +export interface ReplayStream extends Stream { + send(value: T): Operation; + close(value: TClose): Operation; +} + +export function createReplayStream(): ReplayStream { + type Event = IteratorResult; + + const history: Event[] = []; + const subscribers = new Set>(); + let closed = false; + + function replay(queue: Queue, event: Event) { + if (event.done) { + queue.close(event.value); + } else { + queue.add(event.value); + } + } + + function deliver(event: Event) { + if (closed) { + return; + } + closed = event.done === true; + history.push(event); + for (const queue of subscribers) { + replay(queue, event); + } + } + + return { + // deno-lint-ignore require-yield + *send(value: T) { + deliver({ done: false, value }); + }, + // deno-lint-ignore require-yield + *close(value: TClose) { + deliver({ done: true, value }); + }, + *[Symbol.iterator]() { + const queue = createQueue(); + for (const event of history) { + replay(queue, event); + } + subscribers.add(queue); + yield* ensure(() => { + subscribers.delete(queue); + }); + return queue; + }, + }; +} diff --git a/core/src/run-document.ts b/core/src/run-document.ts index ba2feb2..34b1405 100644 --- a/core/src/run-document.ts +++ b/core/src/run-document.ts @@ -8,7 +8,7 @@ * See DEC-005 in specs/decisions.md. */ -import { scoped, spawn, createChannel, withResolvers, until } from "effection"; +import { scoped, spawn, withResolvers, until } from "effection"; import type { Operation, Stream } from "effection"; import { durableRun, @@ -18,6 +18,7 @@ import { } from "@executablemd/durable-streams"; import { exec, readTextFile, stat, cwd } from "@executablemd/runtime"; import type { Workflow, Json } from "@executablemd/durable-streams"; +import { createReplayStream } from "./replay-stream.ts"; import { useDenoCompiler } from "./deno-compiler.ts"; import { useTempFileCompiler } from "./temp-file-compiler.ts"; import type { @@ -140,7 +141,28 @@ function* durableImportComponent( // Markdown component: parse at runtime — deterministic from content const parsed = matter(result.content); const { meta, inputs } = parseFrontmatter(parsed.data as Record); - const bodySegments = scanSegments(parsed.content); + // The markdown body is a verbatim suffix of the raw file, so the body start + // is computed by length — never by content search, which could false-match + // body text repeated inside frontmatter. The invariant check turns any + // gray-matter normalization surprise into a loud error instead of silently + // wrong source positions. + const bodyStart = result.content.length - parsed.content.length; + if (result.content.slice(bodyStart) !== parsed.content) { + throw new Error( + `frontmatter parse did not preserve the markdown body verbatim: ${result.path}`, + ); + } + let baseLine = 1; + for (let i = 0; i < bodyStart; i++) { + if (result.content[i] === "\n") { + baseLine++; + } + } + const bodySegments = scanSegments(parsed.content, { + path: result.path, + baseOffset: bodyStart, + baseLine, + }); return { kind: "markdown" as const, @@ -414,11 +436,15 @@ export function* runDocument(options: RunDocumentOptions): Operation(); + // Replay-safe transport: late subscribers receive every chunk and the + // close value, so subscription readiness before first emission is never + // required (spec §9; see replay-stream.ts). + const channel = createReplayStream(); const { operation, resolve, reject } = withResolvers(); yield* spawn(function* () { let emitted = false; + let emittedText = ""; // Install platform-appropriate compiler middleware // deno-lint-ignore no-explicit-any @@ -433,6 +459,7 @@ export function* runDocument(options: RunDocumentOptions): Operation> 1; + if (lineStarts[mid]! <= offset) { + low = mid; + } else { + high = mid - 1; + } + } + const localLine = low + 1; + const column = offset - lineStarts[low]! + 1; + return { + path: origin?.path, + offset: (origin?.baseOffset ?? 0) + offset, + line: (origin?.baseLine ?? 1) + localLine - 1, + column, + }; +} + /** * Scan raw markdown text into segments. * * Identifies component invocations (``), executable code * blocks (fenced with `exec` in info string), and everything else as text. + * + * When `origin` is provided, component invocations carry `position` values + * expressed in the original file's coordinates. */ -export function scanSegments(text: string): Segment[] { +export function scanSegments(text: string, origin?: SourceOrigin): Segment[] { + const index: PositionIndex = { origin, lineStarts: computeLineStarts(text) }; const segments: Segment[] = []; let pos = 0; let textStart = 0; @@ -128,7 +185,7 @@ export function scanSegments(text: string): Segment[] { // Check for component invocation: `<` followed by uppercase letter if (text[pos] === "<" && pos + 1 < text.length && /[A-Z]/.test(text[pos + 1]!)) { // Make sure we're not inside a fenced code block (handled above) - const component = parseComponentTag(text, pos); + const component = parseComponentTag(text, pos, index); if (component) { // Flush text before component if (pos > textStart) { @@ -281,7 +338,11 @@ interface ParsedComponent { * Returns the parsed component invocation and the position after it, * or null if this is not a valid component tag. */ -function parseComponentTag(text: string, start: number): ParsedComponent | null { +function parseComponentTag( + text: string, + start: number, + index: PositionIndex, +): ParsedComponent | null { let pos = start + 1; // Skip '<' // Parse tag name — must start with uppercase, can contain dots @@ -316,6 +377,7 @@ function parseComponentTag(text: string, start: number): ParsedComponent | null expressions, children: [], selfClosing: true, + position: positionAt(index, start), }, end: pos, }; @@ -326,7 +388,7 @@ function parseComponentTag(text: string, start: number): ParsedComponent | null pos++; // Skip '>' // Parse children until closing tag - const { children, end: childEnd } = parseChildren(text, pos, name); + const { children, end: childEnd } = parseChildren(text, pos, name, index); if (childEnd === -1) return null; return { @@ -337,6 +399,7 @@ function parseComponentTag(text: string, start: number): ParsedComponent | null expressions, children, selfClosing: false, + position: positionAt(index, start), }, end: childEnd, }; @@ -595,7 +658,12 @@ interface ParsedChildren { end: number; // position after closing tag } -function parseChildren(text: string, start: number, tagName: string): ParsedChildren { +function parseChildren( + text: string, + start: number, + tagName: string, + index: PositionIndex, +): ParsedChildren { let pos = start; let textStart = pos; const children: Segment[] = []; @@ -658,7 +726,7 @@ function parseChildren(text: string, start: number, tagName: string): ParsedChil // Check for nested component if (text[pos] === "<" && pos + 1 < text.length && /[A-Z]/.test(text[pos + 1]!)) { - const nested = parseComponentTag(text, pos); + const nested = parseComponentTag(text, pos, index); if (nested) { if (pos > textStart) { pushText(children, text.slice(textStart, pos)); diff --git a/core/src/types.ts b/core/src/types.ts index 534ae60..97659f3 100644 --- a/core/src/types.ts +++ b/core/src/types.ts @@ -5,6 +5,7 @@ * modifier system types, and shared interfaces. */ +import type { Operation } from "effection"; import type { Json as DurableJson, Workflow } from "@executablemd/durable-streams"; // --------------------------------------------------------------------------- @@ -51,6 +52,21 @@ export interface ComponentInvocation { * by substituteContent when projecting children into slots. */ projectedEnv?: { values: Record }; + /** + * Source location of the opening tag in the original file, frontmatter + * included. Absent for dynamically scanned strings (render(markdown)). + */ + position?: SourcePosition; +} + +/** A location in an original source file. Lines and columns are 1-based. */ +export interface SourcePosition { + /** Workspace-relative file path. Undefined for dynamically scanned text. */ + path?: string; + /** Character offset in the original file. */ + offset: number; + line: number; + column: number; } export interface ExecutableCodeBlock { @@ -88,6 +104,34 @@ export interface EvalEnv { values: Record; } +// --------------------------------------------------------------------------- +// Invocation-expansion extension hook +// --------------------------------------------------------------------------- + +/** + * Expansion context offered to `expandInvocation` extensions alongside the + * raw invocation. Exposes exactly what an extension needs: the parent scope's + * interpolation inputs, caller-projected bindings, and incremental expansion + * in the current context. Engine internals (hide set, block counter) stay + * captured inside the `expand` closure. + */ +export interface InvocationContext { + meta: Record; + props: Record; + /** Caller-projected bindings (segment.projectedEnv) for expression evaluation. */ + projectedEnv?: EvalEnv; + /** Incrementally expand segments in the current expansion context. */ + expand(segments: Segment[]): Operation; +} + +/** + * A claimed invocation's replacement segments. `{ segments: [] }` means + * handled-with-no-output — distinct from `undefined` (unhandled). + */ +export interface InvocationHandling { + segments: Segment[]; +} + // --------------------------------------------------------------------------- // Modifier system (spec §3.2–3.5) // --------------------------------------------------------------------------- diff --git a/core/tests/expand-invocation.test.ts b/core/tests/expand-invocation.test.ts new file mode 100644 index 0000000..b81cadc --- /dev/null +++ b/core/tests/expand-invocation.test.ts @@ -0,0 +1,188 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { scoped } from "effection"; +import type { Operation } from "effection"; +import { expandSegments } from "../src/expand.ts"; +import { Component } from "../src/component-api.ts"; +import { scanSegments } from "../src/scanner.ts"; +import { renderSegments } from "../src/render.ts"; +import type { ComponentDefinition, EvalEnv, InvocationContext, Segment } from "../src/types.ts"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeComponent(name: string, body: string): ComponentDefinition { + return { + kind: "markdown", + name, + path: `components/${name}.md`, + meta: {}, + inputs: {}, + bodySegments: scanSegments(body), + }; +} + +function useTestComponents(components: Record): Operation { + return Component.around( + { + // deno-lint-ignore require-yield + *importComponent([name], _next) { + const comp = components[name]; + if (!comp) { + throw new Error(`Component not found: ${name}`); + } + return comp; + }, + }, + { at: "min" }, + ); +} + +function useTestEnv(testEnv: EvalEnv): Operation { + return Component.around({ env: () => testEnv }, { at: "min" }); +} + +// --------------------------------------------------------------------------- +// expandInvocation hook +// --------------------------------------------------------------------------- + +describe("expandInvocation hook", () => { + it("falls through to normal expansion when no extension is installed", function* () { + const output = yield* scoped(function* () { + yield* useTestComponents({ Greeting: makeComponent("Greeting", "Hello world!") }); + yield* useTestEnv({ values: {} }); + const expanded = yield* expandSegments(scanSegments(""), {}, {}, new Set()); + return renderSegments(expanded); + }); + expect(output).toBe("Hello world!"); + }); + + it("lets an extension claim a name and produce segments", function* () { + const output = yield* scoped(function* () { + yield* useTestComponents({}); + yield* useTestEnv({ values: {} }); + yield* Component.around({ + *expandInvocation([invocation, ctx], next) { + if (invocation.name === "Shout") { + const inner = yield* ctx.expand(invocation.children); + return { segments: [{ type: "text", content: renderSegments(inner).toUpperCase() }] }; + } + return yield* next(invocation, ctx); + }, + }); + const expanded = yield* expandSegments( + scanSegments("hi there"), + {}, + {}, + new Set(), + ); + return renderSegments(expanded); + }); + expect(output).toBe("HI THERE"); + }); + + it("delegated names still expand normally alongside claimed names", function* () { + const output = yield* scoped(function* () { + yield* useTestComponents({ Greeting: makeComponent("Greeting", "Hello!") }); + yield* useTestEnv({ values: {} }); + yield* Component.around({ + // deno-lint-ignore require-yield + *expandInvocation([invocation, ctx], next) { + if (invocation.name === "Nothing") { + return { segments: [] }; + } + return yield* next(invocation, ctx); + }, + }); + const expanded = yield* expandSegments( + scanSegments("before"), + {}, + {}, + new Set(), + ); + return renderSegments(expanded); + }); + expect(output).toBe("beforeHello!"); + }); + + it("distinguishes handled-with-no-output from unhandled", function* () { + const seen: string[] = []; + yield* scoped(function* () { + yield* useTestComponents({ Greeting: makeComponent("Greeting", "Hello!") }); + yield* useTestEnv({ values: {} }); + yield* Component.around({ + // deno-lint-ignore require-yield + *expandInvocation([invocation, ctx], next) { + seen.push(invocation.name); + if (invocation.name === "Silent") { + return { segments: [] }; + } + return yield* next(invocation, ctx); + }, + }); + const expanded = yield* expandSegments( + scanSegments(""), + {}, + {}, + new Set(), + ); + expect(renderSegments(expanded)).toBe("Hello!"); + }); + expect(seen).toEqual(["Silent", "Greeting"]); + }); + + it("extension error segments follow the ambient raise policy", function* () { + const raised: string[] = []; + const output = yield* scoped(function* () { + yield* useTestComponents({}); + yield* useTestEnv({ values: {} }); + yield* Component.around({ + *raise([segment], next) { + raised.push(segment.message); + return yield* next(segment); + }, + }); + yield* Component.around({ + // deno-lint-ignore require-yield + *expandInvocation([invocation, ctx], next) { + if (invocation.name === "Broken") { + const error: Segment = { type: "error", message: "broken thing", source: "Broken" }; + return { segments: [error] }; + } + return yield* next(invocation, ctx); + }, + }); + const expanded = yield* expandSegments(scanSegments(""), {}, {}, new Set()); + return renderSegments(expanded); + }); + expect(raised).toEqual(["broken thing"]); + expect(output).toContain("broken thing"); + }); + + it("passes projectedEnv and parent meta/props through the context", function* () { + let observed: InvocationContext | undefined; + yield* scoped(function* () { + yield* useTestComponents({}); + yield* useTestEnv({ values: {} }); + yield* Component.around({ + // deno-lint-ignore require-yield + *expandInvocation([invocation, ctx], next) { + if (invocation.name === "Probe") { + observed = ctx; + return { segments: [] }; + } + return yield* next(invocation, ctx); + }, + }); + yield* expandSegments( + scanSegments(""), + { title: "Doc" }, + { color: "red" }, + new Set(), + ); + }); + expect(observed?.meta).toEqual({ title: "Doc" }); + expect(observed?.props).toEqual({ color: "red" }); + }); +}); diff --git a/core/tests/replay-stream.test.ts b/core/tests/replay-stream.test.ts new file mode 100644 index 0000000..2ca001d --- /dev/null +++ b/core/tests/replay-stream.test.ts @@ -0,0 +1,99 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { scoped, spawn } from "effection"; +import type { Operation, Subscription } from "effection"; +import { createReplayStream } from "../src/replay-stream.ts"; + +function* drain( + subscription: Subscription, +): Operation<{ chunks: string[]; close: string }> { + const chunks: string[] = []; + let next = yield* subscription.next(); + while (!next.done) { + chunks.push(next.value); + next = yield* subscription.next(); + } + return { chunks, close: next.value }; +} + +describe("createReplayStream", () => { + it("replays chunks emitted before the subscription", function* () { + const stream = createReplayStream(); + yield* stream.send("a"); + yield* stream.send("b"); + yield* stream.close("ab"); + + const result = yield* scoped(function* () { + return yield* drain(yield* stream); + }); + expect(result).toEqual({ chunks: ["a", "b"], close: "ab" }); + }); + + it("delivers an event sent at the replay/live boundary exactly once", function* () { + const stream = createReplayStream(); + yield* stream.send("before"); + + const result = yield* scoped(function* () { + const subscription = yield* stream; + // The subscription is attached; a send after attach but before the + // first next() must arrive exactly once, after the replayed history. + yield* stream.send("boundary"); + yield* stream.close("done"); + return yield* drain(subscription); + }); + expect(result).toEqual({ chunks: ["before", "boundary"], close: "done" }); + }); + + it("gives multiple subscribers the full sequence each", function* () { + const stream = createReplayStream(); + yield* stream.send("x"); + + const [first, second] = yield* scoped(function* () { + const one = yield* stream; + const two = yield* stream; + yield* stream.send("y"); + yield* stream.close("xy"); + return [yield* drain(one), yield* drain(two)]; + }); + expect(first).toEqual({ chunks: ["x", "y"], close: "xy" }); + expect(second).toEqual({ chunks: ["x", "y"], close: "xy" }); + }); + + it("serves the close value to a subscriber that arrives after close", function* () { + const stream = createReplayStream(); + yield* stream.send("only"); + yield* stream.close("only"); + + const result = yield* scoped(function* () { + return yield* drain(yield* stream); + }); + expect(result).toEqual({ chunks: ["only"], close: "only" }); + }); + + it("live subscribers receive concurrent sends", function* () { + const stream = createReplayStream(); + + const result = yield* scoped(function* () { + const consumer = yield* spawn(function* () { + return yield* drain(yield* stream); + }); + yield* stream.send("one"); + yield* stream.send("two"); + yield* stream.close("onetwo"); + return yield* consumer; + }); + expect(result).toEqual({ chunks: ["one", "two"], close: "onetwo" }); + }); + + it("ignores events after close", function* () { + const stream = createReplayStream(); + yield* stream.close("final"); + yield* stream.send("late"); + yield* stream.close("later"); + + const result = yield* scoped(function* () { + return yield* drain(yield* stream); + }); + expect(result).toEqual({ chunks: [], close: "final" }); + }); +}); diff --git a/core/tests/source-position.test.ts b/core/tests/source-position.test.ts new file mode 100644 index 0000000..616e33c --- /dev/null +++ b/core/tests/source-position.test.ts @@ -0,0 +1,121 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import { useStubFs } from "@executablemd/runtime/test"; +import { scanSegments } from "../src/scanner.ts"; +import { Component } from "../src/component-api.ts"; +import { runDocument } from "../src/run-document.ts"; +import { collect } from "../src/collect.ts"; +import type { ComponentInvocation, SourcePosition } from "../src/types.ts"; + +function componentsOf(segments: ReturnType): ComponentInvocation[] { + const found: ComponentInvocation[] = []; + for (const segment of segments) { + if (segment.type === "component") { + found.push(segment); + found.push(...componentsOf(segment.children)); + } + } + return found; +} + +describe("source positions", () => { + it("attaches local positions without an origin", function* () { + const [greeting] = componentsOf(scanSegments("line one\n\n")); + expect(greeting?.position).toEqual({ path: undefined, offset: 9, line: 2, column: 1 }); + }); + + it("attaches positions to nested children at original offsets", function* () { + const text = "\n text\n \n\n"; + const [outer, inner] = componentsOf(scanSegments(text)); + expect(outer?.position).toEqual({ path: undefined, offset: 0, line: 1, column: 1 }); + expect(inner?.position).toEqual({ + path: undefined, + offset: text.indexOf("", { path: "Doc.md", baseOffset: 40, baseLine: 5 }), + ); + expect(greeting?.position).toEqual({ + path: "Doc.md", + offset: 41, + line: 6, + column: 1, + }); + }); + + it("computes original-file positions past frontmatter, immune to repeated body text", function* () { + // The frontmatter contains a copy of the body's first line: a content + // search for the body would false-match inside the frontmatter, so only + // the suffix computation yields the correct base. + const doc = [ + "---", + 'title: ""', + "note: intro line", + "---", + "intro line", + "", + "", + ].join("\n"); + + const positions: SourcePosition[] = []; + const stream = new InMemoryStream(); + yield* useStubFs({ "README.md": doc, "components/Probe.md": "probed\n" }); + yield* Component.around({ + *expandInvocation([invocation, ctx], next) { + if (invocation.position) { + positions.push(invocation.position); + } + return yield* next(invocation, ctx); + }, + }); + + const output = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + expect(output).toContain("probed"); + + const probeLine = doc.split("\n").indexOf("") + 1; + expect(positions).toEqual([ + { + path: "README.md", + offset: doc.indexOf("\n") + 1, + line: probeLine, + column: 1, + }, + ]); + }); + + it("positions imported-component invocations in the component's own file", function* () { + const readme = "\n"; + const wrapper = ["---", "title: Wrapper", "---", "before", "", ""].join("\n"); + + const positions: Array<{ name: string; position?: SourcePosition }> = []; + const stream = new InMemoryStream(); + yield* useStubFs({ + "README.md": readme, + "components/Wrapper.md": wrapper, + "components/Probe.md": "probed\n", + }); + yield* Component.around({ + *expandInvocation([invocation, ctx], next) { + positions.push({ name: invocation.name, position: invocation.position }); + return yield* next(invocation, ctx); + }, + }); + + const output = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + expect(output).toContain("probed"); + + const probe = positions.find((p) => p.name === "Probe"); + expect(probe?.position).toEqual({ + path: "components/Wrapper.md", + offset: wrapper.indexOf(""), + line: 5, + column: 1, + }); + }); +}); diff --git a/deno.json b/deno.json index e291aa5..6818dd9 100644 --- a/deno.json +++ b/deno.json @@ -1,5 +1,5 @@ { - "workspace": ["core", "cli", "durable-streams", "runtime", "packages/code-review-agent", "site"], + "workspace": ["core", "cli", "durable-streams", "runtime", "testing", "packages/code-review-agent", "site"], "nodeModulesDir": "auto", "imports": { "@types/node": "npm:@types/node@^24.5.2", @@ -11,6 +11,8 @@ "effection/experimental": "npm:effection@4.1.0-alpha.7/experimental", "@executablemd/durable-streams": "./durable-streams/mod.ts", "@executablemd/runtime": "./runtime/mod.ts", + "@executablemd/testing": "./testing/mod.ts", + "@std/assert": "jsr:@std/assert@^1.0.17", "@executablemd/runtime/test": "./runtime/test/mod.ts", "@executablemd/code-review-agent": "./packages/code-review-agent/mod.ts", "@effectionx/context-api": "npm:@effectionx/context-api@0.6.0", @@ -37,9 +39,9 @@ "build": "deno compile --node-modules-dir=none --exclude-unused-npm --allow-all --include packages/code-review-agent --output dist/xmd cli/src/cli.ts", "gen:publish-workflow": "deno run --allow-all cli/src/cli.ts run scripts/gen-publish-workflow.md", "bump": "deno run -A scripts/bump-version.ts", - "test": "deno test --allow-all core/tests/ durable-streams/tests/ packages/code-review-agent/tests/", - "check": "deno check core/mod.ts", - "lint": "npx oxlint -c .oxlintrc.json core/src/ cli/src/ durable-streams/ runtime/ && npx oxfmt --check core/src/ cli/src/ durable-streams/ runtime/ test-support/", + "test": "deno test --allow-all core/tests/ durable-streams/tests/ testing/tests/ packages/code-review-agent/tests/", + "check": "deno check core/mod.ts testing/mod.ts", + "lint": "npx oxlint -c .oxlintrc.json core/src/ cli/src/ durable-streams/ runtime/ testing/ && npx oxfmt --check core/src/ cli/src/ durable-streams/ runtime/ testing/ test-support/", "review": "deno run --allow-all cli/src/cli.ts run .reviews/ReviewPR.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir core/components -j .reviews/journal.jsonl", "review:local": "deno run --allow-all cli/src/cli.ts run .reviews/ReviewPR.local.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir core/components -j .reviews/journal.local.jsonl", "analyze": "deno run --allow-all cli/src/cli.ts run .reviews/AnalyzeRepo.md --component-dir .reviews/components --component-dir .reviews/policies --component-dir core/components -j .reviews/journal.analyze.jsonl", diff --git a/deno.lock b/deno.lock index 358048b..0120a12 100644 --- a/deno.lock +++ b/deno.lock @@ -52,6 +52,7 @@ "npm:@effectionx/stream-helpers@0.8.3": "0.8.3_effection@4.1.0-alpha.7", "npm:@effectionx/test-adapter@0.7.4": "0.7.4_effection@4.1.0-alpha.7", "npm:@effectionx/timebox@0.4.3": "0.4.3_effection@4.1.0-alpha.7", + "npm:@jsr/std__assert@^1.0.17": "1.0.19", "npm:@opentelemetry/api@^1.9.0": "1.9.1", "npm:@preact/signals@^2.5.1": "2.9.3_preact@10.29.7__preact-render-to-string@6.7.0_preact-render-to-string@6.7.0__preact@10.29.7", "npm:@preact/signals@^2.9.0": "2.9.3_preact@10.29.7__preact-render-to-string@6.7.0_preact-render-to-string@6.7.0__preact@10.29.7", @@ -172,7 +173,10 @@ ] }, "@std/assert@1.0.19": { - "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e" + "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e", + "dependencies": [ + "jsr:@std/internal@^1.0.12" + ] }, "@std/bytes@1.0.6": { "integrity": "f6ac6adbd8ccd99314045f5703e23af0a68d7f7e58364b47d2c7f408aeb5820a" @@ -990,6 +994,17 @@ "@jridgewell/sourcemap-codec" ] }, + "@jsr/std__assert@1.0.19": { + "integrity": "sha512-pEj6RPkGbqlgRmyKwATp4cUs6+ijxtdrv3bq8v1d2I2CEcMEyPaO8cVKro61wGRDH4cNg8Zx6haztvK/9m7gkA==", + "dependencies": [ + "@jsr/std__internal" + ], + "tarball": "https://npm.jsr.io/~/11/@jsr/std__assert/1.0.19.tgz" + }, + "@jsr/std__internal@1.0.12": { + "integrity": "sha512-6xReMW9p+paJgqoFRpOE2nogJFvzPfaLHLIlyADYjKMUcwDyjKZxryIbgcU+gxiTygn8yCjld1HoI0ET4/iZeA==", + "tarball": "https://npm.jsr.io/~/11/@jsr/std__internal/1.0.12.tgz" + }, "@microsoft/fetch-event-source@2.0.1": { "integrity": "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==" }, @@ -2876,6 +2891,7 @@ }, "workspace": { "dependencies": [ + "jsr:@std/assert@^1.0.17", "jsr:@std/testing@1", "npm:@durable-streams/client@~0.2.2", "npm:@effectionx/context-api@0.6.0", @@ -3022,6 +3038,20 @@ "npm:expect@30" ] } + }, + "testing": { + "dependencies": [ + "jsr:@std/assert@^1.0.17" + ], + "packageJson": { + "dependencies": [ + "npm:@effectionx/context-api@0.6.0", + "npm:@effectionx/scope-eval@0.1.3", + "npm:@effectionx/timebox@0.4.3", + "npm:@jsr/std__assert@^1.0.17", + "npm:effection@4.1.0-alpha.7" + ] + } } } } diff --git a/package.json b/package.json index 7321d6c..05b7879 100644 --- a/package.json +++ b/package.json @@ -60,14 +60,15 @@ "test:node": "tsx --tsconfig tsconfig.node.json --test durable-streams/tests/*.test.ts core/tests/frontmatter.test.ts core/tests/heal.test.ts core/tests/scanner.test.ts core/tests/eval-transform.test.ts core/tests/eval-interpolate.test.ts core/tests/document-output-api.test.ts core/tests/output-normalize.test.ts core/tests/output-terminal.test.ts", "test:bun": "bun test durable-streams/tests/ core/tests/frontmatter.test.ts core/tests/heal.test.ts core/tests/scanner.test.ts core/tests/eval-transform.test.ts core/tests/eval-interpolate.test.ts core/tests/document-output-api.test.ts core/tests/output-normalize.test.ts core/tests/output-terminal.test.ts", "test:deno": "deno task test", - "lint": "oxlint -c .oxlintrc.json core/src/ cli/src/ durable-streams/ runtime/ && oxfmt --check core/src/ cli/src/ durable-streams/ runtime/ test-support/", - "fmt": "oxfmt --write core/src/ cli/src/ durable-streams/ runtime/ test-support/ core/tests/ durable-streams/tests/ packages/code-review-agent/" + "lint": "oxlint -c .oxlintrc.json core/src/ cli/src/ durable-streams/ runtime/ testing/ && oxfmt --check core/src/ cli/src/ durable-streams/ runtime/ testing/ test-support/", + "fmt": "oxfmt --write core/src/ cli/src/ durable-streams/ runtime/ testing/ test-support/ core/tests/ durable-streams/tests/ packages/code-review-agent/" }, "workspaces": [ "core", "cli", "durable-streams", "runtime", + "testing", "packages/code-review-agent", "test-support" ] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ecf08ef..6085d80 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,6 +111,9 @@ importers: '@executablemd/durable-streams': specifier: workspace:* version: link:../durable-streams + '@executablemd/testing': + specifier: workspace:* + version: link:../testing configliere: specifier: ^0.2.3 version: 0.2.4 @@ -228,6 +231,27 @@ importers: specifier: ^30.0.0 version: 30.3.0 + testing: + dependencies: + '@effectionx/context-api': + specifier: 0.6.0 + version: 0.6.0(effection@4.1.0-alpha.7) + '@effectionx/scope-eval': + specifier: 0.1.3 + version: 0.1.3(effection@4.1.0-alpha.7) + '@effectionx/timebox': + specifier: 0.4.3 + version: 0.4.3(effection@4.1.0-alpha.7) + '@executablemd/core': + specifier: workspace:* + version: link:../core + '@std/assert': + specifier: npm:@jsr/std__assert@^1.0.17 + version: '@jsr/std__assert@1.0.19' + effection: + specifier: 4.1.0-alpha.7 + version: 4.1.0-alpha.7 + packages: '@babel/code-frame@7.29.0': @@ -488,6 +512,12 @@ packages: '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@jsr/std__assert@1.0.19': + resolution: {integrity: sha512-pEj6RPkGbqlgRmyKwATp4cUs6+ijxtdrv3bq8v1d2I2CEcMEyPaO8cVKro61wGRDH4cNg8Zx6haztvK/9m7gkA==, tarball: https://npm.jsr.io/~/11/@jsr/std__assert/1.0.19.tgz} + + '@jsr/std__internal@1.0.14': + resolution: {integrity: sha512-JT8b/t40WcR9q0GDwRZUooY7aXeXnFal8iidE/5TckdAFtvpVAsaJ71/Xgf1SfoChs41s6CZkuCYM8toaNgdvA==, tarball: https://npm.jsr.io/~/11/@jsr/std__internal/1.0.14.tgz} + '@microsoft/fetch-event-source@2.0.1': resolution: {integrity: sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==} @@ -1493,6 +1523,12 @@ snapshots: '@jridgewell/sourcemap-codec@1.5.5': {} + '@jsr/std__assert@1.0.19': + dependencies: + '@jsr/std__internal': 1.0.14 + + '@jsr/std__internal@1.0.14': {} + '@microsoft/fetch-event-source@2.0.1': {} '@oxfmt/binding-android-arm-eabi@0.41.0': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 1ea908c..b2216e2 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -3,5 +3,6 @@ packages: - cli - durable-streams - runtime + - testing - packages/code-review-agent - test-support diff --git a/specs/release-process-spec.md b/specs/release-process-spec.md index 6dba543..cf8d755 100644 --- a/specs/release-process-spec.md +++ b/specs/release-process-spec.md @@ -47,7 +47,7 @@ sequenceDiagram ## 2. Version lockstep -Every package (`core`, `cli`, `durable-streams`, `runtime`, +Every package (`core`, `cli`, `durable-streams`, `runtime`, `testing`, `packages/code-review-agent`) declares the same version in its `deno.json` and `package.json`. `cli/src/cli.ts` imports `cli/deno.json` and reads `version` from it, so the compiled binary reports the manifest version — the manifests @@ -146,7 +146,7 @@ versions succeed. No dispatch path publishes outside a tag. ## 8. Consumer note -`@executablemd/core`, `@executablemd/runtime`, and `@executablemd/cli` depend -on effection's 4.x prerelease, which npm's peer resolver rejects against -`@effectionx/*` (`^3 || ^4`). Installing them requires `--legacy-peer-deps` -until effection 4 is stable. +`@executablemd/core`, `@executablemd/runtime`, `@executablemd/testing`, and +`@executablemd/cli` depend on effection's 4.x prerelease, which npm's peer +resolver rejects against `@effectionx/*` (`^3 || ^4`). Installing them +requires `--legacy-peer-deps` until effection 4 is stable. diff --git a/specs/testing-spec.md b/specs/testing-spec.md new file mode 100644 index 0000000..471cb8a --- /dev/null +++ b/specs/testing-spec.md @@ -0,0 +1,148 @@ +# Executable.md Testing + +## Motivation + +Executable documents can contain probabilistic behavior. Tests ground that +behavior in observable results and give authors confidence in their documents. + +## Testing Mode + +Tests run only in testing mode. During regular execution, `` and its +entire body are skipped without output, bindings, or side effects. + +`` enables testing mode for its expanded subtree: + +```md + + + +``` + +The CLI command is equivalent to wrapping the entrypoint in ``: + +```sh +xmd test +``` + +Test discovery follows normal component expansion. A test runs only when the +expanded component tree reaches it, including through imports, components, and +conditional rendering. Imported but unrendered tests do not run. + +Ordinary content outside a `` expands normally. An error in that content +aborts the run as an infrastructure error. + +`` emits the naturally expanded report without adding a summary. It +fails after expansion when any test failed or when no tests were discovered. +`xmd test` exits with status `0` when every test passes and status `1` +otherwise. + +Testing uses the standard journal and replay behavior. + +## Atomic Tests + +Atomic tests use ``: + +```md + + + Hello World + + + Hello World + + +``` + +The `name` prop is optional metadata. An unnamed test is identified by its +source location; headings in the body remain ordinary output and are not +inferred as names. + +A test body behaves like any regular component body. Tests run sequentially in +expansion order. Each test runs in a child Effection scope and an isolated +binding environment. It inherits ambient context and bindings, but its context +changes, bindings, and ongoing effects do not escape. Its scope is fully torn +down before the next test starts. + +Each test has a fixed 20-second timeout. A timed-out test is halted, reported as +failed, and fully torn down before execution continues. + +An assertion failure, unexpected error, or teardown error fails only the current +test. Later tests still run. Unexpected errors remain distinct from assertion +failures. Output produced before a failure remains in the report and is followed +by the failure diagnostic. + +Nested `` elements are invalid. Skip, focus, and retry behavior is not +supported. + +## Test API + +`TestApi` controls testing mode. The `testing` operation returns `false` by +default and `true` beneath ``: + +```ts +import { testing } from "@executablemd/testing"; + +const active = yield* testing; +``` + +`TestApi` records completed tests in discovery order. Each result contains its +pass or fail status, optional name, source location, and structured error +details when it failed. Rendered test output is not duplicated in the result. + +## Assertions + +Assertion components use `@std/assert` and follow its function names and +parameter names. The initial components are: + +- `` and `` +- `` and `` +- `` and `` +- `` +- `` +- `` and `` +- `` and `` +- `` and `` + +Props map directly to the corresponding function parameters: `expr` for +truthiness assertions, `actual` and `expected` for comparisons, and optional +`msg` where supported. + +Equality assertions and `` accept either an `expected` +prop or rendered children as the expected string. The two forms are mutually +exclusive. Expected children expand in the current scope and environment, use +the same trailing-whitespace trimming as ``, and do not render +separately. + +Numeric comparisons require an `expected` prop. Match assertions require a +`RegExp` through the `expected` prop. Unary assertions do not accept expected +children. + +Assertion components work inside and outside tests. A failed assertion throws +the `@std/assert` assertion error. Outside a test, that error aborts document +expansion. Inside a test, `` contains and records it. + +Assertions emit Markdown diagnostics in testing mode. During regular execution, +diagnostics are hidden unless `--verbose` is enabled. Failed assertions still +throw when diagnostics are hidden. + +Diagnostics identify the assertion component and outcome. They include the +optional message and relevant actual and expected values. Failure diagnostics +include the underlying assertion detail when available. Their exact Markdown +layout is not prescribed, and formatting arbitrary values must not change the +assertion outcome or introduce a new failure. + +Additional assertion components use the same rules: their names and props map to +an `@std/assert` export, they preserve its comparison and error semantics, and +they use the shared diagnostic behavior. + +## Mocking + +Testing has no separate mocking DSL. Tests install mocks through existing +context API middleware or helpers from `@executablemd/runtime/test`. Middleware +installed within a test applies to subsequent expansion in that test and is +removed with its scope. + +## Unsupported Syntax + +BDD syntax such as `describe`, `it`, `beforeEach`, and `beforeAll` is not +supported. Gherkin syntax such as `Given`, `When`, and `Then` is not supported. diff --git a/testing/deno.json b/testing/deno.json new file mode 100644 index 0000000..a3916cb --- /dev/null +++ b/testing/deno.json @@ -0,0 +1,11 @@ +{ + "name": "@executablemd/testing", + "version": "0.3.1", + "exports": { + ".": "./mod.ts" + }, + "imports": { + "@executablemd/core": "../core/mod.ts", + "@std/assert": "jsr:@std/assert@^1.0.17" + } +} diff --git a/testing/mod.ts b/testing/mod.ts new file mode 100644 index 0000000..c568285 --- /dev/null +++ b/testing/mod.ts @@ -0,0 +1,15 @@ +/** + * @module + * Testing vocabulary for executable.md documents (specs/testing-spec.md). + * + * `` activates testing mode for its expanded subtree, `` + * defines an atomic test, and the assertion components map to `@std/assert`. + * Vocabulary registration (`installTestingVocabulary`) is distinct from + * testing-mode activation (`executeDocument({ testing: true })` or a + * `` element). + */ + +export { Test, testing, record, results, TestFailureError } from "./src/test-api.ts"; +export type { TestApi, TestResult, BoundaryOutcome } from "./src/test-api.ts"; +export { installTestingVocabulary } from "./src/vocabulary.ts"; +export { executeDocument } from "./src/execute.ts"; diff --git a/testing/package.json b/testing/package.json new file mode 100644 index 0000000..4681e52 --- /dev/null +++ b/testing/package.json @@ -0,0 +1,17 @@ +{ + "name": "@executablemd/testing", + "version": "0.3.1", + "description": "Testing vocabulary for executable.md documents: , , and assertion components.", + "type": "module", + "exports": { + ".": "./mod.ts" + }, + "dependencies": { + "@effectionx/context-api": "0.6.0", + "@effectionx/scope-eval": "0.1.3", + "@effectionx/timebox": "0.4.3", + "@executablemd/core": "workspace:*", + "@std/assert": "npm:@jsr/std__assert@^1.0.17", + "effection": "4.1.0-alpha.7" + } +} diff --git a/testing/src/assertions.ts b/testing/src/assertions.ts new file mode 100644 index 0000000..39a3c26 --- /dev/null +++ b/testing/src/assertions.ts @@ -0,0 +1,356 @@ +/** + * Assertion components (specs/testing-spec.md §Assertions). + * + * Each component maps to an `@std/assert` export with the same name and + * parameter names. Expression props evaluate LIVE against the merged binding + * environment — never through JSON serialization, which would destroy + * `RegExp`s, `undefined`, and object identity. + * + * The assertion runs on the raw values BEFORE any diagnostic formatting, so + * formatting arbitrary values (mutating or throwing getters/toJSON/toString) + * can never change the assertion outcome. Diagnostics are built afterwards + * under guarded fallback. + */ + +import { + assert, + assertEquals, + assertExists, + assertFalse, + assertGreater, + assertGreaterOrEqual, + AssertionError, + assertLess, + assertLessOrEqual, + assertMatch, + assertNotEquals, + assertNotMatch, + assertNotStrictEquals, + assertStrictEquals, + assertStringIncludes, +} from "@std/assert"; +import type { Operation } from "effection"; +import { DocumentOutput, env, renderSegments } from "@executablemd/core"; +import type { + ComponentInvocation, + ErrorSegment, + InvocationContext, + Segment, +} from "@executablemd/core"; +import { inTest, testing, verbose } from "./test-api.ts"; + +// --------------------------------------------------------------------------- +// Registry +// --------------------------------------------------------------------------- + +type AssertionKind = + | "unary-truthy" + | "unary-exists" + | "binary-eq" + | "string-includes" + | "match" + | "numeric"; + +export interface AssertionEntry { + name: string; + kind: AssertionKind; + /** Runs the underlying @std/assert function on the resolved raw values. */ + run(values: ResolvedValues): void; + allowsExpectedChildren: boolean; +} + +interface ResolvedValues { + expr?: unknown; + actual?: unknown; + expected?: unknown; + msg?: string; +} + +function entry( + name: string, + kind: AssertionKind, + run: (values: ResolvedValues) => void, +): [string, AssertionEntry] { + const allowsExpectedChildren = kind === "binary-eq" || kind === "string-includes"; + return [name, { name, kind, run, allowsExpectedChildren }]; +} + +export const ASSERTIONS: Map = new Map([ + entry("Assert", "unary-truthy", (v) => assert(v.expr, v.msg)), + entry("AssertFalse", "unary-truthy", (v) => assertFalse(v.expr, v.msg)), + entry("AssertExists", "unary-exists", (v) => assertExists(v.actual, v.msg)), + entry("AssertEquals", "binary-eq", (v) => assertEquals(v.actual, v.expected, v.msg)), + entry("AssertNotEquals", "binary-eq", (v) => assertNotEquals(v.actual, v.expected, v.msg)), + entry("AssertStrictEquals", "binary-eq", (v) => assertStrictEquals(v.actual, v.expected, v.msg)), + entry("AssertNotStrictEquals", "binary-eq", (v) => + assertNotStrictEquals(v.actual, v.expected, v.msg), + ), + entry("AssertStringIncludes", "string-includes", (v) => + assertStringIncludes(coerceString(v.actual), coerceString(v.expected), v.msg), + ), + entry("AssertMatch", "match", (v) => + assertMatch(coerceString(v.actual), requireRegExp(v.expected), v.msg), + ), + entry("AssertNotMatch", "match", (v) => + assertNotMatch(coerceString(v.actual), requireRegExp(v.expected), v.msg), + ), + entry("AssertGreater", "numeric", (v) => assertGreater(v.actual, v.expected, v.msg)), + entry("AssertGreaterOrEqual", "numeric", (v) => + assertGreaterOrEqual(v.actual, v.expected, v.msg), + ), + entry("AssertLess", "numeric", (v) => assertLess(v.actual, v.expected, v.msg)), + entry("AssertLessOrEqual", "numeric", (v) => assertLessOrEqual(v.actual, v.expected, v.msg)), +]); + +function coerceString(value: unknown): string { + if (typeof value === "string") { + return value; + } + throw new AssertionError(`expected a string "actual"/"expected" value, got ${typeof value}`); +} + +function requireRegExp(value: unknown): RegExp { + if (value instanceof RegExp) { + return value; + } + throw new AssertionError( + "match assertions require a RegExp through the expected prop — use expected={/pattern/}", + ); +} + +// --------------------------------------------------------------------------- +// Live expression evaluation +// --------------------------------------------------------------------------- + +const IDENTIFIER_RE = /^[a-zA-Z_$][a-zA-Z0-9_$]*$/; + +function evaluateExpression(expression: string, values: Record): unknown { + const names = Object.keys(values).filter((name) => IDENTIFIER_RE.test(name)); + const fn = new Function(...names, `return (${expression});`); + return fn(...names.map((name) => values[name])); +} + +// --------------------------------------------------------------------------- +// Guarded value formatting — runs only AFTER the assertion outcome is fixed +// --------------------------------------------------------------------------- + +function safeFormat(value: unknown): string { + try { + if (typeof value === "string") { + return JSON.stringify(value); + } + if (value instanceof RegExp) { + return String(value); + } + const json = JSON.stringify(value); + if (json !== undefined) { + return json; + } + return String(value); + } catch { + try { + return String(value); + } catch { + return ""; + } + } +} + +// --------------------------------------------------------------------------- +// Diagnostic-carrying assertion error +// --------------------------------------------------------------------------- + +/** + * An assertion failure enriched with its Markdown diagnostic. Still an + * `AssertionError`, so containment and classification treat it as the + * original @std/assert failure. + */ +export class AssertionDiagnostic extends AssertionError { + override name = "AssertionDiagnostic"; + diagnostic: string; + detail: { actual?: string; expected?: string }; + + constructor(cause: Error, diagnostic: string, detail: { actual?: string; expected?: string }) { + super(cause.message); + this.diagnostic = diagnostic; + this.detail = detail; + this.cause = cause; + } +} + +// --------------------------------------------------------------------------- +// expandAssertion +// --------------------------------------------------------------------------- + +const KIND_PROPS: Record = { + "unary-truthy": { allowed: ["expr", "msg"], required: ["expr"] }, + "unary-exists": { allowed: ["actual", "msg"], required: ["actual"] }, + "binary-eq": { allowed: ["actual", "expected", "msg"], required: ["actual"] }, + "string-includes": { allowed: ["actual", "expected", "msg"], required: ["actual"] }, + match: { allowed: ["actual", "expected", "msg"], required: ["actual", "expected"] }, + numeric: { allowed: ["actual", "expected", "msg"], required: ["actual", "expected"] }, +}; + +function validationError(name: string, message: string): ErrorSegment { + return { type: "error", message: `<${name}> ${message}`, source: name }; +} + +/** + * Expand one assertion component: validate props, resolve raw values, run the + * @std/assert function, then build the diagnostic. Returns diagnostic text on + * a visible pass; throws `AssertionDiagnostic` on failure. + */ +export function* expandAssertion( + assertion: AssertionEntry, + invocation: ComponentInvocation, + ctx: InvocationContext, +): Operation { + const rules = KIND_PROPS[assertion.kind]; + const supplied = [...Object.keys(invocation.props), ...Object.keys(invocation.expressions)]; + + for (const name of supplied) { + if (!rules.allowed.includes(name)) { + return [ + validationError( + assertion.name, + `does not accept a "${name}" prop (allowed: ${rules.allowed.join(", ")}).`, + ), + ]; + } + } + + const hasChildren = !invocation.selfClosing && invocation.children.length > 0; + if (hasChildren && !assertion.allowsExpectedChildren) { + return [validationError(assertion.name, "does not accept expected children.")]; + } + if (hasChildren && supplied.includes("expected")) { + return [ + validationError( + assertion.name, + 'accepts either an "expected" prop or expected children, not both.', + ), + ]; + } + + for (const name of rules.required) { + const suppliedByChildren = name === "expected" && hasChildren; + if (!supplied.includes(name) && !suppliedByChildren) { + return [validationError(assertion.name, `requires the "${name}" prop.`)]; + } + } + if (assertion.kind === "binary-eq" || assertion.kind === "string-includes") { + if (!supplied.includes("expected") && !hasChildren) { + return [validationError(assertion.name, 'requires an "expected" prop or expected children.')]; + } + } + + // Resolve raw values: literal props as-is, expression props evaluated live + // against caller-projected bindings merged under the current environment + // (the same precedence core uses for projected children). + const currentEnv = yield* env; + const merged = { + ...(ctx.projectedEnv?.values ?? {}), + ...(currentEnv?.values ?? {}), + }; + + const resolved: Record = {}; + const resolutionOrder = ["expr", "actual", "expected", "msg"] as const; + for (const name of resolutionOrder) { + if (name in invocation.expressions) { + try { + resolved[name] = evaluateExpression(invocation.expressions[name]!, merged); + } catch (error) { + return [ + validationError( + assertion.name, + `failed to evaluate the "${name}" expression: ${ + error instanceof Error ? error.message : String(error) + }`, + ), + ]; + } + } else if (name in invocation.props) { + resolved[name] = invocation.props[name]; + } + } + + const values: ResolvedValues = { + expr: resolved.expr, + actual: resolved.actual, + expected: resolved.expected, + // Guarded coercion: a hostile toString on a non-string msg must not + // become a new failure before the assertion runs. + msg: + "msg" in resolved + ? typeof resolved.msg === "string" + ? resolved.msg + : safeFormat(resolved.msg) + : undefined, + }; + + if (hasChildren) { + const expanded = yield* ctx.expand(invocation.children); + values.expected = renderSegments(expanded).replace(/\s+$/, ""); + } + + // Run the assertion on the raw values — the outcome is fixed before any + // diagnostic formatting can observe (or mutate) them. + let failure: Error | undefined; + try { + assertion.run(values); + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + } + + const detail: { actual?: string; expected?: string } = {}; + if (assertion.kind === "unary-truthy") { + detail.actual = safeFormat(values.expr); + } else { + detail.actual = safeFormat(values.actual); + if (assertion.kind !== "unary-exists") { + detail.expected = safeFormat(values.expected); + } + } + + if (failure) { + const diagnostic = buildDiagnostic(assertion.name, "failed", values.msg, detail, failure); + const visible = (yield* testing) || (yield* verbose); + const inTestScope = yield* inTest; + if (visible && !inTestScope) { + // Outside a test the throw below aborts expansion before the segment + // could render — emit the diagnostic directly so it reaches the output. + yield* DocumentOutput.operations.output(diagnostic); + } + throw new AssertionDiagnostic(failure, diagnostic, detail); + } + + const visible = (yield* testing) || (yield* verbose); + if (!visible) { + return []; + } + return [{ type: "text", content: buildDiagnostic(assertion.name, "passed", values.msg, detail) }]; +} + +function buildDiagnostic( + name: string, + outcome: "passed" | "failed", + msg: string | undefined, + detail: { actual?: string; expected?: string }, + failure?: Error, +): string { + const icon = outcome === "passed" ? "✅" : "❌"; + const lines = [`> ${icon} **${name}** ${outcome}${msg ? ` — ${msg}` : ""}`]; + if (detail.actual !== undefined) { + lines.push(`> actual: ${detail.actual}`); + } + if (detail.expected !== undefined) { + lines.push(`> expected: ${detail.expected}`); + } + if (failure) { + const message = failure.message.split("\n")[0]; + if (message) { + lines.push(`> ${message}`); + } + } + return `\n${lines.join("\n")}\n`; +} diff --git a/testing/src/execute.ts b/testing/src/execute.ts new file mode 100644 index 0000000..a2b78fb --- /dev/null +++ b/testing/src/execute.ts @@ -0,0 +1,139 @@ +/** + * executeDocument — the single scoped wrapper both CLI paths use + * (specs/testing-spec.md §Testing Mode). + * + * Testing middleware lives and dies with the execution it serves: a bounded + * child task installs the vocabulary, the run-level collectors, and (in + * testing mode) root activation, runs the document inside that scope, and + * evaluates the aggregate outcome only after the inner execution — and + * therefore its closed output stream — completes. When the execution + * finishes, the scope exits and every install is gone. + * + * `output` is a lazy stream delegating to the inner execution's replay-safe + * output — no second channel, nothing dropped for late subscribers. + */ + +import { spawn, withResolvers } from "effection"; +import type { Operation } from "effection"; +import { runDocument } from "@executablemd/core"; +import type { DocumentExecution, RunDocumentOptions } from "@executablemd/core"; +import { Test, TestFailureError } from "./test-api.ts"; +import type { BoundaryOutcome, TestResult } from "./test-api.ts"; +import { installHandlers, installTestingVocabulary } from "./vocabulary.ts"; +import type { TestHandlers } from "./handlers.ts"; + +export interface ExecuteDocumentOptions extends RunDocumentOptions { + /** Activate testing mode at the root (`xmd test` ≡ root ``). */ + testing: boolean; + /** Render assertion diagnostics during regular execution. */ + verbose?: boolean; +} + +interface ExecuteDependencies { + runDocument(options: RunDocumentOptions): Operation; + handlers?: TestHandlers; +} + +/** + * Internal dependency-injection seam: tests inject a throwing `runDocument` + * (pre-publication setup failure) or short-timeout handlers. + */ +export function createExecuteDocument(deps: ExecuteDependencies) { + return function* executeDocument(options: ExecuteDocumentOptions): Operation { + const innerExecution = withResolvers(); + const completion = withResolvers(); + + yield* spawn(function* () { + // One error boundary around setup, runDocument, and completion: any + // failure rejects BOTH resolvers so neither output consumption nor + // `yield* execution` can hang. + try { + if (deps.handlers) { + yield* installHandlers(deps.handlers, { verbose: options.verbose }); + } else { + yield* installTestingVocabulary({ verbose: options.verbose }); + } + + const runResults: TestResult[] = []; + const boundaries: BoundaryOutcome[] = []; + // Run-level collector — ALWAYS installed, so explicit + // boundaries are observable during ordinary runs too. + yield* Test.around({ + // deno-lint-ignore require-yield + *results() { + return runResults; + }, + *record([result], next) { + runResults.push(result); + yield* next(result); + }, + *boundary([outcome], next) { + boundaries.push(outcome); + yield* next(outcome); + }, + }); + if (options.testing) { + yield* Test.around({ testing: () => true }); + } + + const inner = yield* deps.runDocument(options); + innerExecution.resolve(inner); + + // Observing the inner execution keeps this child (and its installs) + // alive; core closes inner.output before the execution settles, so + // the report is fully delivered before any rejection below. + let out: string; + try { + out = yield* inner; + } catch (error) { + completion.reject(error instanceof Error ? error : new Error(String(error))); + return; + } + + const failed = + boundaries.some((b) => b.failed > 0 || b.tests === 0) || + (options.testing && + (runResults.some((r) => r.status === "fail") || runResults.length === 0)); + if (failed) { + completion.reject(new TestFailureError(summarize(runResults, boundaries))); + } else { + completion.resolve(out); + } + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + innerExecution.reject(failure); + completion.reject(failure); + } + }); + + return { + // A Stream IS an Operation: await the published inner + // execution, then delegate to its replay-safe output. + output: { + *[Symbol.iterator]() { + const inner = yield* innerExecution.operation; + return yield* inner.output; + }, + }, + *[Symbol.iterator]() { + return yield* completion.operation; + }, + }; + }; +} + +export const executeDocument = createExecuteDocument({ runDocument }); + +function summarize(results: TestResult[], boundaries: BoundaryOutcome[]): string { + const failed = results.filter((result) => result.status === "fail"); + if (results.length === 0 && boundaries.every((b) => b.tests === 0)) { + return "no tests were discovered"; + } + if (failed.length === 0 && boundaries.some((b) => b.tests === 0)) { + return "a boundary discovered no tests"; + } + const details = failed + .map((result) => ` ${result.name ?? result.location}: ${result.error?.message ?? "failed"}`) + .join("\n"); + return `${failed.length} of ${results.length} tests failed\n${details}`; +} diff --git a/testing/src/handlers.ts b/testing/src/handlers.ts new file mode 100644 index 0000000..cf058d1 --- /dev/null +++ b/testing/src/handlers.ts @@ -0,0 +1,310 @@ +/** + * `` and `` handlers (specs/testing-spec.md). + * + * `createTestHandlers` is the internal dependency-injection seam for the + * fixed 20-second test timeout: the public vocabulary always constructs + * handlers with 20_000; tests construct them directly with a small timeout. + */ + +import { ensure, scoped, spawn, suspend, withResolvers } from "effection"; +import type { Operation, Task } from "effection"; +import { timebox } from "@effectionx/timebox"; +import { unbox, useEvalScope } from "@effectionx/scope-eval"; +import type { EvalScope } from "@effectionx/scope-eval"; +import { AssertionError } from "@std/assert"; +import { Component, env, evalScope } from "@executablemd/core"; +import type { + ComponentInvocation, + ErrorSegment, + EvalEnv, + InvocationContext, + Segment, +} from "@executablemd/core"; +import { Test, boundary, inTest, record, testing } from "./test-api.ts"; +import type { TestResult } from "./test-api.ts"; +import { AssertionDiagnostic, expandAssertion } from "./assertions.ts"; +import type { AssertionEntry } from "./assertions.ts"; + +// --------------------------------------------------------------------------- +// Contained-failure carriers +// --------------------------------------------------------------------------- + +/** An ErrorSegment raised anywhere inside a test body. */ +class RaisedSegmentError extends Error { + override name = "RaisedSegmentError"; + segment: ErrorSegment; + + constructor(segment: ErrorSegment) { + super(segment.message); + this.segment = segment; + } +} + +/** A failure while dismantling an established test scope or lease. */ +class TeardownError extends Error { + override name = "TeardownError"; + + constructor(cause: unknown) { + super(cause instanceof Error ? cause.message : String(cause)); + this.cause = cause; + } +} + +// --------------------------------------------------------------------------- +// Test-owned child EvalScope lease +// --------------------------------------------------------------------------- + +interface EvalScopeLease { + scope: EvalScope; + task: Task; +} + +/** + * Host a child EvalScope in a dedicated task inside the parent EvalScope. + * The child inherits the parent's middleware, but its lifetime belongs to + * the TEST: `parentScope.eval(() => useEvalScope())` alone would tie the + * worker to the parent scope, leaking test-installed middleware into later + * tests. The suspended task keeps the child alive until the lease is halted. + */ +function* leaseChildEvalScope(parentScope: EvalScope): Operation { + const published = withResolvers(); + const boxed = yield* parentScope.eval(function* () { + return yield* spawn(function* () { + published.resolve(yield* useEvalScope()); + yield* suspend(); + }); + }); + return { scope: yield* published.operation, task: unbox(boxed) }; +} + +// --------------------------------------------------------------------------- +// Handlers +// --------------------------------------------------------------------------- + +export interface TestHandlers { + expandTesting(invocation: ComponentInvocation, ctx: InvocationContext): Operation; + expandTest(invocation: ComponentInvocation, ctx: InvocationContext): Operation; + expandAssertion( + assertion: AssertionEntry, + invocation: ComponentInvocation, + ctx: InvocationContext, + ): Operation; +} + +export function createTestHandlers(options: { timeoutMs: number }): TestHandlers { + const { timeoutMs } = options; + + function* expandTesting( + invocation: ComponentInvocation, + ctx: InvocationContext, + ): Operation { + return yield* scoped(function* () { + const local: TestResult[] = []; + yield* Test.around( + { + testing: () => true, + // deno-lint-ignore require-yield + *results() { + return local; + }, + *record([result], next) { + local.push(result); + yield* next(result); + }, + }, + { at: "min" }, + ); + // The report is the naturally expanded subtree — no summary appended. + const report = yield* ctx.expand(invocation.children); + yield* boundary({ + tests: local.length, + failed: local.filter((result) => result.status === "fail").length, + }); + return report; + }); + } + + function* expandTest( + invocation: ComponentInvocation, + ctx: InvocationContext, + ): Operation { + if (!(yield* testing)) { + // Regular execution: the test and its entire body are skipped without + // output, bindings, or side effects. + return []; + } + if (yield* inTest) { + // Returned while the ENCLOSING test's raise interceptor is still + // active — the hook's re-raise fails the current test. + const error: ErrorSegment = { + type: "error", + message: "Nested elements are invalid.", + source: "Test", + }; + return [error]; + } + + const name = typeof invocation.props.name === "string" ? invocation.props.name : undefined; + const location = formatLocation(invocation); + + const parentEnv = yield* env; + const parentScope = yield* evalScope; + if (!parentScope) { + const result = failResult(name, location, { + kind: "error", + message: " requires an eval scope in context.", + }); + yield* record(result); + return [failureDiagnostic(result, { detail: true })]; + } + + // ONE stable binding environment, created before middleware install — + // the accessor returns the same object on every read, so + // writes persist for the assertion that follows. + const testEnv: EvalEnv = { values: { ...(parentEnv?.values ?? {}) } }; + + const testOutput: Segment[] = []; + let bodyError: unknown; + let timedOut = false; + let established = false; + + try { + yield* scoped(function* () { + const lease = yield* leaseChildEvalScope(parentScope); + // Halt the lease during this scope's teardown, before the next test + // can start. A throwing halt propagates as a teardown failure. + yield* ensure(() => lease.task.halt()); + + yield* Component.around( + { + env: () => testEnv, + evalScope: () => lease.scope, + }, + { at: "min" }, + ); + yield* Test.around({ inTest: () => true }, { at: "min" }); + // ErrorSegments fail the test. Outer instrumentation (default "max") + // so nested { at: "min" } policies cannot shadow it: every raise in + // the body — components, regions, code blocks, imports, + // validation, nested — arrives here first. + yield* Component.around({ + // deno-lint-ignore require-yield + *raise([segment]) { + throw new RaisedSegmentError(segment); + }, + }); + established = true; + + const boxed = yield* timebox(timeoutMs, function* () { + for (const child of invocation.children) { + try { + testOutput.push(...(yield* ctx.expand([child]))); + } catch (error) { + bodyError = error; + throw error; + } + } + }); + if (boxed.timeout) { + timedOut = true; + } + }); + } catch (outer) { + if (bodyError === undefined && !timedOut) { + // Setup failures (lease creation, middleware install) are unexpected + // errors; only failures dismantling an ESTABLISHED scope/lease are + // teardown failures. + bodyError = established ? new TeardownError(outer) : outer; + } + } + + const result = classify(name, location, bodyError, timedOut, timeoutMs); + yield* record(result); + + if (result.status === "fail") { + // Containment invariant: a completed test returns only text segments. + // The hook re-raises returned ErrorSegments under the AMBIENT policy — + // after this test's interception scope has ended — so raised segments + // are formatted into the diagnostic instead of returned raw. + if (bodyError instanceof AssertionDiagnostic) { + // The assertion's own diagnostic (built when it threw) follows the + // output produced before the failure, then the test-level line. + testOutput.push({ type: "text", content: bodyError.diagnostic }); + testOutput.push(failureDiagnostic(result, { detail: false })); + } else { + testOutput.push(failureDiagnostic(result, { detail: true })); + } + } + return testOutput; + } + + return { expandTesting, expandTest, expandAssertion }; +} + +// --------------------------------------------------------------------------- +// Result classification and diagnostics +// --------------------------------------------------------------------------- + +function formatLocation(invocation: ComponentInvocation): string { + const position = invocation.position; + if (!position) { + return "unknown"; + } + const at = `${position.line}:${position.column}`; + return position.path ? `${position.path}:${at}` : at; +} + +function failResult( + name: string | undefined, + location: string, + error: NonNullable, +): TestResult { + return { status: "fail", name, location, error }; +} + +function classify( + name: string | undefined, + location: string, + bodyError: unknown, + timedOut: boolean, + timeoutMs: number, +): TestResult { + if (bodyError === undefined && !timedOut) { + return { status: "pass", name, location }; + } + if (timedOut && bodyError === undefined) { + return failResult(name, location, { + kind: "timeout", + message: `test timed out after ${timeoutMs / 1000} seconds`, + }); + } + if (bodyError instanceof AssertionDiagnostic) { + return failResult(name, location, { + kind: "assertion", + message: bodyError.message, + actual: bodyError.detail.actual, + expected: bodyError.detail.expected, + }); + } + if (bodyError instanceof AssertionError) { + return failResult(name, location, { kind: "assertion", message: bodyError.message }); + } + if (bodyError instanceof TeardownError) { + return failResult(name, location, { kind: "teardown", message: bodyError.message }); + } + const message = bodyError instanceof Error ? bodyError.message : String(bodyError); + return failResult(name, location, { kind: "error", message }); +} + +function failureDiagnostic(result: TestResult, options: { detail: boolean }): Segment { + const title = result.name ? `**${result.name}**` : `test at ${result.location}`; + const error = result.error; + const lines = [`> ❌ Test ${title} failed (${error?.kind ?? "error"}): ${error?.message ?? ""}`]; + if (options.detail && error?.actual !== undefined) { + lines.push(`> actual: ${error.actual}`); + } + if (options.detail && error?.expected !== undefined) { + lines.push(`> expected: ${error.expected}`); + } + return { type: "text", content: `\n${lines.join("\n")}\n` }; +} diff --git a/testing/src/test-api.ts b/testing/src/test-api.ts new file mode 100644 index 0000000..ddb3266 --- /dev/null +++ b/testing/src/test-api.ts @@ -0,0 +1,72 @@ +/** + * Test Api — contextual operations for testing mode (specs/testing-spec.md). + * + * `testing` is the activation switch: false by default, true beneath + * `` (or root activation via `executeDocument({ testing: true })`). + * Both perform the identical `Test.around({ testing: () => true })` install, + * which is what makes `xmd test` equivalent to wrapping the entrypoint in + * ``. + * + * `record` delegates outward through nested collectors, so every enclosing + * `` boundary and the run-level collector observe each completed + * test. `boundary` reports each `` element's aggregate outcome. + */ + +import { createApi } from "@effectionx/context-api"; +import type { Operation } from "effection"; + +/** A completed test, in discovery order. Never holds rendered markdown. */ +export interface TestResult { + status: "pass" | "fail"; + name?: string; + /** "path:line:column" ("line:column" for dynamically scanned sources). */ + location: string; + error?: { + kind: "assertion" | "timeout" | "teardown" | "error"; + message: string; + actual?: string; + expected?: string; + }; +} + +/** Aggregate outcome of one `` boundary. */ +export interface BoundaryOutcome { + tests: number; + failed: number; +} + +export interface TestApi { + /** Whether testing mode is active in the current scope. */ + testing: boolean; + /** Whether expansion is currently inside a `` body. */ + inTest: boolean; + /** Whether assertion diagnostics render during regular execution. */ + verbose: boolean; + /** Record a completed test. Collectors delegate outward via `next`. */ + record(result: TestResult): Operation; + /** Completed tests recorded by the nearest collector, discovery order. */ + results(): Operation; + /** Report a `` boundary's aggregate outcome. */ + boundary(outcome: BoundaryOutcome): Operation; +} + +export const Test = createApi("Test", { + testing: false, + inTest: false, + verbose: false, + // deno-lint-ignore require-yield + *record(_result: TestResult): Operation {}, + // deno-lint-ignore require-yield + *results(): Operation { + return []; + }, + // deno-lint-ignore require-yield + *boundary(_outcome: BoundaryOutcome): Operation {}, +}); + +export const { testing, inTest, verbose, record, results, boundary } = Test.operations; + +/** A document execution failed its testing outcome (test failures or zero tests). */ +export class TestFailureError extends Error { + override name = "TestFailureError"; +} diff --git a/testing/src/vocabulary.ts b/testing/src/vocabulary.ts new file mode 100644 index 0000000..203ed57 --- /dev/null +++ b/testing/src/vocabulary.ts @@ -0,0 +1,52 @@ +/** + * Vocabulary registration (specs/testing-spec.md). + * + * Teaches the expansion loop the testing words — ``, ``, and + * the assertion components — via the core `expandInvocation` hook. + * Registration is distinct from activation: installing the vocabulary leaves + * `testing` false, so `` skips and assertions stay usable. + * + * Installs are scope-local; `executeDocument` owns the lifetime for CLI + * runs. Direct core consumers call this inside their own bounded scope. + */ + +import type { Operation } from "effection"; +import { Component } from "@executablemd/core"; +import { Test } from "./test-api.ts"; +import { ASSERTIONS } from "./assertions.ts"; +import { createTestHandlers } from "./handlers.ts"; +import type { TestHandlers } from "./handlers.ts"; + +const TEST_TIMEOUT_MS = 20_000; + +export function* installTestingVocabulary(options?: { verbose?: boolean }): Operation { + yield* installHandlers(createTestHandlers({ timeoutMs: TEST_TIMEOUT_MS }), options); +} + +/** + * Install a specific handler set. Internal seam: tests inject handlers built + * with a short timeout; the public path always uses the fixed 20 seconds. + */ +export function* installHandlers( + handlers: TestHandlers, + options?: { verbose?: boolean }, +): Operation { + if (options?.verbose) { + yield* Test.around({ verbose: () => true }); + } + yield* Component.around({ + *expandInvocation([invocation, ctx], next) { + if (invocation.name === "Testing") { + return { segments: yield* handlers.expandTesting(invocation, ctx) }; + } + if (invocation.name === "Test") { + return { segments: yield* handlers.expandTest(invocation, ctx) }; + } + const assertion = ASSERTIONS.get(invocation.name); + if (assertion) { + return { segments: yield* handlers.expandAssertion(assertion, invocation, ctx) }; + } + return yield* next(invocation, ctx); + }, + }); +} diff --git a/testing/tests/assertions.test.ts b/testing/tests/assertions.test.ts new file mode 100644 index 0000000..2caee6a --- /dev/null +++ b/testing/tests/assertions.test.ts @@ -0,0 +1,180 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { AssertionError } from "@std/assert"; +import { TestFailureError } from "../src/test-api.ts"; +import { failureOf, runDoc } from "./helpers.ts"; + +describe("assertion components", () => { + it("passing assertions in testing mode emit diagnostics", function* () { + const run = yield* runDoc( + { + "README.md": + '\n', + }, + { testing: false }, + ); + expect(run.completion.ok).toBe(true); + expect(run.output).toContain("**AssertEquals** passed"); + expect(run.results).toEqual([{ status: "pass", name: "eq", location: "README.md:1:10" }]); + }); + + it("assertions outside a test pass silently during regular execution", function* () { + const run = yield* runDoc({ + "README.md": "before\n\nafter\n", + }); + expect(run.completion.ok).toBe(true); + expect(run.output).not.toContain("AssertEquals"); + expect(run.output).toContain("before"); + expect(run.output).toContain("after"); + }); + + it("assertions outside a test emit diagnostics with verbose", function* () { + const run = yield* runDoc( + { "README.md": "\n" }, + { verbose: true }, + ); + expect(run.completion.ok).toBe(true); + expect(run.output).toContain("**AssertEquals** passed"); + }); + + it("a failed assertion outside a test aborts the document", function* () { + const run = yield* runDoc({ + "README.md": "before\n\nnever\n", + }); + const error = failureOf(run); + expect(error).toBeInstanceOf(AssertionError); + expect(run.output).not.toContain("never"); + // Diagnostics hidden without verbose — but the assertion still threw. + expect(run.output).not.toContain("AssertEquals"); + }); + + it("a failed assertion outside a test keeps its diagnostic with verbose", function* () { + const run = yield* runDoc( + { "README.md": "before\n\nnever\n" }, + { verbose: true }, + ); + expect(failureOf(run)).toBeInstanceOf(AssertionError); + expect(run.output).toContain("**AssertEquals** failed"); + expect(run.output).toContain("before"); + }); + + it("expected children behave like trimming", function* () { + const doc = [ + "", + '', + "Hello World", + "", + "", + "Hello World", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results.map((r) => r.status)).toEqual(["pass"]); + }); + + it("rejects both expected prop and expected children", function* () { + const doc = + "1\n"; + const run = yield* runDoc({ "README.md": doc }); + expect(failureOf(run)).toBeInstanceOf(TestFailureError); + expect(run.results[0]?.status).toBe("fail"); + expect(run.results[0]?.error?.kind).toBe("error"); + expect(run.results[0]?.error?.message).toContain("not both"); + }); + + it("rejects expected children on unary and numeric assertions", function* () { + const doc = + "x\n"; + const run = yield* runDoc({ "README.md": doc }); + expect(run.results[0]?.error?.message).toContain("expected children"); + }); + + it("match assertions require a real RegExp", function* () { + const doc = '\n'; + const run = yield* runDoc({ "README.md": doc }); + expect(run.results[0]?.status).toBe("fail"); + expect(run.results[0]?.error?.message).toContain("RegExp"); + }); + + it("match assertions accept a RegExp expression", function* () { + const doc = '\n'; + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results[0]?.status).toBe("pass"); + }); + + it("unknown props are rejected per kind", function* () { + const doc = "\n"; + const run = yield* runDoc({ "README.md": doc }); + expect(run.results[0]?.error?.message).toContain('"actual"'); + }); + + it("missing required props are rejected", function* () { + const doc = "\n"; + const run = yield* runDoc({ "README.md": doc }); + expect(run.results[0]?.error?.message).toContain('"expected"'); + }); + + it("assertion expressions see live bindings from eval blocks", function* () { + const doc = [ + "", + "```js eval", + "const answer = { deep: [1, 2, 3] };", + "```", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results[0]?.status).toBe("pass"); + }); + + it("assertion expressions see caller bindings through projection", function* () { + const doc = [ + "```js eval", + 'const fromCaller = "outer-value";', + "```", + "", + '', + "", + "", + ].join("\n"); + const wrap = "projected: \n"; + const run = yield* runDoc({ "README.md": doc, "components/Wrap.md": wrap }); + expect(run.completion.ok).toBe(true); + expect(run.results[0]?.status).toBe("pass"); + }); + + it("formatter-visible toJSON/toString cannot change the outcome", function* () { + const doc = [ + "", + "```js eval", + "const cursed = { toJSON() { throw new Error('evil json'); }, toString() { throw new Error('evil string'); } };", + "```", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results[0]?.status).toBe("pass"); + expect(run.output).toContain("unformattable"); + }); + + it("a throwing getter read at format time cannot change the outcome", function* () { + const doc = [ + "", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results[0]?.status).toBe("pass"); + expect(run.output).toContain("[object Object]"); + }); +}); diff --git a/testing/tests/cli.test.ts b/testing/tests/cli.test.ts new file mode 100644 index 0000000..f1e22dc --- /dev/null +++ b/testing/tests/cli.test.ts @@ -0,0 +1,89 @@ +/** + * CLI integration tests for `xmd test` and `xmd run` (specs/testing-spec.md). + * + * Shells out to `deno run --allow-all cli/src/cli.ts` with piped stdio, so + * exit codes and report output are asserted TTY-independently. + */ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { timebox } from "@effectionx/timebox"; +import { spawn, each } from "effection"; +import type { Operation } from "effection"; +import { exec } from "@effectionx/process"; +import process from "node:process"; + +const TIMEOUT = 30_000; + +interface CliResult { + code: number | undefined; + stdout: string; + stderr: string; +} + +function* runCli(args: string[]): Operation { + const result = yield* timebox(TIMEOUT, function* () { + const proc = yield* exec("deno", { + arguments: ["run", "--allow-all", "cli/src/cli.ts", ...args], + env: process.env as Record, + }); + + const stdoutChunks: string[] = []; + const stderrChunks: string[] = []; + const readStdout = yield* spawn(function* () { + for (const chunk of yield* each(proc.stdout)) { + stdoutChunks.push(new TextDecoder().decode(chunk)); + yield* each.next(); + } + }); + const readStderr = yield* spawn(function* () { + for (const chunk of yield* each(proc.stderr)) { + stderrChunks.push(new TextDecoder().decode(chunk)); + yield* each.next(); + } + }); + + const status = yield* proc.join(); + yield* readStdout; + yield* readStderr; + + return { + code: status.code, + stdout: stdoutChunks.join(""), + stderr: stderrChunks.join(""), + }; + }); + if (result.timeout) { + throw new Error("CLI subprocess timed out"); + } + return result.value; +} + +describe("xmd CLI", () => { + it("test exits 0 and prints the report when every test passes", function* () { + const result = yield* runCli(["test", "testing/tests/fixtures/passing.md"]); + expect(result.code).toBe(0); + expect(result.stdout).toContain("**AssertEquals** passed"); + expect(result.stdout).toContain("Regular content stays."); + }); + + it("test exits 1 and prints the failure diagnostic when a test fails", function* () { + const result = yield* runCli(["test", "testing/tests/fixtures/failing.md"]); + expect(result.code).toBe(1); + expect(result.stdout).toContain("**Assert** failed"); + expect(result.stdout).toContain("Test **bad** failed"); + expect(result.stderr).toContain("tests failed"); + }); + + it("test exits 1 when no tests are discovered", function* () { + const result = yield* runCli(["test", "README.md"]); + expect(result.code).toBe(1); + expect(result.stderr).toContain("no tests were discovered"); + }); + + it("run skips tests entirely and exits 0", function* () { + const result = yield* runCli(["run", "testing/tests/fixtures/failing.md"]); + expect(result.code).toBe(0); + expect(result.stdout).not.toContain("Assert"); + expect(result.stdout).toContain("# Fixture"); + }); +}); diff --git a/testing/tests/execute.test.ts b/testing/tests/execute.test.ts new file mode 100644 index 0000000..b130b47 --- /dev/null +++ b/testing/tests/execute.test.ts @@ -0,0 +1,187 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { scoped, sleep, spawn } from "effection"; +import type { Operation, Subscription } from "effection"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import { useStubFs } from "@executablemd/runtime/test"; +import { API } from "@executablemd/runtime"; +import { Test, TestFailureError, testing } from "../src/test-api.ts"; +import { createExecuteDocument, executeDocument } from "../src/execute.ts"; +import { failureOf, runDoc } from "./helpers.ts"; + +function* drain( + subscription: Subscription, +): Operation<{ chunks: string[]; close: string }> { + const chunks: string[] = []; + let next = yield* subscription.next(); + while (!next.done) { + chunks.push(next.value); + next = yield* subscription.next(); + } + return { chunks, close: next.value }; +} + +describe("executeDocument", () => { + it("a late subscriber still receives all chunks and the close value", function* () { + yield* useStubFs({ "README.md": "hello world\n" }); + const execution = yield* executeDocument({ + docPath: "README.md", + stream: new InMemoryStream(), + testing: false, + }); + // Complete the execution FIRST — only then subscribe. + const value = yield* execution; + const late = yield* scoped(function* () { + return yield* drain(yield* execution.output); + }); + expect(value).toContain("hello world"); + expect(late.close).toBe(value); + expect(late.chunks.join("")).toContain("hello world"); + }); + + it("multiple subscribers each receive the full sequence", function* () { + yield* useStubFs({ "README.md": "alpha\n\nbeta\n" }); + const execution = yield* executeDocument({ + docPath: "README.md", + stream: new InMemoryStream(), + testing: false, + }); + yield* execution; + const [one, two] = yield* scoped(function* () { + return [yield* drain(yield* execution.output), yield* drain(yield* execution.output)]; + }); + expect(one).toEqual(two); + expect(one.close).toContain("alpha"); + }); + + it("consuming only the completion does not deadlock", function* () { + yield* useStubFs({ "README.md": "only completion\n" }); + const execution = yield* executeDocument({ + docPath: "README.md", + stream: new InMemoryStream(), + testing: false, + }); + const value = yield* execution; + expect(value).toContain("only completion"); + }); + + it("a bad docPath is an inner runtime failure after publication", function* () { + yield* useStubFs({}); + const execution = yield* executeDocument({ + docPath: "missing.md", + stream: new InMemoryStream(), + testing: false, + }); + // Output closes (with accumulated output) even though completion rejects. + const drained = yield* scoped(function* () { + return yield* drain(yield* execution.output); + }); + expect(drained.close).toBe(""); + let error: Error | undefined; + try { + yield* execution; + } catch (failure) { + error = failure instanceof Error ? failure : new Error(String(failure)); + } + expect(error?.message).toContain("missing.md"); + expect(error).not.toBeInstanceOf(TestFailureError); + }); + + it("a pre-publication setup failure rejects output and completion", function* () { + const execute = createExecuteDocument({ + // deno-lint-ignore require-yield + *runDocument() { + throw new Error("setup exploded"); + }, + }); + const execution = yield* execute({ + docPath: "README.md", + stream: new InMemoryStream(), + testing: false, + }); + + let outputError: Error | undefined; + try { + yield* scoped(function* () { + yield* drain(yield* execution.output); + }); + } catch (failure) { + outputError = failure instanceof Error ? failure : new Error(String(failure)); + } + let completionError: Error | undefined; + try { + yield* execution; + } catch (failure) { + completionError = failure instanceof Error ? failure : new Error(String(failure)); + } + expect(outputError?.message).toBe("setup exploded"); + expect(completionError?.message).toBe("setup exploded"); + }); + + it("testing middleware does not outlive its execution", function* () { + yield* useStubFs({ + "README.md": "\n", + }); + const first = yield* executeDocument({ + docPath: "README.md", + stream: new InMemoryStream(), + testing: true, + }); + const firstValue = yield* first; + expect(firstValue).toContain("**Assert** passed"); + + // Same caller scope, fresh run WITHOUT testing: the previous run's + // activation and collectors must be gone — the test is skipped. + expect(yield* testing).toBe(false); + const defaults = yield* Test.operations.results(); + expect(defaults).toEqual([]); + const second = yield* runDoc({ "README.md": "\n" }); + expect(second.completion.ok).toBe(true); + expect(second.output).not.toContain("Assert"); + expect(second.results).toEqual([]); + }); + + it("an early caller-scope halt tears down the document and leased eval scope", function* () { + const globalValues = globalThis as Record; + delete globalValues.__executeHaltMarker; + yield* API.Process.around({ + *exec(_args, _next) { + yield* sleep(10_000); + return { exitCode: 0, stdout: "", stderr: "" }; + }, + }); + const doc = [ + "", + '', + "```js persist eval", + "globalThis.__executeHaltMarker = true;", + "yield* spawn(function* () { try { yield* suspend(); } finally { globalThis.__executeHaltMarker = false; } });", + "```", + "```bash exec", + "sleep 600", + "```", + "", + "", + "", + ].join("\n"); + yield* useStubFs({ "README.md": doc }); + + yield* scoped(function* () { + const execution = yield* executeDocument({ + docPath: "README.md", + stream: new InMemoryStream(), + testing: false, + }); + yield* spawn(function* () { + yield* scoped(function* () { + yield* drain(yield* execution.output); + }); + }); + // Give the document time to start the test and spawn its effect, + // then leave the scope — halting everything mid-test. + yield* sleep(200); + }); + + expect(globalValues.__executeHaltMarker).toBe(false); + }); +}); diff --git a/testing/tests/fixtures/failing.md b/testing/tests/fixtures/failing.md new file mode 100644 index 0000000..cb1c815 --- /dev/null +++ b/testing/tests/fixtures/failing.md @@ -0,0 +1,4 @@ +# Fixture + + + diff --git a/testing/tests/fixtures/passing.md b/testing/tests/fixtures/passing.md new file mode 100644 index 0000000..a15f3c1 --- /dev/null +++ b/testing/tests/fixtures/passing.md @@ -0,0 +1,8 @@ +# Fixture + + +Hello World + + + +Regular content stays. diff --git a/testing/tests/helpers.ts b/testing/tests/helpers.ts new file mode 100644 index 0000000..1ea09b1 --- /dev/null +++ b/testing/tests/helpers.ts @@ -0,0 +1,85 @@ +/** + * Document-level test harness: run a stub-fs document through + * executeDocument and observe chunks, close value, completion, and the + * delegated test results/boundaries. + */ + +import type { Operation } from "effection"; +import { forEach } from "@effectionx/stream-helpers"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import { useStubFs } from "@executablemd/runtime/test"; +import type { DocumentExecution } from "@executablemd/core"; +import { executeDocument } from "../src/execute.ts"; +import type { ExecuteDocumentOptions } from "../src/execute.ts"; +import { Test } from "../src/test-api.ts"; +import type { BoundaryOutcome, TestResult } from "../src/test-api.ts"; + +export interface DocRun { + /** Chunks received while streaming. */ + chunks: string[]; + /** The output stream's close value. */ + output: string; + completion: { ok: true; value: string } | { ok: false; error: Error }; + /** Results delegated past the wrapper's run-level collector. */ + results: TestResult[]; + boundaries: BoundaryOutcome[]; +} + +export interface RunDocOptions { + testing?: boolean; + verbose?: boolean; + docPath?: string; + execute?: (options: ExecuteDocumentOptions) => Operation; +} + +export function* runDoc( + files: Record, + options: RunDocOptions = {}, +): Operation { + yield* useStubFs(files); + + const results: TestResult[] = []; + const boundaries: BoundaryOutcome[] = []; + yield* Test.around({ + *record([result], next) { + results.push(result); + yield* next(result); + }, + *boundary([outcome], next) { + boundaries.push(outcome); + yield* next(outcome); + }, + }); + + const execute = options.execute ?? executeDocument; + const execution = yield* execute({ + docPath: options.docPath ?? "README.md", + stream: new InMemoryStream(), + testing: options.testing ?? false, + verbose: options.verbose, + }); + + const chunks: string[] = []; + const output = yield* forEach(function* (chunk: string) { + chunks.push(chunk); + }, execution.output); + + let completion: DocRun["completion"]; + try { + completion = { ok: true, value: yield* execution }; + } catch (error) { + completion = { + ok: false, + error: error instanceof Error ? error : new Error(String(error)), + }; + } + + return { chunks, output, completion, results, boundaries }; +} + +export function failureOf(run: DocRun): Error | undefined { + if (run.completion.ok) { + return undefined; + } + return run.completion.error; +} diff --git a/testing/tests/testing-mode.test.ts b/testing/tests/testing-mode.test.ts new file mode 100644 index 0000000..c88912f --- /dev/null +++ b/testing/tests/testing-mode.test.ts @@ -0,0 +1,304 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { sleep } from "effection"; +import { API } from "@executablemd/runtime"; +import { useFailingExec } from "@executablemd/runtime/test"; +import { runDocument } from "@executablemd/core"; +import { TestFailureError } from "../src/test-api.ts"; +import { createTestHandlers } from "../src/handlers.ts"; +import { createExecuteDocument } from "../src/execute.ts"; +import { failureOf, runDoc } from "./helpers.ts"; + +describe("testing mode", () => { + it("skips entirely during regular execution", function* () { + const execCalls: string[] = []; + yield* API.Process.around({ + // deno-lint-ignore require-yield + *exec([options], _next) { + execCalls.push(options.command.join(" ")); + return { exitCode: 0, stdout: "ran\n", stderr: "" }; + }, + }); + const doc = [ + "before", + "", + "```bash exec", + "echo hi", + "```", + "test body text", + "", + "after", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.output).toContain("before"); + expect(run.output).toContain("after"); + expect(run.output).not.toContain("test body text"); + expect(execCalls).toEqual([]); + expect(run.results).toEqual([]); + }); + + it("explicit runs its subtree during a regular run", function* () { + const doc = '\n'; + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results.map((r) => [r.name, r.status])).toEqual([["t", "pass"]]); + expect(run.boundaries).toEqual([{ tests: 1, failed: 0 }]); + }); + + it("a failing boundary rejects an ordinary run", function* () { + const doc = "\n"; + const run = yield* runDoc({ "README.md": doc }); + expect(failureOf(run)).toBeInstanceOf(TestFailureError); + expect(run.output).toContain("Test"); + }); + + it("an empty boundary rejects an ordinary run", function* () { + const run = yield* runDoc({ "README.md": "no tests here\n" }); + expect(failureOf(run)).toBeInstanceOf(TestFailureError); + expect(run.boundaries).toEqual([{ tests: 0, failed: 0 }]); + }); + + it("root activation runs tests without ", function* () { + const doc = '\n'; + const run = yield* runDoc({ "README.md": doc }, { testing: true }); + expect(run.completion.ok).toBe(true); + expect(run.results.map((r) => [r.name, r.status])).toEqual([["rooted", "pass"]]); + }); + + it("root activation with zero tests fails", function* () { + const run = yield* runDoc({ "README.md": "just text\n" }, { testing: true }); + const error = failureOf(run); + expect(error).toBeInstanceOf(TestFailureError); + expect(error?.message).toContain("no tests were discovered"); + }); + + it("nested delegates results outward", function* () { + const doc = [ + "", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + // Inner boundary reports first; the outer boundary sees the delegated + // result, so neither boundary is empty. + expect(run.boundaries).toEqual([ + { tests: 1, failed: 0 }, + { tests: 1, failed: 0 }, + ]); + expect(run.results).toHaveLength(1); + }); + + it("a failing exec block fails the test; later tests run", function* () { + yield* useFailingExec(3, "command exploded"); + const doc = [ + "", + '', + "```bash exec", + "false", + "```", + "", + '', + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(failureOf(run)).toBeInstanceOf(TestFailureError); + expect(run.results.map((r) => [r.name, r.status, r.error?.kind])).toEqual([ + ["broken", "fail", "error"], + ["fine", "pass", undefined], + ]); + }); + + it("an unresolvable component import fails the test; later tests run", function* () { + const doc = [ + "", + '', + '', + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.results.map((r) => [r.name, r.status, r.error?.kind])).toEqual([ + ["missing", "fail", "error"], + ["fine", "pass", undefined], + ]); + }); + + it("nested fails the enclosing test; later tests run", function* () { + const doc = [ + "", + '', + '', + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.results.map((r) => [r.name, r.status, r.error?.kind])).toEqual([ + ["outer", "fail", "error"], + ["fine", "pass", undefined], + ]); + expect(run.results[0]?.error?.message).toContain("Nested "); + }); + + it("an error raised inside a component-owned region fails the test", function* () { + const comp = "\nregion start\n\n\n"; + const doc = [ + "", + '', + '', + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc, "components/Regioned.md": comp }); + expect(run.results.map((r) => [r.name, r.status])).toEqual([ + ["output-region", "fail"], + ["fine", "pass"], + ]); + }); + + it("a completed failing test inside a documentation region stays contained", function* () { + const comp = "\n\n\n"; + const doc = [ + "", + '', + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc, "components/Regioned.md": comp }); + // The document expands fully — the failure surfaces as a testing + // outcome, not an expansion abort. + expect(failureOf(run)).toBeInstanceOf(TestFailureError); + expect(run.results.map((r) => [r.name, r.status])).toEqual([["contained", "fail"]]); + }); + + it("bindings written in one test are invisible to the next", function* () { + const doc = [ + "```js eval", + "const inherited = 7;", + "```", + "", + '', + "```js eval", + "const leak = 1;", + "```", + 'captured', + "", + "", + '', + '', + "", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results.map((r) => r.status)).toEqual(["pass", "pass"]); + }); + + it("captures persist for the immediately following assertion", function* () { + const doc = [ + "", + 'Hello World', + '', + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results.map((r) => r.status)).toEqual(["pass"]); + }); + + it("effects spawned in one test are torn down before the next begins", function* () { + const globalValues = globalThis as Record; + delete globalValues.__testingLeaseAlive; + const doc = [ + "", + '', + "```js persist eval", + "globalThis.__testingLeaseAlive = true;", + "yield* spawn(function* () { try { yield* suspend(); } finally { globalThis.__testingLeaseAlive = false; } });", + "```", + "", + "", + '', + "", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.completion.ok).toBe(true); + expect(run.results.map((r) => [r.name, r.status])).toEqual([ + ["spawner", "pass"], + ["observer", "pass"], + ]); + }); + + it("output before a failure is kept, followed by the diagnostic", function* () { + const doc = [ + "", + "first line", + "", + "never rendered", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.output).toContain("first line"); + expect(run.output).toContain("**Assert** failed"); + expect(run.output).not.toContain("never rendered"); + const firstAt = run.output.indexOf("first line"); + const diagnosticAt = run.output.indexOf("**Assert** failed"); + expect(firstAt).toBeLessThan(diagnosticAt); + }); + + it("a hanging test times out, is torn down, and later tests run", function* () { + yield* API.Process.around({ + *exec(_args, _next) { + yield* sleep(10_000); + return { exitCode: 0, stdout: "", stderr: "" }; + }, + }); + const execute = createExecuteDocument({ + runDocument, + handlers: createTestHandlers({ timeoutMs: 100 }), + }); + const doc = [ + "", + '', + "```bash exec", + "sleep 60", + "```", + "", + '', + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }, { execute }); + expect(failureOf(run)).toBeInstanceOf(TestFailureError); + expect(run.results.map((r) => [r.name, r.status, r.error?.kind])).toEqual([ + ["hangs", "fail", "timeout"], + ["fine", "pass", undefined], + ]); + }); + + it("unnamed tests are identified by source location", function* () { + const doc = [ + "---", + "title: Located", + "---", + "", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(run.results[0]?.location).toBe("README.md:5:1"); + expect(run.output).toContain("test at README.md:5:1"); + }); +}); From 107c2fc87c8fb7f8d95a44ab85748cc09b752ae5 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:18:21 -0400 Subject: [PATCH 02/14] fix: address review findings on testing implementation - isolated environment now merges caller-projected bindings under the current environment (core's precedence), so tests projected through keep caller eval bindings; regression covers an eval block inside a projected test reading a caller binding - msg is validated as a string by type check alone, never formatted before the assertion runs; regression proves hostile toJSON/toString on a non-string msg cannot replace the outcome - replace prohibited 'as' assertions with a declared union array, filtered env construction, and Reflect.get/deleteProperty - drop two spec-restating comments; lint suppressions and lifecycle/ precedence comments retained --- testing/src/assertions.ts | 23 ++++++++++++++--------- testing/src/handlers.ts | 15 ++++++++++----- testing/tests/assertions.test.ts | 18 ++++++++++++++++++ testing/tests/cli.test.ts | 12 +++++++++++- testing/tests/execute.test.ts | 5 ++--- testing/tests/testing-mode.test.ts | 23 +++++++++++++++++++++-- 6 files changed, 76 insertions(+), 20 deletions(-) diff --git a/testing/src/assertions.ts b/testing/src/assertions.ts index 39a3c26..080d6e3 100644 --- a/testing/src/assertions.ts +++ b/testing/src/assertions.ts @@ -254,7 +254,12 @@ export function* expandAssertion( }; const resolved: Record = {}; - const resolutionOrder = ["expr", "actual", "expected", "msg"] as const; + const resolutionOrder: Array<"expr" | "actual" | "expected" | "msg"> = [ + "expr", + "actual", + "expected", + "msg", + ]; for (const name of resolutionOrder) { if (name in invocation.expressions) { try { @@ -274,18 +279,18 @@ export function* expandAssertion( } } + // @std/assert takes msg as a string; a non-string is rejected by TYPE + // CHECK alone — formatting it here would run hostile toJSON/toString + // side effects before the assertion outcome is established. + if ("msg" in resolved && typeof resolved.msg !== "string") { + return [validationError(assertion.name, 'requires "msg" to be a string when supplied.')]; + } + const values: ResolvedValues = { expr: resolved.expr, actual: resolved.actual, expected: resolved.expected, - // Guarded coercion: a hostile toString on a non-string msg must not - // become a new failure before the assertion runs. - msg: - "msg" in resolved - ? typeof resolved.msg === "string" - ? resolved.msg - : safeFormat(resolved.msg) - : undefined, + msg: typeof resolved.msg === "string" ? resolved.msg : undefined, }; if (hasChildren) { diff --git a/testing/src/handlers.ts b/testing/src/handlers.ts index cf058d1..54608a1 100644 --- a/testing/src/handlers.ts +++ b/testing/src/handlers.ts @@ -114,7 +114,6 @@ export function createTestHandlers(options: { timeoutMs: number }): TestHandlers }, { at: "min" }, ); - // The report is the naturally expanded subtree — no summary appended. const report = yield* ctx.expand(invocation.children); yield* boundary({ tests: local.length, @@ -129,8 +128,6 @@ export function createTestHandlers(options: { timeoutMs: number }): TestHandlers ctx: InvocationContext, ): Operation { if (!(yield* testing)) { - // Regular execution: the test and its entire body are skipped without - // output, bindings, or side effects. return []; } if (yield* inTest) { @@ -160,8 +157,16 @@ export function createTestHandlers(options: { timeoutMs: number }): TestHandlers // ONE stable binding environment, created before middleware install — // the accessor returns the same object on every read, so - // writes persist for the assertion that follows. - const testEnv: EvalEnv = { values: { ...(parentEnv?.values ?? {}) } }; + // writes persist for the assertion that follows. Caller-projected + // bindings merge UNDER the current environment (core's precedence, + // expand.ts §content projection), so a projected through + // still sees the caller's eval bindings. + const testEnv: EvalEnv = { + values: { + ...(ctx.projectedEnv?.values ?? {}), + ...(parentEnv?.values ?? {}), + }, + }; const testOutput: Segment[] = []; let bodyError: unknown; diff --git a/testing/tests/assertions.test.ts b/testing/tests/assertions.test.ts index 2caee6a..cf4c4e5 100644 --- a/testing/tests/assertions.test.ts +++ b/testing/tests/assertions.test.ts @@ -165,6 +165,24 @@ describe("assertion components", () => { expect(run.output).toContain("unformattable"); }); + it("a non-string msg is rejected by type check, never formatted", function* () { + // If msg were formatted before the assertion, the hostile toJSON or + // toString would throw and replace the validation outcome. + const doc = [ + "", + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc }); + expect(failureOf(run)).toBeInstanceOf(TestFailureError); + expect(run.results[0]?.status).toBe("fail"); + expect(run.results[0]?.error?.kind).toBe("error"); + expect(run.results[0]?.error?.message).toContain('"msg"'); + expect(run.results[0]?.error?.message).not.toContain("hostile-json"); + expect(run.results[0]?.error?.message).not.toContain("hostile-string"); + }); + it("a throwing getter read at format time cannot change the outcome", function* () { const doc = [ "", diff --git a/testing/tests/cli.test.ts b/testing/tests/cli.test.ts index f1e22dc..52657b9 100644 --- a/testing/tests/cli.test.ts +++ b/testing/tests/cli.test.ts @@ -20,11 +20,21 @@ interface CliResult { stderr: string; } +function cliEnv(): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if (typeof value === "string") { + env[key] = value; + } + } + return env; +} + function* runCli(args: string[]): Operation { const result = yield* timebox(TIMEOUT, function* () { const proc = yield* exec("deno", { arguments: ["run", "--allow-all", "cli/src/cli.ts", ...args], - env: process.env as Record, + env: cliEnv(), }); const stdoutChunks: string[] = []; diff --git a/testing/tests/execute.test.ts b/testing/tests/execute.test.ts index b130b47..9dea4b6 100644 --- a/testing/tests/execute.test.ts +++ b/testing/tests/execute.test.ts @@ -142,8 +142,7 @@ describe("executeDocument", () => { }); it("an early caller-scope halt tears down the document and leased eval scope", function* () { - const globalValues = globalThis as Record; - delete globalValues.__executeHaltMarker; + Reflect.deleteProperty(globalThis, "__executeHaltMarker"); yield* API.Process.around({ *exec(_args, _next) { yield* sleep(10_000); @@ -182,6 +181,6 @@ describe("executeDocument", () => { yield* sleep(200); }); - expect(globalValues.__executeHaltMarker).toBe(false); + expect(Reflect.get(globalThis, "__executeHaltMarker")).toBe(false); }); }); diff --git a/testing/tests/testing-mode.test.ts b/testing/tests/testing-mode.test.ts index c88912f..17c0c74 100644 --- a/testing/tests/testing-mode.test.ts +++ b/testing/tests/testing-mode.test.ts @@ -200,6 +200,26 @@ describe("testing mode", () => { expect(run.results.map((r) => r.status)).toEqual(["pass", "pass"]); }); + it("a projected through sees caller bindings in its eval blocks", function* () { + const doc = [ + "```js eval", + 'const callerValue = "from-caller";', + "```", + "", + '', + "```js eval", + 'const copied = callerValue + "!";', + "```", + '', + "", + "", + "", + ].join("\n"); + const run = yield* runDoc({ "README.md": doc, "components/Wrap.md": "\n" }); + expect(run.completion.ok).toBe(true); + expect(run.results.map((r) => [r.name, r.status])).toEqual([["projected", "pass"]]); + }); + it("captures persist for the immediately following assertion", function* () { const doc = [ "", @@ -214,8 +234,7 @@ describe("testing mode", () => { }); it("effects spawned in one test are torn down before the next begins", function* () { - const globalValues = globalThis as Record; - delete globalValues.__testingLeaseAlive; + Reflect.deleteProperty(globalThis, "__testingLeaseAlive"); const doc = [ "", '', From 06bbe0a4f2c2192d40aa4a1d7730a3923dfad546 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 21:58:52 -0400 Subject: [PATCH 03/14] refactor: settle the compositional execution architecture core: - rename runDocument to execute (no alias); ExecuteOptions; files renamed - deliver execute through a new test-agnostic Execution context Api - extensions decorate the execution lifecycle with Execution.around middleware instead of wrapping execution functions - DocumentExecution completes with Result: Ok(output) or Err(error); once a handle exists completion never throws - the whole spawned setup and workflow sit in one error boundary that closes output with complete-or-partial text and resolves Err - collect unwraps the Result (string on Ok, throw on Err) @executablemd/testing: - executeDocument removed; useTesting() is scope-local composition: vocabulary + collection + completion-policy middleware + root activation, returning a session whose results operation snapshots discovery order; one session per execution scope, enforced; middleware dies with the scope - vocabulary registration installs an Execution policy so explicit boundaries turn failures or empty boundaries into Err even when root testing is inactive - an existing core Err passes through unchanged; results stay available after failure cli: - xmd run: vocabulary registered, root testing inactive - xmd test: useTesting() composed around the same core execute call - both inspect the completion Result for exit status specs: executable-mdx-spec (execute, Execution Api, Result semantics), testing-spec (useTesting composition and completion semantics) --- cli/src/cli.ts | 36 ++-- core/components/AnthropicProvider.md | 62 +++--- core/components/Instruction.md | 22 +- core/components/LlamafileProvider.md | 53 ++--- core/components/OllamaProvider.md | 44 ++-- core/components/Sample.md | 18 +- core/mod.ts | 16 +- core/src/collect.ts | 18 +- core/src/deno-compiler.ts | 2 +- core/src/{run-document.ts => execute.ts} | 193 ++++++++++-------- core/src/temp-file-compiler.ts | 2 +- core/tests/daemon-integration.test.ts | 44 ++-- core/tests/eval-bindings.test.ts | 12 +- core/tests/eval-durable.test.ts | 20 +- core/tests/eval-persist.test.ts | 14 +- core/tests/eval-return.test.ts | 20 +- core/tests/eval-scope.test.ts | 6 +- core/tests/eval-timeout.test.ts | 6 +- .../{run-document.test.ts => execute.test.ts} | 115 +++++------ core/tests/expression-props.test.ts | 38 ++-- core/tests/find-free-port.test.ts | 18 +- core/tests/function-components.test.ts | 22 +- core/tests/named-slots.test.ts | 14 +- core/tests/provider-integration.test.ts | 48 ++--- core/tests/sample-component.test.ts | 50 ++--- core/tests/smoke.test.ts | 8 +- core/tests/source-position.test.ts | 6 +- core/tests/streaming-emission.test.ts | 10 +- specs/executable-mdx-spec.md | 91 +++++---- specs/testing-spec.md | 31 +++ testing/mod.ts | 20 +- testing/src/execute.ts | 139 ------------- testing/src/test-api.ts | 6 +- testing/src/use-testing.ts | 90 ++++++++ testing/src/vocabulary.ts | 72 ++++++- testing/tests/execute.test.ts | 186 ----------------- testing/tests/helpers.ts | 94 +++++---- testing/tests/testing-mode.test.ts | 11 +- testing/tests/use-testing.test.ts | 190 +++++++++++++++++ 39 files changed, 982 insertions(+), 865 deletions(-) rename core/src/{run-document.ts => execute.ts} (76%) rename core/tests/{run-document.test.ts => execute.test.ts} (93%) delete mode 100644 testing/src/execute.ts create mode 100644 testing/src/use-testing.ts delete mode 100644 testing/tests/execute.test.ts create mode 100644 testing/tests/use-testing.test.ts diff --git a/cli/src/cli.ts b/cli/src/cli.ts index 056b16c..ea3e720 100755 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -25,8 +25,8 @@ import { inspect } from "node:util"; import process from "node:process"; import { program, object, field, cli, commands, type Mods } from "configliere"; import { z } from "zod"; -import { useNormalizedOutput, useTerminalOutput } from "@executablemd/core"; -import { executeDocument, TestFailureError } from "@executablemd/testing"; +import { execute, useNormalizedOutput, useTerminalOutput } from "@executablemd/core"; +import { installTestingVocabulary, TestFailureError, useTesting } from "@executablemd/testing"; import { FileStream } from "./file-stream.ts"; import denoJson from "../deno.json" with { type: "json" }; @@ -217,7 +217,7 @@ function* run( // Output middleware (spec §9). // // Middleware is installed on the DocumentOutput Api via Api.around() before - // runDocument is called. runDocument owns the channel internally — + // execute is called. execute owns the output stream internally — // the CLI just installs transformations and consumes the returned stream. // --------------------------------------------------------------------------- @@ -229,15 +229,20 @@ function* run( yield* useTerminalOutput(); } - // Run the document through the testing wrapper — the vocabulary is - // registered for both commands (assertions work in regular documents), - // while testing MODE activates only for `xmd test`. - const execution = yield* executeDocument({ + // Compose testing around the single core execution entrypoint: both + // commands register the vocabulary (assertions work in regular documents, + // explicit boundaries affect the outcome), while `xmd test` + // additionally activates root testing through a useTesting() session. + if (mode.testing) { + yield* useTesting({ verbose }); + } else { + yield* installTestingVocabulary({ verbose }); + } + + const execution = yield* execute({ docPath, stream, componentDirs: componentDir, - testing: mode.testing, - verbose, }); // Consume the output stream with forEach. @@ -260,15 +265,14 @@ function* run( yield* writer; } - // Observe the execution's outcome AFTER the report finished streaming: + // Inspect the completion Result AFTER the report finished streaming: // test failures, assertion aborts, and any document abort exit nonzero. - try { - yield* execution; - } catch (error) { - if (error instanceof TestFailureError) { - console.error(`\ntests failed: ${error.message}`); + const result = yield* execution; + if (!result.ok) { + if (result.error instanceof TestFailureError) { + console.error(`\ntests failed: ${result.error.message}`); } else { - console.error(error instanceof Error ? error.message : String(error)); + console.error(result.error.message); } yield* exit(1); } diff --git a/core/components/AnthropicProvider.md b/core/components/AnthropicProvider.md index af0d0fc..4915ae2 100644 --- a/core/components/AnthropicProvider.md +++ b/core/components/AnthropicProvider.md @@ -13,38 +13,42 @@ inputs: --- ```ts persist eval -yield* Sample.around({ - *sample([context], next) { - if (context.model !== undefined && context.model !== model) { - return yield* next(context); - } +yield * + Sample.around( + { + *sample([context], next) { + if (context.model !== undefined && context.model !== model) { + return yield* next(context); + } - const messages = []; - if (context.system) { - messages.push({ role: "system", content: context.system }); - } - messages.push({ role: "user", content: context.content }); + const messages = []; + if (context.system) { + messages.push({ role: "system", content: context.system }); + } + messages.push({ role: "user", content: context.content }); - const result = yield* fetch("https://api.anthropic.com/v1/messages", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-api-key": process.env.ANTHROPIC_API_KEY, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 4096, - system: context.system || undefined, - messages: [{ role: "user", content: context.content }], - }), - }) - .expect() - .json(); + const result = yield* fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": process.env.ANTHROPIC_API_KEY, + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model, + max_tokens: 4096, + system: context.system || undefined, + messages: [{ role: "user", content: context.content }], + }), + }) + .expect() + .json(); - return result.content[0].text; - }, -}, { at: 'min' }); + return result.content[0].text; + }, + }, + { at: "min" }, + ); ``` diff --git a/core/components/Instruction.md b/core/components/Instruction.md index f459537..8c35b5d 100644 --- a/core/components/Instruction.md +++ b/core/components/Instruction.md @@ -15,15 +15,19 @@ inputs: --- ```js persist eval -yield* Sample.around({ - *sample([context], next) { - const existing = context.system || ''; - return yield* next({ - ...context, - system: existing ? existing + '\n' + system : system, - }); - }, -}, { at: 'min' }); +yield * + Sample.around( + { + *sample([context], next) { + const existing = context.system || ""; + return yield* next({ + ...context, + system: existing ? existing + "\n" + system : system, + }); + }, + }, + { at: "min" }, + ); ``` diff --git a/core/components/LlamafileProvider.md b/core/components/LlamafileProvider.md index db4aa99..4378ea2 100644 --- a/core/components/LlamafileProvider.md +++ b/core/components/LlamafileProvider.md @@ -22,7 +22,7 @@ inputs: --- ```ts eval -const port = yield* findFreePort(); +const port = yield * findFreePort(); const baseUrl = `http://127.0.0.1:${port}`; ``` @@ -31,35 +31,40 @@ const baseUrl = `http://127.0.0.1:${port}`; ``` ```ts eval -yield* when(function* () { - yield* fetch(`${baseUrl}/health`).expect(); -}); +yield * + when(function* () { + yield* fetch(`${baseUrl}/health`).expect(); + }); ``` ```ts persist eval -yield* Sample.around({ - *sample([context], next) { - if (context.model !== undefined && context.model !== model) { - return yield* next(context); - } +yield * + Sample.around( + { + *sample([context], next) { + if (context.model !== undefined && context.model !== model) { + return yield* next(context); + } - const messages = []; - if (context.system) { - messages.push({ role: "system", content: context.system }); - } - messages.push({ role: "user", content: context.content }); + const messages = []; + if (context.system) { + messages.push({ role: "system", content: context.system }); + } + messages.push({ role: "user", content: context.content }); - const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model, messages, temperature: 0, max_tokens: 2048 }), - }) - .expect() - .json(); + const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model, messages, temperature: 0, max_tokens: 2048 }), + }) + .expect() + .json(); - return result.choices[0].message.content; - }, -}, { at: 'min' }); + return result.choices[0].message.content; + }, + }, + { at: "min" }, + ); ``` diff --git a/core/components/OllamaProvider.md b/core/components/OllamaProvider.md index e69f845..3a344e0 100644 --- a/core/components/OllamaProvider.md +++ b/core/components/OllamaProvider.md @@ -19,29 +19,33 @@ inputs: --- ```ts persist eval -yield* Sample.around({ - *sample([context], next) { - if (context.model !== undefined && context.model !== model) { - return yield* next(context); - } +yield * + Sample.around( + { + *sample([context], next) { + if (context.model !== undefined && context.model !== model) { + return yield* next(context); + } - const messages = []; - if (context.system) { - messages.push({ role: "system", content: context.system }); - } - messages.push({ role: "user", content: context.content }); + const messages = []; + if (context.system) { + messages.push({ role: "system", content: context.system }); + } + messages.push({ role: "user", content: context.content }); - const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model, messages, temperature: 0 }), - }) - .expect() - .json(); + const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model, messages, temperature: 0 }), + }) + .expect() + .json(); - return result.choices[0].message.content; - }, -}, { at: 'min' }); + return result.choices[0].message.content; + }, + }, + { at: "min" }, + ); ``` diff --git a/core/components/Sample.md b/core/components/Sample.md index 0989638..a8f25d1 100644 --- a/core/components/Sample.md +++ b/core/components/Sample.md @@ -30,15 +30,17 @@ inputs: --- ```js persist eval -const childrenOutput = yield* renderChildren(); -const content = childrenOutput || prompt || ''; +const childrenOutput = yield * renderChildren(); +const content = childrenOutput || prompt || ""; -const sampleResult = yield* Sample.operations.sample({ - content, - params: params || undefined, - componentName: 'Sample', - model: model || undefined, -}); +const sampleResult = + yield * + Sample.operations.sample({ + content, + params: params || undefined, + componentName: "Sample", + model: model || undefined, + }); return sampleResult; ``` diff --git a/core/mod.ts b/core/mod.ts index 2529239..030d68f 100644 --- a/core/mod.ts +++ b/core/mod.ts @@ -37,11 +37,7 @@ export { healSegment } from "./src/heal.ts"; export type { Middleware } from "@effectionx/middleware"; export { combine } from "@effectionx/middleware"; -export type { - ModifierFactory, - ModifierMiddleware, - CodeBlockWorkflow, -} from "./src/modifiers.ts"; +export type { ModifierFactory, ModifierMiddleware, CodeBlockWorkflow } from "./src/modifiers.ts"; export { useCodeBlock } from "./src/modifiers.ts"; // --------------------------------------------------------------------------- @@ -108,11 +104,7 @@ export { interpolateEvalBindings } from "./src/eval-interpolate.ts"; export { findFreePort } from "@executablemd/runtime"; export type { TransformResult } from "./src/eval-transform.ts"; -export { - transformBlock, - serializeExports, - isJson, -} from "./src/eval-transform.ts"; +export { transformBlock, serializeExports, isJson } from "./src/eval-transform.ts"; // --------------------------------------------------------------------------- // Output Api & middleware @@ -127,8 +119,8 @@ export { useTerminalOutput } from "./src/output/terminal.ts"; // Document runner // --------------------------------------------------------------------------- -export { runDocument } from "./src/run-document.ts"; -export type { RunDocumentOptions, DocumentExecution } from "./src/run-document.ts"; +export { execute, Execution } from "./src/execute.ts"; +export type { ExecuteOptions, ExecutionApi, DocumentExecution } from "./src/execute.ts"; export { useDenoCompiler } from "./src/deno-compiler.ts"; export { useTempFileCompiler } from "./src/temp-file-compiler.ts"; diff --git a/core/src/collect.ts b/core/src/collect.ts index 2e1a8d7..508a239 100644 --- a/core/src/collect.ts +++ b/core/src/collect.ts @@ -1,24 +1,28 @@ /** * collect — wait for a document execution to complete and return the output. * - * Convenience wrapper that `yield*`s the execution to get the full - * output string. Equivalent to `yield* execution` but reads more - * clearly at call sites. + * Convenience wrapper that unwraps the completion `Result`: + * the output string on `Ok`, a throw on `Err`. * * ```ts - * const output = yield* collect(yield* runDocument(options)); + * const output = yield* collect(yield* execute(options)); * ``` */ import type { Operation } from "effection"; -import type { DocumentExecution } from "./run-document.ts"; +import type { DocumentExecution } from "./execute.ts"; /** * Wait for a document execution to complete and return the full output. + * Throws the failure when the execution completed with `Err`. * - * @param execution - A `DocumentExecution` as returned by `runDocument`. + * @param execution - A `DocumentExecution` as returned by `execute`. * @returns The full output string. */ export function* collect(execution: DocumentExecution): Operation { - return yield* execution; + const result = yield* execution; + if (!result.ok) { + throw result.error; + } + return result.value; } diff --git a/core/src/deno-compiler.ts b/core/src/deno-compiler.ts index e269d6e..d1dbfb7 100644 --- a/core/src/deno-compiler.ts +++ b/core/src/deno-compiler.ts @@ -28,7 +28,7 @@ const STANDARD_IMPORTS = [ /** * Install the Deno data: URI compiler as middleware on the current scope. * - * Must be called inside an Effection scope (e.g., inside `runDocument`'s + * Must be called inside an Effection scope (e.g., inside `execute`'s * spawned task) before any eval blocks execute. */ export function* useDenoCompiler(): Operation { diff --git a/core/src/run-document.ts b/core/src/execute.ts similarity index 76% rename from core/src/run-document.ts rename to core/src/execute.ts index 34b1405..2badf17 100644 --- a/core/src/run-document.ts +++ b/core/src/execute.ts @@ -1,15 +1,18 @@ /** - * Entry point — runDocument (spec §7). + * Entry point — execute (spec §7). * * Wires together the boundary scanner, component import, expansion engine, - * modifier system, and journal infrastructure. + * modifier system, and journal infrastructure. `execute` is delivered + * through the Execution Api so extensions can decorate the execution + * lifecycle as middleware. * * Journal runtime integration is concentrated at execution boundaries. * See DEC-005 in specs/decisions.md. */ -import { scoped, spawn, withResolvers, until } from "effection"; -import type { Operation, Stream } from "effection"; +import { Err, Ok, scoped, spawn, withResolvers, until } from "effection"; +import type { Operation, Result, Stream } from "effection"; +import { createApi } from "@effectionx/context-api"; import { durableRun, createDurableOperation, @@ -59,10 +62,10 @@ import { Stdio } from "@effectionx/process"; import matter from "gray-matter"; // --------------------------------------------------------------------------- -// runDocument options (spec §7.1) +// execute options (spec §7.1) // --------------------------------------------------------------------------- -export interface RunDocumentOptions { +export interface ExecuteOptions { /** Path to the root markdown document (workspace-relative). */ docPath: string; @@ -273,7 +276,7 @@ const silentFactory: ModifierFactory = (_params) => (_args, next) => function* documentWorkflow(): Workflow { // Import root — same pipeline as any component. The provider middleware - // installed by runDocument maps "__root__" to the document path. The + // installed by execute maps "__root__" to the document path. The // ephemeral() wrapper bridges typing only — the import inside remains a // durable, journaled operation. const root = yield* ephemeral(importComponent("__root__")); @@ -353,58 +356,55 @@ function* documentWorkflow(): Workflow { } // --------------------------------------------------------------------------- -// runDocument (spec §7.1) +// execute (spec §7.1) // --------------------------------------------------------------------------- /** * A running document execution. * - * `yield* execution` waits for the workflow to complete and returns - * the full output string (or throws on error). + * `yield* execution` waits for completion and returns a `Result`: + * `Ok(output)` on success, `Err(error)` on document, infrastructure, or + * policy failure. Completion never throws once the handle exists. * - * `execution.output` is a `Stream` for consuming - * chunks as they arrive. The close value is the full output. + * `execution.output` is a `Stream` for consuming chunks as + * they arrive. The close value is the complete rendered output — or the + * partial output rendered before a failure. */ -export interface DocumentExecution extends Operation { - /** Stream of output chunks. Close value is the full output. */ +export interface DocumentExecution extends Operation> { + /** Stream of output chunks. Close value is the full (or partial) output. */ output: Stream; } /** * Execute a markdown document as a durable workflow. * - * Returns a `DocumentExecution` — an operation you can `yield*` to - * get the full output, with a `.output` stream for chunk-by-chunk - * consumption. + * Returns a `DocumentExecution` — an operation you can `yield*` for the + * completion `Result`, with a `.output` stream for chunk-by-chunk + * consumption. Once the handle exists, completion never throws: every + * later failure — document, infrastructure, or policy middleware — closes + * `output` (with the complete or partial rendered text) and resolves + * `Err(error)`. * - * Simple — just get the output: + * Simple — get the outcome: * * ```ts - * const execution = yield* runDocument(options); - * const output = yield* execution; + * const execution = yield* execute(options); + * const result = yield* execution; + * if (result.ok) { + * console.log(result.value); + * } * ``` * * Streaming — consume chunks as they arrive: * * ```ts - * const execution = yield* runDocument(options); + * const execution = yield* execute(options); * const output = yield* forEach(function* (chunk) { * process.stdout.write(chunk); * }, execution.output); * ``` - * - * Error handling: - * - * ```ts - * const execution = yield* runDocument(options); - * try { - * const output = yield* execution; - * } catch (e) { - * // workflow error - * } - * ``` */ -export function* runDocument(options: RunDocumentOptions): Operation { +function* executeDocument(options: ExecuteOptions): Operation { const { docPath, stream, @@ -440,64 +440,66 @@ export function* runDocument(options: RunDocumentOptions): Operation(); - const { operation, resolve, reject } = withResolvers(); + const { operation, resolve } = withResolvers>(); yield* spawn(function* () { let emitted = false; let emittedText = ""; - // Install platform-appropriate compiler middleware - // deno-lint-ignore no-explicit-any - if (typeof (globalThis as any).Deno !== "undefined") { - yield* useDenoCompiler(); - } else { - yield* useTempFileCompiler(); - } + // The ENTIRE setup and workflow sit inside one error boundary: once the + // handle exists, any failure closes output (with whatever rendered so + // far) and resolves Err — completion never throws. + try { + // Install platform-appropriate compiler middleware + // deno-lint-ignore no-explicit-any + if (typeof (globalThis as any).Deno !== "undefined") { + yield* useDenoCompiler(); + } else { + yield* useTempFileCompiler(); + } - // DocumentOutput → channel bridge (innermost middleware — output flows - // through caller-installed normalize/terminal middleware first, then here). - yield* DocumentOutput.around({ - *output([text]) { - emitted = true; - emittedText += text; - yield* channel.send(text); - }, - }); - - yield* Stdio.around({ - *stdout() {}, - *stderr() {}, - }); - - // Create per-document eval scope (spec §3.1). - // Created in the same scope as durableRun so that DurableCtx - // (set by durableRun) is visible to eval code that calls - // renderChildren → importComponent → createDurableOperation. - const rootEvalScope = yield* useEvalScope(); - - // Install the document's runtime Component providers before durableRun - // so the workflow inherits them: component import, modifier execution, - // and the root eval scope. - yield* Component.around( - { - *importComponent([name], _next) { - return yield* durableImportComponent( - name, - name === "__root__" ? docPath : undefined, - componentDirs, - ); + // DocumentOutput → channel bridge (innermost middleware — output flows + // through caller-installed normalize/terminal middleware first, then here). + yield* DocumentOutput.around({ + *output([text]) { + emitted = true; + emittedText += text; + yield* channel.send(text); }, - *applyModifiers([modifiers, context], _next) { - const chain = composeModifierChain(modifiers, context, registry); - return yield* chain(); + }); + + yield* Stdio.around({ + *stdout() {}, + *stderr() {}, + }); + + // Create per-document eval scope (spec §3.1). + // Created in the same scope as durableRun so that DurableCtx + // (set by durableRun) is visible to eval code that calls + // renderChildren → importComponent → createDurableOperation. + const rootEvalScope = yield* useEvalScope(); + + // Install the document's runtime Component providers before durableRun + // so the workflow inherits them: component import, modifier execution, + // and the root eval scope. + yield* Component.around( + { + *importComponent([name], _next) { + return yield* durableImportComponent( + name, + name === "__root__" ? docPath : undefined, + componentDirs, + ); + }, + *applyModifiers([modifiers, context], _next) { + const chain = composeModifierChain(modifiers, context, registry); + return yield* chain(); + }, + evalScope: () => rootEvalScope, }, - evalScope: () => rootEvalScope, - }, - { at: "min" }, - ); + { at: "min" }, + ); - // Run the durable workflow. - try { const output = yield* durableRun(() => documentWorkflow(), { stream, }); @@ -509,12 +511,12 @@ export function* runDocument(options: RunDocumentOptions): Operation; +} + +export const Execution = createApi("Execution", { + *execute(options: ExecuteOptions): Operation { + return yield* executeDocument(options); + }, +}); + +export const { execute } = Execution.operations; diff --git a/core/src/temp-file-compiler.ts b/core/src/temp-file-compiler.ts index 06aa015..63ccc79 100644 --- a/core/src/temp-file-compiler.ts +++ b/core/src/temp-file-compiler.ts @@ -8,7 +8,7 @@ * Standard imports (Effection, executable.md APIs) are captured in the middleware * closure — they are not part of the Compiler API interface. * - * Installed automatically by `runDocument` when running on Node or Bun. + * Installed automatically by `execute` when running on Node or Bun. */ import { call } from "effection"; diff --git a/core/tests/daemon-integration.test.ts b/core/tests/daemon-integration.test.ts index 316a057..56d1ff1 100644 --- a/core/tests/daemon-integration.test.ts +++ b/core/tests/daemon-integration.test.ts @@ -9,7 +9,7 @@ * evalScope.eval(). Tests must NOT rely on the daemon having written * to the filesystem before the next sequential block runs — that is a * race condition. Instead, tests verify deterministic properties: - * - runDocument completes (proves daemon cleanup works) + * - execute completes (proves daemon cleanup works) * - journal shape (no daemon entry) * - output shape (empty output from daemon blocks) * - error propagation via component-scoped daemon + children @@ -18,7 +18,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { race, sleep } from "effection"; import { InMemoryStream } from "@executablemd/durable-streams"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import * as fs from "node:fs"; import * as path from "node:path"; @@ -51,9 +51,9 @@ function writeFiles(dir: string, files: Record): void { describe("Tier Q — Daemon integration", () => { // Q4/Q5: daemon forked into eval scope, process cleaned up on completion - // Verified by: runDocument completes without hanging (daemon terminated), + // Verified by: execute completes without hanging (daemon terminated), // and the daemon block produces no output. - it("Q4/Q5: daemon process forked and cleaned up — runDocument completes", function* () { + it("Q4/Q5: daemon process forked and cleaned up — execute completes", function* () { const tmpDir = makeTempDir(); try { @@ -63,13 +63,13 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), ); - // runDocument completed — daemon was terminated when scope closed. + // execute completed — daemon was terminated when scope closed. // If daemon wasn't cleaned up, this test would hang for 300s. expect(output).toContain("before"); expect(output).toContain("after"); @@ -91,7 +91,7 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -118,7 +118,7 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -141,9 +141,9 @@ describe("Tier Q — Daemon integration", () => { // evalScope.eval(), the DaemonExitError propagates through the eval // scope. At the root document level (no component wrapper), subsequent // blocks may or may not see the error depending on timing. - // Test: daemon that exits immediately still allows runDocument to complete, + // Test: daemon that exits immediately still allows execute to complete, // and the output contains some indication (error or normal completion). - it("Q8: premature daemon exit — runDocument completes without hanging", function* () { + it("Q8: premature daemon exit — execute completes without hanging", function* () { const tmpDir = makeTempDir(); try { @@ -153,13 +153,13 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), ); - // The key property: runDocument completes without hanging. + // The key property: execute completes without hanging. // The daemon exited immediately. The output should contain // surrounding text — the daemon's error may or may not appear // depending on the race between daemon exit and block processing. @@ -196,13 +196,13 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), ); - // runDocument completed successfully — the daemon received a valid + // execute completed successfully — the daemon received a valid // interpolated command. If interpolation failed, {marker} would be // passed verbatim, but the command would still be valid bash. // The key test: no ERROR in output (daemon started successfully). @@ -232,7 +232,7 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -241,7 +241,7 @@ describe("Tier Q — Daemon integration", () => { // The eval block error should appear in output expect(output).toContain("intentional error"); - // runDocument completed without hanging — daemon was cleaned up + // execute completed without hanging — daemon was cleaned up // by structured concurrency when the scope closed. } finally { cleanup(tmpDir); @@ -272,7 +272,7 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); // Golden run const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -284,7 +284,7 @@ describe("Tier Q — Daemon integration", () => { // Replay — eval block replays from journal (restoring tag to env.values), // daemon spawns a fresh process with the restored interpolated value. const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -322,12 +322,12 @@ describe("Tier Q — Daemon integration", () => { const stream = new InMemoryStream(); - // Race runDocument against a short sleep. The sleep wins, - // cancelling the runDocument scope (and the daemon within it). + // Race execute against a short sleep. The sleep wins, + // cancelling the execute scope (and the daemon within it). // If daemon cleanup is broken, race would hang for 300s. const result = yield* race([ collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -335,7 +335,7 @@ describe("Tier Q — Daemon integration", () => { sleep(500), ]); - // sleep(500) returns void, collect(runDocument) returns string. + // sleep(500) returns void, collect(execute) returns string. // If sleep won (expected), result is undefined. // Either way, the test passed — race resolved without hanging. expect(true).toBe(true); diff --git a/core/tests/eval-bindings.test.ts b/core/tests/eval-bindings.test.ts index 2e9018e..cb6d10f 100644 --- a/core/tests/eval-bindings.test.ts +++ b/core/tests/eval-bindings.test.ts @@ -8,7 +8,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs, useEchoExec } from "@executablemd/runtime/test"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; describe("Tier T5 — Binding environment", () => { @@ -23,7 +23,7 @@ describe("Tier T5 — Binding environment", () => { // Should not throw — port is available in block 2 const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -46,7 +46,7 @@ describe("Tier T5 — Binding environment", () => { // Should succeed — x is 2 in block 3 (shadowed by block 2) const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -64,7 +64,7 @@ describe("Tier T5 — Binding environment", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -82,7 +82,7 @@ describe("Tier T5 — Binding environment", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -100,7 +100,7 @@ describe("Tier T5 — Binding environment", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), diff --git a/core/tests/eval-durable.test.ts b/core/tests/eval-durable.test.ts index 17e53e3..65f96c9 100644 --- a/core/tests/eval-durable.test.ts +++ b/core/tests/eval-durable.test.ts @@ -10,7 +10,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs, useEchoExec } from "@executablemd/runtime/test"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; describe("Tier T4 — eval factory and journal integration", () => { @@ -24,7 +24,7 @@ describe("Tier T4 — eval factory and journal integration", () => { yield* useEchoExec(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -46,7 +46,7 @@ describe("Tier T4 — eval factory and journal integration", () => { // Golden run const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -54,7 +54,7 @@ describe("Tier T4 — eval factory and journal integration", () => { // Replay const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -73,7 +73,7 @@ describe("Tier T4 — eval factory and journal integration", () => { yield* useEchoExec(); const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -81,7 +81,7 @@ describe("Tier T4 — eval factory and journal integration", () => { // Replay const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -100,7 +100,7 @@ describe("Tier T4 — eval factory and journal integration", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -120,7 +120,7 @@ describe("Tier T4 — eval factory and journal integration", () => { yield* useEchoExec(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -140,7 +140,7 @@ describe("Tier T4 — eval factory and journal integration", () => { yield* useEchoExec(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -171,7 +171,7 @@ describe("Tier T4 — eval factory and journal integration", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), diff --git a/core/tests/eval-persist.test.ts b/core/tests/eval-persist.test.ts index 67f9a73..acea2ed 100644 --- a/core/tests/eval-persist.test.ts +++ b/core/tests/eval-persist.test.ts @@ -11,7 +11,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs, useEchoExec } from "@executablemd/runtime/test"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import { unbox } from "effection"; import type { Result } from "effection"; @@ -26,7 +26,7 @@ describe("Tier T6 — persist modifier", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -45,7 +45,7 @@ describe("Tier T6 — persist modifier", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -65,7 +65,7 @@ describe("Tier T6 — persist modifier", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -84,14 +84,14 @@ describe("Tier T6 — persist modifier", () => { yield* useEchoExec(); const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), ); const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -145,7 +145,7 @@ describe("Tier T6 — persist modifier", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), diff --git a/core/tests/eval-return.test.ts b/core/tests/eval-return.test.ts index 1621350..f36058c 100644 --- a/core/tests/eval-return.test.ts +++ b/core/tests/eval-return.test.ts @@ -9,7 +9,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs, useEchoExec } from "@executablemd/runtime/test"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; describe("Eval block return value", () => { @@ -22,7 +22,7 @@ describe("Eval block return value", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -40,7 +40,7 @@ describe("Eval block return value", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -58,7 +58,7 @@ describe("Eval block return value", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -77,7 +77,7 @@ describe("Eval block return value", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -95,7 +95,7 @@ describe("Eval block return value", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -114,7 +114,7 @@ describe("Eval block return value", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -132,7 +132,7 @@ describe("Eval block return value", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -152,7 +152,7 @@ describe("Eval block return value", () => { // Golden run const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -162,7 +162,7 @@ describe("Eval block return value", () => { // Replay const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), diff --git a/core/tests/eval-scope.test.ts b/core/tests/eval-scope.test.ts index 09b7628..975e5e9 100644 --- a/core/tests/eval-scope.test.ts +++ b/core/tests/eval-scope.test.ts @@ -8,7 +8,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs, useEchoExec } from "@executablemd/runtime/test"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; describe("Tier T10 — eval-scope hierarchy", () => { @@ -21,7 +21,7 @@ describe("Tier T10 — eval-scope hierarchy", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -43,7 +43,7 @@ describe("Tier T10 — eval-scope hierarchy", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), diff --git a/core/tests/eval-timeout.test.ts b/core/tests/eval-timeout.test.ts index afe8378..80b724a 100644 --- a/core/tests/eval-timeout.test.ts +++ b/core/tests/eval-timeout.test.ts @@ -8,7 +8,7 @@ import { expect } from "@effectionx/bdd/expect"; import { parseDuration } from "../src/modifiers/timeout.ts"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs, useEchoExec } from "@executablemd/runtime/test"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; describe("Tier T7 — timeout modifier", () => { @@ -21,7 +21,7 @@ describe("Tier T7 — timeout modifier", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -62,7 +62,7 @@ describe("Tier T7 — timeout modifier", () => { // Should work with default timeout const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), diff --git a/core/tests/run-document.test.ts b/core/tests/execute.test.ts similarity index 93% rename from core/tests/run-document.test.ts rename to core/tests/execute.test.ts index df92247..ebcc049 100644 --- a/core/tests/run-document.test.ts +++ b/core/tests/execute.test.ts @@ -1,5 +1,5 @@ /** - * Integration tests for runDocument (Tiers B, D, E from spec §11). + * Integration tests for execute (Tiers B, D, E from spec §11). * * Uses API.*.around() middleware for isolation * and InMemoryStream from @executablemd/durable-streams for journaling. @@ -15,7 +15,7 @@ import { useStubFs, useFailingExec } from "@executablemd/runtime/test"; import { API } from "@executablemd/runtime"; import { forEach } from "@effectionx/stream-helpers"; import type { Operation } from "effection"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; // --------------------------------------------------------------------------- @@ -54,7 +54,7 @@ describe("Tier B — durable import", () => { yield* useStubExec(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -88,7 +88,7 @@ describe("Tier B — durable import", () => { // Golden run const firstResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -96,7 +96,7 @@ describe("Tier B — durable import", () => { // Replay — middleware is in scope but durable stream replays from journal const secondResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -114,14 +114,14 @@ describe("Tier B — durable import", () => { yield* useStubExec(); const firstResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), ); const secondResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -138,7 +138,7 @@ describe("Tier B — durable import", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -157,7 +157,7 @@ describe("Tier B — durable import", () => { yield* useStubExec(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: "doc.md", stream, }), @@ -186,7 +186,7 @@ describe("Tier B — durable import", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -221,7 +221,7 @@ describe("Tier B — durable import", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -245,7 +245,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -273,7 +273,7 @@ describe("Tier D — code execution and modifiers", () => { // Golden run const firstResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -281,7 +281,7 @@ describe("Tier D — code execution and modifiers", () => { // Replay — middleware in scope, durable stream replays from journal const secondResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -298,7 +298,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useFailingExec(1, "command not found"); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -317,7 +317,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useStubExec(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -355,7 +355,7 @@ describe("Tier D — code execution and modifiers", () => { }); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -382,7 +382,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -407,7 +407,7 @@ describe("Tier D — code execution and modifiers", () => { // Golden const firstResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -415,7 +415,7 @@ describe("Tier D — code execution and modifiers", () => { // Replay — durable stream replays from journal const secondResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -434,7 +434,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -453,7 +453,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -473,7 +473,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, modifiers: { @@ -504,7 +504,7 @@ describe("Tier D — code execution and modifiers", () => { yield* useStubExec(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, modifiers: { @@ -524,7 +524,7 @@ describe("Tier D — code execution and modifiers", () => { // Tier E — End-to-end tests // --------------------------------------------------------------------------- -describe("runDocument", () => { +describe("execute", () => { // E1: Full document golden run it("E1: full document golden run — root + component + exec", function* () { const stream = new InMemoryStream(); @@ -558,7 +558,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -597,7 +597,7 @@ describe("runDocument", () => { // First run — golden const firstResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -605,7 +605,7 @@ describe("runDocument", () => { // Replay — durable stream replays from journal, middleware not invoked const secondResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -633,7 +633,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -652,7 +652,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -674,7 +674,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -698,7 +698,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -726,7 +726,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -770,7 +770,7 @@ describe("runDocument", () => { // Golden run to get full journal yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -790,7 +790,7 @@ describe("runDocument", () => { execCalled = false; const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream: partialStream, }), @@ -814,7 +814,7 @@ describe("runDocument", () => { // Golden run with just Header yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -827,7 +827,7 @@ describe("runDocument", () => { const newStream = new InMemoryStream(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream: newStream, }), @@ -847,7 +847,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -899,7 +899,7 @@ describe("runDocument", () => { // Golden run const firstResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -910,7 +910,7 @@ describe("runDocument", () => { // Replay — durable stream replays from journal const secondResult = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -936,7 +936,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -964,7 +964,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -992,7 +992,7 @@ describe("runDocument", () => { yield* useStubExec(); const result = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "README.md", stream, }), @@ -1017,7 +1017,7 @@ describe("Component Api dispatch — journal shape", () => { }); yield* useStubExec(); - const output = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + const output = yield* collect(yield* execute({ docPath: "README.md", stream })); expect(output).toContain("note!"); const events = stream.snapshot(); @@ -1052,7 +1052,7 @@ describe("component-declared output — document workflow", () => { "components/Warn.md": "Docs line.\n\n\nWARNING_SHOWN\n\n", }); yield* useStubExec(); - const output = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + const output = yield* collect(yield* execute({ docPath: "README.md", stream })); expect(output).toContain("Intro paragraph."); expect(output).toContain("WARNING_SHOWN"); expect(output).not.toContain("Docs line"); @@ -1061,7 +1061,7 @@ describe("component-declared output — document workflow", () => { it("applies to a root document the same way", function* () { const stream = new InMemoryStream(); yield* useStubFs({ "README.md": "Root docs.\n\n\nROOT_SHOWN\n\n" }); - const output = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + const output = yield* collect(yield* execute({ docPath: "README.md", stream })); expect(output).toContain("ROOT_SHOWN"); expect(output).not.toContain("Root docs"); }); @@ -1070,7 +1070,7 @@ describe("component-declared output — document workflow", () => { const stream = new InMemoryStream(); yield* useStubFs({ "README.md": "Root docs.\n\n\nROOT_SHOWN\n\n" }); - const execution = yield* runDocument({ docPath: "README.md", stream }); + const execution = yield* execute({ docPath: "README.md", stream }); const chunks: string[] = []; const full = yield* forEach(function* (chunk: string) { chunks.push(chunk); @@ -1087,19 +1087,14 @@ describe("component-declared output — document workflow", () => { }); yield* useFailingExec(1, "command not found"); - const execution = yield* runDocument({ docPath: "README.md", stream }); + const execution = yield* execute({ docPath: "README.md", stream }); const chunks: string[] = []; yield* forEach(function* (chunk: string) { chunks.push(chunk); }, execution.output); - let threw = false; - try { - yield* execution; - } catch { - threw = true; - } - expect(threw).toBe(true); + const result = yield* execution; + expect(result.ok).toBe(false); expect(chunks.join("")).not.toContain("SELECTED"); }); @@ -1110,7 +1105,7 @@ describe("component-declared output — document workflow", () => { }); yield* useStubExec(); - const execution = yield* runDocument({ docPath: "README.md", stream }); + const execution = yield* execute({ docPath: "README.md", stream }); const chunks: string[] = []; yield* forEach(function* (chunk: string) { chunks.push(chunk); @@ -1123,7 +1118,7 @@ describe("component-declared output — document workflow", () => { const stream = new InMemoryStream(); yield* useStubFs({ "README.md": "Root docs.\n\n\n" }); - const execution = yield* runDocument({ docPath: "README.md", stream }); + const execution = yield* execute({ docPath: "README.md", stream }); const chunks: string[] = []; const full = yield* forEach(function* (chunk: string) { chunks.push(chunk); @@ -1137,8 +1132,8 @@ describe("component-declared output — document workflow", () => { const stream = new InMemoryStream(); yield* useStubFs({ "README.md": "Root docs.\n\n\nROOT_SHOWN\n\n" }); - const first = yield* collect(yield* runDocument({ docPath: "README.md", stream })); - const second = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + const first = yield* collect(yield* execute({ docPath: "README.md", stream })); + const second = yield* collect(yield* execute({ docPath: "README.md", stream })); expect(second).toBe(first); expect(second).toContain("ROOT_SHOWN"); @@ -1151,7 +1146,7 @@ describe("component-declared output — document workflow", () => { }); yield* useFailingExec(1, "should not run"); - const result = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + const result = yield* collect(yield* execute({ docPath: "README.md", stream })); expect(result).toContain("must be a direct top-level"); diff --git a/core/tests/expression-props.test.ts b/core/tests/expression-props.test.ts index d47149e..bd98f0d 100644 --- a/core/tests/expression-props.test.ts +++ b/core/tests/expression-props.test.ts @@ -8,7 +8,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { parseExpressionValue, scanSegments } from "../src/scanner.ts"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import { InMemoryStream } from "@executablemd/durable-streams"; import * as fs from "node:fs"; @@ -169,7 +169,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -206,7 +206,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -241,7 +241,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -277,7 +277,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -307,7 +307,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -336,7 +336,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -367,7 +367,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -403,7 +403,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -433,7 +433,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -463,7 +463,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -504,7 +504,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -537,7 +537,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -569,7 +569,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -608,7 +608,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -640,14 +640,14 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], }), ); const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -687,7 +687,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -726,7 +726,7 @@ describe("Tier EP — Expression prop evaluation", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], diff --git a/core/tests/find-free-port.test.ts b/core/tests/find-free-port.test.ts index 91f0020..798bfee 100644 --- a/core/tests/find-free-port.test.ts +++ b/core/tests/find-free-port.test.ts @@ -14,7 +14,7 @@ import { InMemoryStream } from "@executablemd/durable-streams"; import { findFreePort } from "@executablemd/runtime"; import { compileBlock } from "../src/eval-context.ts"; import { useDenoCompiler } from "../src/deno-compiler.ts"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import * as fs from "node:fs"; import * as path from "node:path"; @@ -143,7 +143,7 @@ describe("Tier R — Eval module globals", () => { }); // --------------------------------------------------------------------------- -// Tier R — Behavioral integration tests (via runDocument) +// Tier R — Behavioral integration tests (via execute) // --------------------------------------------------------------------------- describe("Tier R — findFreePort in eval blocks", () => { @@ -166,7 +166,7 @@ describe("Tier R — findFreePort in eval blocks", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -203,7 +203,7 @@ describe("Tier R — findFreePort in eval blocks", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -238,7 +238,7 @@ describe("Tier R — findFreePort in eval blocks", () => { const stream = new InMemoryStream(); // Golden run const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -246,7 +246,7 @@ describe("Tier R — findFreePort in eval blocks", () => { // Replay — durableEval returns stored port, findFreePort not invoked const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -293,7 +293,7 @@ describe("Tier R — when in eval blocks", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -332,7 +332,7 @@ describe("Tier R — when in eval blocks", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), @@ -365,7 +365,7 @@ describe("Tier R — when in eval blocks", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, }), diff --git a/core/tests/function-components.test.ts b/core/tests/function-components.test.ts index 1c5a619..bd267a9 100644 --- a/core/tests/function-components.test.ts +++ b/core/tests/function-components.test.ts @@ -6,7 +6,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import { InMemoryStream } from "@executablemd/durable-streams"; import * as fs from "node:fs"; @@ -43,7 +43,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -72,7 +72,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -100,7 +100,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -128,7 +128,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -154,7 +154,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -183,7 +183,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -212,7 +212,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -236,7 +236,7 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -262,14 +262,14 @@ describe("Tier FC — Function components", () => { }); const stream = new InMemoryStream(); const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], }), ); const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], diff --git a/core/tests/named-slots.test.ts b/core/tests/named-slots.test.ts index 30c926c..1fd575f 100644 --- a/core/tests/named-slots.test.ts +++ b/core/tests/named-slots.test.ts @@ -17,7 +17,7 @@ import { scanSegments } from "../src/scanner.ts"; import { renderSegments } from "../src/render.ts"; import { parseFrontmatter } from "../src/frontmatter.ts"; import { validateProps } from "../src/validate.ts"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import { InMemoryStream } from "@executablemd/durable-streams"; import type { Segment, ComponentDefinition, Json, CodeBlockResult } from "../src/types.ts"; @@ -694,7 +694,7 @@ describe("Tier NS-E — renderChildren interaction", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -731,7 +731,7 @@ describe("Tier NS-E — renderChildren interaction", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -769,7 +769,7 @@ describe("Tier NS-E — renderChildren interaction", () => { }); const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -897,7 +897,7 @@ describe("Tier NS-F — Edge cases", () => { }); const stream = new InMemoryStream(); yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -931,7 +931,7 @@ describe("Tier NS-F — Edge cases", () => { }); const stream = new InMemoryStream(); const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -939,7 +939,7 @@ describe("Tier NS-F — Edge cases", () => { ); // Replay — same stream, same output const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], diff --git a/core/tests/provider-integration.test.ts b/core/tests/provider-integration.test.ts index ed780c0..64c1e5f 100644 --- a/core/tests/provider-integration.test.ts +++ b/core/tests/provider-integration.test.ts @@ -14,7 +14,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import { Sample } from "../src/sample-api.ts"; import * as fs from "node:fs"; @@ -139,7 +139,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -168,7 +168,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -211,7 +211,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -227,10 +227,10 @@ describe( }); // S4: Daemon terminated after children expand - // After runDocument completes, the daemon process is not running. - // Verified by: runDocument returns (doesn't hang), proving structured + // After execute completes, the daemon process is not running. + // Verified by: execute returns (doesn't hang), proving structured // concurrency cleaned up the daemon. - it("S4: daemon terminated after children expand — runDocument completes", function* () { + it("S4: daemon terminated after children expand — execute completes", function* () { const tmpDir = makeTempDir(); try { @@ -241,14 +241,14 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], }), ); - // runDocument returned — daemon was cleaned up by structured concurrency. + // execute returned — daemon was cleaned up by structured concurrency. // If daemon wasn't terminated, the process would hang indefinitely. expect(output).toContain("expansion-done"); } finally { @@ -293,7 +293,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -354,7 +354,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -364,7 +364,7 @@ describe( // The daemon crashed during children expansion. // The output should contain some indication of the error // (DaemonExitError) or the child output, depending on timing. - // Key property: runDocument completed (didn't hang). + // Key property: execute completed (didn't hang). expect(output).toBeTruthy(); } finally { cleanup(tmpDir); @@ -399,7 +399,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -472,7 +472,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -537,7 +537,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -602,7 +602,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -663,7 +663,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -694,7 +694,7 @@ describe( // Golden run const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs, @@ -706,7 +706,7 @@ describe( // Replay — eval blocks replay from journal, daemon spawns fresh const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs, @@ -739,7 +739,7 @@ describe( // Golden run const stream1 = new InMemoryStream(); const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream: stream1, componentDirs, @@ -766,7 +766,7 @@ describe( // Fresh stream — provider starts a new daemon, new child runs live const stream2 = new InMemoryStream(); const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream: stream2, componentDirs, @@ -805,7 +805,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -885,7 +885,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -954,7 +954,7 @@ describe( const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], diff --git a/core/tests/sample-component.test.ts b/core/tests/sample-component.test.ts index 0fd7c23..ecd8fd5 100644 --- a/core/tests/sample-component.test.ts +++ b/core/tests/sample-component.test.ts @@ -16,7 +16,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs, useEchoExec } from "@executablemd/runtime/test"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import * as fs from "node:fs"; import * as path from "node:path"; @@ -127,7 +127,7 @@ describe("Tier SC — Sample component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -172,7 +172,7 @@ describe("Tier SC — Sample component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -218,7 +218,7 @@ describe("Tier SC — Sample component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -250,7 +250,7 @@ describe("Tier SC — Sample component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -290,7 +290,7 @@ describe("Tier SC — Sample component", () => { // First run const stream = new InMemoryStream(); const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -301,7 +301,7 @@ describe("Tier SC — Sample component", () => { // Second run (replay) — same stream const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -339,7 +339,7 @@ describe("Tier SC — Sample component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -383,7 +383,7 @@ describe("Tier SC — Sample component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -435,7 +435,7 @@ describe("Tier SC — Sample component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -467,7 +467,7 @@ describe("Tier EO — eval output() function", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -486,7 +486,7 @@ describe("Tier EO — eval output() function", () => { // First run const output1 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -496,7 +496,7 @@ describe("Tier EO — eval output() function", () => { // Replay const output2 = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -515,7 +515,7 @@ describe("Tier EO — eval output() function", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -535,7 +535,7 @@ describe("Tier EO — eval output() function", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -553,7 +553,7 @@ describe("Tier EO — eval output() function", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -588,7 +588,7 @@ describe("Tier RC — renderChildren and render closures", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -617,7 +617,7 @@ describe("Tier RC — renderChildren and render closures", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -646,7 +646,7 @@ describe("Tier RC — renderChildren and render closures", () => { yield* useEchoExec(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "test.md", stream, }), @@ -692,7 +692,7 @@ describe("Tier IN — Instruction component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -730,7 +730,7 @@ describe("Tier IN — Instruction component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -778,7 +778,7 @@ describe("Tier IN — Instruction component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -815,7 +815,7 @@ describe("Tier IN — Instruction component", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -882,7 +882,7 @@ describe("Tier AG — Agent component pattern", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], @@ -967,7 +967,7 @@ describe("Tier AG — Agent component pattern", () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: path.join(tmpDir, "doc.md"), stream, componentDirs: [path.join(tmpDir, "components"), tmpDir], diff --git a/core/tests/smoke.test.ts b/core/tests/smoke.test.ts index d922c43..7147229 100644 --- a/core/tests/smoke.test.ts +++ b/core/tests/smoke.test.ts @@ -12,7 +12,7 @@ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; // --------------------------------------------------------------------------- @@ -24,7 +24,7 @@ describe("smoke test", { sanitizeOps: false, sanitizeResources: false }, () => { const stream = new InMemoryStream(); const output = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "smoke-test/README.md", stream, componentDirs: ["smoke-test", "core/components"], @@ -196,7 +196,7 @@ describe("smoke test", { sanitizeOps: false, sanitizeResources: false }, () => { // Golden run const firstOutput = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "smoke-test/README.md", stream, componentDirs: ["smoke-test", "core/components"], @@ -206,7 +206,7 @@ describe("smoke test", { sanitizeOps: false, sanitizeResources: false }, () => { // Replay — durableRun short-circuits on the Close event in the // journal, returning the stored result without any I/O calls. const secondOutput = yield* collect( - yield* runDocument({ + yield* execute({ docPath: "smoke-test/README.md", stream, componentDirs: ["smoke-test", "core/components"], diff --git a/core/tests/source-position.test.ts b/core/tests/source-position.test.ts index 616e33c..2629893 100644 --- a/core/tests/source-position.test.ts +++ b/core/tests/source-position.test.ts @@ -4,7 +4,7 @@ import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs } from "@executablemd/runtime/test"; import { scanSegments } from "../src/scanner.ts"; import { Component } from "../src/component-api.ts"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; import { collect } from "../src/collect.ts"; import type { ComponentInvocation, SourcePosition } from "../src/types.ts"; @@ -75,7 +75,7 @@ describe("source positions", () => { }, }); - const output = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + const output = yield* collect(yield* execute({ docPath: "README.md", stream })); expect(output).toContain("probed"); const probeLine = doc.split("\n").indexOf("") + 1; @@ -107,7 +107,7 @@ describe("source positions", () => { }, }); - const output = yield* collect(yield* runDocument({ docPath: "README.md", stream })); + const output = yield* collect(yield* execute({ docPath: "README.md", stream })); expect(output).toContain("probed"); const probe = positions.find((p) => p.name === "Probe"); diff --git a/core/tests/streaming-emission.test.ts b/core/tests/streaming-emission.test.ts index 3080c94..3da8891 100644 --- a/core/tests/streaming-emission.test.ts +++ b/core/tests/streaming-emission.test.ts @@ -2,14 +2,14 @@ * Tier SE — Streaming emission tests + Tier BC — Block ID counter (spec §9, §6.1). * * Tests per-segment emission through the document stream returned by - * runDocument, and blockId stability across per-segment expansion calls. + * execute, and blockId stability across per-segment expansion calls. */ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { InMemoryStream } from "@executablemd/durable-streams"; import { forEach } from "@effectionx/stream-helpers"; import { createBlockCounter } from "../src/expand.ts"; -import { runDocument } from "../src/run-document.ts"; +import { execute } from "../src/execute.ts"; describe("Tier BC — Block ID counter", () => { // BC1: Counter increments across calls @@ -36,7 +36,7 @@ describe("Tier SE — Streaming emission", () => { it("SE1: segments emitted in document order via stream", function* () { const chunks: string[] = []; - const execution = yield* runDocument({ + const execution = yield* execute({ docPath: "core/tests/fixtures/streaming/multi-segment.md", stream: new InMemoryStream(), }); @@ -55,7 +55,7 @@ describe("Tier SE — Streaming emission", () => { it("SE10: no empty strings in output chunks", function* () { const chunks: string[] = []; - const execution = yield* runDocument({ + const execution = yield* execute({ docPath: "core/tests/fixtures/streaming/simple.md", stream: new InMemoryStream(), }); @@ -74,7 +74,7 @@ describe("Tier SE — Streaming emission", () => { it("SE9: output() inside durable workflow reaches stream consumer", function* () { const chunks: string[] = []; - const execution = yield* runDocument({ + const execution = yield* execute({ docPath: "core/tests/fixtures/streaming/simple.md", stream: new InMemoryStream(), }); diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index 5273e32..130fb6b 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -106,7 +106,7 @@ The journal records: ### 1.2 Workspace-relative paths All paths stored in a diagnostic trace are **relative to the workspace root** -(the current working directory when `runDocument` is called). This +(the current working directory when `execute` is called). This makes traces easier to compare and avoids leaking absolute local paths. Runtime operations (`readTextFile`, `stat`, `exec`, `glob`) all resolve @@ -451,7 +451,7 @@ registry.set("timeout", timeoutFactory); registry.set("daemon", daemonFactory); ``` -Custom factories can be provided via `RunDocumentOptions.modifiers`. +Custom factories can be provided via `ExecuteOptions.modifiers`. #### Built-in terminal handlers @@ -643,10 +643,10 @@ the block if it overruns. The silent handler discards the output. #### Overriding per-scope Because factories are stored in a registry that can be extended, -custom modifiers can be provided via `RunDocumentOptions`: +custom modifiers can be provided via `ExecuteOptions`: ```typescript -yield* runDocument({ +yield* execute({ docPath: "README.md", stream, runtime, @@ -1134,7 +1134,7 @@ lifetime matches the document's expansion. Resources spawned by `persist` blocks are retained in this scope until expansion completes. The current eval scope is read contextually via the `evalScope` value (§5.5). -The eval scope is created in `runDocument()` (§8.1) **before** +The eval scope is created in `execute()` (§8.1) **before** `durableRun` via `resource(useEvalScope())`. This is critical: `evalScope.eval()` sends to a channel whose processor must be reachable by the Effection scheduler — this only works when both sender @@ -1850,7 +1850,7 @@ returning without calling it. never observe each other's state. Caller-projected content, the current code block, the persistent flag, and function-component content all rely on this. -- `runDocument` installs the document's `importComponent` / +- `execute` installs the document's `importComponent` / `applyModifiers` / root `evalScope` providers before starting the durable workflow, so the whole run inherits them; the journal shape of import, exec, and eval effects is unchanged by contextual @@ -2517,7 +2517,7 @@ A **provider component** is a regular markdown component whose body follows a structured pattern that manages background process lifecycle for its subtree. It composes `eval` + `daemon` + `eval` (readiness) + `eval` (middleware install) + `` into a reusable -component — no framework-level configuration, no `RunDocumentOptions` +component — no framework-level configuration, no `ExecuteOptions` changes. #### Structure @@ -2936,9 +2936,9 @@ only when output is emitted, not what executes, so replay is deterministic. ## 7. Entry point -### 8.1 `runDocument` +### 8.1 `execute` -`runDocument(options)` executes a markdown document as a durable +`execute(options)` executes a markdown document as a durable workflow and returns a `DocumentExecution` handle. Options: - `docPath` — path to the root markdown document @@ -2948,23 +2948,31 @@ workflow and returns a `DocumentExecution` handle. Options: - `modifiers?` — custom modifier factories registered alongside the built-ins (`exec`, `silent`, `eval`, `persist`, `timeout`, `daemon`) -`DocumentExecution` is an `Operation`: `yield* execution` -completes with the document's full output. Its `output` property is a -`Stream` of the chunks emitted during execution -(per-segment for streaming roots, one chunk for buffered `` -roots — §5.4); the stream closes with the full output as its close -value. +`DocumentExecution` is an `Operation>`: `yield* execution` +completes with `Ok(output)` on success and `Err(error)` on document, +infrastructure, or policy failure. Once `execute` has returned a handle, +completion never throws — every later failure closes the output stream +(with the complete or partial rendered output) and resolves `Err`. A +failure before a handle can be created may still throw. Its `output` +property is a replay-safe `Stream` of the chunks emitted +during execution (per-segment for streaming roots, one chunk for buffered +`` roots — §5.4); late and repeated subscribers receive the full +sequence, and the stream closes with the full (or partial) output as its +close value. Execution runs in its own scope. Before the durable workflow starts, -`runDocument` installs the document's scope-local runtime providers — +`execute` installs the document's scope-local runtime providers — the platform compiler, the Component providers for import, modifier execution, and the root eval scope (§5.5), and the output→stream bridge — so nothing leaks onto the caller's scope and the whole run inherits them contextually. -On successful completion, `yield* execution` returns the full output -and the `output` stream closes with it. On failure, the stream closes -and `yield* execution` throws the workflow error. +`execute` is delivered through the `Execution` context Api. The default +provider runs the document; extensions decorate the execution lifecycle +with `Execution.around({ execute })` middleware — observing options, +wrapping the returned handle, or mapping its completion `Result` — without +introducing another execution function. Core itself has no knowledge of +any particular extension. ### 8.2 Usage from standalone code @@ -2973,13 +2981,17 @@ import { run } from "effection"; import { InMemoryStream } from "@executablemd/durable-streams"; await run(function* () { - const execution = yield* runDocument({ + const execution = yield* execute({ docPath: "./README.md", stream: new InMemoryStream(), }); - const output = yield* execution; - console.log(output); + const result = yield* execution; + if (result.ok) { + console.log(result.value); + } else { + console.error(result.error.message); + } }); ``` @@ -3050,7 +3062,7 @@ Three concerns, three mechanisms: | Concern | Mechanism | Where | |---|---|---| | **Transformation** | Middleware (`scope.around`) | `output/normalize.ts`, `output/terminal.ts` | -| **Delivery** | Channel (`createChannel`, internal to `runDocument`) | `run-document.ts` | +| **Delivery** | Channel (`createChannel`, internal to `execute`) | `execute.ts` | | **Consumption** | Stream (`forEach` on `execution.output`) | Caller (`cli.ts`, tests) | Middleware only intercepts and transforms. Buffering and streaming are not @@ -3141,20 +3153,20 @@ export function* useTerminalOutput(): Operation { **File:** `cli/src/cli.ts` (separate `cli` workspace package) The CLI installs output middleware (transforms only — no channel wiring -needed), calls `runDocument` to get a `DocumentExecution`, consumes +needed), calls `execute` to get a `DocumentExecution`, consumes `execution.output` with `forEach` for streaming, and can `yield*` the execution directly to get the full output or catch errors. ```typescript // cli/src/cli.ts import { forEach } from "@effectionx/stream-helpers"; -import { runDocument, useNormalizedOutput, useTerminalOutput } from "@executablemd/core"; +import { execute, useNormalizedOutput, useTerminalOutput } from "@executablemd/core"; function* run(/* ... config params ... */) { if (!raw) yield* useNormalizedOutput(); if (process.stdout.isTTY && !raw) yield* useTerminalOutput(); - const execution = yield* runDocument({ docPath, stream, runtime, ... }); + const execution = yield* execute({ docPath, stream, runtime, ... }); const fullOutput = yield* forEach(function* (chunk: string) { if (process.stdout.isTTY) { @@ -3176,7 +3188,7 @@ function* run(/* ... config params ... */) { output(text) → normalize (middleware, caller-installed) → terminal format (middleware, caller-installed) - → channel.send(text) (internal to runDocument) + → channel.send(text) (internal to execute) → execution.output stream (caller's forEach/collect) ``` @@ -3187,7 +3199,7 @@ User sees cleaned, colorized text streaming segment-by-segment. ``` output(text) → normalize (middleware, caller-installed) - → channel.send(text) (internal to runDocument) + → channel.send(text) (internal to execute) → execution.output stream (caller's forEach/collect) → fullOutput written to stdout at end ``` @@ -3198,7 +3210,7 @@ User gets cleaned raw markdown dumped at end. ``` output(text) - → channel.send(text) (internal to runDocument, no transformation) + → channel.send(text) (internal to execute, no transformation) → execution.output stream (caller's forEach/collect) ``` @@ -3242,16 +3254,17 @@ All middleware and side effects triggered by `output()` (normalization, formatting, channel send) execute on the ephemeral side. No durable state capture occurs in the output pipeline. -The entire workflow runs in a `spawn()` inside `runDocument`. The channel +The entire workflow runs in a `spawn()` inside `execute`. The channel and all execution state (runtime API middleware and eval scope, DocumentOutput→channel bridge) share this spawned scope. The consumer (`forEach`/`collect` on `execution.output`) runs in the **caller's** scope. This cross-boundary communication is safe because scope teardown of the spawned task cancels the producer and closes the channel, which terminates the consumer's forEach loop. The `withResolvers` completion -signal also lives in the spawned scope — `resolve()` and `reject()` are -called from inside the spawn, and the resulting operation is returned to -the caller as part of the `DocumentExecution` handle. +signal also lives in the spawned scope — `resolve(Ok(...))` and +`resolve(Err(...))` are called from inside the spawn, and the resulting +operation is returned to the caller as part of the `DocumentExecution` +handle. ### 9.10 Known issues @@ -3287,7 +3300,7 @@ core/src/ mod.ts Barrel export normalize.ts Whitespace normalization middleware terminal.ts Terminal ANSI formatting middleware - run-document.ts Document runner (owns channel, returns stream) + execute.ts Document runner (owns channel, returns stream) cli/src/ cli.ts CLI entrypoint with forEach stream consumption @@ -3767,7 +3780,7 @@ visible warning blocks, collect into a separate error report). | S1 | Full provider golden run | eval → daemon → when → children → cleanup | | S2 | Port flows from eval to daemon | `{port}` in daemon content matches `findFreePort()` result | | S3 | Children can call sample after daemon ready | `sample` calls in children reach daemon endpoint | -| S4 | Daemon terminated after children expand | After `runDocument` completes, process not running | +| S4 | Daemon terminated after children expand | After `execute` completes, process not running | | S5 | Provider crash during `when` | Daemon exits before ready → `when` fails → `ErrorSegment` | | S6 | Provider crash during children | Daemon exits mid-child-expansion → error propagated | | S7 | Nested providers | Outer + inner provider → both start, inner tears down first | @@ -3821,7 +3834,7 @@ visible warning blocks, collect into a separate error report). | OA7 | Channel close ends consumer | `channel.close()` causes `forEach` to complete | | OA8 | Multiple middleware compose | Normalize → terminal → channel: all three run in order | | OA9 | `ephemeral()` wrapper | `output()` inside durable context produces no journal entry | -| OA10 | runDocument workflow error surfaces through execution | `runDocument` workflow error → `reject(error)` → `yield* execution` throws | +| OA10 | execute workflow error surfaces through execution | `execute` workflow error → `reject(error)` → `yield* execution` throws | ### Tier WN — Whitespace normalization @@ -3941,7 +3954,7 @@ must preserve the trace for diagnosis or remove it before starting a new run. | 41 | `findFreePort` is a standalone VM global using `node:net` | Port allocation is platform I/O; the function uses Effection's `once` + `race` for event handling and `try/finally` for guaranteed cleanup; exposed in the eval sandbox alongside other Effection globals | | 42 | `findFreePort` result journaled with its eval block | The port number is a scalar export; no separate journal-entry type is needed | | 43 | `when` (from `@effectionx/converge`) is the polling VM global | `when` is the exported name from the package; the sandbox already contains it; no rename or addition needed | -| 44 | Provider lifecycle expressed as a component, not a `RunDocumentOptions` field | Scope boundary is visible in the document tree; composable — multiple providers nest naturally via structured concurrency; no framework-level lifecycle hooks required | +| 44 | Provider lifecycle expressed as a component, not a `ExecuteOptions` field | Scope boundary is visible in the document tree; composable — multiple providers nest naturally via structured concurrency; no framework-level lifecycle hooks required | | 45 | Readiness check is a separate `eval` block, not internal to `daemon` | Auditable — strategy visible in the document; replaceable — different daemons have different readiness signals; composable with `when`'s configurable backoff | | 46 | Sample middleware reads `baseUrl` from `env.values` | Avoids a dedicated inference server context key; the binding environment is already the shared state carrier for within-component coordination; scope-correct because a fresh environment is provided per component expansion | | 47 | Each component gets a fresh `EvalEnv` | The component's environment is installed as a scope-local `env` provider around body expansion, so eval blocks within a component share bindings but don't leak into parent or sibling components; critical for provider isolation | @@ -3953,7 +3966,7 @@ must preserve the trace for diagnosis or remove it before starting a new run. | 53 | Sample component calls `Sample.operations.sample()` directly | The enclosing eval operation journals the complete block result | | 54 | Sample component props default to empty string, not undefined | `validateProps` omits optional props with no default from `env.values`, causing `ReferenceError` in eval blocks; empty-string defaults ensure the variables exist; `model \|\| undefined` converts empty to undefined for routing semantics | | 55 | `daemon()` uses `shell: true` | Matches `bash exec` block semantics — the same command string passed to `bash -c` is passed to the shell; handles shell expansions and PATH lookups correctly | -| 56 | Provider installs its own middleware, not a global `useLlamafileSample()` | A single global handler installed before `runDocument()` would execute in the outer scope at call time, where the binding environment has no `baseUrl`; middleware must close over `baseUrl` and `model` at the moment the provider becomes active | +| 56 | Provider installs its own middleware, not a global `useLlamafileSample()` | A single global handler installed before `execute()` would execute in the outer scope at call time, where the binding environment has no `baseUrl`; middleware must close over `baseUrl` and `model` at the moment the provider becomes active | | 57 | Routing key is `model`, not a separate `name` prop | Model identity is the natural key — it unifies "which server to route to" with "which model to request"; a separate `name` prop would require keeping two values in sync with no added expressiveness | | 58 | `context.model === undefined` routes to innermost provider | Omitting a model is the common case for single-provider documents; innermost-wins matches how middleware chains work — handlers installed later sit higher in the chain and are traversed first | | 59 | `callLlamafile()` is a standard import in generated eval modules | Provider components are markdown files — eval blocks are compiled into `data:` URI modules that import executable.md globals from `@executablemd/core`; functions like `callLlamafile`, `callOllama`, `callAnthropic`, `Sample`, `findFreePort`, and `useContent` are available via this import | @@ -3970,7 +3983,7 @@ must preserve the trace for diagnosis or remove it before starting a new run. | 70 | `output()` wrapped in `ephemeral()` | Output emission is a non-durable side effect; journal records durable effects only; output text is derived from journaled expansion results; all middleware/side effects execute on the ephemeral side | | 71 | Middleware installation order: normalize outer, terminal inner, channel innermost | `scope.around` later-installed handlers wrap earlier ones; execution flows outer → inner: normalize → terminal → channel; install order is reverse of execution order; must be documented to prevent reordering | | 72 | `channel.send()` must be `yield*`'d | Ensures backpressure and cancellation safety — no text "in flight" when scope tears down; without `yield*`, buffering issues or silent cancellation may occur | -| 73 | `DocumentExecution` with `withResolvers` | Execution is both an `Operation` (`yield*` for result) and has `.output` stream for chunks; errors surface through `yield* execution` via `withResolvers` `reject()`; eliminates `Result` / `unbox` in consumer code | +| 73 | `DocumentExecution` with `withResolvers` | Execution is both an `Operation>` (`yield*` for the completion Result) and has `.output` stream for chunks; once a handle exists every failure resolves `Err(error)` — completion never throws, so policy middleware (e.g. testing) can map outcomes without exception control flow | | 74 | Function components receive props directly, not wrapped | `function*(props)` not `function*({ props })` — eliminates unnecessary destructuring; props are already validated by the expansion engine before the function is called | | 75 | `useContent()` is contextual, not a function argument | Decouples function components from the expansion engine's API surface; leaf components don't need to ignore an `expandChildren` parameter; Effection-idiomatic — same contextual pattern as `env`/`evalScope`; supports named slots via `useContent("header")` | | 76 | `.md` wins over `.ts` in resolution | Backward compatibility — existing markdown components are not shadowed by TypeScript files added later; explicit — if both exist, the human-readable markdown is preferred | diff --git a/specs/testing-spec.md b/specs/testing-spec.md index 471cb8a..bcf5599 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -89,6 +89,37 @@ const active = yield* testing; pass or fail status, optional name, source location, and structured error details when it failed. Rendered test output is not duplicated in the result. +`useTesting` composes testing around the core execution entrypoint. It +installs the testing vocabulary and collectors, activates testing mode for +the execution, and returns a session whose `results` operation snapshots +completed tests in discovery order: + +```ts +import { execute } from "@executablemd/core"; +import { useTesting } from "@executablemd/testing"; + +const tests = yield* useTesting(); +const execution = yield* execute(options); +const outcome = yield* execution; +const results = yield* tests.results; +``` + +Execution completion is an Effection `Result`: `Ok(output)` on +success, `Err(error)` on document, infrastructure, or testing failure — +completion never throws once the execution handle exists. Under +`useTesting`, an otherwise successful execution completes as +`Err(TestFailureError)` after the output stream closes when any test failed +or no tests were discovered. A failure produced by the document itself +passes through unchanged, and the session's results remain available after +failure. One `useTesting` session applies per execution scope; its +middleware is removed with that scope. `xmd test` composes `useTesting` +around the same `execute` call the `run` command uses. + +Registering the testing vocabulary without `useTesting` leaves testing mode +inactive: `` is skipped, assertion components stay usable, and an +explicit `` boundary still activates its subtree and turns its +failures — or an empty boundary — into an `Err` outcome for the execution. + ## Assertions Assertion components use `@std/assert` and follow its function names and diff --git a/testing/mod.ts b/testing/mod.ts index c568285..691da31 100644 --- a/testing/mod.ts +++ b/testing/mod.ts @@ -4,12 +4,26 @@ * * `` activates testing mode for its expanded subtree, `` * defines an atomic test, and the assertion components map to `@std/assert`. + * + * Composition with core execution: + * + * ```ts + * import { execute } from "@executablemd/core"; + * import { useTesting } from "@executablemd/testing"; + * + * const tests = yield* useTesting(); + * const execution = yield* execute(options); + * const outcome = yield* execution; // Result + * const results = yield* tests.results; + * ``` + * * Vocabulary registration (`installTestingVocabulary`) is distinct from - * testing-mode activation (`executeDocument({ testing: true })` or a - * `` element). + * testing-mode activation (`useTesting()` at the root, or a `` + * element for a subtree). */ export { Test, testing, record, results, TestFailureError } from "./src/test-api.ts"; export type { TestApi, TestResult, BoundaryOutcome } from "./src/test-api.ts"; export { installTestingVocabulary } from "./src/vocabulary.ts"; -export { executeDocument } from "./src/execute.ts"; +export { useTesting } from "./src/use-testing.ts"; +export type { Testing } from "./src/use-testing.ts"; diff --git a/testing/src/execute.ts b/testing/src/execute.ts deleted file mode 100644 index a2b78fb..0000000 --- a/testing/src/execute.ts +++ /dev/null @@ -1,139 +0,0 @@ -/** - * executeDocument — the single scoped wrapper both CLI paths use - * (specs/testing-spec.md §Testing Mode). - * - * Testing middleware lives and dies with the execution it serves: a bounded - * child task installs the vocabulary, the run-level collectors, and (in - * testing mode) root activation, runs the document inside that scope, and - * evaluates the aggregate outcome only after the inner execution — and - * therefore its closed output stream — completes. When the execution - * finishes, the scope exits and every install is gone. - * - * `output` is a lazy stream delegating to the inner execution's replay-safe - * output — no second channel, nothing dropped for late subscribers. - */ - -import { spawn, withResolvers } from "effection"; -import type { Operation } from "effection"; -import { runDocument } from "@executablemd/core"; -import type { DocumentExecution, RunDocumentOptions } from "@executablemd/core"; -import { Test, TestFailureError } from "./test-api.ts"; -import type { BoundaryOutcome, TestResult } from "./test-api.ts"; -import { installHandlers, installTestingVocabulary } from "./vocabulary.ts"; -import type { TestHandlers } from "./handlers.ts"; - -export interface ExecuteDocumentOptions extends RunDocumentOptions { - /** Activate testing mode at the root (`xmd test` ≡ root ``). */ - testing: boolean; - /** Render assertion diagnostics during regular execution. */ - verbose?: boolean; -} - -interface ExecuteDependencies { - runDocument(options: RunDocumentOptions): Operation; - handlers?: TestHandlers; -} - -/** - * Internal dependency-injection seam: tests inject a throwing `runDocument` - * (pre-publication setup failure) or short-timeout handlers. - */ -export function createExecuteDocument(deps: ExecuteDependencies) { - return function* executeDocument(options: ExecuteDocumentOptions): Operation { - const innerExecution = withResolvers(); - const completion = withResolvers(); - - yield* spawn(function* () { - // One error boundary around setup, runDocument, and completion: any - // failure rejects BOTH resolvers so neither output consumption nor - // `yield* execution` can hang. - try { - if (deps.handlers) { - yield* installHandlers(deps.handlers, { verbose: options.verbose }); - } else { - yield* installTestingVocabulary({ verbose: options.verbose }); - } - - const runResults: TestResult[] = []; - const boundaries: BoundaryOutcome[] = []; - // Run-level collector — ALWAYS installed, so explicit - // boundaries are observable during ordinary runs too. - yield* Test.around({ - // deno-lint-ignore require-yield - *results() { - return runResults; - }, - *record([result], next) { - runResults.push(result); - yield* next(result); - }, - *boundary([outcome], next) { - boundaries.push(outcome); - yield* next(outcome); - }, - }); - if (options.testing) { - yield* Test.around({ testing: () => true }); - } - - const inner = yield* deps.runDocument(options); - innerExecution.resolve(inner); - - // Observing the inner execution keeps this child (and its installs) - // alive; core closes inner.output before the execution settles, so - // the report is fully delivered before any rejection below. - let out: string; - try { - out = yield* inner; - } catch (error) { - completion.reject(error instanceof Error ? error : new Error(String(error))); - return; - } - - const failed = - boundaries.some((b) => b.failed > 0 || b.tests === 0) || - (options.testing && - (runResults.some((r) => r.status === "fail") || runResults.length === 0)); - if (failed) { - completion.reject(new TestFailureError(summarize(runResults, boundaries))); - } else { - completion.resolve(out); - } - } catch (error) { - const failure = error instanceof Error ? error : new Error(String(error)); - innerExecution.reject(failure); - completion.reject(failure); - } - }); - - return { - // A Stream IS an Operation: await the published inner - // execution, then delegate to its replay-safe output. - output: { - *[Symbol.iterator]() { - const inner = yield* innerExecution.operation; - return yield* inner.output; - }, - }, - *[Symbol.iterator]() { - return yield* completion.operation; - }, - }; - }; -} - -export const executeDocument = createExecuteDocument({ runDocument }); - -function summarize(results: TestResult[], boundaries: BoundaryOutcome[]): string { - const failed = results.filter((result) => result.status === "fail"); - if (results.length === 0 && boundaries.every((b) => b.tests === 0)) { - return "no tests were discovered"; - } - if (failed.length === 0 && boundaries.some((b) => b.tests === 0)) { - return "a boundary discovered no tests"; - } - const details = failed - .map((result) => ` ${result.name ?? result.location}: ${result.error?.message ?? "failed"}`) - .join("\n"); - return `${failed.length} of ${results.length} tests failed\n${details}`; -} diff --git a/testing/src/test-api.ts b/testing/src/test-api.ts index ddb3266..781189e 100644 --- a/testing/src/test-api.ts +++ b/testing/src/test-api.ts @@ -42,6 +42,8 @@ export interface TestApi { inTest: boolean; /** Whether assertion diagnostics render during regular execution. */ verbose: boolean; + /** Whether a useTesting() session is already active in this scope. */ + sessionActive: boolean; /** Record a completed test. Collectors delegate outward via `next`. */ record(result: TestResult): Operation; /** Completed tests recorded by the nearest collector, discovery order. */ @@ -54,6 +56,7 @@ export const Test = createApi("Test", { testing: false, inTest: false, verbose: false, + sessionActive: false, // deno-lint-ignore require-yield *record(_result: TestResult): Operation {}, // deno-lint-ignore require-yield @@ -64,7 +67,8 @@ export const Test = createApi("Test", { *boundary(_outcome: BoundaryOutcome): Operation {}, }); -export const { testing, inTest, verbose, record, results, boundary } = Test.operations; +export const { testing, inTest, verbose, sessionActive, record, results, boundary } = + Test.operations; /** A document execution failed its testing outcome (test failures or zero tests). */ export class TestFailureError extends Error { diff --git a/testing/src/use-testing.ts b/testing/src/use-testing.ts new file mode 100644 index 0000000..c844ad0 --- /dev/null +++ b/testing/src/use-testing.ts @@ -0,0 +1,90 @@ +/** + * useTesting — scope-local testing composition (specs/testing-spec.md). + * + * One `useTesting()` session per execution scope: it installs the testing + * vocabulary, the collection and completion-policy middleware, and root + * activation, then returns a session handle whose `results` operation + * snapshots completed tests in discovery order. Every install is removed + * with the session's Effection scope. + * + * ```ts + * const tests = yield* useTesting(); + * const execution = yield* execute(options); + * const outcome = yield* execution; // Result + * const results = yield* tests.results; + * ``` + */ + +import type { Operation } from "effection"; +import { Execution } from "@executablemd/core"; +import { sessionActive, Test, TestFailureError } from "./test-api.ts"; +import type { TestResult } from "./test-api.ts"; +import { decorateCompletion, installTestingVocabulary } from "./vocabulary.ts"; + +export interface Testing { + /** Immutable snapshot of completed tests, in discovery order. */ + readonly results: Operation; +} + +export function* useTesting(options?: { verbose?: boolean }): Operation { + if (yield* sessionActive) { + throw new Error( + "useTesting() is already active in this scope — use one session per execution scope", + ); + } + yield* Test.around({ sessionActive: () => true }); + + yield* installTestingVocabulary(options); + + const collected: TestResult[] = []; + yield* Test.around({ + // deno-lint-ignore require-yield + *results() { + return [...collected]; + }, + *record([result], next) { + collected.push(result); + yield* next(result); + }, + }); + + // Root activation — the same install performs for its subtree, + // applied to the whole execution (`xmd test` ≡ root ). + yield* Test.around({ testing: () => true }); + + // Completion policy: an otherwise successful execution becomes + // Err(TestFailureError) after its output closes when tests failed or none + // were discovered. A core Err passes through unchanged, and `results` + // stays available either way. + yield* Execution.around({ + *execute([executeOptions], next) { + const inner = yield* next(executeOptions); + return decorateCompletion(inner, () => { + if (collected.length === 0) { + return new TestFailureError("no tests were discovered"); + } + const failed = collected.filter((result) => result.status === "fail"); + if (failed.length > 0) { + const details = failed + .map( + (result) => + ` ${result.name ?? result.location}: ${result.error?.message ?? "failed"}`, + ) + .join("\n"); + return new TestFailureError( + `${failed.length} of ${collected.length} tests failed\n${details}`, + ); + } + return undefined; + }); + }, + }); + + return { + results: { + *[Symbol.iterator]() { + return Object.freeze([...collected]); + }, + }, + }; +} diff --git a/testing/src/vocabulary.ts b/testing/src/vocabulary.ts index 203ed57..27131a1 100644 --- a/testing/src/vocabulary.ts +++ b/testing/src/vocabulary.ts @@ -2,17 +2,24 @@ * Vocabulary registration (specs/testing-spec.md). * * Teaches the expansion loop the testing words — ``, ``, and - * the assertion components — via the core `expandInvocation` hook. - * Registration is distinct from activation: installing the vocabulary leaves - * `testing` false, so `` skips and assertions stay usable. + * the assertion components — via the core `expandInvocation` hook, and + * decorates the core Execution Api so explicit `` boundaries affect + * the execution outcome even when root testing is inactive. * - * Installs are scope-local; `executeDocument` owns the lifetime for CLI - * runs. Direct core consumers call this inside their own bounded scope. + * Registration is distinct from activation: installing the vocabulary + * leaves `testing` false, so `` skips and assertions stay usable. + * Root activation is `useTesting()`'s job. + * + * Installs are scope-local — call this inside a bounded scope (one CLI + * command invocation, one `scoped()` block). */ +import { Err } from "effection"; import type { Operation } from "effection"; -import { Component } from "@executablemd/core"; -import { Test } from "./test-api.ts"; +import { Component, Execution } from "@executablemd/core"; +import type { DocumentExecution } from "@executablemd/core"; +import { Test, TestFailureError } from "./test-api.ts"; +import type { BoundaryOutcome } from "./test-api.ts"; import { ASSERTIONS } from "./assertions.ts"; import { createTestHandlers } from "./handlers.ts"; import type { TestHandlers } from "./handlers.ts"; @@ -49,4 +56,55 @@ export function* installHandlers( return yield* next(invocation, ctx); }, }); + yield* Execution.around({ + *execute([executeOptions], next) { + // Fresh boundary collection per execution: outcomes reported by + // explicit elements in THIS run decide this run's Result. + const boundaries: BoundaryOutcome[] = []; + yield* Test.around({ + *boundary([outcome], nextBoundary) { + boundaries.push(outcome); + yield* nextBoundary(outcome); + }, + }); + const inner = yield* next(executeOptions); + return decorateCompletion(inner, () => { + const failed = boundaries.filter((b) => b.failed > 0); + if (failed.length > 0) { + return new TestFailureError( + `${failed.reduce((n, b) => n + b.failed, 0)} test(s) failed in `, + ); + } + if (boundaries.some((b) => b.tests === 0)) { + return new TestFailureError("a boundary discovered no tests"); + } + return undefined; + }); + }, + }); +} + +/** + * Map an execution's completion: an `Ok` becomes `Err(failure())` when the + * policy reports one, after the inner completion — and therefore its closed + * output stream — settles. An existing `Err` passes through unchanged. + */ +export function decorateCompletion( + inner: DocumentExecution, + failure: () => Error | undefined, +): DocumentExecution { + return { + output: inner.output, + *[Symbol.iterator]() { + const result = yield* inner; + if (!result.ok) { + return result; + } + const error = failure(); + if (error) { + return Err(error); + } + return result; + }, + }; } diff --git a/testing/tests/execute.test.ts b/testing/tests/execute.test.ts deleted file mode 100644 index 9dea4b6..0000000 --- a/testing/tests/execute.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { describe, it } from "@effectionx/bdd/node"; -import { expect } from "@effectionx/bdd/expect"; -import { scoped, sleep, spawn } from "effection"; -import type { Operation, Subscription } from "effection"; -import { InMemoryStream } from "@executablemd/durable-streams"; -import { useStubFs } from "@executablemd/runtime/test"; -import { API } from "@executablemd/runtime"; -import { Test, TestFailureError, testing } from "../src/test-api.ts"; -import { createExecuteDocument, executeDocument } from "../src/execute.ts"; -import { failureOf, runDoc } from "./helpers.ts"; - -function* drain( - subscription: Subscription, -): Operation<{ chunks: string[]; close: string }> { - const chunks: string[] = []; - let next = yield* subscription.next(); - while (!next.done) { - chunks.push(next.value); - next = yield* subscription.next(); - } - return { chunks, close: next.value }; -} - -describe("executeDocument", () => { - it("a late subscriber still receives all chunks and the close value", function* () { - yield* useStubFs({ "README.md": "hello world\n" }); - const execution = yield* executeDocument({ - docPath: "README.md", - stream: new InMemoryStream(), - testing: false, - }); - // Complete the execution FIRST — only then subscribe. - const value = yield* execution; - const late = yield* scoped(function* () { - return yield* drain(yield* execution.output); - }); - expect(value).toContain("hello world"); - expect(late.close).toBe(value); - expect(late.chunks.join("")).toContain("hello world"); - }); - - it("multiple subscribers each receive the full sequence", function* () { - yield* useStubFs({ "README.md": "alpha\n\nbeta\n" }); - const execution = yield* executeDocument({ - docPath: "README.md", - stream: new InMemoryStream(), - testing: false, - }); - yield* execution; - const [one, two] = yield* scoped(function* () { - return [yield* drain(yield* execution.output), yield* drain(yield* execution.output)]; - }); - expect(one).toEqual(two); - expect(one.close).toContain("alpha"); - }); - - it("consuming only the completion does not deadlock", function* () { - yield* useStubFs({ "README.md": "only completion\n" }); - const execution = yield* executeDocument({ - docPath: "README.md", - stream: new InMemoryStream(), - testing: false, - }); - const value = yield* execution; - expect(value).toContain("only completion"); - }); - - it("a bad docPath is an inner runtime failure after publication", function* () { - yield* useStubFs({}); - const execution = yield* executeDocument({ - docPath: "missing.md", - stream: new InMemoryStream(), - testing: false, - }); - // Output closes (with accumulated output) even though completion rejects. - const drained = yield* scoped(function* () { - return yield* drain(yield* execution.output); - }); - expect(drained.close).toBe(""); - let error: Error | undefined; - try { - yield* execution; - } catch (failure) { - error = failure instanceof Error ? failure : new Error(String(failure)); - } - expect(error?.message).toContain("missing.md"); - expect(error).not.toBeInstanceOf(TestFailureError); - }); - - it("a pre-publication setup failure rejects output and completion", function* () { - const execute = createExecuteDocument({ - // deno-lint-ignore require-yield - *runDocument() { - throw new Error("setup exploded"); - }, - }); - const execution = yield* execute({ - docPath: "README.md", - stream: new InMemoryStream(), - testing: false, - }); - - let outputError: Error | undefined; - try { - yield* scoped(function* () { - yield* drain(yield* execution.output); - }); - } catch (failure) { - outputError = failure instanceof Error ? failure : new Error(String(failure)); - } - let completionError: Error | undefined; - try { - yield* execution; - } catch (failure) { - completionError = failure instanceof Error ? failure : new Error(String(failure)); - } - expect(outputError?.message).toBe("setup exploded"); - expect(completionError?.message).toBe("setup exploded"); - }); - - it("testing middleware does not outlive its execution", function* () { - yield* useStubFs({ - "README.md": "\n", - }); - const first = yield* executeDocument({ - docPath: "README.md", - stream: new InMemoryStream(), - testing: true, - }); - const firstValue = yield* first; - expect(firstValue).toContain("**Assert** passed"); - - // Same caller scope, fresh run WITHOUT testing: the previous run's - // activation and collectors must be gone — the test is skipped. - expect(yield* testing).toBe(false); - const defaults = yield* Test.operations.results(); - expect(defaults).toEqual([]); - const second = yield* runDoc({ "README.md": "\n" }); - expect(second.completion.ok).toBe(true); - expect(second.output).not.toContain("Assert"); - expect(second.results).toEqual([]); - }); - - it("an early caller-scope halt tears down the document and leased eval scope", function* () { - Reflect.deleteProperty(globalThis, "__executeHaltMarker"); - yield* API.Process.around({ - *exec(_args, _next) { - yield* sleep(10_000); - return { exitCode: 0, stdout: "", stderr: "" }; - }, - }); - const doc = [ - "", - '', - "```js persist eval", - "globalThis.__executeHaltMarker = true;", - "yield* spawn(function* () { try { yield* suspend(); } finally { globalThis.__executeHaltMarker = false; } });", - "```", - "```bash exec", - "sleep 600", - "```", - "", - "", - "", - ].join("\n"); - yield* useStubFs({ "README.md": doc }); - - yield* scoped(function* () { - const execution = yield* executeDocument({ - docPath: "README.md", - stream: new InMemoryStream(), - testing: false, - }); - yield* spawn(function* () { - yield* scoped(function* () { - yield* drain(yield* execution.output); - }); - }); - // Give the document time to start the test and spawn its effect, - // then leave the scope — halting everything mid-test. - yield* sleep(200); - }); - - expect(Reflect.get(globalThis, "__executeHaltMarker")).toBe(false); - }); -}); diff --git a/testing/tests/helpers.ts b/testing/tests/helpers.ts index 1ea09b1..857639b 100644 --- a/testing/tests/helpers.ts +++ b/testing/tests/helpers.ts @@ -1,16 +1,19 @@ /** - * Document-level test harness: run a stub-fs document through - * executeDocument and observe chunks, close value, completion, and the - * delegated test results/boundaries. + * Document-level test harness: compose the testing vocabulary (or a full + * useTesting() session) around core execute() inside a bounded scope, and + * observe chunks, close value, completion Result, and the delegated test + * results/boundaries. */ -import type { Operation } from "effection"; +import { scoped } from "effection"; +import type { Operation, Result } from "effection"; import { forEach } from "@effectionx/stream-helpers"; import { InMemoryStream } from "@executablemd/durable-streams"; import { useStubFs } from "@executablemd/runtime/test"; -import type { DocumentExecution } from "@executablemd/core"; -import { executeDocument } from "../src/execute.ts"; -import type { ExecuteDocumentOptions } from "../src/execute.ts"; +import { execute } from "@executablemd/core"; +import { useTesting } from "../src/use-testing.ts"; +import { installHandlers, installTestingVocabulary } from "../src/vocabulary.ts"; +import type { TestHandlers } from "../src/handlers.ts"; import { Test } from "../src/test-api.ts"; import type { BoundaryOutcome, TestResult } from "../src/test-api.ts"; @@ -19,8 +22,8 @@ export interface DocRun { chunks: string[]; /** The output stream's close value. */ output: string; - completion: { ok: true; value: string } | { ok: false; error: Error }; - /** Results delegated past the wrapper's run-level collector. */ + completion: Result; + /** Results delegated past the session/vocabulary collectors. */ results: TestResult[]; boundaries: BoundaryOutcome[]; } @@ -29,52 +32,55 @@ export interface RunDocOptions { testing?: boolean; verbose?: boolean; docPath?: string; - execute?: (options: ExecuteDocumentOptions) => Operation; + /** Inject handlers (e.g. a short timeout) instead of the public set. */ + handlers?: TestHandlers; } export function* runDoc( files: Record, options: RunDocOptions = {}, ): Operation { - yield* useStubFs(files); + return yield* scoped(function* () { + yield* useStubFs(files); - const results: TestResult[] = []; - const boundaries: BoundaryOutcome[] = []; - yield* Test.around({ - *record([result], next) { - results.push(result); - yield* next(result); - }, - *boundary([outcome], next) { - boundaries.push(outcome); - yield* next(outcome); - }, - }); + const results: TestResult[] = []; + const boundaries: BoundaryOutcome[] = []; + yield* Test.around({ + *record([result], next) { + results.push(result); + yield* next(result); + }, + *boundary([outcome], next) { + boundaries.push(outcome); + yield* next(outcome); + }, + }); - const execute = options.execute ?? executeDocument; - const execution = yield* execute({ - docPath: options.docPath ?? "README.md", - stream: new InMemoryStream(), - testing: options.testing ?? false, - verbose: options.verbose, - }); + if (options.handlers) { + yield* installHandlers(options.handlers, { verbose: options.verbose }); + if (options.testing) { + yield* Test.around({ testing: () => true }); + } + } else if (options.testing) { + yield* useTesting({ verbose: options.verbose }); + } else { + yield* installTestingVocabulary({ verbose: options.verbose }); + } - const chunks: string[] = []; - const output = yield* forEach(function* (chunk: string) { - chunks.push(chunk); - }, execution.output); + const execution = yield* execute({ + docPath: options.docPath ?? "README.md", + stream: new InMemoryStream(), + }); - let completion: DocRun["completion"]; - try { - completion = { ok: true, value: yield* execution }; - } catch (error) { - completion = { - ok: false, - error: error instanceof Error ? error : new Error(String(error)), - }; - } + const chunks: string[] = []; + const output = yield* forEach(function* (chunk: string) { + chunks.push(chunk); + }, execution.output); - return { chunks, output, completion, results, boundaries }; + const completion = yield* execution; + + return { chunks, output, completion, results, boundaries }; + }); } export function failureOf(run: DocRun): Error | undefined { diff --git a/testing/tests/testing-mode.test.ts b/testing/tests/testing-mode.test.ts index 17c0c74..9ac646c 100644 --- a/testing/tests/testing-mode.test.ts +++ b/testing/tests/testing-mode.test.ts @@ -3,10 +3,8 @@ import { expect } from "@effectionx/bdd/expect"; import { sleep } from "effection"; import { API } from "@executablemd/runtime"; import { useFailingExec } from "@executablemd/runtime/test"; -import { runDocument } from "@executablemd/core"; import { TestFailureError } from "../src/test-api.ts"; import { createTestHandlers } from "../src/handlers.ts"; -import { createExecuteDocument } from "../src/execute.ts"; import { failureOf, runDoc } from "./helpers.ts"; describe("testing mode", () => { @@ -283,10 +281,6 @@ describe("testing mode", () => { return { exitCode: 0, stdout: "", stderr: "" }; }, }); - const execute = createExecuteDocument({ - runDocument, - handlers: createTestHandlers({ timeoutMs: 100 }), - }); const doc = [ "", '', @@ -298,7 +292,10 @@ describe("testing mode", () => { "", "", ].join("\n"); - const run = yield* runDoc({ "README.md": doc }, { execute }); + const run = yield* runDoc( + { "README.md": doc }, + { handlers: createTestHandlers({ timeoutMs: 100 }) }, + ); expect(failureOf(run)).toBeInstanceOf(TestFailureError); expect(run.results.map((r) => [r.name, r.status, r.error?.kind])).toEqual([ ["hangs", "fail", "timeout"], diff --git a/testing/tests/use-testing.test.ts b/testing/tests/use-testing.test.ts new file mode 100644 index 0000000..01ef560 --- /dev/null +++ b/testing/tests/use-testing.test.ts @@ -0,0 +1,190 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { scoped, sleep, spawn } from "effection"; +import type { Operation, Subscription } from "effection"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import { useStubFs } from "@executablemd/runtime/test"; +import { API } from "@executablemd/runtime"; +import { execute, Execution } from "@executablemd/core"; +import { Test, TestFailureError, testing } from "../src/test-api.ts"; +import { useTesting } from "../src/use-testing.ts"; +import { failureOf, runDoc } from "./helpers.ts"; + +function* drain( + subscription: Subscription, +): Operation<{ chunks: string[]; close: string }> { + const chunks: string[] = []; + let next = yield* subscription.next(); + while (!next.done) { + chunks.push(next.value); + next = yield* subscription.next(); + } + return { chunks, close: next.value }; +} + +describe("useTesting composition", () => { + it("composes around core execute and returns session results", function* () { + const outcome = yield* scoped(function* () { + yield* useStubFs({ + "README.md": '\n', + }); + const tests = yield* useTesting(); + const execution = yield* execute({ docPath: "README.md", stream: new InMemoryStream() }); + const result = yield* execution; + const results = yield* tests.results; + return { result, results }; + }); + expect(outcome.result.ok).toBe(true); + expect(outcome.results.map((r) => [r.name, r.status])).toEqual([["one", "pass"]]); + expect(Object.isFrozen(outcome.results)).toBe(true); + }); + + it("keeps results available after a testing failure", function* () { + const outcome = yield* scoped(function* () { + yield* useStubFs({ + "README.md": '\n', + }); + const tests = yield* useTesting(); + const execution = yield* execute({ docPath: "README.md", stream: new InMemoryStream() }); + const result = yield* execution; + const results = yield* tests.results; + return { result, results }; + }); + expect(outcome.result.ok).toBe(false); + if (!outcome.result.ok) { + expect(outcome.result.error).toBeInstanceOf(TestFailureError); + } + expect(outcome.results.map((r) => [r.name, r.status])).toEqual([["bad", "fail"]]); + }); + + it("preserves a core Err unchanged", function* () { + const result = yield* scoped(function* () { + yield* useStubFs({}); + yield* useTesting(); + const execution = yield* execute({ docPath: "missing.md", stream: new InMemoryStream() }); + return yield* execution; + }); + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error).not.toBeInstanceOf(TestFailureError); + expect(result.error.message).toContain("missing.md"); + } + }); + + it("enforces one session per execution scope", function* () { + let error: Error | undefined; + yield* scoped(function* () { + yield* useTesting(); + try { + yield* useTesting(); + } catch (failure) { + error = failure instanceof Error ? failure : new Error(String(failure)); + } + }); + expect(error?.message).toContain("one session per execution scope"); + }); + + it("removes all session middleware with its scope", function* () { + yield* scoped(function* () { + yield* useStubFs({ "README.md": "\n" }); + yield* useTesting(); + const execution = yield* execute({ docPath: "README.md", stream: new InMemoryStream() }); + const result = yield* execution; + expect(result.ok).toBe(true); + }); + + // Outside the session scope: activation, collectors, and policy are gone. + expect(yield* testing).toBe(false); + expect(yield* Test.operations.results()).toEqual([]); + const second = yield* runDoc({ "README.md": "\n" }); + expect(second.completion.ok).toBe(true); + expect(second.output).not.toContain("Assert"); + expect(second.results).toEqual([]); + }); + + it("completion returns Err instead of throwing for document failures", function* () { + const run = yield* runDoc({}, { docPath: "missing.md" }); + const error = failureOf(run); + expect(error?.message).toContain("missing.md"); + expect(run.output).toBe(""); + }); + + it("a failure before the handle exists may still throw", function* () { + let thrown: Error | undefined; + yield* scoped(function* () { + yield* useStubFs({ "README.md": "hello\n" }); + yield* Execution.around({ + // deno-lint-ignore require-yield + *execute() { + throw new Error("pre-handle setup exploded"); + }, + }); + try { + yield* execute({ docPath: "README.md", stream: new InMemoryStream() }); + } catch (failure) { + thrown = failure instanceof Error ? failure : new Error(String(failure)); + } + }); + expect(thrown?.message).toBe("pre-handle setup exploded"); + }); + + it("a late subscriber still receives all chunks and the close value", function* () { + const { late, value } = yield* scoped(function* () { + yield* useStubFs({ "README.md": "hello world\n" }); + yield* useTesting(); + const execution = yield* execute({ + docPath: "README.md", + stream: new InMemoryStream(), + }); + const result = yield* execution; + const drained = yield* scoped(function* () { + return yield* drain(yield* execution.output); + }); + return { late: drained, value: result }; + }); + // No tests in the doc — the session rejects, but output is intact. + expect(value.ok).toBe(false); + expect(late.close).toContain("hello world"); + expect(late.chunks.join("")).toContain("hello world"); + }); + + it("an early caller-scope halt tears down the document and leased eval scope", function* () { + Reflect.deleteProperty(globalThis, "__useTestingHaltMarker"); + yield* API.Process.around({ + *exec(_args, _next) { + yield* sleep(10_000); + return { exitCode: 0, stdout: "", stderr: "" }; + }, + }); + const doc = [ + "", + '', + "```js persist eval", + "globalThis.__useTestingHaltMarker = true;", + "yield* spawn(function* () { try { yield* suspend(); } finally { globalThis.__useTestingHaltMarker = false; } });", + "```", + "```bash exec", + "sleep 600", + "```", + "", + "", + "", + ].join("\n"); + yield* useStubFs({ "README.md": doc }); + + yield* scoped(function* () { + yield* useTesting(); + const execution = yield* execute({ docPath: "README.md", stream: new InMemoryStream() }); + yield* spawn(function* () { + yield* scoped(function* () { + yield* drain(yield* execution.output); + }); + }); + // Give the document time to start the test and spawn its effect, + // then leave the scope — halting everything mid-test. + yield* sleep(200); + }); + + expect(Reflect.get(globalThis, "__useTestingHaltMarker")).toBe(false); + }); +}); From f378119b9dc8d371405305dd854d376eb3653cde Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:10:56 -0400 Subject: [PATCH 04/14] fix: session/API blockers from architecture review - useTesting() session supports exactly one execute() call - a second call fails clearly before a handle exists (results are cumulative, so a zero-test second document would otherwise inherit the first document's passing outcome); regression added - OA10 decision row updated to Result/Err completion semantics - stale runDocument / run-document.ts / executeDocument references replaced in README, runtime docs and stubs, decisions log, test-api - 'a ExecuteOptions' -> 'an ExecuteOptions' - revert formatting-only churn in core/components/*.md --- README.md | 2 +- core/components/AnthropicProvider.md | 62 +++++++++++++--------------- core/components/Instruction.md | 22 ++++------ core/components/LlamafileProvider.md | 53 +++++++++++------------- core/components/OllamaProvider.md | 44 +++++++++----------- core/components/Sample.md | 18 ++++---- runtime/apis.ts | 2 +- runtime/test/README.md | 6 +-- runtime/test/stubs.ts | 4 +- specs/decisions.md | 4 +- specs/executable-mdx-spec.md | 6 +-- testing/src/test-api.ts | 2 +- testing/src/use-testing.ts | 12 ++++++ testing/tests/use-testing.test.ts | 24 +++++++++++ 14 files changed, 139 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 69f57a7..a468d14 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Each invocation requires a new path. If the path already exists, `xmd` exits wit ## Project layout -- `core/src/run-document.ts` - document entrypoint and durable import pipeline. +- `core/src/execute.ts` - document entrypoint and durable import pipeline. - `core/src/scanner.ts` - boundary scanner for components and executable fences. - `core/src/` - component expansion, eval/exec handling, modifiers, and sampling helpers. - `core/components/` - reusable provider and demo components. diff --git a/core/components/AnthropicProvider.md b/core/components/AnthropicProvider.md index 4915ae2..af0d0fc 100644 --- a/core/components/AnthropicProvider.md +++ b/core/components/AnthropicProvider.md @@ -13,42 +13,38 @@ inputs: --- ```ts persist eval -yield * - Sample.around( - { - *sample([context], next) { - if (context.model !== undefined && context.model !== model) { - return yield* next(context); - } +yield* Sample.around({ + *sample([context], next) { + if (context.model !== undefined && context.model !== model) { + return yield* next(context); + } - const messages = []; - if (context.system) { - messages.push({ role: "system", content: context.system }); - } - messages.push({ role: "user", content: context.content }); + const messages = []; + if (context.system) { + messages.push({ role: "system", content: context.system }); + } + messages.push({ role: "user", content: context.content }); - const result = yield* fetch("https://api.anthropic.com/v1/messages", { - method: "POST", - headers: { - "Content-Type": "application/json", - "x-api-key": process.env.ANTHROPIC_API_KEY, - "anthropic-version": "2023-06-01", - }, - body: JSON.stringify({ - model, - max_tokens: 4096, - system: context.system || undefined, - messages: [{ role: "user", content: context.content }], - }), - }) - .expect() - .json(); - - return result.content[0].text; + const result = yield* fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": process.env.ANTHROPIC_API_KEY, + "anthropic-version": "2023-06-01", }, - }, - { at: "min" }, - ); + body: JSON.stringify({ + model, + max_tokens: 4096, + system: context.system || undefined, + messages: [{ role: "user", content: context.content }], + }), + }) + .expect() + .json(); + + return result.content[0].text; + }, +}, { at: 'min' }); ``` diff --git a/core/components/Instruction.md b/core/components/Instruction.md index 8c35b5d..f459537 100644 --- a/core/components/Instruction.md +++ b/core/components/Instruction.md @@ -15,19 +15,15 @@ inputs: --- ```js persist eval -yield * - Sample.around( - { - *sample([context], next) { - const existing = context.system || ""; - return yield* next({ - ...context, - system: existing ? existing + "\n" + system : system, - }); - }, - }, - { at: "min" }, - ); +yield* Sample.around({ + *sample([context], next) { + const existing = context.system || ''; + return yield* next({ + ...context, + system: existing ? existing + '\n' + system : system, + }); + }, +}, { at: 'min' }); ``` diff --git a/core/components/LlamafileProvider.md b/core/components/LlamafileProvider.md index 4378ea2..db4aa99 100644 --- a/core/components/LlamafileProvider.md +++ b/core/components/LlamafileProvider.md @@ -22,7 +22,7 @@ inputs: --- ```ts eval -const port = yield * findFreePort(); +const port = yield* findFreePort(); const baseUrl = `http://127.0.0.1:${port}`; ``` @@ -31,40 +31,35 @@ const baseUrl = `http://127.0.0.1:${port}`; ``` ```ts eval -yield * - when(function* () { - yield* fetch(`${baseUrl}/health`).expect(); - }); +yield* when(function* () { + yield* fetch(`${baseUrl}/health`).expect(); +}); ``` ```ts persist eval -yield * - Sample.around( - { - *sample([context], next) { - if (context.model !== undefined && context.model !== model) { - return yield* next(context); - } +yield* Sample.around({ + *sample([context], next) { + if (context.model !== undefined && context.model !== model) { + return yield* next(context); + } - const messages = []; - if (context.system) { - messages.push({ role: "system", content: context.system }); - } - messages.push({ role: "user", content: context.content }); + const messages = []; + if (context.system) { + messages.push({ role: "system", content: context.system }); + } + messages.push({ role: "user", content: context.content }); - const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model, messages, temperature: 0, max_tokens: 2048 }), - }) - .expect() - .json(); + const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model, messages, temperature: 0, max_tokens: 2048 }), + }) + .expect() + .json(); - return result.choices[0].message.content; - }, - }, - { at: "min" }, - ); + return result.choices[0].message.content; + }, +}, { at: 'min' }); ``` diff --git a/core/components/OllamaProvider.md b/core/components/OllamaProvider.md index 3a344e0..e69f845 100644 --- a/core/components/OllamaProvider.md +++ b/core/components/OllamaProvider.md @@ -19,33 +19,29 @@ inputs: --- ```ts persist eval -yield * - Sample.around( - { - *sample([context], next) { - if (context.model !== undefined && context.model !== model) { - return yield* next(context); - } +yield* Sample.around({ + *sample([context], next) { + if (context.model !== undefined && context.model !== model) { + return yield* next(context); + } - const messages = []; - if (context.system) { - messages.push({ role: "system", content: context.system }); - } - messages.push({ role: "user", content: context.content }); + const messages = []; + if (context.system) { + messages.push({ role: "system", content: context.system }); + } + messages.push({ role: "user", content: context.content }); - const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ model, messages, temperature: 0 }), - }) - .expect() - .json(); + const result = yield* fetch(`${baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model, messages, temperature: 0 }), + }) + .expect() + .json(); - return result.choices[0].message.content; - }, - }, - { at: "min" }, - ); + return result.choices[0].message.content; + }, +}, { at: 'min' }); ``` diff --git a/core/components/Sample.md b/core/components/Sample.md index a8f25d1..0989638 100644 --- a/core/components/Sample.md +++ b/core/components/Sample.md @@ -30,17 +30,15 @@ inputs: --- ```js persist eval -const childrenOutput = yield * renderChildren(); -const content = childrenOutput || prompt || ""; +const childrenOutput = yield* renderChildren(); +const content = childrenOutput || prompt || ''; -const sampleResult = - yield * - Sample.operations.sample({ - content, - params: params || undefined, - componentName: "Sample", - model: model || undefined, - }); +const sampleResult = yield* Sample.operations.sample({ + content, + params: params || undefined, + componentName: 'Sample', + model: model || undefined, +}); return sampleResult; ``` diff --git a/runtime/apis.ts b/runtime/apis.ts index d9a6e0e..f08aed7 100644 --- a/runtime/apis.ts +++ b/runtime/apis.ts @@ -49,7 +49,7 @@ * * Middleware is **scoped** — it only affects operations within the * current Effection scope and its children. Install before calling - * `runDocument()` or `durableRun()`. + * `execute()` or `durableRun()`. * * ## Test stubs * diff --git a/runtime/test/README.md b/runtime/test/README.md index 21c8ff2..313b586 100644 --- a/runtime/test/README.md +++ b/runtime/test/README.md @@ -26,7 +26,7 @@ Installs an in-memory filesystem. - `stat(path)` reports `exists/isFile` based on whether `path` is a key - `glob()` throws with `"glob not stubbed"` -This is the right default for `runDocument()` tests that want to supply a small +This is the right default for `execute()` tests that want to supply a small virtual document tree inline. ```ts @@ -93,7 +93,7 @@ Typical combinations: These helpers are Effection middleware. They are scoped to the current operation scope and its children. -Install them before calling `runDocument()` or `durableRun()`: +Install them before calling `execute()` or `durableRun()`: ```ts it("renders from stubbed inputs", function* () { @@ -102,7 +102,7 @@ it("renders from stubbed inputs", function* () { yield* useStubFs({ "doc.md": "# Hello\n" }); yield* useEchoExec(); - const execution = yield* runDocument({ docPath: "doc.md", stream }); + const execution = yield* execute({ docPath: "doc.md", stream }); const output = yield* collect(execution); }); ``` diff --git a/runtime/test/stubs.ts b/runtime/test/stubs.ts index 6380a06..c38affa 100644 --- a/runtime/test/stubs.ts +++ b/runtime/test/stubs.ts @@ -3,7 +3,7 @@ * * These helpers install `around()` middleware on the runtime context APIs * to replace real I/O with in-memory implementations. They are scoped to - * the current Effection scope — call them before `runDocument()` or + * the current Effection scope — call them before `execute()` or * `durableRun()` in your test body. * * @example @@ -14,7 +14,7 @@ * yield* useStubFs({ "doc.md": "# Hello\n" }); * yield* useEchoExec(); * - * const execution = yield* runDocument({ docPath: "doc.md", stream }); + * const execution = yield* execute({ docPath: "doc.md", stream }); * const output = yield* execution; * }); * ``` diff --git a/specs/decisions.md b/specs/decisions.md index 2acf57c..9f8dcaf 100644 --- a/specs/decisions.md +++ b/specs/decisions.md @@ -170,13 +170,13 @@ experimental implementation detail. Its API may change. ### Decision Keep stream protocol integration in the execution boundary -(`src/run-document.ts`, `src/eval-handler.ts`, and terminal effect handlers). +(`src/execute.ts`, `src/eval-handler.ts`, and terminal effect handlers). All other modules (scanner, frontmatter, expand, interpolate, validate, render) have zero dependency on the stream package. The expansion engine reaches import and modifier execution through the contextual Component Api (DEC-012) — it doesn't know about durable -effects. The document's providers, installed by `run-document.ts`, are +effects. The document's providers, installed by `execute.ts`, are the only place expansion meets the stream protocol. --- diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index 130fb6b..f179fa3 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -643,7 +643,7 @@ the block if it overruns. The silent handler discards the output. #### Overriding per-scope Because factories are stored in a registry that can be extended, -custom modifiers can be provided via `ExecuteOptions`: +custom modifiers can be provided vian `ExecuteOptions`: ```typescript yield* execute({ @@ -3834,7 +3834,7 @@ visible warning blocks, collect into a separate error report). | OA7 | Channel close ends consumer | `channel.close()` causes `forEach` to complete | | OA8 | Multiple middleware compose | Normalize → terminal → channel: all three run in order | | OA9 | `ephemeral()` wrapper | `output()` inside durable context produces no journal entry | -| OA10 | execute workflow error surfaces through execution | `execute` workflow error → `reject(error)` → `yield* execution` throws | +| OA10 | execute workflow error surfaces through execution | `execute` workflow error → completion resolves `Err(error)` — `yield* execution` returns the `Result`, never throws | ### Tier WN — Whitespace normalization @@ -3954,7 +3954,7 @@ must preserve the trace for diagnosis or remove it before starting a new run. | 41 | `findFreePort` is a standalone VM global using `node:net` | Port allocation is platform I/O; the function uses Effection's `once` + `race` for event handling and `try/finally` for guaranteed cleanup; exposed in the eval sandbox alongside other Effection globals | | 42 | `findFreePort` result journaled with its eval block | The port number is a scalar export; no separate journal-entry type is needed | | 43 | `when` (from `@effectionx/converge`) is the polling VM global | `when` is the exported name from the package; the sandbox already contains it; no rename or addition needed | -| 44 | Provider lifecycle expressed as a component, not a `ExecuteOptions` field | Scope boundary is visible in the document tree; composable — multiple providers nest naturally via structured concurrency; no framework-level lifecycle hooks required | +| 44 | Provider lifecycle expressed as a component, not an `ExecuteOptions` field | Scope boundary is visible in the document tree; composable — multiple providers nest naturally via structured concurrency; no framework-level lifecycle hooks required | | 45 | Readiness check is a separate `eval` block, not internal to `daemon` | Auditable — strategy visible in the document; replaceable — different daemons have different readiness signals; composable with `when`'s configurable backoff | | 46 | Sample middleware reads `baseUrl` from `env.values` | Avoids a dedicated inference server context key; the binding environment is already the shared state carrier for within-component coordination; scope-correct because a fresh environment is provided per component expansion | | 47 | Each component gets a fresh `EvalEnv` | The component's environment is installed as a scope-local `env` provider around body expansion, so eval blocks within a component share bindings but don't leak into parent or sibling components; critical for provider isolation | diff --git a/testing/src/test-api.ts b/testing/src/test-api.ts index 781189e..b2faf21 100644 --- a/testing/src/test-api.ts +++ b/testing/src/test-api.ts @@ -2,7 +2,7 @@ * Test Api — contextual operations for testing mode (specs/testing-spec.md). * * `testing` is the activation switch: false by default, true beneath - * `` (or root activation via `executeDocument({ testing: true })`). + * `` (or root activation via a `useTesting()` session). * Both perform the identical `Test.around({ testing: () => true })` install, * which is what makes `xmd test` equivalent to wrapping the entrypoint in * ``. diff --git a/testing/src/use-testing.ts b/testing/src/use-testing.ts index c844ad0..bea1145 100644 --- a/testing/src/use-testing.ts +++ b/testing/src/use-testing.ts @@ -56,8 +56,20 @@ export function* useTesting(options?: { verbose?: boolean }): Operation // Err(TestFailureError) after its output closes when tests failed or none // were discovered. A core Err passes through unchanged, and `results` // stays available either way. + // + // One execute() per session: results are cumulative across the session, + // so a second document would inherit the first document's outcomes (a + // zero-test document after a passing one would succeed). Fail clearly + // BEFORE a handle exists — the pre-handle throw path. + let executed = false; yield* Execution.around({ *execute([executeOptions], next) { + if (executed) { + throw new Error( + "a useTesting() session supports one execute() call — start a new session for another document", + ); + } + executed = true; const inner = yield* next(executeOptions); return decorateCompletion(inner, () => { if (collected.length === 0) { diff --git a/testing/tests/use-testing.test.ts b/testing/tests/use-testing.test.ts index 01ef560..7ad443c 100644 --- a/testing/tests/use-testing.test.ts +++ b/testing/tests/use-testing.test.ts @@ -71,6 +71,30 @@ describe("useTesting composition", () => { } }); + it("enforces one execute() call per session", function* () { + // Results are cumulative across a session: without the guard, a + // zero-test second document would succeed on the strength of the first + // document's passing test. + let thrown: Error | undefined; + const first = yield* scoped(function* () { + yield* useStubFs({ + "README.md": '\n', + "empty.md": "no tests here\n", + }); + yield* useTesting(); + const execution = yield* execute({ docPath: "README.md", stream: new InMemoryStream() }); + const result = yield* execution; + try { + yield* execute({ docPath: "empty.md", stream: new InMemoryStream() }); + } catch (failure) { + thrown = failure instanceof Error ? failure : new Error(String(failure)); + } + return result; + }); + expect(first.ok).toBe(true); + expect(thrown?.message).toContain("one execute() call"); + }); + it("enforces one session per execution scope", function* () { let error: Error | undefined; yield* scoped(function* () { From 34fb4432c5563c9c0479dc6cbe71551bee2890d9 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:27:00 -0400 Subject: [PATCH 05/14] docs: add Code Rule 10 - no decorative section-divider comments Structure source through names and modules. Enforcement via a local oxlint rule with autofix is tracked in #109. --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index 66e94e2..dc52251 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,7 @@ Do not commit if any check fails. Fix the issue first, then re-run all three. require changes to specs/release-process-spec.md to match. 9. Prefer stateless generators - use a function when calling a function that returns an operation; Do not do this function*(arg) { return yield* generator(arg) } +10. Structure source through names and modules. Do not use decorative section-divider comments. ## PR Process From 980bf09796a798913442e2b5a0c19314414aff5c Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:30:57 -0400 Subject: [PATCH 06/14] docs(spec): fix 'provided vian ExecuteOptions' typo --- specs/executable-mdx-spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/executable-mdx-spec.md b/specs/executable-mdx-spec.md index f179fa3..326342e 100644 --- a/specs/executable-mdx-spec.md +++ b/specs/executable-mdx-spec.md @@ -643,7 +643,7 @@ the block if it overruns. The silent handler discards the output. #### Overriding per-scope Because factories are stored in a registry that can be extended, -custom modifiers can be provided vian `ExecuteOptions`: +custom modifiers can be provided via `ExecuteOptions`: ```typescript yield* execute({ From 5a94bc9326eaa2131b4e86de05af1f177090c52a Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:01:43 -0400 Subject: [PATCH 07/14] fix(testing): preserve testing outcomes across journal replay durableRun returns the stored root result on a full replay without re-expanding the document, so test and boundary collectors stayed empty: a passing session replayed as 'no tests were discovered' and a failing boundary replayed as success. - journal.ts: each completed TestResult and each boundary outcome persists as a testing-owned durable operation (test_result, testing_boundary) during expansion, before the root Close, identified deterministically by source position; payloads validated with type guards on read - vocabulary Execution middleware: on a confirmed full replay (root Close present in ExecuteOptions.stream) the stored records are restored into the current collectors through the same record/boundary operations live expansion uses; live and partial journals hydrate nothing - re-expansion records each result exactly once with completed records replaying in place, and stored results stay authoritative over recomputation - replay regressions: passing/failing/zero-test useTesting live+replay, pass/fail/empty explicit boundaries, partial replay without duplicates, results after replayed failure, unchanged output and zero re-run effects (appendCount 0) - testing spec documents the durable record and replay semantics - @executablemd/durable-streams added as a direct testing dependency --- specs/testing-spec.md | 11 ++- testing/package.json | 1 + testing/src/handlers.ts | 33 +++++-- testing/src/journal.ts | 182 +++++++++++++++++++++++++++++++++++ testing/src/vocabulary.ts | 20 +++- testing/tests/helpers.ts | 5 +- testing/tests/replay.test.ts | 147 ++++++++++++++++++++++++++++ 7 files changed, 387 insertions(+), 12 deletions(-) create mode 100644 testing/src/journal.ts create mode 100644 testing/tests/replay.test.ts diff --git a/specs/testing-spec.md b/specs/testing-spec.md index bcf5599..be6fb3a 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -36,7 +36,16 @@ fails after expansion when any test failed or when no tests were discovered. `xmd test` exits with status `0` when every test passes and status `1` otherwise. -Testing uses the standard journal and replay behavior. +Testing uses the standard journal and replay behavior. Each completed test +and each `` boundary outcome is journaled as a testing-owned +durable operation (`test_result`, `testing_boundary`) while expansion runs, +before the root close event, identified deterministically by source +position. On a full replay of a completed journal the recorded results and +boundary outcomes are restored from the stream into the current collectors +— test bodies and their effects are not rerun — so the testing outcome and +recorded results are preserved. On live and partial runs nothing is +restored: re-expansion records each result exactly once, in discovery +order, with completed records replaying in place. ## Atomic Tests diff --git a/testing/package.json b/testing/package.json index 4681e52..674f45b 100644 --- a/testing/package.json +++ b/testing/package.json @@ -11,6 +11,7 @@ "@effectionx/scope-eval": "0.1.3", "@effectionx/timebox": "0.4.3", "@executablemd/core": "workspace:*", + "@executablemd/durable-streams": "workspace:*", "@std/assert": "npm:@jsr/std__assert@^1.0.17", "effection": "4.1.0-alpha.7" } diff --git a/testing/src/handlers.ts b/testing/src/handlers.ts index 54608a1..8361fcb 100644 --- a/testing/src/handlers.ts +++ b/testing/src/handlers.ts @@ -24,6 +24,7 @@ import { Test, boundary, inTest, record, testing } from "./test-api.ts"; import type { TestResult } from "./test-api.ts"; import { AssertionDiagnostic, expandAssertion } from "./assertions.ts"; import type { AssertionEntry } from "./assertions.ts"; +import { persistBoundaryOutcome, persistTestResult } from "./journal.ts"; // --------------------------------------------------------------------------- // Contained-failure carriers @@ -115,10 +116,16 @@ export function createTestHandlers(options: { timeoutMs: number }): TestHandlers { at: "min" }, ); const report = yield* ctx.expand(invocation.children); - yield* boundary({ - tests: local.length, - failed: local.filter((result) => result.status === "fail").length, - }); + // Journal the outcome before the root Close so a full replay can + // restore it without re-expanding this boundary. + const outcome = yield* persistBoundaryOutcome( + { + tests: local.length, + failed: local.filter((result) => result.status === "fail").length, + }, + formatLocation(invocation), + ); + yield* boundary(outcome); return report; }); } @@ -147,10 +154,12 @@ export function createTestHandlers(options: { timeoutMs: number }): TestHandlers const parentEnv = yield* env; const parentScope = yield* evalScope; if (!parentScope) { - const result = failResult(name, location, { - kind: "error", - message: " requires an eval scope in context.", - }); + const result = yield* persistTestResult( + failResult(name, location, { + kind: "error", + message: " requires an eval scope in context.", + }), + ); yield* record(result); return [failureDiagnostic(result, { detail: true })]; } @@ -223,7 +232,13 @@ export function createTestHandlers(options: { timeoutMs: number }): TestHandlers } } - const result = classify(name, location, bodyError, timedOut, timeoutMs); + // Journal the result before the root Close. On partial replay the + // stored record wins over the recomputation (short-circuited effects + // can change what the re-run observes, e.g. a halted exec no longer + // times out), keeping the original outcome authoritative. + const result = yield* persistTestResult( + classify(name, location, bodyError, timedOut, timeoutMs), + ); yield* record(result); if (result.status === "fail") { diff --git a/testing/src/journal.ts b/testing/src/journal.ts new file mode 100644 index 0000000..6cdce80 --- /dev/null +++ b/testing/src/journal.ts @@ -0,0 +1,182 @@ +/** + * Durable testing records (specs/testing-spec.md §Testing Mode). + * + * Completed tests and explicit `` boundary outcomes are journaled + * as testing-owned durable operations while document expansion runs — + * before the root Close event. On a full replay of a completed journal, + * `durableRun` returns the stored root result without re-expanding the + * document, so no test would ever re-record; the stored records are + * restored from the stream instead. On partial replay the document + * re-expands and each record replays in place, recording exactly once in + * discovery order. + * + * Record identities are deterministic, derived from source position. + */ + +import { createDurableOperation } from "@executablemd/durable-streams"; +import type { DurableStream, Json, Workflow } from "@executablemd/durable-streams"; +import type { Operation } from "effection"; +import type { BoundaryOutcome, TestResult } from "./test-api.ts"; + +const TEST_RESULT = "test_result"; +const TESTING_BOUNDARY = "testing_boundary"; + +export function* persistTestResult(result: TestResult): Workflow { + const stored = yield createDurableOperation( + { type: TEST_RESULT, name: `test:${result.location}` }, + // deno-lint-ignore require-yield + function* (): Operation { + return serializeTestResult(result); + }, + ); + const parsed = parseTestResult(stored); + if (!parsed) { + throw new Error(`journaled test_result for "${result.location}" has an unexpected shape`); + } + return parsed; +} + +export function* persistBoundaryOutcome( + outcome: BoundaryOutcome, + location: string, +): Workflow { + const stored = yield createDurableOperation( + { type: TESTING_BOUNDARY, name: `testing:${location}` }, + // deno-lint-ignore require-yield + function* (): Operation { + return { tests: outcome.tests, failed: outcome.failed }; + }, + ); + const parsed = parseBoundaryOutcome(stored); + if (!parsed) { + throw new Error(`journaled testing_boundary for "${location}" has an unexpected shape`); + } + return parsed; +} + +export interface CompletedRunRecords { + results: TestResult[]; + boundaries: BoundaryOutcome[]; +} + +/** + * Read testing records from a journal that already holds a root Close + * event — the confirmed-full-replay case. Returns undefined for a live or + * partial journal, where expansion itself (re)records each result. + */ +export function* readCompletedRun( + stream: DurableStream, +): Operation { + const events = yield* stream.readAll(); + const completed = events.some((event) => event.type === "close" && event.coroutineId === "root"); + if (!completed) { + return undefined; + } + + const results: TestResult[] = []; + const boundaries: BoundaryOutcome[] = []; + for (const event of events) { + if (event.type !== "yield" || event.result.status !== "ok") { + continue; + } + if (event.description.type === TEST_RESULT) { + const parsed = parseTestResult(event.result.value); + if (parsed) { + results.push(parsed); + } + } else if (event.description.type === TESTING_BOUNDARY) { + const parsed = parseBoundaryOutcome(event.result.value); + if (parsed) { + boundaries.push(parsed); + } + } + } + return { results, boundaries }; +} + +function serializeTestResult(result: TestResult): Json { + const payload: Record = { + status: result.status, + location: result.location, + }; + if (result.name !== undefined) { + payload.name = result.name; + } + if (result.error) { + const error: Record = { + kind: result.error.kind, + message: result.error.message, + }; + if (result.error.actual !== undefined) { + error.actual = result.error.actual; + } + if (result.error.expected !== undefined) { + error.expected = result.error.expected; + } + payload.error = error; + } + return payload; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function optionalString(value: unknown): value is string | undefined { + return value === undefined || typeof value === "string"; +} + +const ERROR_KINDS = ["assertion", "timeout", "teardown", "error"]; + +function parseTestResult(value: unknown): TestResult | undefined { + if (!isRecord(value)) { + return undefined; + } + const { status, location, name, error } = value; + if (status !== "pass" && status !== "fail") { + return undefined; + } + if (typeof location !== "string" || !optionalString(name)) { + return undefined; + } + const result: TestResult = { status, location }; + if (name !== undefined) { + result.name = name; + } + if (error !== undefined) { + if (!isRecord(error)) { + return undefined; + } + const { kind, message, actual, expected } = error; + if (typeof kind !== "string" || !ERROR_KINDS.includes(kind)) { + return undefined; + } + if ( + typeof message !== "string" || + !optionalString(actual) || + !optionalString(expected) || + (kind !== "assertion" && kind !== "timeout" && kind !== "teardown" && kind !== "error") + ) { + return undefined; + } + result.error = { kind, message }; + if (actual !== undefined) { + result.error.actual = actual; + } + if (expected !== undefined) { + result.error.expected = expected; + } + } + return result; +} + +function parseBoundaryOutcome(value: unknown): BoundaryOutcome | undefined { + if (!isRecord(value)) { + return undefined; + } + const { tests, failed } = value; + if (typeof tests !== "number" || typeof failed !== "number" || tests < 0 || failed < 0) { + return undefined; + } + return { tests, failed }; +} diff --git a/testing/src/vocabulary.ts b/testing/src/vocabulary.ts index 27131a1..9cbaa38 100644 --- a/testing/src/vocabulary.ts +++ b/testing/src/vocabulary.ts @@ -18,8 +18,9 @@ import { Err } from "effection"; import type { Operation } from "effection"; import { Component, Execution } from "@executablemd/core"; import type { DocumentExecution } from "@executablemd/core"; -import { Test, TestFailureError } from "./test-api.ts"; +import { boundary, record, Test, TestFailureError } from "./test-api.ts"; import type { BoundaryOutcome } from "./test-api.ts"; +import { readCompletedRun } from "./journal.ts"; import { ASSERTIONS } from "./assertions.ts"; import { createTestHandlers } from "./handlers.ts"; import type { TestHandlers } from "./handlers.ts"; @@ -67,6 +68,23 @@ export function* installHandlers( yield* nextBoundary(outcome); }, }); + // Confirmed full replay: durableRun returns the stored root result + // without re-expanding, so nothing would re-record. Restore the + // journaled testing records into the current collectors — through + // the same record/boundary operations live expansion uses, so every + // session collector and observer sees them in discovery order. A + // live or partial journal (no root Close) hydrates nothing; + // re-expansion records each result exactly once via its durable + // operation. + const replayed = yield* readCompletedRun(executeOptions.stream); + if (replayed) { + for (const result of replayed.results) { + yield* record(result); + } + for (const outcome of replayed.boundaries) { + yield* boundary(outcome); + } + } const inner = yield* next(executeOptions); return decorateCompletion(inner, () => { const failed = boundaries.filter((b) => b.failed > 0); diff --git a/testing/tests/helpers.ts b/testing/tests/helpers.ts index 857639b..84e3214 100644 --- a/testing/tests/helpers.ts +++ b/testing/tests/helpers.ts @@ -9,6 +9,7 @@ import { scoped } from "effection"; import type { Operation, Result } from "effection"; import { forEach } from "@effectionx/stream-helpers"; import { InMemoryStream } from "@executablemd/durable-streams"; +import type { DurableStream } from "@executablemd/durable-streams"; import { useStubFs } from "@executablemd/runtime/test"; import { execute } from "@executablemd/core"; import { useTesting } from "../src/use-testing.ts"; @@ -34,6 +35,8 @@ export interface RunDocOptions { docPath?: string; /** Inject handlers (e.g. a short timeout) instead of the public set. */ handlers?: TestHandlers; + /** Supply a journal stream (e.g. for replay scenarios). */ + stream?: DurableStream; } export function* runDoc( @@ -69,7 +72,7 @@ export function* runDoc( const execution = yield* execute({ docPath: options.docPath ?? "README.md", - stream: new InMemoryStream(), + stream: options.stream ?? new InMemoryStream(), }); const chunks: string[] = []; diff --git a/testing/tests/replay.test.ts b/testing/tests/replay.test.ts new file mode 100644 index 0000000..c25e966 --- /dev/null +++ b/testing/tests/replay.test.ts @@ -0,0 +1,147 @@ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import type { DurableEvent } from "@executablemd/durable-streams"; +import { API } from "@executablemd/runtime"; +import { TestFailureError } from "../src/test-api.ts"; +import { failureOf, runDoc } from "./helpers.ts"; +import type { DocRun, RunDocOptions } from "./helpers.ts"; +import type { Operation } from "effection"; + +interface LiveAndReplay { + live: DocRun; + replay: DocRun; + replayStream: InMemoryStream; +} + +/** + * Run a document live on a fresh journal, then run it again on a copy of + * the completed journal — a confirmed full replay. + */ +function* liveThenReplay( + files: Record, + options: RunDocOptions = {}, +): Operation { + const liveStream = new InMemoryStream(); + const live = yield* runDoc(files, { ...options, stream: liveStream }); + const events = yield* liveStream.readAll(); + const replayStream = new InMemoryStream(events); + const replay = yield* runDoc(files, { ...options, stream: replayStream }); + return { live, replay, replayStream }; +} + +describe("testing replay", () => { + it("a passing useTesting run preserves its outcome and results on full replay", function* () { + const { live, replay, replayStream } = yield* liveThenReplay( + { "README.md": '\n' }, + { testing: true }, + ); + expect(live.completion.ok).toBe(true); + expect(replay.completion.ok).toBe(true); + expect(replay.results).toEqual(live.results); + expect(replay.results.map((r) => [r.name, r.status])).toEqual([["one", "pass"]]); + // Full replay appends nothing to the journal. + expect(replayStream.appendCount).toBe(0); + }); + + it("a failing useTesting run stays failed on full replay, with results available", function* () { + const { live, replay } = yield* liveThenReplay( + { "README.md": '\n' }, + { testing: true }, + ); + expect(failureOf(live)).toBeInstanceOf(TestFailureError); + expect(failureOf(replay)).toBeInstanceOf(TestFailureError); + expect(replay.results).toEqual(live.results); + expect(replay.results.map((r) => [r.name, r.status, r.error?.kind])).toEqual([ + ["bad", "fail", "assertion"], + ]); + }); + + it("a zero-test useTesting run stays failed on full replay", function* () { + const { live, replay } = yield* liveThenReplay( + { "README.md": "just text\n" }, + { testing: true }, + ); + expect(failureOf(live)?.message).toContain("no tests were discovered"); + expect(failureOf(replay)?.message).toContain("no tests were discovered"); + expect(replay.results).toEqual([]); + }); + + it("explicit boundaries preserve pass, fail, and empty outcomes on replay", function* () { + const passing = yield* liveThenReplay({ + "README.md": "\n", + }); + expect(passing.live.completion.ok).toBe(true); + expect(passing.replay.completion.ok).toBe(true); + expect(passing.replay.boundaries).toEqual([{ tests: 1, failed: 0 }]); + + const failing = yield* liveThenReplay({ + "README.md": "\n", + }); + expect(failureOf(failing.live)).toBeInstanceOf(TestFailureError); + expect(failureOf(failing.replay)).toBeInstanceOf(TestFailureError); + expect(failing.replay.boundaries).toEqual([{ tests: 1, failed: 1 }]); + + const empty = yield* liveThenReplay({ + "README.md": "no tests\n", + }); + expect(failureOf(empty.live)).toBeInstanceOf(TestFailureError); + expect(failureOf(empty.replay)).toBeInstanceOf(TestFailureError); + expect(empty.replay.boundaries).toEqual([{ tests: 0, failed: 0 }]); + }); + + it("partial replay records each result exactly once, in discovery order", function* () { + const files = { + "README.md": [ + '', + '', + "", + ].join("\n"), + }; + const liveStream = new InMemoryStream(); + const live = yield* runDoc(files, { testing: true, stream: liveStream }); + expect(live.completion.ok).toBe(true); + + // Drop the root Close: the journal is now a partial trace, so the + // document re-expands with completed durable records replaying in place. + const events = yield* liveStream.readAll(); + const partial = events.filter((event: DurableEvent) => event.type !== "close"); + const resumed = yield* runDoc(files, { + testing: true, + stream: new InMemoryStream(partial), + }); + expect(resumed.completion.ok).toBe(true); + expect(resumed.results.map((r) => [r.name, r.status])).toEqual([ + ["first", "pass"], + ["second", "pass"], + ]); + }); + + it("full replay leaves output unchanged and reruns no document effects", function* () { + const execCalls: string[] = []; + yield* API.Process.around({ + // deno-lint-ignore require-yield + *exec([options], _next) { + execCalls.push(options.command.join(" ")); + return { exitCode: 0, stdout: "ran\n", stderr: "" }; + }, + }); + const files = { + "README.md": [ + "", + "```bash exec", + "echo hi", + "```", + "", + "", + "", + ].join("\n"), + }; + const { live, replay, replayStream } = yield* liveThenReplay(files); + expect(live.completion.ok).toBe(true); + expect(replay.completion.ok).toBe(true); + expect(execCalls).toHaveLength(1); + expect(replay.output).toBe(live.output); + expect(replayStream.appendCount).toBe(0); + }); +}); From 9591610a71c91eb659406bcc6aafc7645f709a60 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:27:03 -0400 Subject: [PATCH 08/14] chore: regenerate lockfiles and publish workflow for testing's durable-streams dependency --- .github/workflows/publish-packages.yml | 2 +- bun.lock | 1 + pnpm-lock.yaml | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 5a42247..8cf5276 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -83,7 +83,7 @@ jobs: version: ${{ needs.version.outputs.value }} testing: - needs: [version, core] + needs: [version, core, durable-streams] uses: ./.github/workflows/publish-one.yml with: package: testing diff --git a/bun.lock b/bun.lock index 39c3244..f429c0f 100644 --- a/bun.lock +++ b/bun.lock @@ -121,6 +121,7 @@ "@effectionx/scope-eval": "0.1.3", "@effectionx/timebox": "0.4.3", "@executablemd/core": "workspace:*", + "@executablemd/durable-streams": "workspace:*", "@std/assert": "npm:@jsr/std__assert@^1.0.17", "effection": "4.1.0-alpha.7", }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6085d80..faec0b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -245,6 +245,9 @@ importers: '@executablemd/core': specifier: workspace:* version: link:../core + '@executablemd/durable-streams': + specifier: workspace:* + version: link:../durable-streams '@std/assert': specifier: npm:@jsr/std__assert@^1.0.17 version: '@jsr/std__assert@1.0.19' From fe36a3c37a7c28bbd4259dca9a13e03a8bbade26 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:30:44 -0400 Subject: [PATCH 09/14] refactor(smoke): make the smoke document self-testing via the testing API - smoke-test/README.md captures the entire guide once into a root binding and re-emits it, so regular xmd run output stays semantically unchanged and every document effect runs exactly once; seven named, top-level, sibling bodies (Components, Execution, Captures, Evaluation, Providers, Output regions, Durability) inspect the captured guide with AssertStringIncludes / AssertMatch / AssertNotMatch - no wrapper, so regular execution skips them - harness moves to testing/tests/smoke.test.ts (core stays independent of @executablemd/testing) and shrinks to infrastructure: useTesting() composed around core execute(), the exact seven embedded tests asserted in discovery order all passing, journal size checked host-side, and a fresh-session full replay asserted equal in Result and TestResult snapshot with zero appended journal events --- core/tests/smoke.test.ts | 218 ------------------------------------ smoke-test/README.md | 106 ++++++++++++++++++ testing/tests/smoke.test.ts | 68 +++++++++++ 3 files changed, 174 insertions(+), 218 deletions(-) delete mode 100644 core/tests/smoke.test.ts create mode 100644 testing/tests/smoke.test.ts diff --git a/core/tests/smoke.test.ts b/core/tests/smoke.test.ts deleted file mode 100644 index 7147229..0000000 --- a/core/tests/smoke.test.ts +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Smoke test — runs the full smoke-test document (smoke-test/README.md) - * through the entire pipeline using the real Node.js runtime. Verifies - * the output contains expected content from every feature: frontmatter - * interpolation, component expansion, nested components, dotted names, - * executable code blocks, silent modifier, props, Content slot, markdown - * healing, non-executable passthrough, eval blocks with shared bindings, - * binding capture (`as` and ``), persist modifier, timeout modifier, - * daemon modifier, sample modifier, - * bracket params, provider pattern, and nested providers. - */ -import { describe, it } from "@effectionx/bdd/node"; -import { expect } from "@effectionx/bdd/expect"; -import { InMemoryStream } from "@executablemd/durable-streams"; -import { execute } from "../src/execute.ts"; -import { collect } from "../src/collect.ts"; - -// --------------------------------------------------------------------------- -// Smoke test -// --------------------------------------------------------------------------- - -describe("smoke test", { sanitizeOps: false, sanitizeResources: false }, () => { - it("runs the full smoke-test document and produces expected output", function* () { - const stream = new InMemoryStream(); - - const output = yield* collect( - yield* execute({ - docPath: "smoke-test/README.md", - stream, - componentDirs: ["smoke-test", "core/components"], - }), - ); - - // ----- Root frontmatter interpolation ----- - expect(output).toContain("# Executable MDX"); - expect(output).toContain("version **0.1.0**"); - expect(output).toContain("https://github.com/thefrontside/effectionx"); - - // ----- Section component (Content slot + meta interpolation) ----- - expect(output).toContain("§ What is Executable MDX?"); - expect(output).toContain("§ Components"); - expect(output).toContain("§ Nested Components"); - expect(output).toContain("§ Executable Code Blocks"); - expect(output).toContain("§ Props and Interpolation"); - expect(output).toContain("§ Markdown Healing"); - expect(output).toContain("§ Durability"); - expect(output).toContain("§ Smoke Test Summary"); - - // ----- Note component (props interpolation + defaults) ----- - // Default level=info - expect(output).toContain("📝 **info:** This note uses the default level (info)."); - // Overridden level=warning - expect(output).toContain("📝 **warning:** This note overrides the level to warning."); - - // ----- Feature component (nested — Feature contains Note) ----- - expect(output).toContain("**Recursive Expansion**"); - expect(output).toContain("Components expand bottom-up"); - expect(output).toContain("This note was generated inside the Feature component."); - - // ----- Dotted component name (Tips.Formatting → Tips/Formatting.md) ----- - expect(output).toContain("💡 **Formatting tip:**"); - expect(output).toContain(""); - - // ----- Executable code blocks ----- - // find count — should produce a number - expect(output).toMatch(/\d+/); - // echo block - expect(output).toContain("Hello from a durable workflow"); - - // ----- Silent exec — output suppressed ----- - // The silent block content should NOT appear in output - expect(output).not.toContain("This output is journaled but not shown in the document"); - - // ----- Non-executable code block (yaml passthrough) ----- - expect(output).toContain("# This is just a code block"); - expect(output).toContain("type: string"); - - // ----- PropDemo component ----- - expect(output).toContain('"Hey, world!"'); - - // ----- Badge component (no inputs, meta interpolation) ----- - expect(output).toContain("✓ verified"); - - // ----- Markdown healing ----- - // The unclosed bold before should be healed - // "**bold before the component" should get closed before the boundary - // Both text segments should be independently valid markdown - - // ----- Smoke test summary table ----- - expect(output).toContain("| Feature"); - expect(output).toContain("Root frontmatter"); - expect(output).toContain("Component with props"); - expect(output).toContain("Content slot"); - expect(output).toContain("Nested expansion"); - expect(output).toContain("Dotted component name"); - expect(output).toContain("exec modifier"); - expect(output).toContain("silent modifier"); - expect(output).toContain("Markdown healing"); - - // ----- Binding capture section ----- - expect(output).toContain("§ Binding Capture"); - expect(output).toContain("Capture values:"); - expect(output).toContain("component binding from Fragment"); - expect(output).toContain("| inline binding from Capture"); - expect(output).not.toContain("Hidden capture should not render inline."); - - // ----- Capture with CSS selector ----- - // select="code[lang=json]" extracts only the JSON code block value, - // ignoring surrounding prose text - expect(output).toContain('Selected JSON: ["alpha","bravo",42]'); - expect(output).not.toContain("Some prose before the data"); - - // ----- In-Process Evaluation section ----- - expect(output).toContain("§ In-Process Evaluation"); - - // Eval blocks produce no rendered output — their bindings are invisible - expect(output).not.toContain("Hello from eval"); - expect(output).not.toContain("with 3 numbers"); - - // persist eval — resource survival via spawn + when convergence - // The persist block spawns a task; the next block converges on it. - // Neither produces rendered output. - expect(output).not.toContain("serverReady"); // eval binding — no output - expect(output).toContain("kept the task alive"); // prose explains persist - - // timeout eval block — produces no output - expect(output).not.toContain("startedAt"); - - // findFreePort + eval binding interpolation - // The eval block allocates a port; the exec block uses {port} syntax. - // The output should contain "Server would start on port " - expect(output).toMatch(/Server would start on port \d+/); - - // But exec blocks in the same document still produce output - expect(output).toContain("Exec blocks are independent of eval bindings"); - - // Eval summary table entries - expect(output).toContain("eval modifier"); - expect(output).toContain("persist modifier"); - expect(output).toContain("persist resource survival"); - expect(output).toContain("timeout modifier"); - expect(output).toContain("eval + exec coexistence"); - expect(output).toContain("findFreePort VM global"); - expect(output).toContain("eval binding interpolation"); - - // ----- Background Processes section (daemon) ----- - expect(output).toContain("§ Background Processes"); - // The daemon server responds with "daemon-ok" - expect(output).toContain("daemon-ok"); - - // ----- Sample Component section ----- - expect(output).toContain("§ Sample Component"); - // Self-closing with prompt — StubProvider returns [response-from-sample-stub] - expect(output).toContain("[response-from-sample-stub]"); - // With children — children rendered then sampled - // The children text should NOT appear raw (it's consumed by the Sample component) - - // ----- Smoke test summary table — new entries ----- - expect(output).toContain("daemon modifier"); - expect(output).toContain("provider pattern"); - expect(output).toContain("per-component eval scope"); - expect(output).toContain("props in env.values"); - expect(output).toContain("Sample component"); - expect(output).toContain("output() function"); - expect(output).toContain("renderChildren() closure"); - expect(output).toContain("Instruction component"); - expect(output).toContain("composable instructions"); - expect(output).toContain("component as capture"); - expect(output).toContain("Capture directive"); - expect(output).toContain("Capture select"); - - // ----- Instruction Component section ----- - expect(output).toContain("§ Instruction Component"); - // Instruction wraps Sample — response includes system prompt text - expect(output).toContain("[response-from-instruction-stub|system:You are a helpful pirate.]"); - - // ----- Component-declared Output section ----- - expect(output).toContain("§ Component-declared Output"); - // The inside renders (its `when` binding was computed by - // preceding documentation). - expect(output).toContain("OUTPUTDEMO_SELECTED"); - // The component's documentation prose is suppressed. - expect(output).not.toContain("OUTPUTDEMO_DOC_LEAK"); - - // ----- Durability section ----- - expect(output).toContain("Run at:"); - - // ----- Journal should have events ----- - const events = stream.snapshot(); - // At minimum: root import + Section(x9) + Note(x5 total) + Feature + Badge + Formatting + PropDemo + exec blocks + eval blocks - expect(events.length).toBeGreaterThan(10); - }); - - it("replay produces identical output without re-reading files", function* () { - const stream = new InMemoryStream(); - - // Golden run - const firstOutput = yield* collect( - yield* execute({ - docPath: "smoke-test/README.md", - stream, - componentDirs: ["smoke-test", "core/components"], - }), - ); - - // Replay — durableRun short-circuits on the Close event in the - // journal, returning the stored result without any I/O calls. - const secondOutput = yield* collect( - yield* execute({ - docPath: "smoke-test/README.md", - stream, - componentDirs: ["smoke-test", "core/components"], - }), - ); - - expect(secondOutput).toBe(firstOutput); - }); -}); diff --git a/smoke-test/README.md b/smoke-test/README.md index 88dc3b3..b6e6ef6 100644 --- a/smoke-test/README.md +++ b/smoke-test/README.md @@ -4,6 +4,8 @@ version: 0.1.0 repo: https://github.com/thefrontside/effectionx --- + + # {meta.title} This document is both a guide and a smoke test. Every feature described @@ -518,3 +520,107 @@ EOF If you can read this table, every feature worked. + + + +{guide} + + + + + + + + + + + + + + + + + + + +"} /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/tests/smoke.test.ts b/testing/tests/smoke.test.ts new file mode 100644 index 0000000..19ea010 --- /dev/null +++ b/testing/tests/smoke.test.ts @@ -0,0 +1,68 @@ +/** + * Smoke harness — infrastructure only. The smoke-test document tests + * itself: its guide is captured into a root binding and inspected by the + * embedded bodies (smoke-test/README.md). This harness composes + * useTesting() around core execute(), asserts the exact embedded tests + * pass in discovery order, keeps the journal-size check host-side, and + * verifies a full replay reproduces the identical outcome without + * appending journal events. + */ +import { describe, it } from "@effectionx/bdd/node"; +import { expect } from "@effectionx/bdd/expect"; +import { scoped } from "effection"; +import type { Operation, Result } from "effection"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import { execute } from "@executablemd/core"; +import type { ExecuteOptions } from "@executablemd/core"; +import { useTesting } from "../src/use-testing.ts"; +import type { TestResult } from "../src/test-api.ts"; + +const EMBEDDED_TESTS = [ + "Components", + "Execution", + "Captures", + "Evaluation", + "Providers", + "Output regions", + "Durability", +]; + +interface SmokeSession { + result: Result; + results: readonly TestResult[]; +} + +function* runSmokeSession(options: ExecuteOptions): Operation { + return yield* scoped(function* () { + const tests = yield* useTesting(); + const execution = yield* execute(options); + const result = yield* execution; + return { result, results: yield* tests.results }; + }); +} + +describe("smoke test", { sanitizeOps: false, sanitizeResources: false }, () => { + it("smoke document passes its embedded tests live and on replay", function* () { + const stream = new InMemoryStream(); + const options: ExecuteOptions = { + docPath: "smoke-test/README.md", + stream, + componentDirs: ["smoke-test", "core/components"], + }; + + const live = yield* runSmokeSession(options); + if (!live.result.ok) { + throw live.result.error; + } + expect(live.results.map((entry) => [entry.name, entry.status])).toEqual( + EMBEDDED_TESTS.map((name) => [name, "pass"]), + ); + expect(stream.snapshot().length).toBeGreaterThan(10); + const liveAppendCount = stream.appendCount; + + const replay = yield* runSmokeSession(options); + expect(replay.result).toEqual(live.result); + expect(replay.results).toEqual(live.results); + expect(stream.appendCount).toBe(liveAppendCount); + }); +}); From 29eb984c951448f8c9980f46d442497c8718a70a Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:48:07 -0400 Subject: [PATCH 10/14] refactor(smoke): embed tests beside the features they verify The smoke guide splits into self-testing feature documents under smoke-test/Guide/. Each document captures its own rendered content into a local binding, re-emits it, and carries a sibling named asserting against that capture - no wrapper, no nesting, no re-executed effects. README.md composes the sixteen feature documents in guide order and keeps the root-frontmatter behavior and its test, since that behavior belongs to the root document. Expression Props and Text Interpolation share the itemCount binding, so they stay together in Guide/Interpolation.md (with its own title meta so the {meta.title} demo renders identically). The harness now expects the exact seventeen embedded test names in discovery order; execution, journal, result, and replay checks remain the only TypeScript assertions. --- smoke-test/Guide/Captures.md | 57 +++ smoke-test/Guide/Components.md | 41 ++ smoke-test/Guide/Daemons.md | 51 +++ smoke-test/Guide/Durability.md | 29 ++ smoke-test/Guide/Evaluation.md | 99 +++++ smoke-test/Guide/Execution.md | 53 +++ smoke-test/Guide/Healing.md | 26 ++ smoke-test/Guide/Instructions.md | 35 ++ smoke-test/Guide/Interpolation.md | 63 +++ smoke-test/Guide/NamedSlots.md | 37 ++ smoke-test/Guide/NestedComponents.md | 32 ++ smoke-test/Guide/OutputRegions.md | 23 + smoke-test/Guide/Overview.md | 19 + smoke-test/Guide/Props.md | 26 ++ smoke-test/Guide/Sampling.md | 33 ++ smoke-test/Guide/Summary.md | 96 +++++ smoke-test/README.md | 621 ++------------------------- testing/tests/smoke.test.ts | 29 +- 18 files changed, 767 insertions(+), 603 deletions(-) create mode 100644 smoke-test/Guide/Captures.md create mode 100644 smoke-test/Guide/Components.md create mode 100644 smoke-test/Guide/Daemons.md create mode 100644 smoke-test/Guide/Durability.md create mode 100644 smoke-test/Guide/Evaluation.md create mode 100644 smoke-test/Guide/Execution.md create mode 100644 smoke-test/Guide/Healing.md create mode 100644 smoke-test/Guide/Instructions.md create mode 100644 smoke-test/Guide/Interpolation.md create mode 100644 smoke-test/Guide/NamedSlots.md create mode 100644 smoke-test/Guide/NestedComponents.md create mode 100644 smoke-test/Guide/OutputRegions.md create mode 100644 smoke-test/Guide/Overview.md create mode 100644 smoke-test/Guide/Props.md create mode 100644 smoke-test/Guide/Sampling.md create mode 100644 smoke-test/Guide/Summary.md diff --git a/smoke-test/Guide/Captures.md b/smoke-test/Guide/Captures.md new file mode 100644 index 0000000..87bcded --- /dev/null +++ b/smoke-test/Guide/Captures.md @@ -0,0 +1,57 @@ + + +
+ +Binding capture routes rendered output into the eval binding environment +instead of writing it at the invocation site. + +Component-level capture uses `as="name"`: + +component binding from Fragment + +Inline capture uses the built-in `` directive: + +inline binding from Capture + + +Captured bindings are available to later executable blocks: + +```bash exec +echo "Capture values: {capturedFromComponent} | {capturedInline}" +``` + +Capture with CSS selector extracts specific content from rendered output: + + +Some prose before the data. + +```json +["alpha","bravo",42] +``` + +More prose after. + + +```bash exec +printf 'Selected JSON: %s\n' '{capturedJson}' +``` + +This Note is captured but intentionally not rendered inline: + + + +
+ + + +{rendered} + + + + + + + + + + diff --git a/smoke-test/Guide/Components.md b/smoke-test/Guide/Components.md new file mode 100644 index 0000000..ad8d3b0 --- /dev/null +++ b/smoke-test/Guide/Components.md @@ -0,0 +1,41 @@ + + +
+ +A component is a markdown file with a declared interface. The component's +frontmatter defines its own metadata and the props it accepts +via `inputs`. Here's the frontmatter from the Note component used below: + +```yaml +# components/Note.md frontmatter +emoji: 📝 +inputs: + level: info + message: + type: string + required: true +``` + +Components are invoked with JSX syntax. Props must match the declared +inputs — undeclared props are rejected, required props must be provided, +and defaults fill in for omitted optional props. + + + + + +Components can wrap content using the `` slot. The Section +component wrapping this text works exactly that way — it receives a +title prop and renders its children inside a headed section. + +
+ + + +{rendered} + + + + + + diff --git a/smoke-test/Guide/Daemons.md b/smoke-test/Guide/Daemons.md new file mode 100644 index 0000000..850c3f1 --- /dev/null +++ b/smoke-test/Guide/Daemons.md @@ -0,0 +1,51 @@ + + +
+ +The `daemon` modifier starts a long-running process that survives across +subsequent blocks. Combined with `when()` for readiness polling, this +implements the provider pattern: start a service, wait until it's ready, +then run children against it. + +The eval block below allocates a port, the daemon block starts a Node +HTTP server on it, and the readiness block polls until the server +responds: + +```js eval +const daemonPort = yield * findFreePort(); +const daemonUrl = "http://127.0.0.1:" + daemonPort; +``` + +```bash daemon exec +node -e "require('http').createServer((q,s)=>{s.writeHead(200);s.end('daemon-ok')}).listen({daemonPort},'127.0.0.1')" +``` + +```js eval +yield * + when( + function* () { + yield* fetch(daemonUrl + "/health").expect(); + }, + { timeout: 5000, interval: 50 }, + ); +``` + +The daemon is alive — let's verify by hitting it: + +```bash exec +curl -s http://127.0.0.1:{daemonPort} +``` + +When this section ends, the daemon process is terminated by structured +concurrency — no manual cleanup needed. + +
+ + + +{rendered} + + + + + diff --git a/smoke-test/Guide/Durability.md b/smoke-test/Guide/Durability.md new file mode 100644 index 0000000..9833903 --- /dev/null +++ b/smoke-test/Guide/Durability.md @@ -0,0 +1,29 @@ + + +
+ +Every component import and code execution is recorded in a journal. +If this document's execution crashes mid-way, re-running it replays +completed operations from the journal and continues from where it +left off — no command is re-executed, no file is re-read. + +```bash exec +echo "Run at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" +``` + +Run this document twice. The timestamp above will be the same both +times — it was journaled on the first run and replayed on the second. + +If a component file changes between runs, the replay guard detects +the stale content hash and halts replay, forcing a fresh execution. + +
+ + + +{rendered} + + + + + diff --git a/smoke-test/Guide/Evaluation.md b/smoke-test/Guide/Evaluation.md new file mode 100644 index 0000000..56d4121 --- /dev/null +++ b/smoke-test/Guide/Evaluation.md @@ -0,0 +1,99 @@ + + +
+ +Eval blocks run JavaScript **in-process** as Effection generator operations. +Unlike `exec` blocks (which run shell commands in a subprocess), `eval` +blocks execute in the same process, sharing a binding environment across +blocks within a component. + +Bindings declared in one eval block are available in subsequent blocks: + +```js eval +const greeting = "Hello from eval"; +const numbers = [1, 2, 3]; +``` + +The bindings from the previous block are available here: + +```js eval +const message = `${greeting} with ${numbers.length} numbers`; +``` + +Eval blocks produce **no rendered output** — they exist for bindings +and side effects. The output from eval blocks is empty, so nothing +appears between this text and the next section. + +The `persist` modifier extends a block's resource lifetime from the +block scope to the component scope. Without `persist`, spawned tasks +and resources are torn down when the eval block completes. With it, +they survive for all subsequent blocks in the component. + +The block below spawns a background task that sets `status.ready` +after a short delay. Because it uses `persist`, the task stays alive: + +```js persist eval +const status = { ready: false }; +yield * + spawn(function* () { + yield* sleep(10); + status.ready = true; + }); +``` + +The next block converges on the spawned task using `when()`. This +only works because `persist` kept the task alive across the block +boundary — without it, the task would have been torn down: + +```js eval +yield * + when(function* () { + if (!status.ready) throw new Error("not ready"); + }); +const serverReady = status.ready; +``` + +The `timeout` modifier cancels the block if it does not complete within +the specified duration. Accepted units: `ms`, `s`, `m`. If the block +times out, an error is recorded in the output and execution halts. + +```js timeout=30s eval +const startedAt = Date.now(); +``` + +The `findFreePort` VM global finds an available TCP port using the OS: + +```js eval +const port = yield * findFreePort(); +``` + +Eval binding interpolation substitutes bare `{name}` references in code +block content with values from the eval binding environment. The port +allocated above flows into subsequent blocks via `{port}`: + +```bash exec +echo "Server would start on port {port}" +``` + +Eval and exec blocks coexist independently in the same document: + +```bash exec +echo "Exec blocks are independent of eval bindings" +``` + +
+ + + +{rendered} + + + + + + + + + + + diff --git a/smoke-test/Guide/Execution.md b/smoke-test/Guide/Execution.md new file mode 100644 index 0000000..f0403b5 --- /dev/null +++ b/smoke-test/Guide/Execution.md @@ -0,0 +1,53 @@ + + +
+ +Code blocks with `exec` in the info string are executed as shell +commands. The output replaces the code block in the rendered document. + +How many markdown files are in the smoke-test directory? + +```bash exec +find smoke-test -name '*.md' | wc -l | tr -d ' ' +``` + +The info string is a **middleware chain** read left-to-right. Each +word after the language is a modifier handler that wraps the next. + +`exec` alone — runs the command, shows stdout: + +```bash exec +echo "Hello from a durable workflow" +``` + +`silent exec` — runs and journals the command, but suppresses +output. Useful for setup steps: + +```bash silent exec +echo "This output is journaled but not shown in the document" +``` + +Non-executable code blocks are passed through as regular markdown. +This block has no `exec` modifier, so it's just syntax-highlighted text: + +```yaml +# This is just a code block — not executed +inputs: + name: + type: string +``` + +
+ + + +{rendered} + + + + + + + + + diff --git a/smoke-test/Guide/Healing.md b/smoke-test/Guide/Healing.md new file mode 100644 index 0000000..6389e74 --- /dev/null +++ b/smoke-test/Guide/Healing.md @@ -0,0 +1,26 @@ + + +
+ +Components and executable code blocks are **semantic boundaries**. +Markdown constructs cannot span them. Each text segment is healed +independently using remend. + +For example, if bold is opened before a component but not closed, +remend closes it before expansion proceeds. This prevents unclosed +markers from bleeding into component output. + +The text below opens \*\*bold before the component + +and continues after. Each segment is independently valid markdown. + +
+ + + +{rendered} + + + + + diff --git a/smoke-test/Guide/Instructions.md b/smoke-test/Guide/Instructions.md new file mode 100644 index 0000000..7d2a9ab --- /dev/null +++ b/smoke-test/Guide/Instructions.md @@ -0,0 +1,35 @@ + + +
+ +The `` component surfaces the system prompt as visible, +composable document content. Instead of hiding the LLM's instructions +inside provider internals, authors wrap Sample calls with +`` to define what the LLM is told. + +Without an instruction, the provider uses a hardcoded default system +prompt. With ``, the author's text replaces that default: + + + + + + + + + +Instructions accumulate through nesting. Agent components install +instruction middleware via `persist eval` blocks that enrich the +`SampleContext.system` field. When present, the system prompt is +passed through to the provider. + +
+ + + +{rendered} + + + + + diff --git a/smoke-test/Guide/Interpolation.md b/smoke-test/Guide/Interpolation.md new file mode 100644 index 0000000..2dfa9ff --- /dev/null +++ b/smoke-test/Guide/Interpolation.md @@ -0,0 +1,63 @@ +--- +title: Executable MDX +--- + + + +
+ +Expression props pass runtime values from eval blocks to child +components. Unlike string attributes, expression props resolve +at expansion time against the eval binding environment. + +```js eval +const dynamicGreeting = "Howdy"; +const dynamicSubject = "expression props"; +const itemCount = 3; +``` + +The values computed above flow into PropDemo via expression props: + + + +JSON literals resolve at scan time — no eval block needed: + + + +
+ +
+ +Eval bindings also resolve in **prose text**, not just code blocks. Values +computed in eval blocks flow naturally into surrounding text without +needing a template literal inside an eval block. + +```js eval +const textPort = 49821; +const textHost = "127.0.0.1"; +``` + +The server is running at {textHost}:{textPort}. + +Both `{meta.*}` / `{props.*}` and bare `{name}` work in the same text. +Meta values resolve first, then eval bindings fill in remaining references. +The document title is {meta.title} and the text port is {textPort}. + +Escaped braces produce literal output: \{textPort} stays as-is. + +If a bare reference has no matching binding, it passes through verbatim: +{undefinedBinding} is not resolved. + +Non-string values are coerced via `String()`. The count from the +Expression Props section is {itemCount}. + +
+ + + +{rendered} + + + + + diff --git a/smoke-test/Guide/NamedSlots.md b/smoke-test/Guide/NamedSlots.md new file mode 100644 index 0000000..e2770c3 --- /dev/null +++ b/smoke-test/Guide/NamedSlots.md @@ -0,0 +1,37 @@ + + +
+ +Components can render caller-provided content in multiple distinct +regions using named slots. The `slot` prop on child components directs +content to matching `` positions in the +component body. + + + + **Left column** content via named slot. + + + **Right column** content via named slot. + + + +Named slots compose with the existing content slot. Children without +a `slot` prop go to the default slot: + + + + + + +
+ + + +{rendered} + + + + + + diff --git a/smoke-test/Guide/NestedComponents.md b/smoke-test/Guide/NestedComponents.md new file mode 100644 index 0000000..8417c6b --- /dev/null +++ b/smoke-test/Guide/NestedComponents.md @@ -0,0 +1,32 @@ + + +
+ +Components can reference other components. When a component's body +contains another component invocation, the system resolves, imports, and +expands it recursively — with cycle detection to prevent infinite loops. + + + +Dotted names map to directory paths. The component below lives at +`components/Tips/Formatting.md`: + + + +
+ + + +{rendered} + + + + + + + +"} /> + diff --git a/smoke-test/Guide/OutputRegions.md b/smoke-test/Guide/OutputRegions.md new file mode 100644 index 0000000..3e73e83 --- /dev/null +++ b/smoke-test/Guide/OutputRegions.md @@ -0,0 +1,23 @@ + + +
+ +A component can declare which region of its body renders using ``. +Everything outside the region is documentation that executes (its eval blocks +run, its captures populate bindings) but never renders into the consumer. +`OutputDemo` computes a binding in documentation, then renders a `` +inside `` that depends on it. + + + +
+ + + +{rendered} + + + + + + diff --git a/smoke-test/Guide/Overview.md b/smoke-test/Guide/Overview.md new file mode 100644 index 0000000..071abd7 --- /dev/null +++ b/smoke-test/Guide/Overview.md @@ -0,0 +1,19 @@ + + +
+ +Executable MDX treats markdown files as **durable workflows**. A document +can contain component invocations that expand other markdown files, and +code blocks that execute shell commands. Every I/O operation is recorded +in a journal so that execution survives crashes and replays from where +it left off. + +
+ + + +{rendered} + + + + diff --git a/smoke-test/Guide/Props.md b/smoke-test/Guide/Props.md new file mode 100644 index 0000000..8918380 --- /dev/null +++ b/smoke-test/Guide/Props.md @@ -0,0 +1,26 @@ + + +
+ +Every component has access to two namespaces for interpolation: + +- `{meta.key}` — the component's own frontmatter values +- `{props.key}` — values passed by the caller via JSX props + +The Section component's frontmatter defines `emoji: §` which it +prepends to each title via `{meta.emoji}`. The Note component +uses `{meta.emoji}` for its 📝 prefix and `{props.message}` for +the caller-provided text. + + + +
+ + + +{rendered} + + + + + diff --git a/smoke-test/Guide/Sampling.md b/smoke-test/Guide/Sampling.md new file mode 100644 index 0000000..f69459e --- /dev/null +++ b/smoke-test/Guide/Sampling.md @@ -0,0 +1,33 @@ + + +
+ +The `` component captures its children's rendered output (or +accepts a `prompt` prop) and routes it through the Sample Api for LLM +processing. It uses `output()` to produce rendered output and +`renderChildren()` to capture children. + +Self-closing mode — prompt sent directly to the provider: + + + + + +With children — children are rendered first, then sampled: + + +This is child content to be processed. + + + + +
+ + + +{rendered} + + + + + diff --git a/smoke-test/Guide/Summary.md b/smoke-test/Guide/Summary.md new file mode 100644 index 0000000..08dcdbb --- /dev/null +++ b/smoke-test/Guide/Summary.md @@ -0,0 +1,96 @@ + + +
+ +This document exercises every feature of the system: + +```bash exec +cat <<'EOF' +| Feature | Exercised by | +|---------------------------|-----------------------------------------| +| Root frontmatter | Title and version in opening paragraph | +| Component with props |
, | +| Required props | (message is required) | +| Default props | uses level=info by default | +| Content slot |
wraps children via | +| Nested expansion | Section > Feature > Note (3 levels) | +| Dotted component name | | +| exec modifier | Multiple bash exec blocks | +| silent modifier | bash silent exec block | +| Non-executable code | yaml block (passthrough) | +| Markdown healing | Unclosed bold before | +| No-inputs component | accepts zero props | +| meta interpolation | {meta.emoji} in Section and Note | +| props interpolation | {props.title}, {props.message}, etc. | +| Props passthrough | | +| Expression props | | +| component as capture | ... | +| Capture directive | ... | +| Capture select | ... | +| Durability | Timestamp stable across reruns | +| eval modifier | js eval blocks with shared bindings | +| persist modifier | js persist eval block, resource lifetime| +| persist resource survival | spawn in persist eval + when() converge | +| timeout modifier | js timeout=30s eval block | +| eval + exec coexistence | Both modifier types in same document | +| findFreePort VM global | yield* findFreePort() in eval block | +| eval binding interpolation| {port} in exec block from eval binding | +| daemon modifier | bash daemon exec starts background proc | +| daemon + when readiness | Daemon server polled until ready | +| provider pattern | StubProvider installs Sample middleware | +| per-component eval scope | Each provider gets isolated middleware | +| props in env.values | model prop available in eval blocks | +| Sample component | , with children | +| output() function | Sample component calls output() | +| renderChildren() closure | Sample component captures children | +| Named slots | with slot="left"/slot="right" | +| Fragment passthrough | wraps raw text | +| Instruction component | wraps Sample calls | +| composable instructions | Instructions enrich SampleContext.system | +| Text interpolation | {textHost}:{textPort} in prose text | +| Text + meta coexistence | {meta.title} and {textPort} in same text| +| Escaped text bindings | \{textPort} produces literal braces | +| Verbatim unresolved | {undefinedBinding} left as-is | +| Non-string text coercion | {itemCount} coerced via String() | +EOF +``` + +If you can read this table, every feature worked. + +
+ + + +{rendered} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smoke-test/README.md b/smoke-test/README.md index b6e6ef6..6a98d8e 100644 --- a/smoke-test/README.md +++ b/smoke-test/README.md @@ -4,623 +4,56 @@ version: 0.1.0 repo: https://github.com/thefrontside/effectionx --- - + # {meta.title} This document is both a guide and a smoke test. Every feature described here is exercised by the document itself — if it renders correctly, the system works. This is version **{meta.version}** of {meta.title}, -built from the source at [{meta.repo}]({meta.repo}). +built from the source at [{meta.repo}]({meta.repo}). Each chapter below +is a self-testing feature document: it captures its own rendered +content, re-emits it, and carries a sibling test that inspects the +capture. -
- -Executable MDX treats markdown files as **durable workflows**. A document -can contain component invocations that expand other markdown files, and -code blocks that execute shell commands. Every I/O operation is recorded -in a journal so that execution survives crashes and replays from where -it left off. - -
- -
- -A component is a markdown file with a declared interface. The component's -frontmatter defines its own metadata and the props it accepts -via `inputs`. Here's the frontmatter from the Note component used below: - -```yaml -# components/Note.md frontmatter -emoji: 📝 -inputs: - level: info - message: - type: string - required: true -``` - -Components are invoked with JSX syntax. Props must match the declared -inputs — undeclared props are rejected, required props must be provided, -and defaults fill in for omitted optional props. - - - - - -Components can wrap content using the `` slot. The Section -component wrapping this text works exactly that way — it receives a -title prop and renders its children inside a headed section. - -
- -
- -Components can reference other components. When a component's body -contains another component invocation, the system resolves, imports, and -expands it recursively — with cycle detection to prevent infinite loops. - - - -Dotted names map to directory paths. The component below lives at -`components/Tips/Formatting.md`: - - - -
- -
- -Code blocks with `exec` in the info string are executed as shell -commands. The output replaces the code block in the rendered document. - -How many markdown files are in the smoke-test directory? - -```bash exec -find smoke-test -name '*.md' | wc -l | tr -d ' ' -``` - -The info string is a **middleware chain** read left-to-right. Each -word after the language is a modifier handler that wraps the next. - -`exec` alone — runs the command, shows stdout: - -```bash exec -echo "Hello from a durable workflow" -``` - -`silent exec` — runs and journals the command, but suppresses -output. Useful for setup steps: - -```bash silent exec -echo "This output is journaled but not shown in the document" -``` - -Non-executable code blocks are passed through as regular markdown. -This block has no `exec` modifier, so it's just syntax-highlighted text: - -```yaml -# This is just a code block — not executed -inputs: - name: - type: string -``` - -
- -
- -Every component has access to two namespaces for interpolation: - -- `{meta.key}` — the component's own frontmatter values -- `{props.key}` — values passed by the caller via JSX props - -The Section component's frontmatter defines `emoji: §` which it -prepends to each title via `{meta.emoji}`. The Note component -uses `{meta.emoji}` for its 📝 prefix and `{props.message}` for -the caller-provided text. - - - -
- -
- -Expression props pass runtime values from eval blocks to child -components. Unlike string attributes, expression props resolve -at expansion time against the eval binding environment. - -```js eval -const dynamicGreeting = "Howdy"; -const dynamicSubject = "expression props"; -const itemCount = 3; -``` - -The values computed above flow into PropDemo via expression props: - - - -JSON literals resolve at scan time — no eval block needed: - - - -
- -
- -Binding capture routes rendered output into the eval binding environment -instead of writing it at the invocation site. - -Component-level capture uses `as="name"`: - -component binding from Fragment - -Inline capture uses the built-in `` directive: - -inline binding from Capture -Captured bindings are available to later executable blocks: - -```bash exec -echo "Capture values: {capturedFromComponent} | {capturedInline}" -``` - -Capture with CSS selector extracts specific content from rendered output: - - -Some prose before the data. - -```json -["alpha","bravo",42] -``` - -More prose after. - - -```bash exec -printf 'Selected JSON: %s\n' '{capturedJson}' -``` - -This Note is captured but intentionally not rendered inline: - - - -
- -
- -Components and executable code blocks are **semantic boundaries**. -Markdown constructs cannot span them. Each text segment is healed -independently using remend. - -For example, if bold is opened before a component but not closed, -remend closes it before expansion proceeds. This prevents unclosed -markers from bleeding into component output. - -The text below opens \*\*bold before the component - -and continues after. Each segment is independently valid markdown. - -
- -
- -Eval blocks run JavaScript **in-process** as Effection generator operations. -Unlike `exec` blocks (which run shell commands in a subprocess), `eval` -blocks execute in the same process, sharing a binding environment across -blocks within a component. - -Bindings declared in one eval block are available in subsequent blocks: - -```js eval -const greeting = "Hello from eval"; -const numbers = [1, 2, 3]; -``` - -The bindings from the previous block are available here: - -```js eval -const message = `${greeting} with ${numbers.length} numbers`; -``` - -Eval blocks produce **no rendered output** — they exist for bindings -and side effects. The output from eval blocks is empty, so nothing -appears between this text and the next section. - -The `persist` modifier extends a block's resource lifetime from the -block scope to the component scope. Without `persist`, spawned tasks -and resources are torn down when the eval block completes. With it, -they survive for all subsequent blocks in the component. +{intro} -The block below spawns a background task that sets `status.ready` -after a short delay. Because it uses `persist`, the task stays alive: - -```js persist eval -const status = { ready: false }; -yield * - spawn(function* () { - yield* sleep(10); - status.ready = true; - }); -``` - -The next block converges on the spawned task using `when()`. This -only works because `persist` kept the task alive across the block -boundary — without it, the task would have been torn down: - -```js eval -yield * - when(function* () { - if (!status.ready) throw new Error("not ready"); - }); -const serverReady = status.ready; -``` - -The `timeout` modifier cancels the block if it does not complete within -the specified duration. Accepted units: `ms`, `s`, `m`. If the block -times out, an error is recorded in the output and execution halts. - -```js timeout=30s eval -const startedAt = Date.now(); -``` - -The `findFreePort` VM global finds an available TCP port using the OS: - -```js eval -const port = yield * findFreePort(); -``` - -Eval binding interpolation substitutes bare `{name}` references in code -block content with values from the eval binding environment. The port -allocated above flows into subsequent blocks via `{port}`: - -```bash exec -echo "Server would start on port {port}" -``` - -Eval and exec blocks coexist independently in the same document: - -```bash exec -echo "Exec blocks are independent of eval bindings" -``` - -
- -
- -Eval bindings also resolve in **prose text**, not just code blocks. Values -computed in eval blocks flow naturally into surrounding text without -needing a template literal inside an eval block. - -```js eval -const textPort = 49821; -const textHost = "127.0.0.1"; -``` - -The server is running at {textHost}:{textPort}. - -Both `{meta.*}` / `{props.*}` and bare `{name}` work in the same text. -Meta values resolve first, then eval bindings fill in remaining references. -The document title is {meta.title} and the text port is {textPort}. - -Escaped braces produce literal output: \{textPort} stays as-is. - -If a bare reference has no matching binding, it passes through verbatim: -{undefinedBinding} is not resolved. - -Non-string values are coerced via `String()`. The count from the -Expression Props section is {itemCount}. - -
- -
- -The `daemon` modifier starts a long-running process that survives across -subsequent blocks. Combined with `when()` for readiness polling, this -implements the provider pattern: start a service, wait until it's ready, -then run children against it. - -The eval block below allocates a port, the daemon block starts a Node -HTTP server on it, and the readiness block polls until the server -responds: - -```js eval -const daemonPort = yield * findFreePort(); -const daemonUrl = "http://127.0.0.1:" + daemonPort; -``` - -```bash daemon exec -node -e "require('http').createServer((q,s)=>{s.writeHead(200);s.end('daemon-ok')}).listen({daemonPort},'127.0.0.1')" -``` - -```js eval -yield * - when( - function* () { - yield* fetch(daemonUrl + "/health").expect(); - }, - { timeout: 5000, interval: 50 }, - ); -``` - -The daemon is alive — let's verify by hitting it: - -```bash exec -curl -s http://127.0.0.1:{daemonPort} -``` - -When this section ends, the daemon process is terminated by structured -concurrency — no manual cleanup needed. - -
- -
- -The `` component captures its children's rendered output (or -accepts a `prompt` prop) and routes it through the Sample Api for LLM -processing. It uses `output()` to produce rendered output and -`renderChildren()` to capture children. - -Self-closing mode — prompt sent directly to the provider: - - - - - -With children — children are rendered first, then sampled: - - -This is child content to be processed. - - - - -
- -
- -Components can render caller-provided content in multiple distinct -regions using named slots. The `slot` prop on child components directs -content to matching `` positions in the -component body. - - - - **Left column** content via named slot. - - - **Right column** content via named slot. - - - -Named slots compose with the existing content slot. Children without -a `slot` prop go to the default slot: - - - - - - -
- -
- -The `` component surfaces the system prompt as visible, -composable document content. Instead of hiding the LLM's instructions -inside provider internals, authors wrap Sample calls with -`` to define what the LLM is told. - -Without an instruction, the provider uses a hardcoded default system -prompt. With ``, the author's text replaces that default: - - - - - - - - - -Instructions accumulate through nesting. Agent components install -instruction middleware via `persist eval` blocks that enrich the -`SampleContext.system` field. When present, the system prompt is -passed through to the provider. - -
- -
+ + + + + -Every component import and code execution is recorded in a journal. -If this document's execution crashes mid-way, re-running it replays -completed operations from the journal and continues from where it -left off — no command is re-executed, no file is re-read. + -```bash exec -echo "Run at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" -``` + -Run this document twice. The timestamp above will be the same both -times — it was journaled on the first run and replayed on the second. + -If a component file changes between runs, the replay guard detects -the stale content hash and halts replay, forcing a fresh execution. + -
+ -
+ -A component can declare which region of its body renders using ``. -Everything outside the region is documentation that executes (its eval blocks -run, its captures populate bindings) but never renders into the consumer. -`OutputDemo` computes a binding in documentation, then renders a `` -inside `` that depends on it. + - + -
+ -
+ -This document exercises every feature of the system: + -```bash exec -cat <<'EOF' -| Feature | Exercised by | -|---------------------------|-----------------------------------------| -| Root frontmatter | Title and version in opening paragraph | -| Component with props |
, | -| Required props | (message is required) | -| Default props | uses level=info by default | -| Content slot |
wraps children via | -| Nested expansion | Section > Feature > Note (3 levels) | -| Dotted component name | | -| exec modifier | Multiple bash exec blocks | -| silent modifier | bash silent exec block | -| Non-executable code | yaml block (passthrough) | -| Markdown healing | Unclosed bold before | -| No-inputs component | accepts zero props | -| meta interpolation | {meta.emoji} in Section and Note | -| props interpolation | {props.title}, {props.message}, etc. | -| Props passthrough | | -| Expression props | | -| component as capture | ... | -| Capture directive | ... | -| Capture select | ... | -| Durability | Timestamp stable across reruns | -| eval modifier | js eval blocks with shared bindings | -| persist modifier | js persist eval block, resource lifetime| -| persist resource survival | spawn in persist eval + when() converge | -| timeout modifier | js timeout=30s eval block | -| eval + exec coexistence | Both modifier types in same document | -| findFreePort VM global | yield* findFreePort() in eval block | -| eval binding interpolation| {port} in exec block from eval binding | -| daemon modifier | bash daemon exec starts background proc | -| daemon + when readiness | Daemon server polled until ready | -| provider pattern | StubProvider installs Sample middleware | -| per-component eval scope | Each provider gets isolated middleware | -| props in env.values | model prop available in eval blocks | -| Sample component | , with children | -| output() function | Sample component calls output() | -| renderChildren() closure | Sample component captures children | -| Named slots | with slot="left"/slot="right" | -| Fragment passthrough | wraps raw text | -| Instruction component | wraps Sample calls | -| composable instructions | Instructions enrich SampleContext.system | -| Text interpolation | {textHost}:{textPort} in prose text | -| Text + meta coexistence | {meta.title} and {textPort} in same text| -| Escaped text bindings | \{textPort} produces literal braces | -| Verbatim unresolved | {undefinedBinding} left as-is | -| Non-string text coercion | {itemCount} coerced via String() | -EOF -``` + -If you can read this table, every feature worked. + -
+ - + -{guide} - - - - - - - - - - - - - - - - - - - -"} /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + diff --git a/testing/tests/smoke.test.ts b/testing/tests/smoke.test.ts index 19ea010..b5cdbbb 100644 --- a/testing/tests/smoke.test.ts +++ b/testing/tests/smoke.test.ts @@ -1,11 +1,12 @@ /** - * Smoke harness — infrastructure only. The smoke-test document tests - * itself: its guide is captured into a root binding and inspected by the - * embedded bodies (smoke-test/README.md). This harness composes - * useTesting() around core execute(), asserts the exact embedded tests - * pass in discovery order, keeps the journal-size check host-side, and - * verifies a full replay reproduces the identical outcome without - * appending journal events. + * Smoke harness — infrastructure only. The smoke guide is composed from + * self-testing feature documents (smoke-test/Guide/*.md): each captures + * its own rendered content, re-emits it, and carries a sibling + * that inspects the capture; README.md keeps only the root-frontmatter + * behavior and its test. This harness composes useTesting() around core + * execute(), asserts the exact embedded tests pass in discovery order, + * keeps the journal-size check host-side, and verifies a full replay + * reproduces the identical outcome without appending journal events. */ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; @@ -18,13 +19,23 @@ import { useTesting } from "../src/use-testing.ts"; import type { TestResult } from "../src/test-api.ts"; const EMBEDDED_TESTS = [ + "Root frontmatter", + "Overview", "Components", + "Nested components", "Execution", + "Props", + "Interpolation", "Captures", + "Healing", "Evaluation", - "Providers", - "Output regions", + "Daemons", + "Sampling", + "Named slots", + "Instructions", "Durability", + "Output regions", + "Summary", ]; interface SmokeSession { From bab1510b4448c87f48e42fedda3f5731ff75cedb Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:08:28 -0400 Subject: [PATCH 11/14] refactor(smoke): assert smallest observable values at their production sites Every embedded test now connects a specific captured result to its exact expected value instead of searching the whole rendered section: - demos are captured at their production site and re-emitted immediately; assertions use AssertEquals/AssertStrictEquals for deterministic values, anchored AssertMatch for genuinely dynamic ones (file count, timestamp, allocated port), and AssertNotMatch only for deliberately suppressed content - README asserts the exact rendered heading, version statement, and repo link; Components adds a Content-slot sentinel Section (hoisted to doc level - Section cannot nest inside Section); Execution isolates file count, visible exec, empty silent output, and yaml passthrough; Interpolation splits into Expression props and Text interpolation tests asserting exact interpolated lines; Captures asserts the four bindings directly plus equality-based site captures proving captured-only content never rendered; Healing uses a genuinely unclosed ** marker and asserts the exact healed boundary; Evaluation asserts live bindings (message, serverReady, startedAt, port) and the port-derived exec line, with the eval-block region asserted to render prose only; Daemons equals the curl response to daemon-ok; Sampling proves which content reached the provider (StubProvider now echoes context.content); Instructions asserts the exact system prompt; NamedSlots asserts complete table shapes; OutputRegions isolates output - the nested-Instruction accumulation demo is impossible (engine cycle detection rejects Instruction within Instruction), so the composition claim is removed per the review's fallback - Overview and Summary tests removed (prose-only / tautological) - harness updated to the sixteen exact test names in discovery order and stripped of its explanatory header --- smoke-test/Guide/Captures.md | 47 ++++++++++-------------- smoke-test/Guide/Components.md | 30 ++++++++------- smoke-test/Guide/Daemons.md | 20 +++++----- smoke-test/Guide/Durability.md | 17 ++++----- smoke-test/Guide/Evaluation.md | 42 +++++++++------------ smoke-test/Guide/Execution.md | 32 ++++++++++------ smoke-test/Guide/Healing.md | 25 +++++-------- smoke-test/Guide/Instructions.md | 33 ++++++----------- smoke-test/Guide/Interpolation.md | 55 +++++++++++++++++----------- smoke-test/Guide/NamedSlots.md | 33 +++++++---------- smoke-test/Guide/NestedComponents.md | 26 +++++-------- smoke-test/Guide/OutputRegions.md | 17 +++------ smoke-test/Guide/Overview.md | 10 ----- smoke-test/Guide/Props.md | 15 +++----- smoke-test/Guide/Sampling.md | 33 ++++++++--------- smoke-test/Guide/Summary.md | 42 +-------------------- smoke-test/README.md | 25 ++++++------- smoke-test/StubProvider.md | 2 +- testing/tests/smoke.test.ts | 15 +------- 19 files changed, 209 insertions(+), 310 deletions(-) diff --git a/smoke-test/Guide/Captures.md b/smoke-test/Guide/Captures.md index 87bcded..059d731 100644 --- a/smoke-test/Guide/Captures.md +++ b/smoke-test/Guide/Captures.md @@ -1,5 +1,3 @@ - -
Binding capture routes rendered output into the eval binding environment @@ -14,14 +12,11 @@ Inline capture uses the built-in `` directive: inline binding from Capture -Captured bindings are available to later executable blocks: - -```bash exec -echo "Capture values: {capturedFromComponent} | {capturedInline}" -``` - -Capture with CSS selector extracts specific content from rendered output: +Capture with CSS selector extracts specific content from rendered output. +The site below renders only its explanatory sentence — the captured prose +and JSON never render inline: +Selecting from rich content: Some prose before the data. @@ -30,28 +25,24 @@ Some prose before the data. ``` More prose after. - - -```bash exec -printf 'Selected JSON: %s\n' '{capturedJson}' -``` + +{jsonSite} This Note is captured but intentionally not rendered inline: - - -
- - - -{rendered} +Hidden note site: + +{hiddenNoteSite} - - - - - - - + + + + 📝 **info:** Hidden capture should not render inline.\n"} /> + + + + + +
diff --git a/smoke-test/Guide/Components.md b/smoke-test/Guide/Components.md index ad8d3b0..5a86ffa 100644 --- a/smoke-test/Guide/Components.md +++ b/smoke-test/Guide/Components.md @@ -1,4 +1,10 @@ - +The Section component receives a title prop and renders its children +inside a headed section via the `` slot. The sentinel below +proves the slot renders caller content in place: + +
SENTINEL-CONTENT
 + +{slotSentinel}
@@ -20,22 +26,18 @@ Components are invoked with JSX syntax. Props must match the declared inputs — undeclared props are rejected, required props must be provided, and defaults fill in for omitted optional props. - + - +{noteDefault} -Components can wrap content using the `` slot. The Section -component wrapping this text works exactly that way — it receives a -title prop and renders its children inside a headed section. - -
+ - - -{rendered} +{noteOverride} - - - + 📝 **info:** This note uses the default level (info)."} /> + 📝 **warning:** This note overrides the level to warning."} /> + + +
diff --git a/smoke-test/Guide/Daemons.md b/smoke-test/Guide/Daemons.md index 850c3f1..e76cb55 100644 --- a/smoke-test/Guide/Daemons.md +++ b/smoke-test/Guide/Daemons.md @@ -1,5 +1,3 @@ - -
The `daemon` modifier starts a long-running process that survives across @@ -30,22 +28,22 @@ yield * ); ``` -The daemon is alive — let's verify by hitting it: +The daemon is alive — the response below is the evidence that startup, +readiness polling, and the daemon's lifetime all worked: + ```bash exec curl -s http://127.0.0.1:{daemonPort} ``` + + +{daemonResponse} When this section ends, the daemon process is terminated by structured concurrency — no manual cleanup needed. -
- - - -{rendered} - - - + + +
diff --git a/smoke-test/Guide/Durability.md b/smoke-test/Guide/Durability.md index 9833903..75bf4cf 100644 --- a/smoke-test/Guide/Durability.md +++ b/smoke-test/Guide/Durability.md @@ -1,5 +1,3 @@ - -
Every component import and code execution is recorded in a journal. @@ -7,9 +5,13 @@ If this document's execution crashes mid-way, re-running it replays completed operations from the journal and continues from where it left off — no command is re-executed, no file is re-read. + ```bash exec echo "Run at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" ``` + + +{runStamp} Run this document twice. The timestamp above will be the same both times — it was journaled on the first run and replayed on the second. @@ -17,13 +19,8 @@ times — it was journaled on the first run and replayed on the second. If a component file changes between runs, the replay guard detects the stale content hash and halts replay, forcing a fresh execution. -
- - - -{rendered} - - - + + +
diff --git a/smoke-test/Guide/Evaluation.md b/smoke-test/Guide/Evaluation.md index 56d4121..d763e14 100644 --- a/smoke-test/Guide/Evaluation.md +++ b/smoke-test/Guide/Evaluation.md @@ -1,14 +1,12 @@ - -
Eval blocks run JavaScript **in-process** as Effection generator operations. Unlike `exec` blocks (which run shell commands in a subprocess), `eval` blocks execute in the same process, sharing a binding environment across -blocks within a component. - -Bindings declared in one eval block are available in subsequent blocks: +blocks within a component. Eval blocks produce **no rendered output** — +they exist for bindings and side effects: + ```js eval const greeting = "Hello from eval"; const numbers = [1, 2, 3]; @@ -19,10 +17,9 @@ The bindings from the previous block are available here: ```js eval const message = `${greeting} with ${numbers.length} numbers`; ``` + -Eval blocks produce **no rendered output** — they exist for bindings -and side effects. The output from eval blocks is empty, so nothing -appears between this text and the next section. +{evalRendered} The `persist` modifier extends a block's resource lifetime from the block scope to the component scope. Without `persist`, spawned tasks @@ -54,8 +51,7 @@ const serverReady = status.ready; ``` The `timeout` modifier cancels the block if it does not complete within -the specified duration. Accepted units: `ms`, `s`, `m`. If the block -times out, an error is recorded in the output and execution halts. +the specified duration. Accepted units: `ms`, `s`, `m`. ```js timeout=30s eval const startedAt = Date.now(); @@ -71,9 +67,13 @@ Eval binding interpolation substitutes bare `{name}` references in code block content with values from the eval binding environment. The port allocated above flows into subsequent blocks via `{port}`: + ```bash exec echo "Server would start on port {port}" ``` + + +{portEcho} Eval and exec blocks coexist independently in the same document: @@ -81,19 +81,13 @@ Eval and exec blocks coexist independently in the same document: echo "Exec blocks are independent of eval bindings" ``` -
- - - -{rendered} - - - - - - - - - + + + + + + + + diff --git a/smoke-test/Guide/Execution.md b/smoke-test/Guide/Execution.md index f0403b5..a17a105 100644 --- a/smoke-test/Guide/Execution.md +++ b/smoke-test/Guide/Execution.md @@ -1,5 +1,3 @@ - -
Code blocks with `exec` in the info string are executed as shell @@ -7,47 +5,57 @@ commands. The output replaces the code block in the rendered document. How many markdown files are in the smoke-test directory? + ```bash exec find smoke-test -name '*.md' | wc -l | tr -d ' ' ``` + + +{fileCount} The info string is a **middleware chain** read left-to-right. Each word after the language is a modifier handler that wraps the next. `exec` alone — runs the command, shows stdout: + ```bash exec echo "Hello from a durable workflow" ``` + + +{visibleExec} `silent exec` — runs and journals the command, but suppresses output. Useful for setup steps: + ```bash silent exec echo "This output is journaled but not shown in the document" ``` + + +{silentExec} Non-executable code blocks are passed through as regular markdown. This block has no `exec` modifier, so it's just syntax-highlighted text: + ```yaml # This is just a code block — not executed inputs: name: type: string ``` - -
- -{rendered} +{yamlBlock} - - - - - - + + + + + + diff --git a/smoke-test/Guide/Healing.md b/smoke-test/Guide/Healing.md index 6389e74..b566a45 100644 --- a/smoke-test/Guide/Healing.md +++ b/smoke-test/Guide/Healing.md @@ -1,26 +1,19 @@ - -
Components and executable code blocks are **semantic boundaries**. Markdown constructs cannot span them. Each text segment is healed -independently using remend. - -For example, if bold is opened before a component but not closed, -remend closes it before expansion proceeds. This prevents unclosed -markers from bleeding into component output. +independently using remend: if bold is opened before a component but not +closed, remend closes it before expansion proceeds, so unclosed markers +cannot bleed into component output. -The text below opens \*\*bold before the component +The text below opens **bold before the component -and continues after. Each segment is independently valid markdown. - -
+and continues after. - - -{rendered} +{healed} - - + + + diff --git a/smoke-test/Guide/Instructions.md b/smoke-test/Guide/Instructions.md index 7d2a9ab..731035a 100644 --- a/smoke-test/Guide/Instructions.md +++ b/smoke-test/Guide/Instructions.md @@ -1,35 +1,24 @@ - -
The `` component surfaces the system prompt as visible, composable document content. Instead of hiding the LLM's instructions inside provider internals, authors wrap Sample calls with -`` to define what the LLM is told. - -Without an instruction, the provider uses a hardcoded default system -prompt. With ``, the author's text replaces that default: +`` to define what the LLM is told. The stub +provider echoes the system prompt it received, so each response proves +what the provider was told. - + - - - - -Instructions accumulate through nesting. Agent components install -instruction middleware via `persist eval` blocks that enrich the -`SampleContext.system` field. When present, the system prompt is -passed through to the provider. - -
- - + -{rendered} +{instructionResponse} - - + + + + + diff --git a/smoke-test/Guide/Interpolation.md b/smoke-test/Guide/Interpolation.md index 2dfa9ff..5abaeac 100644 --- a/smoke-test/Guide/Interpolation.md +++ b/smoke-test/Guide/Interpolation.md @@ -2,8 +2,6 @@ title: Executable MDX --- - -
Expression props pass runtime values from eval blocks to child @@ -18,11 +16,26 @@ const itemCount = 3; The values computed above flow into PropDemo via expression props: - + + +{expressionPropDemo} JSON literals resolve at scan time — no eval block needed: - + + +{jsonPropNote} + +Non-string values are coerced via `String()` when interpolated in text: + +The count computed above is {itemCount}. +{coercionLine} + + + 📝 **info:** Props were successfully passed through to this component."} /> + 📝 **info:** JSON props: count={42}, verbose={true}"} /> + +
@@ -37,27 +50,27 @@ const textPort = 49821; const textHost = "127.0.0.1"; ``` -The server is running at {textHost}:{textPort}. +The server is running at {textHost}:{textPort}. +{hostLine} Both `{meta.*}` / `{props.*}` and bare `{name}` work in the same text. -Meta values resolve first, then eval bindings fill in remaining references. -The document title is {meta.title} and the text port is {textPort}. +Meta values resolve first, then eval bindings fill in remaining references +— here the enclosing Section's `{meta.emoji}` meets an eval binding: -Escaped braces produce literal output: \{textPort} stays as-is. +Meta and bindings coexist: section emoji {meta.emoji} and text port {textPort}. +{metaLine} -If a bare reference has no matching binding, it passes through verbatim: -{undefinedBinding} is not resolved. +Escaped braces produce literal output: \{textPort} stays as-is. +{escapedLine} -Non-string values are coerced via `String()`. The count from the -Expression Props section is {itemCount}. +If a bare reference has no matching binding, it passes through verbatim: {undefinedBinding} is not resolved. +{unresolvedLine} - - - - -{rendered} - - - - + + + + + + + diff --git a/smoke-test/Guide/NamedSlots.md b/smoke-test/Guide/NamedSlots.md index e2770c3..b3b1b53 100644 --- a/smoke-test/Guide/NamedSlots.md +++ b/smoke-test/Guide/NamedSlots.md @@ -1,5 +1,3 @@ - -
Components can render caller-provided content in multiple distinct @@ -7,31 +5,26 @@ regions using named slots. The `slot` prop on child components directs content to matching `` positions in the component body. - - - **Left column** content via named slot. - - - **Right column** content via named slot. - - + + **Left column** content via named slot. + **Right column** content via named slot. + + +{slotTable} Named slots compose with the existing content slot. Children without a `slot` prop go to the default slot: - + - - -
+ - - -{rendered} +{slotTableNotes} - - - + + 📝 **info:** This note is in the left column.\n | \n> 📝 **info:** This note is in the right column.\n |"} /> + + diff --git a/smoke-test/Guide/NestedComponents.md b/smoke-test/Guide/NestedComponents.md index 8417c6b..68ee002 100644 --- a/smoke-test/Guide/NestedComponents.md +++ b/smoke-test/Guide/NestedComponents.md @@ -1,32 +1,26 @@ - -
Components can reference other components. When a component's body contains another component invocation, the system resolves, imports, and expands it recursively — with cycle detection to prevent infinite loops. - +/> + +{featureOutput} Dotted names map to directory paths. The component below lives at `components/Tips/Formatting.md`: - - -
+ - - -{rendered} +{formattingTip} - - - - - -"} /> + 📝 **info:** This note was generated inside the Feature component."} /> +` inside your component\nbody to mark where the caller's children appear. If your component doesn't\ninclude ``, children are silently discarded."} /> + + diff --git a/smoke-test/Guide/OutputRegions.md b/smoke-test/Guide/OutputRegions.md index 3e73e83..7a31e70 100644 --- a/smoke-test/Guide/OutputRegions.md +++ b/smoke-test/Guide/OutputRegions.md @@ -1,5 +1,3 @@ - -
A component can declare which region of its body renders using ``. @@ -8,16 +6,13 @@ run, its captures populate bindings) but never renders into the consumer. `OutputDemo` computes a binding in documentation, then renders a `` inside `` that depends on it. - - -
- - + -{rendered} +{outputDemo} - - - + + + + diff --git a/smoke-test/Guide/Overview.md b/smoke-test/Guide/Overview.md index 071abd7..5551e21 100644 --- a/smoke-test/Guide/Overview.md +++ b/smoke-test/Guide/Overview.md @@ -1,5 +1,3 @@ - -
Executable MDX treats markdown files as **durable workflows**. A document @@ -9,11 +7,3 @@ in a journal so that execution survives crashes and replays from where it left off.
- - - -{rendered} - - - - diff --git a/smoke-test/Guide/Props.md b/smoke-test/Guide/Props.md index 8918380..b85a71b 100644 --- a/smoke-test/Guide/Props.md +++ b/smoke-test/Guide/Props.md @@ -1,5 +1,3 @@ - -
Every component has access to two namespaces for interpolation: @@ -12,15 +10,12 @@ prepends to each title via `{meta.emoji}`. The Note component uses `{meta.emoji}` for its 📝 prefix and `{props.message}` for the caller-provided text. - - -
- - + -{rendered} +{propDemo} - - + 📝 **info:** Props were successfully passed through to this component."} /> + + diff --git a/smoke-test/Guide/Sampling.md b/smoke-test/Guide/Sampling.md index f69459e..1e34045 100644 --- a/smoke-test/Guide/Sampling.md +++ b/smoke-test/Guide/Sampling.md @@ -1,33 +1,30 @@ - -
The `` component captures its children's rendered output (or accepts a `prompt` prop) and routes it through the Sample Api for LLM -processing. It uses `output()` to produce rendered output and -`renderChildren()` to capture children. - -Self-closing mode — prompt sent directly to the provider: +processing. The stub provider echoes the content it received, so each +response proves which content reached the provider. - +Self-closing mode — prompt sent directly to the provider: -With children — children are rendered first, then sampled: + - -This is child content to be processed. - +{promptResponse} - +With children — children are rendered first, then sampled; the child +content is consumed by the provider rather than rendered directly: -
+This is child content to be processed. - - -{rendered} +{childrenResponse} - - + + + + + + diff --git a/smoke-test/Guide/Summary.md b/smoke-test/Guide/Summary.md index 08dcdbb..0b0a0b9 100644 --- a/smoke-test/Guide/Summary.md +++ b/smoke-test/Guide/Summary.md @@ -1,11 +1,9 @@ - -
This document exercises every feature of the system: ```bash exec -cat <<'EOF' +cat <<'TABLE' | Feature | Exercised by | |---------------------------|-----------------------------------------| | Root frontmatter | Title and version in opening paragraph | @@ -52,45 +50,9 @@ cat <<'EOF' | Escaped text bindings | \{textPort} produces literal braces | | Verbatim unresolved | {undefinedBinding} left as-is | | Non-string text coercion | {itemCount} coerced via String() | -EOF +TABLE ``` If you can read this table, every feature worked.
- - - -{rendered} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/smoke-test/README.md b/smoke-test/README.md index 6a98d8e..a5b7ee0 100644 --- a/smoke-test/README.md +++ b/smoke-test/README.md @@ -4,26 +4,25 @@ version: 0.1.0 repo: https://github.com/thefrontside/effectionx --- - - -# {meta.title} +# {meta.title} +{introHeading} This document is both a guide and a smoke test. Every feature described here is exercised by the document itself — if it renders correctly, -the system works. This is version **{meta.version}** of {meta.title}, -built from the source at [{meta.repo}]({meta.repo}). Each chapter below -is a self-testing feature document: it captures its own rendered -content, re-emits it, and carries a sibling test that inspects the -capture. +the system works. Each chapter below is a self-testing feature document: +it captures each demonstrated result at its production site, re-emits +it, and carries a sibling test that inspects the capture. - +This is version **{meta.version}** of {meta.title}. +{introVersion} -{intro} +Built from the source at [{meta.repo}]({meta.repo}). +{introRepo} - - - + + + diff --git a/smoke-test/StubProvider.md b/smoke-test/StubProvider.md index 687aa4d..d71b11e 100644 --- a/smoke-test/StubProvider.md +++ b/smoke-test/StubProvider.md @@ -15,7 +15,7 @@ yield* Sample.around({ return yield* next(context); } const sys = context.system ? '|system:' + context.system : ''; - return '[response-from-' + model + sys + ']'; + return '[response-from-' + model + sys + '|content:' + context.content + ']'; }, }, { at: 'min' }); ``` diff --git a/testing/tests/smoke.test.ts b/testing/tests/smoke.test.ts index b5cdbbb..d5e0406 100644 --- a/testing/tests/smoke.test.ts +++ b/testing/tests/smoke.test.ts @@ -1,13 +1,3 @@ -/** - * Smoke harness — infrastructure only. The smoke guide is composed from - * self-testing feature documents (smoke-test/Guide/*.md): each captures - * its own rendered content, re-emits it, and carries a sibling - * that inspects the capture; README.md keeps only the root-frontmatter - * behavior and its test. This harness composes useTesting() around core - * execute(), asserts the exact embedded tests pass in discovery order, - * keeps the journal-size check host-side, and verifies a full replay - * reproduces the identical outcome without appending journal events. - */ import { describe, it } from "@effectionx/bdd/node"; import { expect } from "@effectionx/bdd/expect"; import { scoped } from "effection"; @@ -20,12 +10,12 @@ import type { TestResult } from "../src/test-api.ts"; const EMBEDDED_TESTS = [ "Root frontmatter", - "Overview", "Components", "Nested components", "Execution", "Props", - "Interpolation", + "Expression props", + "Text interpolation", "Captures", "Healing", "Evaluation", @@ -35,7 +25,6 @@ const EMBEDDED_TESTS = [ "Instructions", "Durability", "Output regions", - "Summary", ]; interface SmokeSession { From b3cf9e3b0769e9a1106d01a2d2165f04c8965979 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:21:24 -0400 Subject: [PATCH 12/14] refactor(smoke): run each complete scenario inside its atomic Every embedded test now owns its full scenario - setup, action, capture, and assertion live inside the same , connected directly to the captured actual value: - guide prose stays outside tests; test-owned demos moved inside, so regular execution renders the prose guide while xmd test runs every scenario (test examples are intentionally skipped during regular runs) - the daemon test contains port allocation, daemon startup, readiness polling, and the curl assertion, so teardown is bounded by the atomic test scope - feature-group tests split into 36 individually named atomic tests, one behavioral outcome each; no test depends on bindings or effects from a sibling (each defines its own eval bindings and providers) - capture re-emissions removed - assertion diagnostics carry the actual and expected values - harness updated to the 36 exact names in discovery order --- smoke-test/Guide/Captures.md | 41 ++++++--------- smoke-test/Guide/Components.md | 33 +++++------- smoke-test/Guide/Daemons.md | 25 +++------ smoke-test/Guide/Durability.md | 20 +++----- smoke-test/Guide/Evaluation.md | 77 +++++++++++----------------- smoke-test/Guide/Execution.md | 40 +++++---------- smoke-test/Guide/Healing.md | 9 ++-- smoke-test/Guide/Instructions.md | 15 ++---- smoke-test/Guide/Interpolation.md | 77 ++++++++++++++-------------- smoke-test/Guide/NamedSlots.md | 21 +++----- smoke-test/Guide/NestedComponents.md | 20 +++----- smoke-test/Guide/OutputRegions.md | 9 ++-- smoke-test/Guide/Props.md | 9 ++-- smoke-test/Guide/Sampling.md | 29 ++++------- smoke-test/README.md | 34 ++++++------ testing/tests/smoke.test.ts | 52 +++++++++++++------ 16 files changed, 219 insertions(+), 292 deletions(-) diff --git a/smoke-test/Guide/Captures.md b/smoke-test/Guide/Captures.md index 059d731..7613615 100644 --- a/smoke-test/Guide/Captures.md +++ b/smoke-test/Guide/Captures.md @@ -1,21 +1,28 @@
Binding capture routes rendered output into the eval binding environment -instead of writing it at the invocation site. +instead of writing it at the invocation site. Component-level capture +uses `as="name"` on any component invocation; inline capture uses the +built-in `` directive; and `` extracts +specific nodes from rendered content with a CSS selector. -Component-level capture uses `as="name"`: - -component binding from Fragment +
-Inline capture uses the built-in `` directive: + +Hidden note site: + + 📝 **info:** Hidden capture should not render inline.\n"} /> + + + + inline binding from Capture + + -Capture with CSS selector extracts specific content from rendered output. -The site below renders only its explanatory sentence — the captured prose -and JSON never render inline: - + Selecting from rich content: Some prose before the data. @@ -26,23 +33,7 @@ Some prose before the data. More prose after. -{jsonSite} - -This Note is captured but intentionally not rendered inline: - -Hidden note site: - -{hiddenNoteSite} - - - - - 📝 **info:** Hidden capture should not render inline.\n"} /> - - - - diff --git a/smoke-test/Guide/Components.md b/smoke-test/Guide/Components.md index 5a86ffa..166ec31 100644 --- a/smoke-test/Guide/Components.md +++ b/smoke-test/Guide/Components.md @@ -1,16 +1,8 @@ -The Section component receives a title prop and renders its children -inside a headed section via the `` slot. The sentinel below -proves the slot renders caller content in place: - -
SENTINEL-CONTENT
 - -{slotSentinel} -
A component is a markdown file with a declared interface. The component's frontmatter defines its own metadata and the props it accepts -via `inputs`. Here's the frontmatter from the Note component used below: +via `inputs`. Here's the frontmatter from the Note component tested below: ```yaml # components/Note.md frontmatter @@ -24,20 +16,23 @@ inputs: Components are invoked with JSX syntax. Props must match the declared inputs — undeclared props are rejected, required props must be provided, -and defaults fill in for omitted optional props. +and defaults fill in for omitted optional props. Components wrap caller +content through the `` slot, the way the Section component +around this text renders its children inside a headed section. - +
-{noteDefault} + + + 📝 **info:** This note uses the default level (info)."} /> + + - -{noteOverride} - - - 📝 **info:** This note uses the default level (info)."} /> 📝 **warning:** This note overrides the level to warning."} /> - - + +
SENTINEL-CONTENT
 + + diff --git a/smoke-test/Guide/Daemons.md b/smoke-test/Guide/Daemons.md index e76cb55..c1a11ea 100644 --- a/smoke-test/Guide/Daemons.md +++ b/smoke-test/Guide/Daemons.md @@ -3,21 +3,21 @@ The `daemon` modifier starts a long-running process that survives across subsequent blocks. Combined with `when()` for readiness polling, this implements the provider pattern: start a service, wait until it's ready, -then run children against it. +then run against it. The test below allocates a port, starts a Node HTTP +server as a daemon, polls until it responds, and asserts the response — +when the test's scope closes, structured concurrency terminates the +daemon with no manual cleanup. -The eval block below allocates a port, the daemon block starts a Node -HTTP server on it, and the readiness block polls until the server -responds: + + ```js eval const daemonPort = yield * findFreePort(); const daemonUrl = "http://127.0.0.1:" + daemonPort; ``` - ```bash daemon exec node -e "require('http').createServer((q,s)=>{s.writeHead(200);s.end('daemon-ok')}).listen({daemonPort},'127.0.0.1')" ``` - ```js eval yield * when( @@ -27,23 +27,10 @@ yield * { timeout: 5000, interval: 50 }, ); ``` - -The daemon is alive — the response below is the evidence that startup, -readiness polling, and the daemon's lifetime all worked: - ```bash exec curl -s http://127.0.0.1:{daemonPort} ``` - -{daemonResponse} - -When this section ends, the daemon process is terminated by structured -concurrency — no manual cleanup needed. - - - - diff --git a/smoke-test/Guide/Durability.md b/smoke-test/Guide/Durability.md index 75bf4cf..493b3ae 100644 --- a/smoke-test/Guide/Durability.md +++ b/smoke-test/Guide/Durability.md @@ -3,24 +3,18 @@ Every component import and code execution is recorded in a journal. If this document's execution crashes mid-way, re-running it replays completed operations from the journal and continues from where it -left off — no command is re-executed, no file is re-read. +left off — no command is re-executed, no file is re-read. The +timestamp asserted below is journaled on the first run; the harness +proves it is replayed rather than regenerated by asserting live and +replay outcomes are identical with zero appended journal events. + + + ```bash exec echo "Run at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" ``` - -{runStamp} - -Run this document twice. The timestamp above will be the same both -times — it was journaled on the first run and replayed on the second. - -If a component file changes between runs, the replay guard detects -the stale content hash and halts replay, forcing a fresh execution. - - - - diff --git a/smoke-test/Guide/Evaluation.md b/smoke-test/Guide/Evaluation.md index d763e14..7d3613a 100644 --- a/smoke-test/Guide/Evaluation.md +++ b/smoke-test/Guide/Evaluation.md @@ -4,31 +4,35 @@ Eval blocks run JavaScript **in-process** as Effection generator operations. Unlike `exec` blocks (which run shell commands in a subprocess), `eval` blocks execute in the same process, sharing a binding environment across blocks within a component. Eval blocks produce **no rendered output** — -they exist for bindings and side effects: +they exist for bindings and side effects. The `persist` modifier extends +a block's resource lifetime from the block scope to the component scope, +the `timeout` modifier cancels a block that overruns its duration, the +`findFreePort` VM global allocates a free TCP port, and bare `{name}` +references interpolate eval bindings into other code blocks. + + + ```js eval +const quietValue = 1; +``` + + + + + +```js eval const greeting = "Hello from eval"; const numbers = [1, 2, 3]; ``` - -The bindings from the previous block are available here: - ```js eval const message = `${greeting} with ${numbers.length} numbers`; ``` - - -{evalRendered} - -The `persist` modifier extends a block's resource lifetime from the -block scope to the component scope. Without `persist`, spawned tasks -and resources are torn down when the eval block completes. With it, -they survive for all subsequent blocks in the component. - -The block below spawns a background task that sets `status.ready` -after a short delay. Because it uses `persist`, the task stays alive: + + + ```js persist eval const status = { ready: false }; yield * @@ -37,11 +41,6 @@ yield * status.ready = true; }); ``` - -The next block converges on the spawned task using `when()`. This -only works because `persist` kept the task alive across the block -boundary — without it, the task would have been torn down: - ```js eval yield * when(function* () { @@ -49,45 +48,31 @@ yield * }); const serverReady = status.ready; ``` + + -The `timeout` modifier cancels the block if it does not complete within -the specified duration. Accepted units: `ms`, `s`, `m`. - + ```js timeout=30s eval const startedAt = Date.now(); ``` + + -The `findFreePort` VM global finds an available TCP port using the OS: - + ```js eval const port = yield * findFreePort(); ``` + + -Eval binding interpolation substitutes bare `{name}` references in code -block content with values from the eval binding environment. The port -allocated above flows into subsequent blocks via `{port}`: - + +```js eval +const port = yield * findFreePort(); +``` ```bash exec echo "Server would start on port {port}" ``` - -{portEcho} - -Eval and exec blocks coexist independently in the same document: - -```bash exec -echo "Exec blocks are independent of eval bindings" -``` - - - - - - - - - diff --git a/smoke-test/Guide/Execution.md b/smoke-test/Guide/Execution.md index a17a105..7d9ca25 100644 --- a/smoke-test/Guide/Execution.md +++ b/smoke-test/Guide/Execution.md @@ -2,44 +2,39 @@ Code blocks with `exec` in the info string are executed as shell commands. The output replaces the code block in the rendered document. +For example, counting the markdown files in this suite: -How many markdown files are in the smoke-test directory? - - ```bash exec find smoke-test -name '*.md' | wc -l | tr -d ' ' ``` - - -{fileCount} The info string is a **middleware chain** read left-to-right. Each -word after the language is a modifier handler that wraps the next. +word after the language is a modifier handler that wraps the next: +`exec` alone runs the command and shows stdout, `silent exec` runs and +journals the command but suppresses output, and blocks without `exec` +are passed through as regular markdown. -`exec` alone — runs the command, shows stdout: + + ```bash exec echo "Hello from a durable workflow" ``` + + -{visibleExec} - -`silent exec` — runs and journals the command, but suppresses -output. Useful for setup steps: - + ```bash silent exec echo "This output is journaled but not shown in the document" ``` + + -{silentExec} - -Non-executable code blocks are passed through as regular markdown. -This block has no `exec` modifier, so it's just syntax-highlighted text: - + ```yaml # This is just a code block — not executed @@ -48,14 +43,5 @@ inputs: type: string ``` - -{yamlBlock} - - - - - - - diff --git a/smoke-test/Guide/Healing.md b/smoke-test/Guide/Healing.md index b566a45..5f3579f 100644 --- a/smoke-test/Guide/Healing.md +++ b/smoke-test/Guide/Healing.md @@ -6,14 +6,11 @@ independently using remend: if bold is opened before a component but not closed, remend closes it before expansion proceeds, so unclosed markers cannot bleed into component output. + + + The text below opens **bold before the component and continues after. - -{healed} - - - - diff --git a/smoke-test/Guide/Instructions.md b/smoke-test/Guide/Instructions.md index 731035a..18dcd0e 100644 --- a/smoke-test/Guide/Instructions.md +++ b/smoke-test/Guide/Instructions.md @@ -4,21 +4,16 @@ The `` component surfaces the system prompt as visible, composable document content. Instead of hiding the LLM's instructions inside provider internals, authors wrap Sample calls with `` to define what the LLM is told. The stub -provider echoes the system prompt it received, so each response proves +provider echoes the system prompt it received, so the response proves what the provider was told. - + + + - -{instructionResponse} - - - - - - + diff --git a/smoke-test/Guide/Interpolation.md b/smoke-test/Guide/Interpolation.md index 5abaeac..987d38f 100644 --- a/smoke-test/Guide/Interpolation.md +++ b/smoke-test/Guide/Interpolation.md @@ -6,71 +6,72 @@ title: Executable MDX Expression props pass runtime values from eval blocks to child components. Unlike string attributes, expression props resolve -at expansion time against the eval binding environment. +at expansion time against the eval binding environment. JSON literals +resolve at scan time — no eval block needed — and non-string values +coerce via `String()` when interpolated in text. + + + ```js eval const dynamicGreeting = "Howdy"; const dynamicSubject = "expression props"; -const itemCount = 3; ``` - -The values computed above flow into PropDemo via expression props: - + 📝 **info:** Props were successfully passed through to this component."} /> + -{expressionPropDemo} - -JSON literals resolve at scan time — no eval block needed: - + + 📝 **info:** JSON props: count={42}, verbose={true}"} /> + -{jsonPropNote} - -Non-string values are coerced via `String()` when interpolated in text: - + +```js eval +const itemCount = 3; +``` The count computed above is {itemCount}. -{coercionLine} - - - 📝 **info:** Props were successfully passed through to this component."} /> - 📝 **info:** JSON props: count={42}, verbose={true}"} /> - -
Eval bindings also resolve in **prose text**, not just code blocks. Values computed in eval blocks flow naturally into surrounding text without -needing a template literal inside an eval block. +needing a template literal inside an eval block. Both `{meta.*}` / +`{props.*}` and bare `{name}` work in the same text: meta values resolve +first, then eval bindings fill in remaining references. Escaped braces +stay literal, and references with no matching binding pass through +verbatim. +
+ + ```js eval const textPort = 49821; const textHost = "127.0.0.1"; ``` - The server is running at {textHost}:{textPort}. -{hostLine} - -Both `{meta.*}` / `{props.*}` and bare `{name}` work in the same text. -Meta values resolve first, then eval bindings fill in remaining references -— here the enclosing Section's `{meta.emoji}` meets an eval binding: + + -Meta and bindings coexist: section emoji {meta.emoji} and text port {textPort}. -{metaLine} + +```js eval +const textPort = 49821; +``` +The document title is {meta.title} and the text port is {textPort}. + + + +```js eval +const textPort = 49821; +``` Escaped braces produce literal output: \{textPort} stays as-is. -{escapedLine} + + + If a bare reference has no matching binding, it passes through verbatim: {undefinedBinding} is not resolved. -{unresolvedLine} - - - - - - - diff --git a/smoke-test/Guide/NamedSlots.md b/smoke-test/Guide/NamedSlots.md index b3b1b53..1c2c360 100644 --- a/smoke-test/Guide/NamedSlots.md +++ b/smoke-test/Guide/NamedSlots.md @@ -3,28 +3,23 @@ Components can render caller-provided content in multiple distinct regions using named slots. The `slot` prop on child components directs content to matching `` positions in the -component body. +component body, and children without a `slot` prop go to the default +slot. + + + **Left column** content via named slot. **Right column** content via named slot. + + -{slotTable} - -Named slots compose with the existing content slot. Children without -a `slot` prop go to the default slot: - + - -{slotTableNotes} - - - 📝 **info:** This note is in the left column.\n | \n> 📝 **info:** This note is in the right column.\n |"} /> - - diff --git a/smoke-test/Guide/NestedComponents.md b/smoke-test/Guide/NestedComponents.md index 68ee002..35fdd3e 100644 --- a/smoke-test/Guide/NestedComponents.md +++ b/smoke-test/Guide/NestedComponents.md @@ -3,24 +3,20 @@ Components can reference other components. When a component's body contains another component invocation, the system resolves, imports, and expands it recursively — with cycle detection to prevent infinite loops. +Dotted names map to directory paths: `` lives at +`components/Tips/Formatting.md`. + + + + 📝 **info:** This note was generated inside the Feature component."} /> + -{featureOutput} - -Dotted names map to directory paths. The component below lives at -`components/Tips/Formatting.md`: - + - -{formattingTip} - - - 📝 **info:** This note was generated inside the Feature component."} /> ` inside your component\nbody to mark where the caller's children appear. If your component doesn't\ninclude ``, children are silently discarded."} /> - - diff --git a/smoke-test/Guide/OutputRegions.md b/smoke-test/Guide/OutputRegions.md index 7a31e70..7d1a801 100644 --- a/smoke-test/Guide/OutputRegions.md +++ b/smoke-test/Guide/OutputRegions.md @@ -6,13 +6,10 @@ run, its captures populate bindings) but never renders into the consumer. `OutputDemo` computes a binding in documentation, then renders a `` inside `` that depends on it. - - -{outputDemo} + - + + - - diff --git a/smoke-test/Guide/Props.md b/smoke-test/Guide/Props.md index b85a71b..0693a40 100644 --- a/smoke-test/Guide/Props.md +++ b/smoke-test/Guide/Props.md @@ -10,12 +10,9 @@ prepends to each title via `{meta.emoji}`. The Note component uses `{meta.emoji}` for its 📝 prefix and `{props.message}` for the caller-provided text. - - -{propDemo} + - + + 📝 **info:** Props were successfully passed through to this component."} /> - - diff --git a/smoke-test/Guide/Sampling.md b/smoke-test/Guide/Sampling.md index 1e34045..e5fef82 100644 --- a/smoke-test/Guide/Sampling.md +++ b/smoke-test/Guide/Sampling.md @@ -3,28 +3,21 @@ The `` component captures its children's rendered output (or accepts a `prompt` prop) and routes it through the Sample Api for LLM processing. The stub provider echoes the content it received, so each -response proves which content reached the provider. +response proves which content reached the provider — and, in children +mode, that the child content was consumed rather than rendered directly. - - -Self-closing mode — prompt sent directly to the provider: + + + - -{promptResponse} - -With children — children are rendered first, then sampled; the child -content is consumed by the provider rather than rendered directly: - -This is child content to be processed. - -{childrenResponse} - - - + + + +This is child content to be processed. + - - + diff --git a/smoke-test/README.md b/smoke-test/README.md index a5b7ee0..f73ac88 100644 --- a/smoke-test/README.md +++ b/smoke-test/README.md @@ -4,25 +4,23 @@ version: 0.1.0 repo: https://github.com/thefrontside/effectionx --- -# {meta.title} -{introHeading} - -This document is both a guide and a smoke test. Every feature described -here is exercised by the document itself — if it renders correctly, -the system works. Each chapter below is a self-testing feature document: -it captures each demonstrated result at its production site, re-emits -it, and carries a sibling test that inspects the capture. - -This is version **{meta.version}** of {meta.title}. -{introVersion} - -Built from the source at [{meta.repo}]({meta.repo}). -{introRepo} +# {meta.title} + +This document is both a guide and a smoke test. Each chapter explains a +feature in prose and carries atomic tests that run the demonstrated +scenario end to end — setup, action, capture, and assertion live inside +each test, so regular execution renders the guide while `xmd test` runs +every scenario. This is version **{meta.version}** of {meta.title}, +built from the source at [{meta.repo}]({meta.repo}). + + +# {meta.title} + + - - - - + +This is version **{meta.version}** of {meta.title}, built from the source at [{meta.repo}]({meta.repo}). + diff --git a/testing/tests/smoke.test.ts b/testing/tests/smoke.test.ts index d5e0406..37faa41 100644 --- a/testing/tests/smoke.test.ts +++ b/testing/tests/smoke.test.ts @@ -9,22 +9,42 @@ import { useTesting } from "../src/use-testing.ts"; import type { TestResult } from "../src/test-api.ts"; const EMBEDDED_TESTS = [ - "Root frontmatter", - "Components", - "Nested components", - "Execution", - "Props", - "Expression props", - "Text interpolation", - "Captures", - "Healing", - "Evaluation", - "Daemons", - "Sampling", - "Named slots", - "Instructions", - "Durability", - "Output regions", + "Root frontmatter interpolates into the heading", + "Root frontmatter interpolates into prose", + "Note renders its default level", + "Note renders an overridden level", + "Section renders children through its Content slot", + "Feature expands its nested Note", + "Dotted component names resolve to directory paths", + "exec renders command stdout", + "silent exec suppresses rendered output", + "Non-executable code blocks pass through verbatim", + "Props interpolate into the component body", + "Expression props resolve from eval bindings", + "JSON literal props resolve at scan time", + "Non-string bindings coerce in text", + "Eval bindings resolve in prose text", + "Meta and eval bindings share the same text", + "Escaped braces stay literal", + "Unresolved references pass through verbatim", + "Component as-capture binds without rendering inline", + "Capture binds inline content", + "Capture select extracts the matching node", + "Unclosed bold heals at a component boundary", + "Eval blocks render no output", + "Eval blocks share bindings", + "Persist keeps spawned tasks alive across blocks", + "Timeout-bounded eval blocks complete", + "findFreePort allocates a free port", + "Eval bindings interpolate into exec blocks", + "A daemon serves requests until its scope closes", + "Sample sends its prompt to the provider", + "Sample consumes children as content", + "Named slots place content in their regions", + "Named slots compose with the default slot", + "Instruction sets the provider system prompt", + "Exec output is journaled with the run timestamp", + "Output regions render only the selected region", ]; interface SmokeSession { From 8ce9430b31edb373312feee3e2163d94511a1f61 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:26:47 -0400 Subject: [PATCH 13/14] ci: smoke-test the compiled xmd binary end to end New distinct 'smoke' CI job: pinned Deno, deno task build, then the ./dist/xmd binary runs 'test smoke-test/README.md' directly with --raw, failing on the binary's exit code. The Deno suite keeps the focused API replay tests; the compiled binary is the authoritative end-to-end check. The binary initially failed every eval block using when()/fetch(): STANDARD_IMPORTS resolve at runtime from generated eval modules, so 'deno compile --exclude-unused-npm' pruned @effectionx/converge and @effectionx/fetch from the binary. The compiler modules that inject STANDARD_IMPORTS now statically anchor those packages, keeping them in any compile graph that embeds core. --- .github/workflows/ci.yml | 19 +++++++++++++++++++ core/src/deno-compiler.ts | 5 +++++ core/src/temp-file-compiler.ts | 5 +++++ 3 files changed, 29 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04fb792..e95de9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,25 @@ jobs: - name: Test run: deno task test + smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: denoland/setup-deno@e95548e56dfa95d4e1a28d6f422fafe75c4c26fb # v2.0.3 + with: + deno-version: v2.9.1 + + - name: Build the xmd binary + run: deno task build + + - name: Smoke test the compiled binary + run: | + ./dist/xmd test smoke-test/README.md \ + --component-dir smoke-test \ + --component-dir core/components \ + --raw + site: runs-on: ubuntu-latest steps: diff --git a/core/src/deno-compiler.ts b/core/src/deno-compiler.ts index d1dbfb7..0ef1cea 100644 --- a/core/src/deno-compiler.ts +++ b/core/src/deno-compiler.ts @@ -12,6 +12,11 @@ import { call } from "effection"; import type { Operation } from "effection"; import { API } from "@executablemd/runtime"; +// STANDARD_IMPORTS below resolve at runtime from generated eval modules; +// without these static anchors, `deno compile --exclude-unused-npm` prunes +// the packages from the binary and every eval block using them fails. +import "@effectionx/converge"; +import "@effectionx/fetch"; /** * Standard import statements prepended to every generated eval module. diff --git a/core/src/temp-file-compiler.ts b/core/src/temp-file-compiler.ts index 63ccc79..7391a9b 100644 --- a/core/src/temp-file-compiler.ts +++ b/core/src/temp-file-compiler.ts @@ -14,6 +14,11 @@ import { call } from "effection"; import type { Operation } from "effection"; import { API } from "@executablemd/runtime"; +// STANDARD_IMPORTS below resolve at runtime from generated eval modules; +// without these static anchors, `deno compile --exclude-unused-npm` prunes +// the packages from the binary and every eval block using them fails. +import "@effectionx/converge"; +import "@effectionx/fetch"; import { writeFile, unlink, mkdir } from "node:fs/promises"; import { resolve } from "node:path"; import { randomUUID } from "node:crypto"; From 283510729c1ba016abf72cefddbfae139ee90e94 Mon Sep 17 00:00:00 2001 From: Taras Mankovski <74687+taras@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:35:33 -0400 Subject: [PATCH 14/14] ci: lint and format the testing package in the lint job --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e95de9e..e9593c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,10 @@ jobs: - run: pnpm install - name: Lint - run: pnpm exec oxlint -c .oxlintrc.json core/src/ cli/src/ durable-streams/ durable-effects/ runtime/ + run: pnpm exec oxlint -c .oxlintrc.json core/src/ cli/src/ durable-streams/ durable-effects/ runtime/ testing/ - name: Format - run: pnpm exec oxfmt --check core/src/ cli/src/ durable-streams/ durable-effects/ runtime/ vendor/ test-support/ + run: pnpm exec oxfmt --check core/src/ cli/src/ durable-streams/ durable-effects/ runtime/ testing/ vendor/ test-support/ test-deno: runs-on: ubuntu-latest