From ce04be658e32e842b1f03c4526dd348dd30d4b75 Mon Sep 17 00:00:00 2001 From: Ryan Dombrowski Date: Wed, 5 Aug 2026 09:31:57 -0400 Subject: [PATCH] fix(build): explain failures and persist accepted work safely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Release hardening for the Phase 3 Build loop: the three blockers that stand between chat and being the public-facing first experience. One cohesive change — the failure model, the accept path, and the refine gate are the same user story (a turn either explains itself or persists itself), and they share the same fold, route, and tests. BLOCKER 1 (#41) — structured failure presentation. A failed turn now says WHY, from the fields the pipeline already reports and never from message wording: S1/S2 -> attempt.gates[].errors (those gates carry no findings) S3 -> attempt.findings[]: rule id, message, location, and the OWNER-AUTHORED rationale, verbatim emit -> emitted.refusal, else emitted.validations[].gates[].errors adapter-> attempt.adapterError, with an actionable headline exhaustion -> named as such, with the last attempt's reasons composer-core gains buildFailure() (kind + headline + stoppedAt + reasons), and the view renders every reason with its gate, code, target, message and rationale, keeps the full audit report in a disclosure, and never reduces a failure to a bare outcome. Successful gates stay visible — the strip shows where the pipeline stopped. agent-client no longer flattens a 4xx to "agent replied N": a refusal carrying `findings` keeps them, and a refused Accept renders those findings in place. Vocabulary gaps keep their own explicit panel (the Component Workshop hook), distinguishable from malformed output because it is derived from S2 evidence only. Accept and Refine are offered only for a finished, passing turn with a surface (canAcceptTurn / canRefineTurn). Also: the fold now reconciles its streamed attempts against the audit report, so gap detection and failure presentation read ONE authoritative source even if a progress event is dropped mid-stream. BLOCKER 2 (#42) — safe worked-example persistence. Identity is minted from the contract ON DISK (`nextExampleId`), never from a page-local counter, so a reload or a second tab cannot collide with saved work; an EXPLICIT id that already exists is refused 409 with a pathed finding rather than overwritten; the write is append-only. Provenance is truthful: `examplePromptFor` walks the turn's parent chain and stores the ORIGINAL ask plus a deterministic record of the refinements ("a status screen — refined: make the title clearer"), so the saved pair describes the accepted surface rather than the last edit instruction. #43 (narrow) — Refine is no longer offered for failed, refused, or adapter-error turns; scripted no longer borrows another intent's worked example (it refuses honestly and the view states the absence, while a model may still run without few-shot context); and the scripted refinement marker is MONOTONIC, so consecutive refinements can never be byte-identical no-ops reported as success. Fail-first: composer-core 10/10 failed before the failure model existed; agent 4/4 failed before minting/collision/intent-honesty; 8 new DOM specs failed before the UI carried reasons. Now: composer-core 50/50, agent 44/44, DOM 42/42, exhibit 106 passed/4 skips, composer production 13/13, typecheck clean, both builds green. Co-Authored-By: Claude Opus 5 --- apps/agent/src/project.test.ts | 70 ++++++ apps/agent/src/project.ts | 75 +++++-- apps/composer/app/agent-client.ts | 22 +- apps/composer/app/state.tsx | 47 +++- apps/composer/app/views/build-view.tsx | 88 +++++++- e2e/composer-build-a11y.spec.ts | 2 +- e2e/composer-build.spec.ts | 202 +++++++++++++++++- packages/composer-core/src/build.ts | 187 ++++++++++++++++ .../composer-core/src/composer-core.test.ts | 186 +++++++++++++++- packages/composer-core/src/index.ts | 7 + 10 files changed, 845 insertions(+), 41 deletions(-) diff --git a/apps/agent/src/project.test.ts b/apps/agent/src/project.test.ts index d323ed1..1679890 100644 --- a/apps/agent/src/project.test.ts +++ b/apps/agent/src/project.test.ts @@ -454,3 +454,73 @@ describe("accepting a build result (/project/save-example, fail-closed)", () => expect(JSON.stringify(surfaceOfRun(next.events).surface)).toBe(JSON.stringify(refined)); }); }); + + +describe("safe worked-example persistence (#42) and honest scripted absence (#43)", () => { + const contractOf = () => JSON.parse(readFileSync(join(root, "acme-ui.dspack.json"), "utf8")); + const surfaceOf = () => structuredClone(contractOf().examples[0].surface); + + it("mints a collision-free id from the contract on disk when the client supplies none", async () => { + const before = contractOf().examples.map((e: any) => e.id); + const a = await call("save-example", { path: root, example: { intent: "status-report", prompt: "first ask", surface: surfaceOf() } }); + expect(a.status).toBe(200); + expect(a.payload.example.id).toMatch(/^ex\.chat-\d+$/); + expect(before).not.toContain(a.payload.example.id); + + const b = await call("save-example", { path: root, example: { intent: "status-report", prompt: "second ask", surface: surfaceOf() } }); + expect(b.status).toBe(200); + expect(b.payload.example.id).not.toBe(a.payload.example.id); // distinct across accepts + + // Both survive; every pre-existing example is byte-identical. + const doc = contractOf(); + expect(doc.examples.map((e: any) => e.id)).toEqual(expect.arrayContaining([a.payload.example.id, b.payload.example.id, ...before])); + for (const id of before) { + expect(JSON.stringify(doc.examples.find((e: any) => e.id === id))).toBe( + JSON.stringify(JSON.parse(readFileSync(join(root, "acme-ui.dspack.json"), "utf8")).examples.find((e: any) => e.id === id)), + ); + } + }); + + it("REFUSES an explicit id that already exists rather than overwriting it", async () => { + const existing = contractOf().examples[0]; + const before = JSON.stringify(existing); + const { status, payload } = await call("save-example", { + path: root, + example: { id: existing.id, intent: "status-report", prompt: "hostile overwrite", surface: surfaceOf() }, + }); + expect(status).toBe(409); + expect(payload.findings[0].code).toBe("example-exists"); + expect(payload.findings[0].message).toContain(existing.id); + // Untouched, byte for byte. + expect(JSON.stringify(contractOf().examples.find((e: any) => e.id === existing.id))).toBe(before); + }); + + it("the newly accepted example is consumable as few-shot and scripted plays the latest without replacing older ones", async () => { + const refined = surfaceOf(); + refined.root.children[0].children[0].text = "Rollout status"; + const saved = await call("save-example", { + path: root, + example: { intent: "status-report", prompt: "a status screen — refined: say Rollout status", surface: refined }, + }); + expect(saved.status).toBe(200); + const doc = contractOf(); + const { compileContext } = await import("@aestheticfunction/dspack-gen/core"); + const context = compileContext(doc, "status-report"); + expect(context.fewshot.some((m: any) => m.role === "assistant" && m.content.includes("Rollout status"))).toBe(true); + // Older examples still present and still served. + expect(doc.examples.length).toBeGreaterThan(1); + expect(context.fewshot.length).toBeGreaterThanOrEqual(doc.examples.filter((e: any) => e.intent === "status-report").length); + }); + + it("an intent with no matching example never borrows another intent's: scripted refuses honestly", async () => { + const doc = contractOf(); + doc.intents = [...doc.intents, { id: "onboarding", description: "Welcome a new operator." }]; + const { writeFileSync } = await import("node:fs"); + writeFileSync(join(root, "acme-ui.dspack.json"), JSON.stringify(doc, null, 2) + "\n"); + + const { status, payload } = await call("run", { path: root, prompt: "an onboarding screen", intent: "onboarding", modelRef: "scripted" }); + expect(status).toBe(400); + expect(String(payload.error)).toMatch(/onboarding/); + expect(String(payload.error)).toMatch(/worked example/i); + }); +}); diff --git a/apps/agent/src/project.ts b/apps/agent/src/project.ts index fb125b1..c0a6e7d 100644 --- a/apps/agent/src/project.ts +++ b/apps/agent/src/project.ts @@ -481,9 +481,22 @@ function scriptedRunAdapter(example: { surface: unknown }, conversation: Convers try { const refined = JSON.parse(priorRaw) as Record; const textNode = firstTextNode(refined); - if (textNode && !textNode.text.endsWith(" (refined)")) textNode.text = `${textNode.text} (refined)`; - else if (!textNode) (refined as { id?: string }).id = "refined"; - return new ScriptedAdapter([{ output: refined }]); + if (textNode) { + // MONOTONIC, never idempotent: successive refinements must each + // produce a genuinely different surface, or the twin would report + // a byte-identical no-op as a successful refinement (#43). + const existing = /^(.*?)(?: \(refined(?: (\d+))?\))$/.exec(textNode.text); + const base = existing ? existing[1] : textNode.text; + const next = existing ? Number(existing[2] ?? 1) + 1 : 1; + textNode.text = next === 1 ? `${base} (refined)` : `${base} (refined ${next})`; + } else { + const previous = /^refined(?: (\d+))?$/.exec(String((refined as { id?: string }).id ?? "")); + const next = previous ? Number(previous[1] ?? 1) + 1 : 1; + (refined as { id?: string }).id = next === 1 ? "refined" : `refined ${next}`; + } + // Three entries so a refinement can survive bounded repair too — a + // refinement run must never die with a script-exhaustion error. + return new ScriptedAdapter([{ output: refined }, { output: refined }, { output: refined }]); } catch { // Fall through: an unparseable prior surface behaves like a fresh run. } @@ -512,12 +525,18 @@ async function runProject(ctx: ProjectContext, body: Record, re const conversation = parseConversation(props.conversation); const examples = (contract.examples as Array<{ intent: string; surface: unknown }> | undefined) ?? []; - // LAST match: accepted chat results join the corpus at the end, and the - // deterministic twin plays the owner's latest accepted example. - const matching = examples.filter((e) => e.intent === intent); - const example = matching.at(-1) ?? examples.at(-1); + // LAST match FOR THIS INTENT: accepted chat results join the corpus at the + // end, and the deterministic twin plays the owner's latest accepted + // example. Never borrow another intent's example — a screen built for a + // different intent is not a deterministic stand-in, it is a wrong answer + // reported as a right one (#43). + const example = examples.filter((e) => e.intent === intent).at(-1); if (modelRef === "scripted" && !example) { - throw new ProjectError(400, "scripted mode needs at least one worked example in the contract"); + throw new ProjectError( + 400, + `scripted mode replays this intent's own worked example, and '${intent}' has none yet. ` + + `Author one in Scenarios, or run with a model — generation works from the scoped contract without few-shot context.`, + ); } const adapter = modelRef === "scripted" @@ -553,6 +572,16 @@ async function runProject(ctx: ProjectContext, body: Record, re } +/** The next free `ex.chat-N` for this contract (monotonic, gap-tolerant). */ +function nextExampleId(existing: string[]): string { + let n = 0; + for (const id of existing) { + const match = /^ex\.chat-(\d+)$/.exec(id); + if (match) n = Math.max(n, Number(match[1])); + } + return `ex.chat-${n + 1}`; +} + /** * Accept a build result as a governed worked example — the ONLY save format * for chat-accepted surfaces, and fail-closed SERVER-SIDE: a disabled @@ -565,15 +594,33 @@ async function runProject(ctx: ProjectContext, body: Record, re async function saveExample(ctx: ProjectContext, body: Record) { const raw = body.example as Record | undefined; if (!raw || typeof raw !== "object") throw new ProjectError(400, "example is required"); - const id = String(raw.id ?? ""); - if (!/^ex\.[a-z0-9][a-z0-9-]*$/.test(id)) { - throw new ProjectError(400, "example.id must be kebab-case with the 'ex.' prefix"); - } if (!raw.surface || typeof raw.surface !== "object") throw new ProjectError(400, "example.surface must be a surface document"); const prompt = String(raw.prompt ?? ""); if (!prompt) throw new ProjectError(400, "example.prompt is required (the ask that produced this surface)"); const contract = readJson(ctx.contractPath) as Record; + const existing = ((contract.examples as Array<{ id?: unknown }> | undefined) ?? []).map((e) => String(e?.id ?? "")); + + // Identity is derived from the contract ON DISK, never from a page-local + // counter: a browser that reloaded (or a second tab) cannot mint an id + // that collides with work already saved. An EXPLICIT id that already + // exists is refused outright — accepting a build result never overwrites + // an existing worked example, least of all an owner-authored one (#42). + const requested = raw.id === undefined ? "" : String(raw.id); + if (requested && !/^ex\.[a-z0-9][a-z0-9-]*$/.test(requested)) { + throw new ProjectError(400, "example.id must be kebab-case with the 'ex.' prefix"); + } + if (requested && existing.includes(requested)) { + return { + status: 409, + payload: { + ok: false, + findings: [finding("document", "example-exists", "error", "example.id", `'${requested}' already exists in this contract; accepting would overwrite it. Choose another id, or leave it blank to mint the next free one.`)], + }, + }; + } + const id = requested || nextExampleId(existing); + const intents = ((contract.intents as Array<{ id: string }> | undefined) ?? []).map((i) => i.id); const intent = String(raw.intent ?? ""); if (!intents.includes(intent)) { @@ -606,9 +653,7 @@ async function saveExample(ctx: ProjectContext, body: Record) { }; const document = structuredClone(contract); const examples = ((document.examples as unknown[] | undefined) ?? []) as Array<{ id: string }>; - const at = examples.findIndex((e) => e.id === id); - if (at >= 0) examples[at] = entry as never; - else examples.push(entry as never); + examples.push(entry as never); // append-only: the id was proven free above document.examples = examples; // The same guarded write as /project/save: ledger preserved, harness clean. diff --git a/apps/composer/app/agent-client.ts b/apps/composer/app/agent-client.ts index 31d17fc..0dc1a82 100644 --- a/apps/composer/app/agent-client.ts +++ b/apps/composer/app/agent-client.ts @@ -28,7 +28,14 @@ export interface ConnectPayload { extraSurfaces: Array<{ name: string; surface: unknown }>; } -export type AgentResult = { ok: true; value: T } | { ok: false; error: string }; +/** + * A refusal keeps its STRUCTURED evidence: routes that answer 4xx with a + * findings array (the fail-closed accept gate) must not be flattened to + * "agent replied 422" — the gate reasons are the whole point (#41). + */ +export type AgentResult = + | { ok: true; value: T } + | { ok: false; error: string; findings?: ComposerFinding[]; status?: number }; const DEFAULT_AGENT = process.env.NEXT_PUBLIC_AGENT_URL ?? "http://localhost:8787"; @@ -45,7 +52,15 @@ async function post(route: string, body: unknown): Promise> { signal: AbortSignal.timeout(180_000), }); const payload = await res.json(); - if (!res.ok) return { ok: false, error: String(payload.error ?? `agent replied ${res.status}`) }; + if (!res.ok) { + const findings = Array.isArray(payload?.findings) ? (payload.findings as ComposerFinding[]) : undefined; + return { + ok: false, + status: res.status, + ...(findings ? { findings } : {}), + error: String(payload?.error ?? (findings?.length ? findings.map((f) => f.message).join("; ") : `agent replied ${res.status}`)), + }; + } return { ok: true, value: payload as T }; } catch (e) { return { ok: false, error: e instanceof Error ? e.message : String(e) }; @@ -148,7 +163,8 @@ export function streamProjectRun( } export interface AcceptedExample { - id: string; + /** Omit to let the agent mint a collision-free id from the contract (#42). */ + id?: string; intent: string; name?: string; prompt: string; diff --git a/apps/composer/app/state.tsx b/apps/composer/app/state.tsx index 5581f97..9e88dad 100644 --- a/apps/composer/app/state.tsx +++ b/apps/composer/app/state.tsx @@ -13,6 +13,8 @@ import { addTombstone, applyFreshFact, buildReadiness, + canRefineTurn, + examplePromptFor, foldBuildEvents, ledgerStatus, removeTombstone, @@ -52,10 +54,14 @@ export interface BuildTurn { modelRef: string; /** True when this turn refined the previous surface (seed supplied). */ refinement: boolean; + /** The turn this one refined, for truthful example provenance (#42). */ + parentId?: number; progress: BuildTurnProgress; /** Component ids the ask needed but the owner has not approved (S2 evidence). */ gaps: string[]; accepted?: string; // the saved example id + /** Structured findings from a refused Accept, rendered in place (#41). */ + acceptFindings?: ComposerFinding[]; } export interface ComposerState { @@ -96,7 +102,8 @@ export interface ComposerState { /** Setup completeness for building; reason names the exact remaining work. */ readiness: BuildReadiness; runBuild: (input: { prompt: string; intent: string; modelRef: string; refine?: boolean }) => Promise; - acceptBuildTurn: (turnId: number, exampleId: string) => Promise; + /** Accept a turn as a worked example; the agent mints the id (#42). */ + acceptBuildTurn: (turnId: number, exampleId?: string) => Promise; clearBuildThread: () => void; } @@ -514,9 +521,12 @@ export function ComposerProvider({ children }: { children: ReactNode }) { return; } if (buildBusy) return; - const prior = input.refine ? [...buildTurns].reverse().find((t) => t.progress.surface) : undefined; + // Only a completed, passing turn can seed a refinement (#43): a failed + // turn still carries its last attempt's surface, and seeding that + // regenerates from something the contract already rejected. + const prior = input.refine ? [...buildTurns].reverse().find((t) => canRefineTurn(t.progress)) : undefined; if (input.refine && !prior) { - setNotice("Nothing to refine yet — run a build first."); + setNotice("Nothing to refine yet — refinement starts from a completed build that passed its gates."); return; } setBuildBusy(true); @@ -527,6 +537,7 @@ export function ComposerProvider({ children }: { children: ReactNode }) { intent: input.intent, modelRef: input.modelRef, refinement: !!prior, + ...(prior ? { parentId: prior.id } : {}), progress: { status: "streaming", attempts: [] }, gaps: [], }; @@ -577,7 +588,7 @@ export function ComposerProvider({ children }: { children: ReactNode }) { * format — and immediately joins that intent's few-shot corpus. */ const acceptBuildTurn = useCallback( - async (turnId: number, exampleId: string) => { + async (turnId: number, exampleId?: string) => { if (!contract || decisionLock.current) return; const turn = buildTurns.find((t) => t.id === turnId); if (!turn?.progress.surface || turn.progress.outcome !== "passed") { @@ -587,19 +598,32 @@ export function ComposerProvider({ children }: { children: ReactNode }) { decisionLock.current = true; setBusy("accepting build result"); try { + // Truthful provenance: walk back to the ORIGINAL ask and record the + // refinements that shaped this surface (#42). + const chain: string[] = []; + for (let t: BuildTurn | undefined = turn; t; t = t.parentId ? buildTurns.find((x) => x.id === t!.parentId) : undefined) { + chain.unshift(t.prompt); + } + const prompt = examplePromptFor(chain); const result = await agentSaveExample(projectPath, { - id: exampleId, + ...(exampleId ? { id: exampleId } : {}), // omitted ⇒ the agent mints a collision-free id intent: turn.intent, - name: `Chat: ${turn.prompt.slice(0, 60)}`, - prompt: turn.prompt, + name: `Chat: ${chain[0].slice(0, 60)}`, + prompt, surface: turn.progress.surface, }); if (!result.ok) { - setNotice(`Accept failed: ${result.error}`); + // Structured gate reasons, never a bare HTTP status (#41). + const detail = result.findings?.length + ? result.findings.map((f) => `${f.gate} ${f.code}: ${f.message}`).join(" · ") + : result.error; + setNotice(`Accept refused: ${detail}`); + setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, acceptFindings: result.findings ?? [] } : t))); return; } if (!result.value.ok) { - setNotice(`Accept refused by the gates: ${result.value.findings.map((f) => f.message).join("; ").slice(0, 300)}`); + setNotice(`Accept refused: ${result.value.findings.map((f) => `${f.gate} ${f.code}: ${f.message}`).join(" · ").slice(0, 400)}`); + setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, acceptFindings: result.value.findings } : t))); return; } if (result.value.ledger) setLedger(result.value.ledger); @@ -611,8 +635,9 @@ export function ComposerProvider({ children }: { children: ReactNode }) { doc.examples = examples; setContract(doc); recomputeEmit(doc, profile); - setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, accepted: exampleId } : t))); - setNotice(`Accepted as worked example '${exampleId}' — it now seeds generation for '${turn.intent}'.`); + const savedId = result.value.example?.id ?? exampleId ?? ""; + setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, accepted: savedId, acceptFindings: undefined } : t))); + setNotice(`Accepted as worked example '${savedId}' — it now seeds generation for '${turn.intent}'.`); } finally { decisionLock.current = false; setBusy(null); diff --git a/apps/composer/app/views/build-view.tsx b/apps/composer/app/views/build-view.tsx index 007e360..e46479f 100644 --- a/apps/composer/app/views/build-view.tsx +++ b/apps/composer/app/views/build-view.tsx @@ -16,6 +16,7 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { A2uiCanvas } from "@dspack-studio/a2ui-ingest"; import { wireframeRegistryFor } from "@dspack-studio/wireframe-renderers"; import { shadcnRegistry } from "@dspack-studio/shadcn-renderers"; +import { buildFailure, canAcceptTurn, canRefineTurn } from "@dspack-studio/composer-core"; import type { BuildTurn } from "../state"; import { useComposer } from "../state"; import { browserEmit } from "../validation"; @@ -47,7 +48,9 @@ function TurnCanvas({ turn }: { turn: BuildTurn }) { function TurnBlock({ turn }: { turn: BuildTurn }) { const { acceptBuildTurn, buildBusy, busy } = useComposer(); - const [exampleId, setExampleId] = useState(`ex.chat-${turn.id}`); + // Blank by default: identity is minted from the contract ON DISK, so a + // reload or a second tab can never collide with saved work (#42). + const [exampleId, setExampleId] = useState(""); const locked = buildBusy || busy !== null; // The Accept button unmounts on success; a keyboard user's focus must // land on the confirmation, never fall to the document body. @@ -92,6 +95,67 @@ function TurnBlock({ turn }: { turn: BuildTurn }) { {turn.progress.error &&
  • {turn.progress.error}
    • } + {(() => { + const failure = buildFailure(turn.progress); + if (!failure) return null; + return ( +
    +

    {failure.headline}

    +

    + stopped at {failure.stoppedAt} +

    +
      + {failure.reasons.map((reason, i) => ( +
    • + + {[reason.gate, reason.code].filter(Boolean).join(" ")} + {reason.target ? ` · ${reason.target}` : ""} + +
      + {reason.message} + {reason.rationale && ( + <> +
      + + Why this rule exists: {reason.rationale} + + + )} +
      • + ))} + {failure.reasons.length === 0 && ( +
    • No structured reason was reported; the full audit report is below.
      • + )} +
    +
    + full audit report + 
    +                {JSON.stringify(turn.progress.report ?? {}, null, 1)}
+
    +
    +
    + ); + })()} + + {turn.acceptFindings && turn.acceptFindings.length > 0 && ( +
    +

    + The agent refused to save this surface as a worked example. +

    +
      + {turn.acceptFindings.map((f, i) => ( +
    • + + {f.gate} {f.code} + {f.target ? ` · ${f.target}` : ""} + {" "} + {f.message} +
      • + ))} +
    +
    + )} + {turn.gaps.length > 0 && (

    Vocabulary gap: this ask needs {turn.gaps.map((g) => `'${g}'`).join(", ")}, which no approved component provides. Building @@ -102,20 +166,21 @@ function TurnBlock({ turn }: { turn: BuildTurn }) { - {turn.progress.status === "finished" && turn.progress.outcome === "passed" && !turn.accepted && ( + {canAcceptTurn(turn.progress) && !turn.accepted && (

    setExampleId(e.target.value)} - aria-label={`Example id for turn ${turn.id}`} + placeholder="(agent mints a free id)" + aria-label={`Example id for turn ${turn.id} — leave blank to let the agent mint a collision-free id`} style={{ fontFamily: "var(--mono)", fontSize: 12, background: "var(--bg-1)", border: "1px solid var(--line)", color: "var(--fg)", padding: "5px 8px", borderRadius: 2 }} data-testid={`build-example-id-${turn.id}`} />
    + {examplesForIntent === 0 && ( +

    + No worked example for {activeIntent} yet. scripted replays this intent's own example, so it + cannot run; a model generates from the scoped contract with no few-shot context. Accepting a build here creates the + first one. +

    + )} +
    { await connect(page, project.root); await expect(page.getByTestId("nav-build")).toBeEnabled(); await page.getByTestId("nav-build").click(); + // Wait for the Build surface itself: readiness is derived from the live + // browser emit, so the view can briefly render its not-ready panel. If it + // never opens, report WHY rather than timing out on a missing locator. + const prompt = page.getByTestId("build-prompt"); + const notReady = page.getByTestId("build-not-ready"); + const needsAgent = page.getByTestId("build-needs-agent"); + await expect + .poll(async () => { + if (await prompt.count()) return "ready"; + if (await notReady.count()) return `not ready: ${await notReady.innerText()}`; + if (await needsAgent.count()) return "needs agent (demo mode)"; + return "no build panel rendered"; + }, { timeout: 20_000 }) + .toBe("ready"); return project; } @@ -93,12 +108,14 @@ test("Accept persists the worked example, survives reload, and the next run rece await expect(page.getByTestId("build-canvas-2")).toContainText("(refined)", { timeout: 30_000 }); await page.getByTestId("build-accept-2").click(); - await expect(page.getByTestId("build-accepted-2")).toContainText("ex.chat-2"); + await expect(page.getByTestId("build-accepted-2")).toContainText(/ex\.chat-\d+/); // Persisted on disk with the ledger intact. const doc = project.contract(); - const saved = doc.examples.find((e: any) => e.id === "ex.chat-2"); + const saved = doc.examples.find((e: any) => /^ex\.chat-\d+$/.test(e.id)); expect(saved.intent).toBe("status-report"); + // Truthful provenance: the ORIGINAL ask leads, the refinement is recorded. + expect(saved.prompt).toBe("a deployment status screen — refined: make the title clearer"); expect(JSON.stringify(saved.surface)).toContain("(refined)"); expect(doc.metadata["x-bootstrap"]).toBeDefined(); @@ -106,7 +123,7 @@ test("Accept persists the worked example, survives reload, and the next run rece await page.reload(); await connect(page, project.root); await page.getByTestId("nav-scenarios").click(); - await expect(page.locator("body")).toContainText("ex.chat-2"); + await expect(page.locator("body")).toContainText(saved.id); // Few-shot round-trip, user-visible: a fresh scripted run now converges on // the LATEST accepted example — the accepted result feeds the next run. @@ -179,5 +196,182 @@ test("double submission and double acceptance are locked", async ({ page }) => { await accept.dispatchEvent("click").catch(() => undefined); await expect(page.getByTestId("build-accepted-1")).toBeVisible(); const doc = project.contract(); - expect(doc.examples.filter((e: any) => e.id === "ex.chat-1")).toHaveLength(1); // exactly once + expect(doc.examples.filter((e: any) => /^ex\.chat-\d+$/.test(e.id))).toHaveLength(1); // exactly once +}); + + +test("an S3 governance failure shows the exact rule and the owner's rationale (#41)", async ({ page }) => { + const project = demoProject(); + // A governed violation the contract's OWN rule catches: the demo's rule + // set is authored, so we drive the corpus into violating it. + const doc = project.contract(); + // A GOVERNANCE-only violation (rule.status-report.info-card-required, + // severity must, with the owner's authored rationale): a status-report + // surface whose root is not the required InfoCard. Emit stays clean, so + // Build remains open — S3 is what rejects it. + const rule = doc.rules.find((r: any) => r.id === "rule.status-report.info-card-required"); + expect(rule?.rationale, "the demo contract must ship an authored rationale").toBeTruthy(); + doc.examples[0].surface = { + dspackSurface: "0.1", + system: doc.name, + intent: "status-report", + root: { component: "note-field", id: "notes", props: { label: "Operator notes", resizable: true } }, + }; + project.writeContract(doc); + await connect(page, project.root); + await expect(page.getByTestId("nav-build")).toBeEnabled(); + await page.getByTestId("nav-build").click(); + await expect(page.getByTestId("build-prompt")).toBeVisible(); + await page.getByTestId("build-prompt").fill("a status screen"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-outcome-1")).toContainText("failed", { timeout: 30_000 }); + + const failure = page.getByTestId("build-failure-1"); + await expect(failure).toBeVisible(); + await expect(failure).toContainText("stopped at"); + // The exact rule id, its message, and the OWNER'S rationale, verbatim. + await expect(failure).toContainText("rule.status-report.info-card-required"); + await expect(page.getByTestId("build-rationale-1")).toContainText(rule.rationale); + // The full report stays available for inspection. + await expect(failure.getByText("full audit report")).toBeVisible(); + // Neither action is offered for a turn without a valid surface. + await expect(page.getByTestId("build-accept-1")).toHaveCount(0); + await expect(page.getByTestId("build-refine")).toBeDisabled(); +}); + +test("an emit refusal shows the emitter's verbatim evidence, not a bare failed-gate (#41)", async ({ page }) => { + const project = await ready(page); + // The demo's authored casualty: a surface using it is lint-clean but the + // emitter refuses it with its written reason. + const doc = project.contract(); + const casualty = JSON.parse(readFileSync(`${project.root}/surfaces/uses-casualty.dsurface.json`, "utf8")); + doc.examples[0].surface = casualty; + project.writeContract(doc); + await page.getByTestId("build-prompt").fill("a screen using the casualty"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-outcome-1")).toContainText("failed-gate", { timeout: 30_000 }); + + const failure = page.getByTestId("build-failure-1"); + await expect(failure).toContainText(/declared casualty/i); + await expect(failure).toContainText(/steps is an array prop/); // the authored reason, verbatim + await expect(page.getByTestId("build-accept-1")).toHaveCount(0); +}); + +test("an adapter failure explains itself actionably (#41)", async ({ page }) => { + const project = demoProject(); + await connect(page, project.root); + await page.getByTestId("nav-build").click(); + // Select a model ref the agent will try to reach and fail on. + await page.getByTestId("build-model").selectOption({ index: 0 }).catch(() => undefined); + await page.evaluate(() => { + const select = document.querySelector('[data-testid="build-model"]') as HTMLSelectElement; + const option = document.createElement("option"); + option.value = "ollama:definitely-not-a-model"; + option.textContent = "ollama:definitely-not-a-model"; + select.appendChild(option); + select.value = "ollama:definitely-not-a-model"; + select.dispatchEvent(new Event("change", { bubbles: true })); + }); + await page.getByTestId("build-prompt").fill("anything"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-failure-1")).toBeVisible({ timeout: 60_000 }); + const failure = page.getByTestId("build-failure-1"); + await expect(failure).toContainText(/model provider|stream ended/i); + const text = await failure.innerText(); + expect(text).not.toMatch(/^outcome: failed-adapter$/m); +}); + +test("a refused Accept renders the agent's findings, not an HTTP status (#41)", async ({ page }) => { + const project = await ready(page); + await runScripted(page, "a deployment status screen"); + // Sabotage the corpus AFTER generating: the accept gate re-lints and refuses. + const doc = project.contract(); + doc.components["not-approved-anymore"] = undefined; + delete doc.components["tag-pill"]; // the generated surface now references unknown vocabulary + project.writeContract(doc); + + await page.getByTestId("build-accept-1").click(); + const findings = page.getByTestId("build-accept-findings-1"); + await expect(findings).toBeVisible(); + await expect(findings).toContainText(/S2|S1|document/); + await expect(page.getByTestId("notice")).not.toContainText("agent replied"); + await expect(page.getByTestId("notice")).toContainText(/refused/i); +}); + +test("two accepts across a reload mint distinct ids and preserve both examples (#42)", async ({ page }) => { + const project = await ready(page); + const before = project.contract().examples.map((e: any) => e.id); + await runScripted(page, "a deployment status screen"); + await page.getByTestId("build-accept-1").click(); + await expect(page.getByTestId("build-accepted-1")).toBeVisible(); + const first = project.contract().examples.at(-1).id; + + // A reload resets every page-local counter — identity must not depend on it. + await page.reload(); + await connect(page, project.root); + await page.getByTestId("nav-build").click(); + await runScripted(page, "another status screen"); + await page.getByTestId("build-accept-1").click(); + await expect(page.getByTestId("build-accepted-1")).toBeVisible(); + + const ids = project.contract().examples.map((e: any) => e.id); + const second = ids.at(-1); + expect(second).not.toBe(first); + expect(ids).toEqual(expect.arrayContaining([...before, first, second])); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); + +test("an explicit id collision is refused, never an overwrite (#42)", async ({ page }) => { + const project = await ready(page); + const existing = project.contract().examples[0]; + const before = JSON.stringify(existing); + await runScripted(page, "a deployment status screen"); + await page.getByTestId("build-example-id-1").fill(existing.id); + await page.getByTestId("build-accept-1").click(); + + await expect(page.getByTestId("build-accept-findings-1")).toContainText(existing.id); + await expect(page.getByTestId("build-accepted-1")).toHaveCount(0); + expect(JSON.stringify(project.contract().examples[0])).toBe(before); // byte-identical +}); + +test("an intent with no example cannot borrow another intent's (#43)", async ({ page }) => { + const project = await ready(page); + const doc = project.contract(); + doc.intents = [...doc.intents, { id: "onboarding", description: "Welcome a new operator." }]; + project.writeContract(doc); + // Reconnect so the view sees the new intent, then select it. + await connect(page, project.root); + await page.getByTestId("nav-build").click(); + await page.getByTestId("build-intent").selectOption("onboarding"); + await expect(page.getByTestId("build-no-fewshot")).toContainText("onboarding"); + + await page.getByTestId("build-prompt").fill("an onboarding screen"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-status")).toContainText(/latest outcome|turn/, { timeout: 30_000 }); + // Honest refusal, and nothing from the other intent was rendered. + await expect(page.getByTestId("build-canvas-1")).toHaveCount(0); + await expect(page.locator("body")).toContainText(/worked example/i); +}); + +test("two consecutive refinements each use the immediately prior surface, non-vacuously (#43)", async ({ page }) => { + const project = await ready(page); + await runScripted(page, "a deployment status screen"); + const first = await page.getByTestId("build-canvas-1").innerText(); + + await page.getByTestId("build-prompt").fill("make the title clearer"); + await page.getByTestId("build-refine").click(); + await expect(page.getByTestId("build-canvas-2")).toBeVisible({ timeout: 30_000 }); + const second = await page.getByTestId("build-canvas-2").innerText(); + + await page.getByTestId("build-prompt").fill("and say it once more"); + await page.getByTestId("build-refine").click(); + await expect(page.getByTestId("build-canvas-3")).toBeVisible({ timeout: 30_000 }); + const third = await page.getByTestId("build-canvas-3").innerText(); + + // Turn 3 built on turn 2 — and is NOT a byte-identical no-op reported as success. + expect(second).not.toBe(first); + expect(third, "the second refinement must not be a silent no-op").not.toBe(second); + await expect(page.getByTestId("build-outcome-3")).toContainText("passed"); + // All three remain in the thread for comparison and audit. + await expect(page.getByTestId("build-canvas-1")).toBeVisible(); }); diff --git a/packages/composer-core/src/build.ts b/packages/composer-core/src/build.ts index 264ac74..09d4c76 100644 --- a/packages/composer-core/src/build.ts +++ b/packages/composer-core/src/build.ts @@ -109,6 +109,25 @@ export function foldBuildEvents(events: Array>): BuildTurnPr progress.outcome = String(value.outcome ?? ""); progress.exitCode = Number(value.exitCode ?? -1); progress.report = value.report as Record; + // The report is the artifact of record: reconcile the streamed + // attempts against it so every consumer (gap detection, failure + // presentation) reads ONE authoritative source, even if a progress + // event was dropped mid-stream. Repair messages already folded are + // kept — they are indexed the same way. + const reported = (value.report?.attempts ?? []) as Array>; + if (reported.length > 0) { + progress.attempts = reported.map((attempt, i) => { + const reportedGates = (attempt.gates ?? []) as TurnGate[]; + const streamed = progress.attempts[i]; + return { + index: Number(attempt.index ?? i), + // Prefer the report's gates; fall back to what streamed, so a + // reconciliation never LOSES detail either direction. + gates: reportedGates.length > 0 ? reportedGates : (streamed?.gates ?? []), + ...(streamed?.repair ? { repair: streamed.repair } : {}), + }; + }); + } const surface = value.report?.attempts?.at?.(-1)?.surface; if (surface && typeof surface === "object") progress.surface = surface as Record; } else if (name === "dspack.error") { @@ -143,3 +162,171 @@ export function vocabularyGap(progress: BuildTurnProgress): string[] { } return [...ids]; } + + +/* ------------------------------------------------------------------ */ +/* Structured failure presentation (#41). */ +/* */ +/* A failed turn must say WHY, from the structured fields the pipeline */ +/* already reports — never a bare outcome word, and never a cause */ +/* inferred from message wording. Every reason is lifted verbatim: */ +/* S1/S2 -> attempt.gates[].errors (those gates carry no findings) */ +/* S3 -> attempt.findings[] (ruleId + message + the */ +/* owner-authored rationale) */ +/* emit -> emitted.refusal, else emitted.validations[].gates[] */ +/* adapter-> attempt.adapterError */ +/* The full report stays on the turn for inspection either way. */ +/* ------------------------------------------------------------------ */ + +export type BuildFailureKind = "lint" | "repair-exhausted" | "emit-refusal" | "emit-gate" | "adapter" | "unknown"; + +export interface BuildFailureReason { + /** S1 | S2 | S3 | A1 | A2 | A3 when the reason belongs to a gate. */ + gate?: string; + /** Rule id, emitter gate name, or other structured code. */ + code?: string; + /** Surface path, component id, or catalog version the reason points at. */ + target?: string; + message: string; + /** Owner-authored rationale — S3 findings only; never synthesized. */ + rationale?: string; +} + +export interface BuildFailure { + kind: BuildFailureKind; + /** One honest sentence naming what stopped the run. */ + headline: string; + /** Where the pipeline stopped, e.g. "attempt 2 · S3 governance". */ + stoppedAt: string; + reasons: BuildFailureReason[]; +} + +const GATE_LABEL: Record = { + S1: "S1 surface schema", + S2: "S2 contract vocabulary", + S3: "S3 governance", +}; + +/** Reasons from one attempt's gates and findings, in gate order. */ +function attemptReasons(attempt: Record | undefined): BuildFailureReason[] { + if (!attempt) return []; + const reasons: BuildFailureReason[] = []; + for (const gate of (attempt.gates ?? []) as TurnGate[]) { + if (gate.status !== "FAIL") continue; + if (gate.gate === "S3") continue; // S3 speaks through findings + for (const error of gate.errors ?? []) { + reasons.push({ gate: gate.gate, code: gate.name, message: error }); + } + } + for (const finding of (attempt.findings ?? []) as Array>) { + if (finding.level && finding.level !== "error") continue; + reasons.push({ + gate: "S3", + code: String(finding.ruleId ?? ""), + target: String(finding.location?.path ?? finding.location?.component ?? ""), + message: String(finding.message ?? ""), + ...(finding.rationale ? { rationale: String(finding.rationale) } : {}), + }); + } + return reasons; +} + +/** + * The structured failure for a finished turn, or null when it passed (or has + * not finished). Never invents a cause: an outcome with no structured + * evidence returns kind "unknown" and says so plainly. + */ +export function buildFailure(progress: BuildTurnProgress): BuildFailure | null { + if (progress.status === "error") { + return { + kind: "adapter", + headline: "The run did not complete — the agent stream ended early.", + stoppedAt: "stream", + reasons: [{ message: progress.error ?? "the stream ended without a result" }], + }; + } + if (!progress.outcome || progress.outcome === "passed") return null; + + const report = (progress.report ?? {}) as Record; + const attempts = (report.attempts ?? []) as Array>; + const last = attempts.at(-1); + const emitted = report.emitted as Record | undefined; + + if (progress.outcome === "failed-adapter") { + const message = String(last?.adapterError ?? "the model adapter failed without a message"); + return { + kind: "adapter", + headline: "The model provider could not produce a result — nothing was generated.", + stoppedAt: `attempt ${(last?.index ?? 0) + 1} · provider`, + reasons: [{ code: report.generation?.adapterId, message }], + }; + } + + if (progress.outcome === "failed-gate") { + if (emitted?.refusal) { + return { + kind: "emit-refusal", + headline: "The surface passed governance but the emitter refused it.", + stoppedAt: "emit · refusal", + reasons: [{ gate: "emit", message: String(emitted.refusal) }], + }; + } + const reasons: BuildFailureReason[] = []; + for (const validation of (emitted?.validations ?? []) as Array>) { + for (const gate of (validation.gates ?? []) as Array>) { + if (gate.pass) continue; + for (const error of (gate.errors ?? ["gate failed"]) as string[]) { + reasons.push({ gate: String(gate.gate), code: String(gate.name ?? ""), target: validation.a2uiVersion ? `a2ui@${validation.a2uiVersion}` : undefined, message: error }); + } + } + } + if (reasons.length > 0) { + return { + kind: "emit-gate", + headline: "The surface passed governance but failed the catalog gates.", + stoppedAt: `emit · ${reasons[0].gate}`, + reasons, + }; + } + } + + const reasons = attemptReasons(last); + const failedGate = ((last?.gates ?? []) as TurnGate[]).find((g) => g.status === "FAIL"); + const stoppedAt = `attempt ${(last?.index ?? 0) + 1} · ${failedGate ? GATE_LABEL[failedGate.gate] ?? failedGate.gate : "generation"}`; + const exhausted = progress.outcome === "failed-lint-exhausted" && attempts.length > 1; + if (reasons.length === 0) { + return { + kind: "unknown", + headline: `The run ended as ${progress.outcome} without structured evidence — the full report is below.`, + stoppedAt, + reasons: [], + }; + } + return { + kind: exhausted ? "repair-exhausted" : "lint", + headline: exhausted + ? `Bounded repair was exhausted after ${attempts.length} attempts — the last attempt still violates the contract.` + : "The generated surface does not satisfy the contract.", + stoppedAt, + reasons, + }; +} + +/** A turn offers Accept/Refine only when it finished, passed, and has a surface. */ +export function canAcceptTurn(progress: BuildTurnProgress): boolean { + return progress.status === "finished" && progress.outcome === "passed" && !!progress.surface; +} + +export const canRefineTurn = canAcceptTurn; + +/** + * The worked example's prompt: the ORIGINAL build request, plus a concise + * deterministic record of the refinements that shaped the accepted surface. + * Truthful provenance — a reader (and the few-shot corpus) sees the ask that + * produced this result, not merely the last edit instruction. + */ +export function examplePromptFor(chain: string[]): string { + const [original, ...refinements] = chain.map((p) => p.trim()).filter(Boolean); + if (!original) return ""; + return refinements.length === 0 ? original : `${original} — refined: ${refinements.join("; ")}`; +} diff --git a/packages/composer-core/src/composer-core.test.ts b/packages/composer-core/src/composer-core.test.ts index 99a5304..8846710 100644 --- a/packages/composer-core/src/composer-core.test.ts +++ b/packages/composer-core/src/composer-core.test.ts @@ -32,7 +32,15 @@ import { unresolvedErrors, } from "./findings"; import { COMPOSER_ADAPTERS, composerAdapter } from "./adapters"; -import { buildReadiness, foldBuildEvents, vocabularyGap } from "./build"; +import { + buildFailure, + buildReadiness, + canAcceptTurn, + canRefineTurn, + examplePromptFor, + foldBuildEvents, + vocabularyGap, +} from "./build"; const fixture = (name: string) => JSON.parse(readFileSync(fileURLToPath(new URL(`../fixtures/${name}`, import.meta.url)), "utf8")); @@ -516,3 +524,179 @@ describe("folding a streamed build run", () => { expect(turn.error).toBe("agent gone"); }); }); + + +describe("structured build failures (#41)", () => { + const run = (outcome: string, report: Record, extra: Array> = []) => + foldBuildEvents([ + { type: "RUN_STARTED" }, + { type: "STEP_STARTED", stepName: "attempt-0" }, + ...extra, + { type: "CUSTOM", name: "dspack.audit", value: { outcome, exitCode: outcome === "passed" ? 0 : 2, report } }, + { type: "RUN_FINISHED" }, + ]); + + it("an S3 governance failure names the rule, its message, its location, and the owner's rationale", () => { + const f = buildFailure( + run("failed-lint-exhausted", { + attempts: [ + { + index: 0, + surface: { root: {} }, + gates: [ + { gate: "S1", name: "surface-schema", status: "PASS" }, + { gate: "S2", name: "contract-vocabulary", status: "PASS" }, + { gate: "S3", name: "governance", status: "FAIL" }, + ], + findings: [ + { + ruleId: "rule.destructive-requires-alertdialog", + level: "error", + message: "destructive action without an AlertDialog", + rationale: "Irreversible actions must be confirmed deliberately.", + location: { path: "$.root.children[0]", component: "action-button" }, + }, + ], + }, + ], + }), + ); + expect(f?.kind).toBe("lint"); + expect(f?.stoppedAt).toMatch(/attempt 1.*S3/); + expect(f?.reasons).toHaveLength(1); + expect(f?.reasons[0]).toMatchObject({ + gate: "S3", + code: "rule.destructive-requires-alertdialog", + target: "$.root.children[0]", + message: "destructive action without an AlertDialog", + rationale: "Irreversible actions must be confirmed deliberately.", + }); + }); + + it("an S1/S2 failure reports its gate errors verbatim (no findings exist for those gates)", () => { + const f = buildFailure( + run("failed-lint-exhausted", { + attempts: [ + { + index: 0, + gates: [ + { gate: "S1", name: "surface-schema", status: "PASS" }, + { gate: "S2", name: "contract-vocabulary", status: "FAIL", errors: ["component 'nope' is not contract vocabulary"] }, + { gate: "S3", name: "governance", status: "SKIPPED" }, + ], + findings: [], + }, + ], + }), + ); + expect(f?.kind).toBe("lint"); + expect(f?.reasons[0]).toMatchObject({ gate: "S2", message: "component 'nope' is not contract vocabulary" }); + expect(f?.reasons[0].rationale).toBeUndefined(); + }); + + it("an emit refusal renders the emitter's verbatim refusal, not a bare outcome", () => { + const f = buildFailure( + run("failed-gate", { + attempts: [{ index: 0, surface: { root: {} }, gates: [{ gate: "S1", name: "surface-schema", status: "PASS" }], findings: [] }], + emitted: { target: "a2ui", warnings: [], validations: [], refusal: "component 'mini-stepper' is a declared casualty (cannot-represent): steps is free-form." }, + }), + ); + expect(f?.kind).toBe("emit-refusal"); + expect(f?.reasons[0].message).toContain("declared casualty"); + expect(f?.stoppedAt).toMatch(/emit/i); + }); + + it("an emit A-gate failure names the gate and its errors", () => { + const f = buildFailure( + run("failed-gate", { + attempts: [{ index: 0, surface: { root: {} }, gates: [], findings: [] }], + emitted: { + target: "a2ui", + warnings: [], + validations: [{ a2uiVersion: "0.9.1", gates: [{ gate: "A3", name: "instance", pass: false, errors: ["TextField requires label"] }] }], + }, + }), + ); + expect(f?.kind).toBe("emit-gate"); + expect(f?.reasons[0]).toMatchObject({ gate: "A3", message: "TextField requires label" }); + }); + + it("an adapter failure carries the typed error and an actionable explanation", () => { + const f = buildFailure(run("failed-adapter", { attempts: [{ index: 0, adapterError: "fetch failed: connect ECONNREFUSED 127.0.0.1:11434" }] })); + expect(f?.kind).toBe("adapter"); + expect(f?.reasons[0].message).toContain("ECONNREFUSED"); + expect(f?.headline).toMatch(/model|provider/i); + }); + + it("repair exhaustion is named as such, with the last attempt's reasons", () => { + const gates = [{ gate: "S2", name: "contract-vocabulary", status: "FAIL", errors: ["component 'x' is not contract vocabulary"] }]; + const f = buildFailure( + run("failed-lint-exhausted", { + attempts: [ + { index: 0, gates, findings: [] }, + { index: 1, gates, findings: [] }, + { index: 2, gates, findings: [] }, + ], + repairMessages: ["r1", "r2"], + }), + ); + expect(f?.kind).toBe("repair-exhausted"); + expect(f?.headline).toMatch(/repair/i); + expect(f?.reasons[0].gate).toBe("S2"); + }); + + it("a passing run has no failure, and successful gates are never hidden", () => { + const progress = run("passed", { attempts: [{ index: 0, surface: { root: { component: "x" } }, gates: [{ gate: "S1", name: "s", status: "PASS" }], findings: [] }] }); + expect(buildFailure(progress)).toBeNull(); + expect(progress.attempts[0].gates).toHaveLength(1); // the fold keeps every gate + }); + + it("Refine and Accept are offered only for a completed run with a valid surface", () => { + const passed = run("passed", { attempts: [{ index: 0, surface: { root: {} }, gates: [], findings: [] }] }); + expect(canAcceptTurn(passed)).toBe(true); + expect(canRefineTurn(passed)).toBe(true); + + // A failed turn may still carry a surface from its last attempt — neither action applies. + const failed = run("failed-lint-exhausted", { attempts: [{ index: 0, surface: { root: {} }, gates: [], findings: [] }] }); + expect(canAcceptTurn(failed)).toBe(false); + expect(canRefineTurn(failed)).toBe(false); + + const adapter = run("failed-adapter", { attempts: [{ index: 0, adapterError: "boom" }] }); + expect(canAcceptTurn(adapter)).toBe(false); + expect(canRefineTurn(adapter)).toBe(false); + + const streaming = foldBuildEvents([{ type: "RUN_STARTED" }]); + expect(canAcceptTurn(streaming)).toBe(false); + expect(canRefineTurn(streaming)).toBe(false); + }); + + it("a vocabulary gap stays distinguishable from malformed model output", () => { + const gap = run("failed-lint-exhausted", { + attempts: [{ index: 0, gates: [{ gate: "S2", name: "contract-vocabulary", status: "FAIL", errors: ["component 'timeline' is not contract vocabulary"] }], findings: [] }], + }); + expect(vocabularyGap(gap)).toEqual(["timeline"]); + expect(buildFailure(gap)?.kind).toBe("lint"); + + const malformed = run("failed-lint-exhausted", { + attempts: [{ index: 0, gates: [{ gate: "S1", name: "surface-schema", status: "FAIL", errors: ["(root) must be object"] }], findings: [] }], + }); + expect(vocabularyGap(malformed)).toEqual([]); + expect(buildFailure(malformed)?.reasons[0].gate).toBe("S1"); + }); +}); + +describe("worked-example prompt provenance (#42)", () => { + it("keeps the ORIGINAL request as the prompt and records refinements deterministically", () => { + expect(examplePromptFor(["a deployment status screen"])).toBe("a deployment status screen"); + expect(examplePromptFor(["a deployment status screen", "make the title clearer"])).toBe( + "a deployment status screen — refined: make the title clearer", + ); + expect(examplePromptFor(["a status screen", "make the title clearer", "add the region"])).toBe( + "a status screen — refined: make the title clearer; add the region", + ); + // Deterministic and stable: same chain, same string. + expect(examplePromptFor(["a", "b"])).toBe(examplePromptFor(["a", "b"])); + // Never merely the last instruction. + expect(examplePromptFor(["original", "last"])).toContain("original"); + }); +}); diff --git a/packages/composer-core/src/index.ts b/packages/composer-core/src/index.ts index bf6d6ba..c0f6dad 100644 --- a/packages/composer-core/src/index.ts +++ b/packages/composer-core/src/index.ts @@ -47,10 +47,17 @@ export { } from "./adapters"; export { buildReadiness, + buildFailure, + canAcceptTurn, + canRefineTurn, + examplePromptFor, foldBuildEvents, vocabularyGap, type BuildReadiness, type BuildTurnProgress, + type BuildFailure, + type BuildFailureKind, + type BuildFailureReason, type TurnAttempt, type TurnGate, } from "./build";