diff --git a/apps/agent/src/project.test.ts b/apps/agent/src/project.test.ts index d323ed1..1679890 100644 --- a/apps/agent/src/project.test.ts +++ b/apps/agent/src/project.test.ts @@ -454,3 +454,73 @@ describe("accepting a build result (/project/save-example, fail-closed)", () => expect(JSON.stringify(surfaceOfRun(next.events).surface)).toBe(JSON.stringify(refined)); }); }); + + +describe("safe worked-example persistence (#42) and honest scripted absence (#43)", () => { + const contractOf = () => JSON.parse(readFileSync(join(root, "acme-ui.dspack.json"), "utf8")); + const surfaceOf = () => structuredClone(contractOf().examples[0].surface); + + it("mints a collision-free id from the contract on disk when the client supplies none", async () => { + const before = contractOf().examples.map((e: any) => e.id); + const a = await call("save-example", { path: root, example: { intent: "status-report", prompt: "first ask", surface: surfaceOf() } }); + expect(a.status).toBe(200); + expect(a.payload.example.id).toMatch(/^ex\.chat-\d+$/); + expect(before).not.toContain(a.payload.example.id); + + const b = await call("save-example", { path: root, example: { intent: "status-report", prompt: "second ask", surface: surfaceOf() } }); + expect(b.status).toBe(200); + expect(b.payload.example.id).not.toBe(a.payload.example.id); // distinct across accepts + + // Both survive; every pre-existing example is byte-identical. + const doc = contractOf(); + expect(doc.examples.map((e: any) => e.id)).toEqual(expect.arrayContaining([a.payload.example.id, b.payload.example.id, ...before])); + for (const id of before) { + expect(JSON.stringify(doc.examples.find((e: any) => e.id === id))).toBe( + JSON.stringify(JSON.parse(readFileSync(join(root, "acme-ui.dspack.json"), "utf8")).examples.find((e: any) => e.id === id)), + ); + } + }); + + it("REFUSES an explicit id that already exists rather than overwriting it", async () => { + const existing = contractOf().examples[0]; + const before = JSON.stringify(existing); + const { status, payload } = await call("save-example", { + path: root, + example: { id: existing.id, intent: "status-report", prompt: "hostile overwrite", surface: surfaceOf() }, + }); + expect(status).toBe(409); + expect(payload.findings[0].code).toBe("example-exists"); + expect(payload.findings[0].message).toContain(existing.id); + // Untouched, byte for byte. + expect(JSON.stringify(contractOf().examples.find((e: any) => e.id === existing.id))).toBe(before); + }); + + it("the newly accepted example is consumable as few-shot and scripted plays the latest without replacing older ones", async () => { + const refined = surfaceOf(); + refined.root.children[0].children[0].text = "Rollout status"; + const saved = await call("save-example", { + path: root, + example: { intent: "status-report", prompt: "a status screen — refined: say Rollout status", surface: refined }, + }); + expect(saved.status).toBe(200); + const doc = contractOf(); + const { compileContext } = await import("@aestheticfunction/dspack-gen/core"); + const context = compileContext(doc, "status-report"); + expect(context.fewshot.some((m: any) => m.role === "assistant" && m.content.includes("Rollout status"))).toBe(true); + // Older examples still present and still served. + expect(doc.examples.length).toBeGreaterThan(1); + expect(context.fewshot.length).toBeGreaterThanOrEqual(doc.examples.filter((e: any) => e.intent === "status-report").length); + }); + + it("an intent with no matching example never borrows another intent's: scripted refuses honestly", async () => { + const doc = contractOf(); + doc.intents = [...doc.intents, { id: "onboarding", description: "Welcome a new operator." }]; + const { writeFileSync } = await import("node:fs"); + writeFileSync(join(root, "acme-ui.dspack.json"), JSON.stringify(doc, null, 2) + "\n"); + + const { status, payload } = await call("run", { path: root, prompt: "an onboarding screen", intent: "onboarding", modelRef: "scripted" }); + expect(status).toBe(400); + expect(String(payload.error)).toMatch(/onboarding/); + expect(String(payload.error)).toMatch(/worked example/i); + }); +}); diff --git a/apps/agent/src/project.ts b/apps/agent/src/project.ts index fb125b1..c0a6e7d 100644 --- a/apps/agent/src/project.ts +++ b/apps/agent/src/project.ts @@ -481,9 +481,22 @@ function scriptedRunAdapter(example: { surface: unknown }, conversation: Convers try { const refined = JSON.parse(priorRaw) as Record; const textNode = firstTextNode(refined); - if (textNode && !textNode.text.endsWith(" (refined)")) textNode.text = `${textNode.text} (refined)`; - else if (!textNode) (refined as { id?: string }).id = "refined"; - return new ScriptedAdapter([{ output: refined }]); + if (textNode) { + // MONOTONIC, never idempotent: successive refinements must each + // produce a genuinely different surface, or the twin would report + // a byte-identical no-op as a successful refinement (#43). + const existing = /^(.*?)(?: \(refined(?: (\d+))?\))$/.exec(textNode.text); + const base = existing ? existing[1] : textNode.text; + const next = existing ? Number(existing[2] ?? 1) + 1 : 1; + textNode.text = next === 1 ? `${base} (refined)` : `${base} (refined ${next})`; + } else { + const previous = /^refined(?: (\d+))?$/.exec(String((refined as { id?: string }).id ?? "")); + const next = previous ? Number(previous[1] ?? 1) + 1 : 1; + (refined as { id?: string }).id = next === 1 ? "refined" : `refined ${next}`; + } + // Three entries so a refinement can survive bounded repair too — a + // refinement run must never die with a script-exhaustion error. + return new ScriptedAdapter([{ output: refined }, { output: refined }, { output: refined }]); } catch { // Fall through: an unparseable prior surface behaves like a fresh run. } @@ -512,12 +525,18 @@ async function runProject(ctx: ProjectContext, body: Record, re const conversation = parseConversation(props.conversation); const examples = (contract.examples as Array<{ intent: string; surface: unknown }> | undefined) ?? []; - // LAST match: accepted chat results join the corpus at the end, and the - // deterministic twin plays the owner's latest accepted example. - const matching = examples.filter((e) => e.intent === intent); - const example = matching.at(-1) ?? examples.at(-1); + // LAST match FOR THIS INTENT: accepted chat results join the corpus at the + // end, and the deterministic twin plays the owner's latest accepted + // example. Never borrow another intent's example — a screen built for a + // different intent is not a deterministic stand-in, it is a wrong answer + // reported as a right one (#43). + const example = examples.filter((e) => e.intent === intent).at(-1); if (modelRef === "scripted" && !example) { - throw new ProjectError(400, "scripted mode needs at least one worked example in the contract"); + throw new ProjectError( + 400, + `scripted mode replays this intent's own worked example, and '${intent}' has none yet. ` + + `Author one in Scenarios, or run with a model — generation works from the scoped contract without few-shot context.`, + ); } const adapter = modelRef === "scripted" @@ -553,6 +572,16 @@ async function runProject(ctx: ProjectContext, body: Record, re } +/** The next free `ex.chat-N` for this contract (monotonic, gap-tolerant). */ +function nextExampleId(existing: string[]): string { + let n = 0; + for (const id of existing) { + const match = /^ex\.chat-(\d+)$/.exec(id); + if (match) n = Math.max(n, Number(match[1])); + } + return `ex.chat-${n + 1}`; +} + /** * Accept a build result as a governed worked example — the ONLY save format * for chat-accepted surfaces, and fail-closed SERVER-SIDE: a disabled @@ -565,15 +594,33 @@ async function runProject(ctx: ProjectContext, body: Record, re async function saveExample(ctx: ProjectContext, body: Record) { const raw = body.example as Record | undefined; if (!raw || typeof raw !== "object") throw new ProjectError(400, "example is required"); - const id = String(raw.id ?? ""); - if (!/^ex\.[a-z0-9][a-z0-9-]*$/.test(id)) { - throw new ProjectError(400, "example.id must be kebab-case with the 'ex.' prefix"); - } if (!raw.surface || typeof raw.surface !== "object") throw new ProjectError(400, "example.surface must be a surface document"); const prompt = String(raw.prompt ?? ""); if (!prompt) throw new ProjectError(400, "example.prompt is required (the ask that produced this surface)"); const contract = readJson(ctx.contractPath) as Record; + const existing = ((contract.examples as Array<{ id?: unknown }> | undefined) ?? []).map((e) => String(e?.id ?? "")); + + // Identity is derived from the contract ON DISK, never from a page-local + // counter: a browser that reloaded (or a second tab) cannot mint an id + // that collides with work already saved. An EXPLICIT id that already + // exists is refused outright — accepting a build result never overwrites + // an existing worked example, least of all an owner-authored one (#42). + const requested = raw.id === undefined ? "" : String(raw.id); + if (requested && !/^ex\.[a-z0-9][a-z0-9-]*$/.test(requested)) { + throw new ProjectError(400, "example.id must be kebab-case with the 'ex.' prefix"); + } + if (requested && existing.includes(requested)) { + return { + status: 409, + payload: { + ok: false, + findings: [finding("document", "example-exists", "error", "example.id", `'${requested}' already exists in this contract; accepting would overwrite it. Choose another id, or leave it blank to mint the next free one.`)], + }, + }; + } + const id = requested || nextExampleId(existing); + const intents = ((contract.intents as Array<{ id: string }> | undefined) ?? []).map((i) => i.id); const intent = String(raw.intent ?? ""); if (!intents.includes(intent)) { @@ -606,9 +653,7 @@ async function saveExample(ctx: ProjectContext, body: Record) { }; const document = structuredClone(contract); const examples = ((document.examples as unknown[] | undefined) ?? []) as Array<{ id: string }>; - const at = examples.findIndex((e) => e.id === id); - if (at >= 0) examples[at] = entry as never; - else examples.push(entry as never); + examples.push(entry as never); // append-only: the id was proven free above document.examples = examples; // The same guarded write as /project/save: ledger preserved, harness clean. diff --git a/apps/composer/app/agent-client.ts b/apps/composer/app/agent-client.ts index 31d17fc..0dc1a82 100644 --- a/apps/composer/app/agent-client.ts +++ b/apps/composer/app/agent-client.ts @@ -28,7 +28,14 @@ export interface ConnectPayload { extraSurfaces: Array<{ name: string; surface: unknown }>; } -export type AgentResult = { ok: true; value: T } | { ok: false; error: string }; +/** + * A refusal keeps its STRUCTURED evidence: routes that answer 4xx with a + * findings array (the fail-closed accept gate) must not be flattened to + * "agent replied 422" — the gate reasons are the whole point (#41). + */ +export type AgentResult = + | { ok: true; value: T } + | { ok: false; error: string; findings?: ComposerFinding[]; status?: number }; const DEFAULT_AGENT = process.env.NEXT_PUBLIC_AGENT_URL ?? "http://localhost:8787"; @@ -45,7 +52,15 @@ async function post(route: string, body: unknown): Promise> { signal: AbortSignal.timeout(180_000), }); const payload = await res.json(); - if (!res.ok) return { ok: false, error: String(payload.error ?? `agent replied ${res.status}`) }; + if (!res.ok) { + const findings = Array.isArray(payload?.findings) ? (payload.findings as ComposerFinding[]) : undefined; + return { + ok: false, + status: res.status, + ...(findings ? { findings } : {}), + error: String(payload?.error ?? (findings?.length ? findings.map((f) => f.message).join("; ") : `agent replied ${res.status}`)), + }; + } return { ok: true, value: payload as T }; } catch (e) { return { ok: false, error: e instanceof Error ? e.message : String(e) }; @@ -148,7 +163,8 @@ export function streamProjectRun( } export interface AcceptedExample { - id: string; + /** Omit to let the agent mint a collision-free id from the contract (#42). */ + id?: string; intent: string; name?: string; prompt: string; diff --git a/apps/composer/app/state.tsx b/apps/composer/app/state.tsx index 5581f97..9e88dad 100644 --- a/apps/composer/app/state.tsx +++ b/apps/composer/app/state.tsx @@ -13,6 +13,8 @@ import { addTombstone, applyFreshFact, buildReadiness, + canRefineTurn, + examplePromptFor, foldBuildEvents, ledgerStatus, removeTombstone, @@ -52,10 +54,14 @@ export interface BuildTurn { modelRef: string; /** True when this turn refined the previous surface (seed supplied). */ refinement: boolean; + /** The turn this one refined, for truthful example provenance (#42). */ + parentId?: number; progress: BuildTurnProgress; /** Component ids the ask needed but the owner has not approved (S2 evidence). */ gaps: string[]; accepted?: string; // the saved example id + /** Structured findings from a refused Accept, rendered in place (#41). */ + acceptFindings?: ComposerFinding[]; } export interface ComposerState { @@ -96,7 +102,8 @@ export interface ComposerState { /** Setup completeness for building; reason names the exact remaining work. */ readiness: BuildReadiness; runBuild: (input: { prompt: string; intent: string; modelRef: string; refine?: boolean }) => Promise; - acceptBuildTurn: (turnId: number, exampleId: string) => Promise; + /** Accept a turn as a worked example; the agent mints the id (#42). */ + acceptBuildTurn: (turnId: number, exampleId?: string) => Promise; clearBuildThread: () => void; } @@ -514,9 +521,12 @@ export function ComposerProvider({ children }: { children: ReactNode }) { return; } if (buildBusy) return; - const prior = input.refine ? [...buildTurns].reverse().find((t) => t.progress.surface) : undefined; + // Only a completed, passing turn can seed a refinement (#43): a failed + // turn still carries its last attempt's surface, and seeding that + // regenerates from something the contract already rejected. + const prior = input.refine ? [...buildTurns].reverse().find((t) => canRefineTurn(t.progress)) : undefined; if (input.refine && !prior) { - setNotice("Nothing to refine yet — run a build first."); + setNotice("Nothing to refine yet — refinement starts from a completed build that passed its gates."); return; } setBuildBusy(true); @@ -527,6 +537,7 @@ export function ComposerProvider({ children }: { children: ReactNode }) { intent: input.intent, modelRef: input.modelRef, refinement: !!prior, + ...(prior ? { parentId: prior.id } : {}), progress: { status: "streaming", attempts: [] }, gaps: [], }; @@ -577,7 +588,7 @@ export function ComposerProvider({ children }: { children: ReactNode }) { * format — and immediately joins that intent's few-shot corpus. */ const acceptBuildTurn = useCallback( - async (turnId: number, exampleId: string) => { + async (turnId: number, exampleId?: string) => { if (!contract || decisionLock.current) return; const turn = buildTurns.find((t) => t.id === turnId); if (!turn?.progress.surface || turn.progress.outcome !== "passed") { @@ -587,19 +598,32 @@ export function ComposerProvider({ children }: { children: ReactNode }) { decisionLock.current = true; setBusy("accepting build result"); try { + // Truthful provenance: walk back to the ORIGINAL ask and record the + // refinements that shaped this surface (#42). + const chain: string[] = []; + for (let t: BuildTurn | undefined = turn; t; t = t.parentId ? buildTurns.find((x) => x.id === t!.parentId) : undefined) { + chain.unshift(t.prompt); + } + const prompt = examplePromptFor(chain); const result = await agentSaveExample(projectPath, { - id: exampleId, + ...(exampleId ? { id: exampleId } : {}), // omitted ⇒ the agent mints a collision-free id intent: turn.intent, - name: `Chat: ${turn.prompt.slice(0, 60)}`, - prompt: turn.prompt, + name: `Chat: ${chain[0].slice(0, 60)}`, + prompt, surface: turn.progress.surface, }); if (!result.ok) { - setNotice(`Accept failed: ${result.error}`); + // Structured gate reasons, never a bare HTTP status (#41). + const detail = result.findings?.length + ? result.findings.map((f) => `${f.gate} ${f.code}: ${f.message}`).join(" · ") + : result.error; + setNotice(`Accept refused: ${detail}`); + setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, acceptFindings: result.findings ?? [] } : t))); return; } if (!result.value.ok) { - setNotice(`Accept refused by the gates: ${result.value.findings.map((f) => f.message).join("; ").slice(0, 300)}`); + setNotice(`Accept refused: ${result.value.findings.map((f) => `${f.gate} ${f.code}: ${f.message}`).join(" · ").slice(0, 400)}`); + setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, acceptFindings: result.value.findings } : t))); return; } if (result.value.ledger) setLedger(result.value.ledger); @@ -611,8 +635,9 @@ export function ComposerProvider({ children }: { children: ReactNode }) { doc.examples = examples; setContract(doc); recomputeEmit(doc, profile); - setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, accepted: exampleId } : t))); - setNotice(`Accepted as worked example '${exampleId}' — it now seeds generation for '${turn.intent}'.`); + const savedId = result.value.example?.id ?? exampleId ?? ""; + setBuildTurns((prev) => prev.map((t) => (t.id === turnId ? { ...t, accepted: savedId, acceptFindings: undefined } : t))); + setNotice(`Accepted as worked example '${savedId}' — it now seeds generation for '${turn.intent}'.`); } finally { decisionLock.current = false; setBusy(null); diff --git a/apps/composer/app/views/build-view.tsx b/apps/composer/app/views/build-view.tsx index 007e360..e46479f 100644 --- a/apps/composer/app/views/build-view.tsx +++ b/apps/composer/app/views/build-view.tsx @@ -16,6 +16,7 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { A2uiCanvas } from "@dspack-studio/a2ui-ingest"; import { wireframeRegistryFor } from "@dspack-studio/wireframe-renderers"; import { shadcnRegistry } from "@dspack-studio/shadcn-renderers"; +import { buildFailure, canAcceptTurn, canRefineTurn } from "@dspack-studio/composer-core"; import type { BuildTurn } from "../state"; import { useComposer } from "../state"; import { browserEmit } from "../validation"; @@ -47,7 +48,9 @@ function TurnCanvas({ turn }: { turn: BuildTurn }) { function TurnBlock({ turn }: { turn: BuildTurn }) { const { acceptBuildTurn, buildBusy, busy } = useComposer(); - const [exampleId, setExampleId] = useState(`ex.chat-${turn.id}`); + // Blank by default: identity is minted from the contract ON DISK, so a + // reload or a second tab can never collide with saved work (#42). + const [exampleId, setExampleId] = useState(""); const locked = buildBusy || busy !== null; // The Accept button unmounts on success; a keyboard user's focus must // land on the confirmation, never fall to the document body. @@ -92,6 +95,67 @@ function TurnBlock({ turn }: { turn: BuildTurn }) { {turn.progress.error &&
  • {turn.progress.error}
    • } + {(() => { + const failure = buildFailure(turn.progress); + if (!failure) return null; + return ( +
    +

    {failure.headline}

    +

    + stopped at {failure.stoppedAt} +

    +
      + {failure.reasons.map((reason, i) => ( +
    • + + {[reason.gate, reason.code].filter(Boolean).join(" ")} + {reason.target ? ` · ${reason.target}` : ""} + +
      + {reason.message} + {reason.rationale && ( + <> +
      + + Why this rule exists: {reason.rationale} + + + )} +
      • + ))} + {failure.reasons.length === 0 && ( +
    • No structured reason was reported; the full audit report is below.
      • + )} +
    +
    + full audit report + 
    +                {JSON.stringify(turn.progress.report ?? {}, null, 1)}
+
    +
    +
    + ); + })()} + + {turn.acceptFindings && turn.acceptFindings.length > 0 && ( +
    +

    + The agent refused to save this surface as a worked example. +

    +
      + {turn.acceptFindings.map((f, i) => ( +
    • + + {f.gate} {f.code} + {f.target ? ` · ${f.target}` : ""} + {" "} + {f.message} +
      • + ))} +
    +
    + )} + {turn.gaps.length > 0 && (

    Vocabulary gap: this ask needs {turn.gaps.map((g) => `'${g}'`).join(", ")}, which no approved component provides. Building @@ -102,20 +166,21 @@ function TurnBlock({ turn }: { turn: BuildTurn }) { - {turn.progress.status === "finished" && turn.progress.outcome === "passed" && !turn.accepted && ( + {canAcceptTurn(turn.progress) && !turn.accepted && (

    setExampleId(e.target.value)} - aria-label={`Example id for turn ${turn.id}`} + placeholder="(agent mints a free id)" + aria-label={`Example id for turn ${turn.id} — leave blank to let the agent mint a collision-free id`} style={{ fontFamily: "var(--mono)", fontSize: 12, background: "var(--bg-1)", border: "1px solid var(--line)", color: "var(--fg)", padding: "5px 8px", borderRadius: 2 }} data-testid={`build-example-id-${turn.id}`} />
    + {examplesForIntent === 0 && ( +

    + No worked example for {activeIntent} yet. scripted replays this intent's own example, so it + cannot run; a model generates from the scoped contract with no few-shot context. Accepting a build here creates the + first one. +

    + )} +
    { await connect(page, project.root); await expect(page.getByTestId("nav-build")).toBeEnabled(); await page.getByTestId("nav-build").click(); + // Wait for the Build surface itself: readiness is derived from the live + // browser emit, so the view can briefly render its not-ready panel. If it + // never opens, report WHY rather than timing out on a missing locator. + const prompt = page.getByTestId("build-prompt"); + const notReady = page.getByTestId("build-not-ready"); + const needsAgent = page.getByTestId("build-needs-agent"); + await expect + .poll(async () => { + if (await prompt.count()) return "ready"; + if (await notReady.count()) return `not ready: ${await notReady.innerText()}`; + if (await needsAgent.count()) return "needs agent (demo mode)"; + return "no build panel rendered"; + }, { timeout: 20_000 }) + .toBe("ready"); return project; } @@ -93,12 +108,14 @@ test("Accept persists the worked example, survives reload, and the next run rece await expect(page.getByTestId("build-canvas-2")).toContainText("(refined)", { timeout: 30_000 }); await page.getByTestId("build-accept-2").click(); - await expect(page.getByTestId("build-accepted-2")).toContainText("ex.chat-2"); + await expect(page.getByTestId("build-accepted-2")).toContainText(/ex\.chat-\d+/); // Persisted on disk with the ledger intact. const doc = project.contract(); - const saved = doc.examples.find((e: any) => e.id === "ex.chat-2"); + const saved = doc.examples.find((e: any) => /^ex\.chat-\d+$/.test(e.id)); expect(saved.intent).toBe("status-report"); + // Truthful provenance: the ORIGINAL ask leads, the refinement is recorded. + expect(saved.prompt).toBe("a deployment status screen — refined: make the title clearer"); expect(JSON.stringify(saved.surface)).toContain("(refined)"); expect(doc.metadata["x-bootstrap"]).toBeDefined(); @@ -106,7 +123,7 @@ test("Accept persists the worked example, survives reload, and the next run rece await page.reload(); await connect(page, project.root); await page.getByTestId("nav-scenarios").click(); - await expect(page.locator("body")).toContainText("ex.chat-2"); + await expect(page.locator("body")).toContainText(saved.id); // Few-shot round-trip, user-visible: a fresh scripted run now converges on // the LATEST accepted example — the accepted result feeds the next run. @@ -179,5 +196,182 @@ test("double submission and double acceptance are locked", async ({ page }) => { await accept.dispatchEvent("click").catch(() => undefined); await expect(page.getByTestId("build-accepted-1")).toBeVisible(); const doc = project.contract(); - expect(doc.examples.filter((e: any) => e.id === "ex.chat-1")).toHaveLength(1); // exactly once + expect(doc.examples.filter((e: any) => /^ex\.chat-\d+$/.test(e.id))).toHaveLength(1); // exactly once +}); + + +test("an S3 governance failure shows the exact rule and the owner's rationale (#41)", async ({ page }) => { + const project = demoProject(); + // A governed violation the contract's OWN rule catches: the demo's rule + // set is authored, so we drive the corpus into violating it. + const doc = project.contract(); + // A GOVERNANCE-only violation (rule.status-report.info-card-required, + // severity must, with the owner's authored rationale): a status-report + // surface whose root is not the required InfoCard. Emit stays clean, so + // Build remains open — S3 is what rejects it. + const rule = doc.rules.find((r: any) => r.id === "rule.status-report.info-card-required"); + expect(rule?.rationale, "the demo contract must ship an authored rationale").toBeTruthy(); + doc.examples[0].surface = { + dspackSurface: "0.1", + system: doc.name, + intent: "status-report", + root: { component: "note-field", id: "notes", props: { label: "Operator notes", resizable: true } }, + }; + project.writeContract(doc); + await connect(page, project.root); + await expect(page.getByTestId("nav-build")).toBeEnabled(); + await page.getByTestId("nav-build").click(); + await expect(page.getByTestId("build-prompt")).toBeVisible(); + await page.getByTestId("build-prompt").fill("a status screen"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-outcome-1")).toContainText("failed", { timeout: 30_000 }); + + const failure = page.getByTestId("build-failure-1"); + await expect(failure).toBeVisible(); + await expect(failure).toContainText("stopped at"); + // The exact rule id, its message, and the OWNER'S rationale, verbatim. + await expect(failure).toContainText("rule.status-report.info-card-required"); + await expect(page.getByTestId("build-rationale-1")).toContainText(rule.rationale); + // The full report stays available for inspection. + await expect(failure.getByText("full audit report")).toBeVisible(); + // Neither action is offered for a turn without a valid surface. + await expect(page.getByTestId("build-accept-1")).toHaveCount(0); + await expect(page.getByTestId("build-refine")).toBeDisabled(); +}); + +test("an emit refusal shows the emitter's verbatim evidence, not a bare failed-gate (#41)", async ({ page }) => { + const project = await ready(page); + // The demo's authored casualty: a surface using it is lint-clean but the + // emitter refuses it with its written reason. + const doc = project.contract(); + const casualty = JSON.parse(readFileSync(`${project.root}/surfaces/uses-casualty.dsurface.json`, "utf8")); + doc.examples[0].surface = casualty; + project.writeContract(doc); + await page.getByTestId("build-prompt").fill("a screen using the casualty"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-outcome-1")).toContainText("failed-gate", { timeout: 30_000 }); + + const failure = page.getByTestId("build-failure-1"); + await expect(failure).toContainText(/declared casualty/i); + await expect(failure).toContainText(/steps is an array prop/); // the authored reason, verbatim + await expect(page.getByTestId("build-accept-1")).toHaveCount(0); +}); + +test("an adapter failure explains itself actionably (#41)", async ({ page }) => { + const project = demoProject(); + await connect(page, project.root); + await page.getByTestId("nav-build").click(); + // Select a model ref the agent will try to reach and fail on. + await page.getByTestId("build-model").selectOption({ index: 0 }).catch(() => undefined); + await page.evaluate(() => { + const select = document.querySelector('[data-testid="build-model"]') as HTMLSelectElement; + const option = document.createElement("option"); + option.value = "ollama:definitely-not-a-model"; + option.textContent = "ollama:definitely-not-a-model"; + select.appendChild(option); + select.value = "ollama:definitely-not-a-model"; + select.dispatchEvent(new Event("change", { bubbles: true })); + }); + await page.getByTestId("build-prompt").fill("anything"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-failure-1")).toBeVisible({ timeout: 60_000 }); + const failure = page.getByTestId("build-failure-1"); + await expect(failure).toContainText(/model provider|stream ended/i); + const text = await failure.innerText(); + expect(text).not.toMatch(/^outcome: failed-adapter$/m); +}); + +test("a refused Accept renders the agent's findings, not an HTTP status (#41)", async ({ page }) => { + const project = await ready(page); + await runScripted(page, "a deployment status screen"); + // Sabotage the corpus AFTER generating: the accept gate re-lints and refuses. + const doc = project.contract(); + doc.components["not-approved-anymore"] = undefined; + delete doc.components["tag-pill"]; // the generated surface now references unknown vocabulary + project.writeContract(doc); + + await page.getByTestId("build-accept-1").click(); + const findings = page.getByTestId("build-accept-findings-1"); + await expect(findings).toBeVisible(); + await expect(findings).toContainText(/S2|S1|document/); + await expect(page.getByTestId("notice")).not.toContainText("agent replied"); + await expect(page.getByTestId("notice")).toContainText(/refused/i); +}); + +test("two accepts across a reload mint distinct ids and preserve both examples (#42)", async ({ page }) => { + const project = await ready(page); + const before = project.contract().examples.map((e: any) => e.id); + await runScripted(page, "a deployment status screen"); + await page.getByTestId("build-accept-1").click(); + await expect(page.getByTestId("build-accepted-1")).toBeVisible(); + const first = project.contract().examples.at(-1).id; + + // A reload resets every page-local counter — identity must not depend on it. + await page.reload(); + await connect(page, project.root); + await page.getByTestId("nav-build").click(); + await runScripted(page, "another status screen"); + await page.getByTestId("build-accept-1").click(); + await expect(page.getByTestId("build-accepted-1")).toBeVisible(); + + const ids = project.contract().examples.map((e: any) => e.id); + const second = ids.at(-1); + expect(second).not.toBe(first); + expect(ids).toEqual(expect.arrayContaining([...before, first, second])); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); + +test("an explicit id collision is refused, never an overwrite (#42)", async ({ page }) => { + const project = await ready(page); + const existing = project.contract().examples[0]; + const before = JSON.stringify(existing); + await runScripted(page, "a deployment status screen"); + await page.getByTestId("build-example-id-1").fill(existing.id); + await page.getByTestId("build-accept-1").click(); + + await expect(page.getByTestId("build-accept-findings-1")).toContainText(existing.id); + await expect(page.getByTestId("build-accepted-1")).toHaveCount(0); + expect(JSON.stringify(project.contract().examples[0])).toBe(before); // byte-identical +}); + +test("an intent with no example cannot borrow another intent's (#43)", async ({ page }) => { + const project = await ready(page); + const doc = project.contract(); + doc.intents = [...doc.intents, { id: "onboarding", description: "Welcome a new operator." }]; + project.writeContract(doc); + // Reconnect so the view sees the new intent, then select it. + await connect(page, project.root); + await page.getByTestId("nav-build").click(); + await page.getByTestId("build-intent").selectOption("onboarding"); + await expect(page.getByTestId("build-no-fewshot")).toContainText("onboarding"); + + await page.getByTestId("build-prompt").fill("an onboarding screen"); + await page.getByTestId("build-run").click(); + await expect(page.getByTestId("build-status")).toContainText(/latest outcome|turn/, { timeout: 30_000 }); + // Honest refusal, and nothing from the other intent was rendered. + await expect(page.getByTestId("build-canvas-1")).toHaveCount(0); + await expect(page.locator("body")).toContainText(/worked example/i); +}); + +test("two consecutive refinements each use the immediately prior surface, non-vacuously (#43)", async ({ page }) => { + const project = await ready(page); + await runScripted(page, "a deployment status screen"); + const first = await page.getByTestId("build-canvas-1").innerText(); + + await page.getByTestId("build-prompt").fill("make the title clearer"); + await page.getByTestId("build-refine").click(); + await expect(page.getByTestId("build-canvas-2")).toBeVisible({ timeout: 30_000 }); + const second = await page.getByTestId("build-canvas-2").innerText(); + + await page.getByTestId("build-prompt").fill("and say it once more"); + await page.getByTestId("build-refine").click(); + await expect(page.getByTestId("build-canvas-3")).toBeVisible({ timeout: 30_000 }); + const third = await page.getByTestId("build-canvas-3").innerText(); + + // Turn 3 built on turn 2 — and is NOT a byte-identical no-op reported as success. + expect(second).not.toBe(first); + expect(third, "the second refinement must not be a silent no-op").not.toBe(second); + await expect(page.getByTestId("build-outcome-3")).toContainText("passed"); + // All three remain in the thread for comparison and audit. + await expect(page.getByTestId("build-canvas-1")).toBeVisible(); }); diff --git a/packages/composer-core/src/build.ts b/packages/composer-core/src/build.ts index 264ac74..09d4c76 100644 --- a/packages/composer-core/src/build.ts +++ b/packages/composer-core/src/build.ts @@ -109,6 +109,25 @@ export function foldBuildEvents(events: Array>): BuildTurnPr progress.outcome = String(value.outcome ?? ""); progress.exitCode = Number(value.exitCode ?? -1); progress.report = value.report as Record; + // The report is the artifact of record: reconcile the streamed + // attempts against it so every consumer (gap detection, failure + // presentation) reads ONE authoritative source, even if a progress + // event was dropped mid-stream. Repair messages already folded are + // kept — they are indexed the same way. + const reported = (value.report?.attempts ?? []) as Array>; + if (reported.length > 0) { + progress.attempts = reported.map((attempt, i) => { + const reportedGates = (attempt.gates ?? []) as TurnGate[]; + const streamed = progress.attempts[i]; + return { + index: Number(attempt.index ?? i), + // Prefer the report's gates; fall back to what streamed, so a + // reconciliation never LOSES detail either direction. + gates: reportedGates.length > 0 ? reportedGates : (streamed?.gates ?? []), + ...(streamed?.repair ? { repair: streamed.repair } : {}), + }; + }); + } const surface = value.report?.attempts?.at?.(-1)?.surface; if (surface && typeof surface === "object") progress.surface = surface as Record; } else if (name === "dspack.error") { @@ -143,3 +162,171 @@ export function vocabularyGap(progress: BuildTurnProgress): string[] { } return [...ids]; } + + +/* ------------------------------------------------------------------ */ +/* Structured failure presentation (#41). */ +/* */ +/* A failed turn must say WHY, from the structured fields the pipeline */ +/* already reports — never a bare outcome word, and never a cause */ +/* inferred from message wording. Every reason is lifted verbatim: */ +/* S1/S2 -> attempt.gates[].errors (those gates carry no findings) */ +/* S3 -> attempt.findings[] (ruleId + message + the */ +/* owner-authored rationale) */ +/* emit -> emitted.refusal, else emitted.validations[].gates[] */ +/* adapter-> attempt.adapterError */ +/* The full report stays on the turn for inspection either way. */ +/* ------------------------------------------------------------------ */ + +export type BuildFailureKind = "lint" | "repair-exhausted" | "emit-refusal" | "emit-gate" | "adapter" | "unknown"; + +export interface BuildFailureReason { + /** S1 | S2 | S3 | A1 | A2 | A3 when the reason belongs to a gate. */ + gate?: string; + /** Rule id, emitter gate name, or other structured code. */ + code?: string; + /** Surface path, component id, or catalog version the reason points at. */ + target?: string; + message: string; + /** Owner-authored rationale — S3 findings only; never synthesized. */ + rationale?: string; +} + +export interface BuildFailure { + kind: BuildFailureKind; + /** One honest sentence naming what stopped the run. */ + headline: string; + /** Where the pipeline stopped, e.g. "attempt 2 · S3 governance". */ + stoppedAt: string; + reasons: BuildFailureReason[]; +} + +const GATE_LABEL: Record = { + S1: "S1 surface schema", + S2: "S2 contract vocabulary", + S3: "S3 governance", +}; + +/** Reasons from one attempt's gates and findings, in gate order. */ +function attemptReasons(attempt: Record | undefined): BuildFailureReason[] { + if (!attempt) return []; + const reasons: BuildFailureReason[] = []; + for (const gate of (attempt.gates ?? []) as TurnGate[]) { + if (gate.status !== "FAIL") continue; + if (gate.gate === "S3") continue; // S3 speaks through findings + for (const error of gate.errors ?? []) { + reasons.push({ gate: gate.gate, code: gate.name, message: error }); + } + } + for (const finding of (attempt.findings ?? []) as Array>) { + if (finding.level && finding.level !== "error") continue; + reasons.push({ + gate: "S3", + code: String(finding.ruleId ?? ""), + target: String(finding.location?.path ?? finding.location?.component ?? ""), + message: String(finding.message ?? ""), + ...(finding.rationale ? { rationale: String(finding.rationale) } : {}), + }); + } + return reasons; +} + +/** + * The structured failure for a finished turn, or null when it passed (or has + * not finished). Never invents a cause: an outcome with no structured + * evidence returns kind "unknown" and says so plainly. + */ +export function buildFailure(progress: BuildTurnProgress): BuildFailure | null { + if (progress.status === "error") { + return { + kind: "adapter", + headline: "The run did not complete — the agent stream ended early.", + stoppedAt: "stream", + reasons: [{ message: progress.error ?? "the stream ended without a result" }], + }; + } + if (!progress.outcome || progress.outcome === "passed") return null; + + const report = (progress.report ?? {}) as Record; + const attempts = (report.attempts ?? []) as Array>; + const last = attempts.at(-1); + const emitted = report.emitted as Record | undefined; + + if (progress.outcome === "failed-adapter") { + const message = String(last?.adapterError ?? "the model adapter failed without a message"); + return { + kind: "adapter", + headline: "The model provider could not produce a result — nothing was generated.", + stoppedAt: `attempt ${(last?.index ?? 0) + 1} · provider`, + reasons: [{ code: report.generation?.adapterId, message }], + }; + } + + if (progress.outcome === "failed-gate") { + if (emitted?.refusal) { + return { + kind: "emit-refusal", + headline: "The surface passed governance but the emitter refused it.", + stoppedAt: "emit · refusal", + reasons: [{ gate: "emit", message: String(emitted.refusal) }], + }; + } + const reasons: BuildFailureReason[] = []; + for (const validation of (emitted?.validations ?? []) as Array>) { + for (const gate of (validation.gates ?? []) as Array>) { + if (gate.pass) continue; + for (const error of (gate.errors ?? ["gate failed"]) as string[]) { + reasons.push({ gate: String(gate.gate), code: String(gate.name ?? ""), target: validation.a2uiVersion ? `a2ui@${validation.a2uiVersion}` : undefined, message: error }); + } + } + } + if (reasons.length > 0) { + return { + kind: "emit-gate", + headline: "The surface passed governance but failed the catalog gates.", + stoppedAt: `emit · ${reasons[0].gate}`, + reasons, + }; + } + } + + const reasons = attemptReasons(last); + const failedGate = ((last?.gates ?? []) as TurnGate[]).find((g) => g.status === "FAIL"); + const stoppedAt = `attempt ${(last?.index ?? 0) + 1} · ${failedGate ? GATE_LABEL[failedGate.gate] ?? failedGate.gate : "generation"}`; + const exhausted = progress.outcome === "failed-lint-exhausted" && attempts.length > 1; + if (reasons.length === 0) { + return { + kind: "unknown", + headline: `The run ended as ${progress.outcome} without structured evidence — the full report is below.`, + stoppedAt, + reasons: [], + }; + } + return { + kind: exhausted ? "repair-exhausted" : "lint", + headline: exhausted + ? `Bounded repair was exhausted after ${attempts.length} attempts — the last attempt still violates the contract.` + : "The generated surface does not satisfy the contract.", + stoppedAt, + reasons, + }; +} + +/** A turn offers Accept/Refine only when it finished, passed, and has a surface. */ +export function canAcceptTurn(progress: BuildTurnProgress): boolean { + return progress.status === "finished" && progress.outcome === "passed" && !!progress.surface; +} + +export const canRefineTurn = canAcceptTurn; + +/** + * The worked example's prompt: the ORIGINAL build request, plus a concise + * deterministic record of the refinements that shaped the accepted surface. + * Truthful provenance — a reader (and the few-shot corpus) sees the ask that + * produced this result, not merely the last edit instruction. + */ +export function examplePromptFor(chain: string[]): string { + const [original, ...refinements] = chain.map((p) => p.trim()).filter(Boolean); + if (!original) return ""; + return refinements.length === 0 ? original : `${original} — refined: ${refinements.join("; ")}`; +} diff --git a/packages/composer-core/src/composer-core.test.ts b/packages/composer-core/src/composer-core.test.ts index 99a5304..8846710 100644 --- a/packages/composer-core/src/composer-core.test.ts +++ b/packages/composer-core/src/composer-core.test.ts @@ -32,7 +32,15 @@ import { unresolvedErrors, } from "./findings"; import { COMPOSER_ADAPTERS, composerAdapter } from "./adapters"; -import { buildReadiness, foldBuildEvents, vocabularyGap } from "./build"; +import { + buildFailure, + buildReadiness, + canAcceptTurn, + canRefineTurn, + examplePromptFor, + foldBuildEvents, + vocabularyGap, +} from "./build"; const fixture = (name: string) => JSON.parse(readFileSync(fileURLToPath(new URL(`../fixtures/${name}`, import.meta.url)), "utf8")); @@ -516,3 +524,179 @@ describe("folding a streamed build run", () => { expect(turn.error).toBe("agent gone"); }); }); + + +describe("structured build failures (#41)", () => { + const run = (outcome: string, report: Record, extra: Array> = []) => + foldBuildEvents([ + { type: "RUN_STARTED" }, + { type: "STEP_STARTED", stepName: "attempt-0" }, + ...extra, + { type: "CUSTOM", name: "dspack.audit", value: { outcome, exitCode: outcome === "passed" ? 0 : 2, report } }, + { type: "RUN_FINISHED" }, + ]); + + it("an S3 governance failure names the rule, its message, its location, and the owner's rationale", () => { + const f = buildFailure( + run("failed-lint-exhausted", { + attempts: [ + { + index: 0, + surface: { root: {} }, + gates: [ + { gate: "S1", name: "surface-schema", status: "PASS" }, + { gate: "S2", name: "contract-vocabulary", status: "PASS" }, + { gate: "S3", name: "governance", status: "FAIL" }, + ], + findings: [ + { + ruleId: "rule.destructive-requires-alertdialog", + level: "error", + message: "destructive action without an AlertDialog", + rationale: "Irreversible actions must be confirmed deliberately.", + location: { path: "$.root.children[0]", component: "action-button" }, + }, + ], + }, + ], + }), + ); + expect(f?.kind).toBe("lint"); + expect(f?.stoppedAt).toMatch(/attempt 1.*S3/); + expect(f?.reasons).toHaveLength(1); + expect(f?.reasons[0]).toMatchObject({ + gate: "S3", + code: "rule.destructive-requires-alertdialog", + target: "$.root.children[0]", + message: "destructive action without an AlertDialog", + rationale: "Irreversible actions must be confirmed deliberately.", + }); + }); + + it("an S1/S2 failure reports its gate errors verbatim (no findings exist for those gates)", () => { + const f = buildFailure( + run("failed-lint-exhausted", { + attempts: [ + { + index: 0, + gates: [ + { gate: "S1", name: "surface-schema", status: "PASS" }, + { gate: "S2", name: "contract-vocabulary", status: "FAIL", errors: ["component 'nope' is not contract vocabulary"] }, + { gate: "S3", name: "governance", status: "SKIPPED" }, + ], + findings: [], + }, + ], + }), + ); + expect(f?.kind).toBe("lint"); + expect(f?.reasons[0]).toMatchObject({ gate: "S2", message: "component 'nope' is not contract vocabulary" }); + expect(f?.reasons[0].rationale).toBeUndefined(); + }); + + it("an emit refusal renders the emitter's verbatim refusal, not a bare outcome", () => { + const f = buildFailure( + run("failed-gate", { + attempts: [{ index: 0, surface: { root: {} }, gates: [{ gate: "S1", name: "surface-schema", status: "PASS" }], findings: [] }], + emitted: { target: "a2ui", warnings: [], validations: [], refusal: "component 'mini-stepper' is a declared casualty (cannot-represent): steps is free-form." }, + }), + ); + expect(f?.kind).toBe("emit-refusal"); + expect(f?.reasons[0].message).toContain("declared casualty"); + expect(f?.stoppedAt).toMatch(/emit/i); + }); + + it("an emit A-gate failure names the gate and its errors", () => { + const f = buildFailure( + run("failed-gate", { + attempts: [{ index: 0, surface: { root: {} }, gates: [], findings: [] }], + emitted: { + target: "a2ui", + warnings: [], + validations: [{ a2uiVersion: "0.9.1", gates: [{ gate: "A3", name: "instance", pass: false, errors: ["TextField requires label"] }] }], + }, + }), + ); + expect(f?.kind).toBe("emit-gate"); + expect(f?.reasons[0]).toMatchObject({ gate: "A3", message: "TextField requires label" }); + }); + + it("an adapter failure carries the typed error and an actionable explanation", () => { + const f = buildFailure(run("failed-adapter", { attempts: [{ index: 0, adapterError: "fetch failed: connect ECONNREFUSED 127.0.0.1:11434" }] })); + expect(f?.kind).toBe("adapter"); + expect(f?.reasons[0].message).toContain("ECONNREFUSED"); + expect(f?.headline).toMatch(/model|provider/i); + }); + + it("repair exhaustion is named as such, with the last attempt's reasons", () => { + const gates = [{ gate: "S2", name: "contract-vocabulary", status: "FAIL", errors: ["component 'x' is not contract vocabulary"] }]; + const f = buildFailure( + run("failed-lint-exhausted", { + attempts: [ + { index: 0, gates, findings: [] }, + { index: 1, gates, findings: [] }, + { index: 2, gates, findings: [] }, + ], + repairMessages: ["r1", "r2"], + }), + ); + expect(f?.kind).toBe("repair-exhausted"); + expect(f?.headline).toMatch(/repair/i); + expect(f?.reasons[0].gate).toBe("S2"); + }); + + it("a passing run has no failure, and successful gates are never hidden", () => { + const progress = run("passed", { attempts: [{ index: 0, surface: { root: { component: "x" } }, gates: [{ gate: "S1", name: "s", status: "PASS" }], findings: [] }] }); + expect(buildFailure(progress)).toBeNull(); + expect(progress.attempts[0].gates).toHaveLength(1); // the fold keeps every gate + }); + + it("Refine and Accept are offered only for a completed run with a valid surface", () => { + const passed = run("passed", { attempts: [{ index: 0, surface: { root: {} }, gates: [], findings: [] }] }); + expect(canAcceptTurn(passed)).toBe(true); + expect(canRefineTurn(passed)).toBe(true); + + // A failed turn may still carry a surface from its last attempt — neither action applies. + const failed = run("failed-lint-exhausted", { attempts: [{ index: 0, surface: { root: {} }, gates: [], findings: [] }] }); + expect(canAcceptTurn(failed)).toBe(false); + expect(canRefineTurn(failed)).toBe(false); + + const adapter = run("failed-adapter", { attempts: [{ index: 0, adapterError: "boom" }] }); + expect(canAcceptTurn(adapter)).toBe(false); + expect(canRefineTurn(adapter)).toBe(false); + + const streaming = foldBuildEvents([{ type: "RUN_STARTED" }]); + expect(canAcceptTurn(streaming)).toBe(false); + expect(canRefineTurn(streaming)).toBe(false); + }); + + it("a vocabulary gap stays distinguishable from malformed model output", () => { + const gap = run("failed-lint-exhausted", { + attempts: [{ index: 0, gates: [{ gate: "S2", name: "contract-vocabulary", status: "FAIL", errors: ["component 'timeline' is not contract vocabulary"] }], findings: [] }], + }); + expect(vocabularyGap(gap)).toEqual(["timeline"]); + expect(buildFailure(gap)?.kind).toBe("lint"); + + const malformed = run("failed-lint-exhausted", { + attempts: [{ index: 0, gates: [{ gate: "S1", name: "surface-schema", status: "FAIL", errors: ["(root) must be object"] }], findings: [] }], + }); + expect(vocabularyGap(malformed)).toEqual([]); + expect(buildFailure(malformed)?.reasons[0].gate).toBe("S1"); + }); +}); + +describe("worked-example prompt provenance (#42)", () => { + it("keeps the ORIGINAL request as the prompt and records refinements deterministically", () => { + expect(examplePromptFor(["a deployment status screen"])).toBe("a deployment status screen"); + expect(examplePromptFor(["a deployment status screen", "make the title clearer"])).toBe( + "a deployment status screen — refined: make the title clearer", + ); + expect(examplePromptFor(["a status screen", "make the title clearer", "add the region"])).toBe( + "a status screen — refined: make the title clearer; add the region", + ); + // Deterministic and stable: same chain, same string. + expect(examplePromptFor(["a", "b"])).toBe(examplePromptFor(["a", "b"])); + // Never merely the last instruction. + expect(examplePromptFor(["original", "last"])).toContain("original"); + }); +}); diff --git a/packages/composer-core/src/index.ts b/packages/composer-core/src/index.ts index bf6d6ba..c0f6dad 100644 --- a/packages/composer-core/src/index.ts +++ b/packages/composer-core/src/index.ts @@ -47,10 +47,17 @@ export { } from "./adapters"; export { buildReadiness, + buildFailure, + canAcceptTurn, + canRefineTurn, + examplePromptFor, foldBuildEvents, vocabularyGap, type BuildReadiness, type BuildTurnProgress, + type BuildFailure, + type BuildFailureKind, + type BuildFailureReason, type TurnAttempt, type TurnGate, } from "./build";