Add <Syntax> to list available XMD symbols (#758)

deno.json

@@ -58,7 +58,8 @@
     "verify:clean": "deno run --allow-all --node-modules-dir=none --cached-only --frozen scripts/verify-clean.ts",
     "deps": "deno run --allow-all scripts/deps.ts",
     "deps:target": "deno run --allow-all scripts/deps-target.ts",
-    "build": "deno task build:web && deno compile --node-modules-dir=none --cached-only --frozen --exclude-unused-npm --allow-all --include packages/code-review-agent --include packages/cli/src/documents --output dist/xmd packages/cli/src/compiled.ts",
+    "validate:docs": "deno run --allow-all scripts/validate-documentation.ts",
+    "build": "deno task validate:docs && deno task build:web && deno compile --node-modules-dir=none --cached-only --frozen --exclude-unused-npm --allow-all --include packages/code-review-agent --include packages/cli/src/documents --include packages/core/src/components/components.md --include packages/core/src/agent/components.md --include packages/cli/src/components.md --include packages/testing/src/components.md --include packages/web/src/components.md --include packages/workflow/src/composition/components.md --output dist/xmd packages/cli/src/compiled.ts",
     "build:web": "deno run --allow-all --node-modules-dir=none --cached-only --frozen scripts/preflight.ts scripts/build-web-client.ts",
     "gen:publish-workflow": "deno run --allow-all packages/cli/src/deno.ts run scripts/gen-publish-workflow.md",
     "bump": "deno run -A scripts/bump-version.ts",

packages/cli/src/authorship-profile.ts

@@ -45,7 +45,7 @@ import {
   useTerminalOutput,
 } from "@executablemd/core";
 import type { AgentProviderOptions, Json } from "@executablemd/core";
-import type { DeclaredMarkdownComponent } from "@executablemd/core/host";
+import type { SyntaxSymbolsProvider, DeclaredMarkdownComponent } from "@executablemd/core/host";
 import { executeInstalled, installInvocationAgentProvider } from "@executablemd/core/host";
 import { createAcpxProvider } from "@executablemd/acp";
 import type { AcpxProviderDependencies } from "@executablemd/acp";
@@ -163,11 +163,21 @@ export interface AuthorshipProfile {
    * The `<Plan>` declaration this command runs under.
    *
    * Built by the command, from the packaged Component's bytes, before the adapter
-   * root is imported. It carries the sealed surface, the sealed verbosity, the
-   * catalog this invocation will build and the Agent context it settled — none of
-   * which is a prop the adapter could supply or a document could reach.
+   * root is imported. It carries the sealed surface, the sealed verbosity and the
+   * Agent context it settled — none of which is a prop the adapter could supply
+   * or a document could reach.
    */
   declaration: DeclaredMarkdownComponent;
+  /**
+   * The symbols this authorship describes.
+   *
+   * The `run` profile's, because a Plan is a program a later `xmd run` executes:
+   * deriving them from this execution — which searches no repository and
+   * refuses almost every capability — would describe a vocabulary the approved
+   * program would not have. Captured with the rest of the installation, before
+   * any installed code, middleware or document code runs.
+   */
+  symbols: SyntaxSymbolsProvider;
 }
 
 /** What building the constrained provider needs, and nothing more. */
@@ -226,7 +236,7 @@ export interface PlanAuthorshipPolicy {
  *
  * Not a description built beside the installation but the installation itself:
  * the same value registers the provider, installs the invocation options, and
- * is handed to a trusted host as its observation. There is nothing for a report
+ * is handed to a trusted host as its reference. There is nothing for a report
  * to disagree with, because there is no second report.
  */
 export interface PlanProviderAssembly {
@@ -275,7 +285,7 @@ export function planAgentContext(
     defaultAgent: stack.defaultAgent,
     *installProvider(invocation: PlanAuthorshipInvocation): Operation<PlanProviderAssembly> {
       // One assembly, used for every installation and handed back as the
-      // observation. Nothing is reconstructed afterward, so a report cannot
+      // reference. Nothing is reconstructed afterward, so a report cannot
       // describe an arrangement other than the one installed.
       const installed: PlanProviderAssembly = {
         provider: "acpx",
@@ -319,7 +329,7 @@ export interface AuthorshipFrame {
   readonly session: string;
   /** The exact authored label a trusted child host may address privately. */
   readonly authoredSession?: string;
-  observe?(observation: PlanAuthorshipObservation): Operation<void>;
+  observe?(reference: PlanAuthorshipObservation): Operation<void>;
   installElicitation(): Operation<void>;
 }
 
@@ -456,6 +466,13 @@ export function* runPlanCommandDocument(profile: AuthorshipProfile): Operation<R
           {
             components: agentIdentityComponents(),
             declarations: [profile.declaration],
+            // What a Plan may write is the `run` profile's vocabulary, not this
+            // authorship execution's — the adapter searches no repository, the
+            // Component installs a ceiling that refuses almost everything, and
+            // symbols derived from *that* would describe a program nobody can
+            // run. Captured before any installation runs, so no prop, binding,
+            // middleware answer or projected content reaches it.
+            symbols: profile.symbols,
           },
         ],
       );

packages/cli/src/cli.ts

@@ -67,7 +67,6 @@ import {
   inspectDocument,
   agentIdentityComponents,
   installAgentComponents,
-  registerComponents,
   retainedSource,
   rootSourcePath,
   useNormalizedOutput,
@@ -99,7 +98,7 @@ import { timeout as runTimeout } from "@executablemd/runtime";
 import { installRunAgentStack, resolveAgentStack, resolveAuthorshipStack } from "./agent-stack.ts";
 import { planComponentDeclaration } from "./plan-component.ts";
 import { planAgentContext } from "./authorship-profile.ts";
-import { VERBOSE_REGISTRATION } from "./verbose-component.ts";
+import { useVerboseComponent } from "./verbose-component.ts";
 import type { AgentStack } from "./agent-stack.ts";
 import { reportFailure } from "./report.ts";
 import { TIMEOUT_FLAGS, resolvePlanTimeout, resolveRunTimeouts } from "./timeouts.ts";
@@ -128,7 +127,12 @@ import { runPlan } from "./plan.ts";
 import { runUpgrade } from "./upgrade.ts";
 import type { UpgradeAssembly } from "./upgrade.ts";
 import { componentSearchPath, resolveTestTarget } from "./test-target.ts";
-import { renderSyntaxJson, renderSyntaxMarkdown, syntaxCatalog } from "./syntax.ts";
+import {
+  renderSyntaxDocumentation,
+  renderSyntaxJson,
+  renderSyntaxMarkdown,
+  syntaxSymbols,
+} from "./syntax.ts";
 import { deliverWhole } from "./stdout-delivery.ts";
 import { testingExecutionHost } from "./testing-host.ts";
 import type { ChildPlanDeclaration } from "./testing-host.ts";
@@ -149,7 +153,7 @@ import type { HostWorkflowInstaller, WorkflowHost, WorkflowStart } from "./workf
 import { runWorkflowManagement } from "./workflow-management.ts";
 import { establishDefinition } from "./workflow-definition.ts";
 import type { EstablishedDefinition } from "./workflow-definition.ts";
-import { COMPOSITION_REGISTRATIONS, useWorkflowServiceDenial } from "@executablemd/workflow";
+import { useCompositionComponents, useWorkflowServiceDenial } from "@executablemd/workflow";
 import denoJson from "../deno.json" with { type: "json" };
 
 const SECRET_DETECTION_OPTION = "--secret-detection";
@@ -356,12 +360,18 @@ const testConfig = object({
  * `run` and `test` declare, and explicit values replace the defaults.
  */
 const syntaxConfig = object({
+  component: {
+    description:
+      "component to describe in full — `xmd syntax Elicit` renders its catalog metadata " +
+      "and long-form documentation instead of the compact catalog",
+    ...field(z.string().optional(), cli.argument()),
+  },
   include: {
     description: "component search directory",
     ...field(z.array(z.string()), field.default(["components", "."]), field.array()),
   },
   json: {
-    description: "write the catalog as version-1 JSON instead of markdown",
+    description: "write the catalog as version-2 JSON instead of markdown",
     ...field(z.boolean(), field.default(false)),
   },
 });
@@ -910,12 +920,12 @@ export function* installDocumentComponents(mode: DocumentMode, verbose: boolean)
   // `<Verbose>` is registered at all.
   yield* Config.around({ verbose: () => verbose }, { at: "min" });
 
-  // The repository-composition vocabulary, as ordinary shadowable defaults.
-  // Registering it installs no provider, discovers no repository, acquires no
-  // lock and reaches no network: what a name *does* is decided by whichever
-  // provider the command installed, and a runtime that installs none still
-  // resolves every one of these.
-  yield* registerComponents(COMPOSITION_REGISTRATIONS);
+  // The repository-composition vocabulary, as ordinary shadowable defaults,
+  // with the documentation that describes it. Bootstrapping it installs no
+  // provider, discovers no repository, acquires no lock and reaches no network:
+  // what a name *does* is decided by whichever provider the command installed,
+  // and a runtime that installs none still resolves every one of these.
+  yield* useCompositionComponents();
 
   // Compose testing around the single core execution entrypoint: both
   // commands register the components (assertions work in regular documents,
@@ -928,7 +938,7 @@ export function* installDocumentComponents(mode: DocumentMode, verbose: boolean)
     yield* installTestAgentComponents();
     yield* installAgentComponents();
   } else {
-    yield* registerComponents([VERBOSE_REGISTRATION]);
+    yield* useVerboseComponent();
     yield* installTestingComponents({ verbose });
   }
 
@@ -1026,11 +1036,6 @@ function* runDocument(
         ? {}
         : { observeAuthorship: request.observeAuthorship }),
       installElicitation: request.installElicitation,
-      // Rendered when a `<Plan>` first asks, not before: an ordinary run that
-      // writes none never builds a catalog it has no reader for.
-      *catalog() {
-        return renderSyntaxMarkdown(yield* syntaxCatalog(include));
-      },
     });
 
   const plan = yield* planDeclaration({
@@ -2511,7 +2516,7 @@ function* dispatch(
         },
         {
           ...(sessions === undefined ? {} : { sessions }),
-          catalog: syntaxCatalog,
+          symbols: syntaxSymbols,
           // The two facts about this process's own stderr that nothing further
           // in may go and read: whether it is a terminal, and whether it took
           // what it was handed. The approved Plan's sinks are stdout and
@@ -2635,8 +2640,22 @@ function* dispatch(
       // the catalog would read as complete.
       let rendered: string;
       try {
-        const catalog = yield* syntaxCatalog(command.config.include);
-        rendered = command.config.json ? renderSyntaxJson(catalog) : renderSyntaxMarkdown(catalog);
+        const named = command.config.component;
+        if (named === undefined) {
+          // The compact list of symbols, unchanged: routine discovery output and
+          // every default Plan prompt read it, and long documentation would make
+          // both unnecessarily large.
+          const catalog = yield* syntaxSymbols(command.config.include);
+          rendered = command.config.json
+            ? renderSyntaxJson(catalog)
+            : renderSyntaxMarkdown(catalog);
+        } else {
+          // The same selection, index and renderer `<Syntax names={…}>` uses, so
+          // the command and the component cannot describe one component two
+          // ways. JSON stays the compact projection; it is the symbols' shape,
+          // and documentation is prose rather than a symbol member.
+          rendered = yield* renderSyntaxDocumentation(command.config.include, [named]);
+        }
       } catch (error) {
         console.error(describeError(error));
         yield* exit(1);

packages/cli/src/components.md

@@ -0,0 +1,24 @@
+Long-form documentation for the components the `xmd` command registers.
+
+One component, and it exists because a document usually has two audiences: the
+person running it, who wants the result, and the person debugging it, who wants
+to know how it got there.
+
+## Verbose
+
+Expands its content only when run verbosity is on.
+
+```mdx
+<Verbose>
+Resolved {documents.length} documents from {include}.
+</Verbose>
+```
+
+`--verbose` turns it on for the whole run, and a component may override
+verbosity for its own content. When verbosity is off the content is not
+expanded at all — so anything expensive inside it costs nothing on an ordinary
+run, and this is a place to put detail rather than a place to hide it.
+
+`as` captures the rendered verbose text, or an **empty string** when verbosity
+is off. That is what lets a document build a diagnostic once and use it in more
+than one place without branching on the flag itself.

packages/cli/src/documents/Plan.md

@@ -71,6 +71,7 @@ word a person reads, for the same reason.
 Getting the available XMD components and constructs and setting up the planning session.
 </PlanProgress>
 
+<Syntax as="syntax" />
 <PlanInputs session={props.session} instruction={prompt} as="inputs" />
 
 ## Say what this surface calls things
@@ -159,7 +160,7 @@ out.
 
 Everything you may use is described below. Use nothing that is not here.
 
-{inputs.syntax}
+{syntax}
 
 Reply with the Plan source and nothing else. No enclosing code fence, no
 explanation before or after it.

packages/cli/src/plan-component.ts

@@ -36,6 +36,11 @@
  * So they resolve only while canonical core is expanding these exact bytes:
  * not from the caller's root, not from the Prompt the caller projected, not from
  * a sibling `<Plan>`, and not from anything middleware can answer.
+ *
+ * `<Syntax />` is not among them. What a document may write here is a public
+ * question with a public answer, and canonical core owns both — so `Plan.md`
+ * writes the same component any other document writes, and the catalog the Agent
+ * is shown is the one an operator can print.
  */
 
 import { createHash } from "node:crypto";
@@ -206,8 +211,6 @@ export interface PlanComponentAssembly {
   observeAuthorship?(observation: PlanAuthorshipObservation): Operation<void>;
   /** Who answers the review question. */
   installElicitation(): Operation<void>;
-  /** The run profile's rendered vocabulary, as the first Agent turn receives it. */
-  catalog(): Operation<string>;
   /**
    * How this host decides whether one candidate is structurally a program.
    *
@@ -223,22 +226,22 @@ export interface PlanComponentAssembly {
 const INPUTS_RETURNS = {
   type: "object",
   properties: {
-    syntax: { type: "string" },
     session: { type: "string" },
     surface: { type: "string" },
     durable: { type: "boolean" },
     authoredSession: { type: "string" },
   },
-  required: ["syntax", "session", "surface", "durable"],
+  required: ["session", "surface", "durable"],
   additionalProperties: false,
 };
 
 /**
- * What the frozen inputs are given: the caller's optional session name, and the
+ * What the frozen inputs are given: the caller's optional session name and the
  * prompt this invocation is about.
  *
- * The prompt is here so that the first durable record of the invocation is
- * about a question as well as a catalog. Only its digest is kept.
+ * Only the prompt's digest is retained here. The vocabulary the Agent is shown
+ * is public `<Syntax />`'s, with a durable record of its own, so each protocol
+ * can be read and reconciled independently.
  */
 const INPUTS_PROPS = {
   type: "object",
@@ -432,14 +435,15 @@ function* uninvocable(): Operation<never> {
 /**
  * Freeze this invocation's authorship inputs, and retain them.
  *
- * The catalog is an observation the first Agent turn is built from, so it is
- * journaled: a continuation restores what the run actually showed the agent
- * rather than rebuilding one from a working tree that has moved. The instruction
- * identity beside it is what makes a continuation answerable at all: this is the
- * first durable record of the invocation, so comparing it here refuses a Plan
- * asked for different instructions before a directory, a provider, a turn, a
- * review or an admission exists — the only place that can refuse without having
- * already done some of the work it would be refusing.
+ * The vocabulary the Agent is shown is not here. `<Syntax />` is a public
+ * component canonical core owns, it retains its own observation, and `Plan.md`
+ * binds it directly — so the catalog and the question are two records that can
+ * be read and reconciled independently rather than one that has to be read
+ * whole.
+ *
+ * The instruction identity here makes a continuation answerable at all:
+ * comparing it refuses a Plan asked under another prompt before a directory, a
+ * provider, a turn, a review or an admission exists.
  *
  * The session placement is derived here too, from the durable identity canonical
  * execution minted for this exact expansion — which is what makes two `<Plan>`
@@ -469,7 +473,7 @@ function planInputs(assembly: PlanComponentAssembly): IdentityComponent {
         const instruction = sourceDigest(String(props.instruction));
 
         const frozen = yield* durablePlanOperation<Json>(`plan:inputs:${id}`, function* () {
-          return { syntax: yield* assembly.catalog(), instruction };
+          return { instruction };
         });
 
         // A history is input, so it is parsed rather than trusted.
@@ -482,7 +486,6 @@ function planInputs(assembly: PlanComponentAssembly): IdentityComponent {
         }
 
         return {
-          syntax: retained.syntax,
           session,
           surface: assembly.surface,
           durable: durability(assembly, authored),
@@ -692,14 +695,13 @@ function checkDraft(validate: StructuralValidation): IdentityComponent {
 
 /** What the frozen inputs retained, or nothing when the record is not one. */
 interface RetainedInputs {
-  readonly syntax: string;
   readonly instruction: string;
 }
 
 /**
  * The frozen inputs a record holds, read as a closed protocol.
  *
- * Exactly two members, both strings. A record missing one, carrying a member
+ * Exactly one string member. A record missing it, carrying a member
  * this version does not know, or holding one of the wrong type is a record this
  * version cannot read — not one to fill in a default for, because every default
  * here is a guess about what an earlier run actually asked.
@@ -708,14 +710,14 @@ function readInputs(value: Json): RetainedInputs | undefined {
   if (typeof value !== "object" || value === null || Array.isArray(value)) {
     return undefined;
   }
-  const { syntax, instruction } = value;
-  if (Object.keys(value).length !== 2) {
+  const { instruction } = value;
+  if (Object.keys(value).length !== 1) {
     return undefined;
   }
-  if (typeof syntax !== "string" || typeof instruction !== "string") {
+  if (typeof instruction !== "string") {
     return undefined;
   }
-  return { syntax, instruction };
+  return { instruction };
 }
 
 /**