Add public catalog API source for toolkit docs generation

.github/workflows/generate-toolkit-docs.md

@@ -5,7 +5,7 @@ This workflow regenerates toolkit JSON and opens a PR with the changes. It can b
 ## What it does
 
 1. Builds the toolkit docs generator.
-2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine tool metadata and summary endpoints.
+2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine public catalog endpoints.
 3. Syncs integrations sidebar navigation from the generated JSON.
 4. Creates or updates a PR on the stable `automation/toolkit-docs` branch if any files changed. Later runs overwrite that open PR with the latest generated docs.
 
@@ -14,7 +14,6 @@ This workflow regenerates toolkit JSON and opens a PR with the changes. It can b
 Required secrets:
 
 - `ENGINE_API_URL`
-- `ENGINE_API_KEY`
 - `ANTHROPIC_API_KEY`
 
 Optional secrets:

.github/workflows/generate-toolkit-docs.yml

@@ -65,9 +65,8 @@ jobs:
             --skip-unchanged \
             --preserve-last-known-good \
             --verbose \
-            --api-source tool-metadata \
+            --api-source public-catalog \
             --tool-metadata-url "$ENGINE_API_URL" \
-            --tool-metadata-key "$ENGINE_API_KEY" \
             --llm-provider anthropic \
             --llm-model "$ANTHROPIC_MODEL" \
             --llm-api-key "$ANTHROPIC_API_KEY" \
@@ -86,7 +85,6 @@ jobs:
         working-directory: toolkit-docs-generator
         env:
           ENGINE_API_URL: ${{ secrets.ENGINE_API_URL }}
-          ENGINE_API_KEY: ${{ secrets.ENGINE_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           ANTHROPIC_MODEL: ${{ secrets.ANTHROPIC_MODEL || 'claude-sonnet-4-6' }}
           # Stronger model for the secret-coherence editor. Keeps

toolkit-docs-generator/ARCHITECTURE.md

@@ -19,7 +19,7 @@ flowchart TD
     manual["Manual run<br/>workflow_dispatch"] --> generate
     porter["Porter deploy succeeded<br/>repository_dispatch"] --> generate
 
-    engine["Engine API<br/>/v1/tool_metadata"] -->|"tools, parameters, auth, secrets"| generate
+    engine["Engine public catalog<br/>/v1/public/tool_catalog + /v1/public/tools"] -->|"tools, parameters, auth, secrets"| generate
     previous["data/toolkits/*.json<br/>previous run"] -->|"signatures and curation hashes"| generate
 
     generate["generate --all --skip-unchanged"] --> changed{"Changed since<br/>last run?"}
@@ -66,8 +66,9 @@ it. The sidebar sync writes navigation only, and never touches toolkit JSON.
 
 ### Data sources
 
-- `EngineApiSource` fetches tool metadata from the Engine API.
-- `ArcadeApiSource` fetches tool metadata from the Arcade API.
+- `PublicCatalogApiSource` fetches tool metadata from the Engine public catalog API.
+- `EngineApiSource` fetches tool metadata from the authenticated Engine API (deprecated).
+- `ArcadeApiSource` fetches tool metadata from the Arcade API (deprecated).
 - `DesignSystemMetadataSource` loads toolkit metadata from `@arcadeai/design-system`.
 - `MarkdownCurationSource` compiles documentation chunks, import declarations,
   and subpages from the configured curation directory. When configured, that
@@ -138,7 +139,8 @@ public, read-only values configured through these Vercel environment variables:
 
 ## Key files
 
-- `src/sources/engine-api.ts` — tool metadata from Engine API
+- `src/sources/public-catalog-api.ts` — tool metadata from Engine public catalog
+- `src/sources/engine-api.ts` — tool metadata from authenticated Engine API (deprecated)
 - `src/sources/markdown-curation.ts` — Markdown and MDX curation compiler
   ([format reference](CURATION.md))
 - `src/sources/toolkit-data-source.ts` — unified data source

toolkit-docs-generator/README.md

@@ -19,8 +19,8 @@ The generator merges three inputs into one JSON output per toolkit:
 
 It also reads the previous output when you use `--skip-unchanged` or `--previous-output`.
 
-When `--skip-unchanged` runs against the tool metadata API, the generator fetches
-one complete snapshot from `/v1/tool_metadata`. It reuses that snapshot for
+When `--skip-unchanged` runs against the public catalog API, the generator fetches
+one complete snapshot from `/v1/public/tool_catalog` and `/v1/public/tools`. It reuses that snapshot for
 change detection, progress calculation, and generation so a run cannot compare
 different API states. Only changed toolkits are regenerated.
 
@@ -30,13 +30,13 @@ The workflow file is `/.github/workflows/generate-toolkit-docs.yml`.
 It runs these steps:
 
 1. Type-check and test the toolkit docs generator.
-2. Generate toolkit JSON using `toolkit-docs-generator` and the Engine API.
+2. Generate toolkit JSON using `toolkit-docs-generator` and the Engine public catalog API.
 3. Sync sidebar navigation from `toolkit-docs-generator/data/toolkits` to the `_meta.tsx` files.
 4. Create or update a pull request if there are changes.
 
 Required secrets:
 
-- `ENGINE_API_URL`, `ENGINE_API_KEY`
+- `ENGINE_API_URL`
 - `ANTHROPIC_API_KEY` for examples, summaries, and secret-coherence edits
 
 Optional secrets:
@@ -231,8 +231,8 @@ deletes it and rebuilds `index.json`.
 - `--all` generate all toolkits
 - `--providers` generate a subset of toolkits
 - `--skip-unchanged` only write changed toolkits
-- `--api-source` select `tool-metadata` (default with Engine creds), `list-tools`
-  (only with the explicit flag), or `mock`
+- `--api-source` select `public-catalog` (default with `ENGINE_API_URL`), `tool-metadata`
+  (deprecated; requires `ENGINE_API_KEY`), `list-tools` (deprecated), or `mock`
 - `--previous-output` compare against a previous output directory
 - `--custom-sections` load an authoritative Markdown/MDX curation directory
 - `--skip-examples`, `--skip-summary` disable LLM steps

toolkit-docs-generator/src/cli/api-source.ts

@@ -1,34 +1,36 @@
-export type ApiSource = "list-tools" | "tool-metadata" | "mock";
+export type ApiSource =
+  | "public-catalog"
+  | "list-tools"
+  | "tool-metadata"
+  | "mock";
 
 type ApiSourceOptions = {
   apiSource?: string;
   toolMetadataUrl?: string;
   toolMetadataKey?: string;
 };
 
-export const resolveApiSource = (options: ApiSourceOptions): ApiSource => {
-  // Explicit source takes precedence
-  if (options.apiSource) {
-    const source = options.apiSource.toLowerCase();
-    if (source === "list-tools") {
-      return "list-tools";
-    }
-    if (source === "engine") {
-      return "tool-metadata";
-    }
-    if (source === "tool-metadata") {
-      return "tool-metadata";
-    }
-    if (source === "mock") {
-      return "mock";
-    }
-    throw new Error(
-      `Invalid --api-source "${options.apiSource}". Use "list-tools", "tool-metadata", or "mock".`
-    );
+const EXPLICIT_API_SOURCES: Record<string, ApiSource> = {
+  "public-catalog": "public-catalog",
+  public: "public-catalog",
+  "list-tools": "list-tools",
+  engine: "tool-metadata",
+  "tool-metadata": "tool-metadata",
+  mock: "mock",
+};
+
+const resolveExplicitApiSource = (apiSource: string): ApiSource => {
+  const resolved = EXPLICIT_API_SOURCES[apiSource.toLowerCase()];
+  if (resolved) {
+    return resolved;
   }
 
-  // Auto-detect based on provided Engine credentials only.
-  // List-tools endpoint must be explicitly selected via --api-source list-tools.
+  throw new Error(
+    `Invalid --api-source "${apiSource}". Use "public-catalog", "list-tools", "tool-metadata", or "mock".`
+  );
+};
+
+const resolveAutoDetectedApiSource = (options: ApiSourceOptions): ApiSource => {
   const hasToolMetadataKey = !!(
     options.toolMetadataKey ?? process.env.ENGINE_API_KEY
   );
@@ -39,5 +41,21 @@ export const resolveApiSource = (options: ApiSourceOptions): ApiSource => {
   if (hasToolMetadataKey && hasToolMetadataUrl) {
     return "tool-metadata";
   }
+
+  if (hasToolMetadataUrl) {
+    return "public-catalog";
+  }
+
   return "mock";
 };
+
+export const resolveApiSource = (options: ApiSourceOptions): ApiSource => {
+  if (options.apiSource) {
+    return resolveExplicitApiSource(options.apiSource);
+  }
+
+  return resolveAutoDetectedApiSource(options);
+};
+
+export const isDeprecatedApiSource = (apiSource: ApiSource): boolean =>
+  apiSource === "tool-metadata" || apiSource === "list-tools";