diff --git a/.github/workflows/generate-toolkit-docs.md b/.github/workflows/generate-toolkit-docs.md index 4ef6fbdd4..19d18ba95 100644 --- a/.github/workflows/generate-toolkit-docs.md +++ b/.github/workflows/generate-toolkit-docs.md @@ -5,7 +5,7 @@ This workflow regenerates toolkit JSON and opens a PR with the changes. It can b ## What it does 1. Builds the toolkit docs generator. -2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine tool metadata and summary endpoints. +2. Generates toolkit JSON in `toolkit-docs-generator/data/toolkits` using the Engine public catalog endpoints. 3. Syncs integrations sidebar navigation from the generated JSON. 4. Creates or updates a PR on the stable `automation/toolkit-docs` branch if any files changed. Later runs overwrite that open PR with the latest generated docs. @@ -14,7 +14,6 @@ This workflow regenerates toolkit JSON and opens a PR with the changes. It can b Required secrets: - `ENGINE_API_URL` -- `ENGINE_API_KEY` - `ANTHROPIC_API_KEY` Optional secrets: diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index b1030f89a..bbed4a76b 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -65,9 +65,8 @@ jobs: --skip-unchanged \ --preserve-last-known-good \ --verbose \ - --api-source tool-metadata \ + --api-source public-catalog \ --tool-metadata-url "$ENGINE_API_URL" \ - --tool-metadata-key "$ENGINE_API_KEY" \ --llm-provider anthropic \ --llm-model "$ANTHROPIC_MODEL" \ --llm-api-key "$ANTHROPIC_API_KEY" \ @@ -86,7 +85,6 @@ jobs: working-directory: toolkit-docs-generator env: ENGINE_API_URL: ${{ secrets.ENGINE_API_URL }} - ENGINE_API_KEY: ${{ secrets.ENGINE_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} ANTHROPIC_MODEL: ${{ secrets.ANTHROPIC_MODEL || 'claude-sonnet-4-6' }} # Stronger model for the secret-coherence editor. Keeps diff --git a/toolkit-docs-generator/ARCHITECTURE.md b/toolkit-docs-generator/ARCHITECTURE.md index e66401cac..c1a697b78 100644 --- a/toolkit-docs-generator/ARCHITECTURE.md +++ b/toolkit-docs-generator/ARCHITECTURE.md @@ -19,7 +19,7 @@ flowchart TD manual["Manual run
workflow_dispatch"] --> generate porter["Porter deploy succeeded
repository_dispatch"] --> generate - engine["Engine API
/v1/tool_metadata"] -->|"tools, parameters, auth, secrets"| generate + engine["Engine public catalog
/v1/public/tool_catalog + /v1/public/tools"] -->|"tools, parameters, auth, secrets"| generate previous["data/toolkits/*.json
previous run"] -->|"signatures and curation hashes"| generate generate["generate --all --skip-unchanged"] --> changed{"Changed since
last run?"} @@ -66,8 +66,9 @@ it. The sidebar sync writes navigation only, and never touches toolkit JSON. ### Data sources -- `EngineApiSource` fetches tool metadata from the Engine API. -- `ArcadeApiSource` fetches tool metadata from the Arcade API. +- `PublicCatalogApiSource` fetches tool metadata from the Engine public catalog API. +- `EngineApiSource` fetches tool metadata from the authenticated Engine API (deprecated). +- `ArcadeApiSource` fetches tool metadata from the Arcade API (deprecated). - `DesignSystemMetadataSource` loads toolkit metadata from `@arcadeai/design-system`. - `MarkdownCurationSource` compiles documentation chunks, import declarations, and subpages from the configured curation directory. When configured, that @@ -138,7 +139,8 @@ public, read-only values configured through these Vercel environment variables: ## Key files -- `src/sources/engine-api.ts` — tool metadata from Engine API +- `src/sources/public-catalog-api.ts` — tool metadata from Engine public catalog +- `src/sources/engine-api.ts` — tool metadata from authenticated Engine API (deprecated) - `src/sources/markdown-curation.ts` — Markdown and MDX curation compiler ([format reference](CURATION.md)) - `src/sources/toolkit-data-source.ts` — unified data source diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index bc971cb0f..1f6d0f6a5 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -19,8 +19,8 @@ The generator merges three inputs into one JSON output per toolkit: It also reads the previous output when you use `--skip-unchanged` or `--previous-output`. -When `--skip-unchanged` runs against the tool metadata API, the generator fetches -one complete snapshot from `/v1/tool_metadata`. It reuses that snapshot for +When `--skip-unchanged` runs against the public catalog API, the generator fetches +one complete snapshot from `/v1/public/tool_catalog` and `/v1/public/tools`. It reuses that snapshot for change detection, progress calculation, and generation so a run cannot compare different API states. Only changed toolkits are regenerated. @@ -30,13 +30,13 @@ The workflow file is `/.github/workflows/generate-toolkit-docs.yml`. It runs these steps: 1. Type-check and test the toolkit docs generator. -2. Generate toolkit JSON using `toolkit-docs-generator` and the Engine API. +2. Generate toolkit JSON using `toolkit-docs-generator` and the Engine public catalog API. 3. Sync sidebar navigation from `toolkit-docs-generator/data/toolkits` to the `_meta.tsx` files. 4. Create or update a pull request if there are changes. Required secrets: -- `ENGINE_API_URL`, `ENGINE_API_KEY` +- `ENGINE_API_URL` - `ANTHROPIC_API_KEY` for examples, summaries, and secret-coherence edits Optional secrets: @@ -231,8 +231,8 @@ deletes it and rebuilds `index.json`. - `--all` generate all toolkits - `--providers` generate a subset of toolkits - `--skip-unchanged` only write changed toolkits -- `--api-source` select `tool-metadata` (default with Engine creds), `list-tools` - (only with the explicit flag), or `mock` +- `--api-source` select `public-catalog` (default with `ENGINE_API_URL`), `tool-metadata` + (deprecated; requires `ENGINE_API_KEY`), `list-tools` (deprecated), or `mock` - `--previous-output` compare against a previous output directory - `--custom-sections` load an authoritative Markdown/MDX curation directory - `--skip-examples`, `--skip-summary` disable LLM steps diff --git a/toolkit-docs-generator/src/cli/api-source.ts b/toolkit-docs-generator/src/cli/api-source.ts index 3304a9bf8..c52cfca01 100644 --- a/toolkit-docs-generator/src/cli/api-source.ts +++ b/toolkit-docs-generator/src/cli/api-source.ts @@ -1,4 +1,8 @@ -export type ApiSource = "list-tools" | "tool-metadata" | "mock"; +export type ApiSource = + | "public-catalog" + | "list-tools" + | "tool-metadata" + | "mock"; type ApiSourceOptions = { apiSource?: string; @@ -6,29 +10,27 @@ type ApiSourceOptions = { toolMetadataKey?: string; }; -export const resolveApiSource = (options: ApiSourceOptions): ApiSource => { - // Explicit source takes precedence - if (options.apiSource) { - const source = options.apiSource.toLowerCase(); - if (source === "list-tools") { - return "list-tools"; - } - if (source === "engine") { - return "tool-metadata"; - } - if (source === "tool-metadata") { - return "tool-metadata"; - } - if (source === "mock") { - return "mock"; - } - throw new Error( - `Invalid --api-source "${options.apiSource}". Use "list-tools", "tool-metadata", or "mock".` - ); +const EXPLICIT_API_SOURCES: Record = { + "public-catalog": "public-catalog", + public: "public-catalog", + "list-tools": "list-tools", + engine: "tool-metadata", + "tool-metadata": "tool-metadata", + mock: "mock", +}; + +const resolveExplicitApiSource = (apiSource: string): ApiSource => { + const resolved = EXPLICIT_API_SOURCES[apiSource.toLowerCase()]; + if (resolved) { + return resolved; } - // Auto-detect based on provided Engine credentials only. - // List-tools endpoint must be explicitly selected via --api-source list-tools. + throw new Error( + `Invalid --api-source "${apiSource}". Use "public-catalog", "list-tools", "tool-metadata", or "mock".` + ); +}; + +const resolveAutoDetectedApiSource = (options: ApiSourceOptions): ApiSource => { const hasToolMetadataKey = !!( options.toolMetadataKey ?? process.env.ENGINE_API_KEY ); @@ -39,5 +41,21 @@ export const resolveApiSource = (options: ApiSourceOptions): ApiSource => { if (hasToolMetadataKey && hasToolMetadataUrl) { return "tool-metadata"; } + + if (hasToolMetadataUrl) { + return "public-catalog"; + } + return "mock"; }; + +export const resolveApiSource = (options: ApiSourceOptions): ApiSource => { + if (options.apiSource) { + return resolveExplicitApiSource(options.apiSource); + } + + return resolveAutoDetectedApiSource(options); +}; + +export const isDeprecatedApiSource = (apiSource: ApiSource): boolean => + apiSource === "tool-metadata" || apiSource === "list-tools"; diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index 97269e35d..8dde372a2 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -58,8 +58,9 @@ import { createCachedToolkitDataSource, createEngineToolkitDataSource, createMockToolkitDataSource, - type IToolkitDataSource, + createPublicCatalogToolkitDataSource, type ToolkitData, + type ToolkitDataSource, } from "../sources/toolkit-data-source"; import { type MergedToolkit, @@ -83,7 +84,11 @@ import { readFailedToolsReport, writeFailedToolsReport, } from "../utils/run-logs"; -import { type ApiSource, resolveApiSource } from "./api-source"; +import { + type ApiSource, + isDeprecatedApiSource, + resolveApiSource, +} from "./api-source"; import { cleanupExcludedToolkitOutput } from "./exclusion-cleanup"; import { assertSafeCurrentToolkitSnapshot, @@ -507,6 +512,21 @@ const resolveListToolsConfig = (options: ToolkitDataSourceOptions) => { }; }; +const resolvePublicCatalogConfig = (options: ToolkitDataSourceOptions) => { + const baseUrl = options.toolMetadataUrl ?? process.env.ENGINE_API_URL; + + if (!baseUrl) { + return null; + } + + return { + baseUrl, + ...(options.toolMetadataPageSize + ? { toolsPageSize: options.toolMetadataPageSize } + : {}), + }; +}; + const resolveToolMetadataConfig = (options: ToolkitDataSourceOptions) => { const baseUrl = options.toolMetadataUrl ?? process.env.ENGINE_API_URL; const apiKey = options.toolMetadataKey ?? process.env.ENGINE_API_KEY; @@ -524,62 +544,122 @@ const resolveToolMetadataConfig = (options: ToolkitDataSourceOptions) => { }; }; -const createToolkitDataSourceForApi = ( - apiSource: ApiSource, +const warnDeprecatedApiSource = (apiSource: ApiSource): void => { + if (!isDeprecatedApiSource(apiSource)) { + return; + } + + console.warn( + chalk.yellow( + `Warning: --api-source ${apiSource} is deprecated. Use "public-catalog" instead.` + ) + ); +}; + +const createPublicCatalogToolkitSource = ( + options: ToolkitDataSourceOptions, + metadataSource: ReturnType, + verbose: boolean +): ToolkitDataSource => { + const config = resolvePublicCatalogConfig(options); + if (!config) { + throw new Error( + "Public catalog API requires --tool-metadata-url (or ENGINE_API_URL environment variable)." + ); + } + if (verbose) { + console.log( + chalk.dim( + `Using /v1/public/tool_catalog + /v1/public/tools: ${config.baseUrl}` + ) + ); + } + return createPublicCatalogToolkitDataSource({ + publicCatalog: config, + metadataSource, + }); +}; + +const createListToolsToolkitSource = ( options: ToolkitDataSourceOptions, metadataSource: ReturnType, - mockDataDir: string, verbose: boolean, spinner?: ReturnType -): IToolkitDataSource => { - if (apiSource === "list-tools") { - const config = resolveListToolsConfig(options); - if (!config) { - throw new Error( - "List tools API requires --list-tools-key (or ARCADE_API_KEY environment variable)." - ); - } - if (verbose) { - console.log(chalk.dim(`Using /v1/tools endpoint: ${config.baseUrl}`)); - } - // Add progress callback for API pagination - const onProgress = spinner - ? (fetched: number, total: number) => { - spinner.text = `Fetching tools from API... ${fetched}/${total}`; - } - : undefined; - return createArcadeToolkitDataSource({ - arcade: { ...config, onProgress }, - metadataSource, - }); +): ToolkitDataSource => { + const config = resolveListToolsConfig(options); + if (!config) { + throw new Error( + "List tools API requires --list-tools-key (or ARCADE_API_KEY environment variable)." + ); } - - if (apiSource === "tool-metadata") { - const config = resolveToolMetadataConfig(options); - if (!config) { - throw new Error( - "Tool metadata API requires --tool-metadata-url and --tool-metadata-key." - ); - } - if (verbose) { - console.log( - chalk.dim(`Using /v1/tool_metadata endpoint: ${config.baseUrl}`) - ); - } - return createEngineToolkitDataSource({ - engine: config, - metadataSource, - }); + if (verbose) { + console.log(chalk.dim(`Using /v1/tools endpoint: ${config.baseUrl}`)); } + const onProgress = spinner + ? (fetched: number, total: number) => { + spinner.text = `Fetching tools from API... ${fetched}/${total}`; + } + : undefined; + return createArcadeToolkitDataSource({ + arcade: { ...config, onProgress }, + metadataSource, + }); +}; +const createToolMetadataToolkitSource = ( + options: ToolkitDataSourceOptions, + metadataSource: ReturnType, + verbose: boolean +): ToolkitDataSource => { + const config = resolveToolMetadataConfig(options); + if (!config) { + throw new Error( + "Tool metadata API requires --tool-metadata-url and --tool-metadata-key." + ); + } if (verbose) { - console.log(chalk.dim(`Using mock data: ${mockDataDir}`)); + console.log( + chalk.dim(`Using /v1/tool_metadata endpoint: ${config.baseUrl}`) + ); } - return createMockToolkitDataSource({ - dataDir: mockDataDir, + return createEngineToolkitDataSource({ + engine: config, + metadataSource, }); }; +const createToolkitDataSourceForApi = ( + apiSource: ApiSource, + options: ToolkitDataSourceOptions, + metadataSource: ReturnType, + mockDataDir: string, + verbose: boolean, + spinner?: ReturnType +): ToolkitDataSource => { + warnDeprecatedApiSource(apiSource); + + switch (apiSource) { + case "public-catalog": + return createPublicCatalogToolkitSource(options, metadataSource, verbose); + case "list-tools": + return createListToolsToolkitSource( + options, + metadataSource, + verbose, + spinner + ); + case "tool-metadata": + return createToolMetadataToolkitSource(options, metadataSource, verbose); + default: + if (verbose) { + console.log(chalk.dim(`Using mock data: ${mockDataDir}`)); + } + return createMockToolkitDataSource({ + dataDir: mockDataDir, + }); + } +}; + const normalizeToolkitKey = (toolkitId: string): string => toolkitId.toLowerCase(); @@ -831,7 +911,7 @@ program .option("--metadata-file ", "Path to metadata JSON file") .option( "--api-source ", - 'API source: "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata), or "mock" (default: auto-detect)' + 'API source: "public-catalog" (/v1/public/*), "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata, deprecated), or "mock" (default: auto-detect)' ) .option( "--list-tools-url ", @@ -846,10 +926,13 @@ program "List tools API page size", (value) => Number.parseInt(value, 10) ) - .option("--tool-metadata-url ", "Tool metadata API URL") + .option( + "--tool-metadata-url ", + "Engine API base URL (or ENGINE_API_URL env)" + ) .option( "--tool-metadata-key ", - "Tool metadata API key (or ENGINE_API_KEY env)" + "Tool metadata API key (or ENGINE_API_KEY env; deprecated, only for tool-metadata source)" ) .option( "--tool-metadata-page-size ", @@ -1954,7 +2037,7 @@ program .option("--metadata-file ", "Path to metadata JSON file") .option( "--api-source ", - 'API source: "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata), or "mock" (default: auto-detect)' + 'API source: "public-catalog" (/v1/public/*), "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata, deprecated), or "mock" (default: auto-detect)' ) .option( "--list-tools-url ", @@ -1969,10 +2052,13 @@ program "List tools API page size", (value) => Number.parseInt(value, 10) ) - .option("--tool-metadata-url ", "Tool metadata API URL") + .option( + "--tool-metadata-url ", + "Engine API base URL (or ENGINE_API_URL env)" + ) .option( "--tool-metadata-key ", - "Tool metadata API key (or ENGINE_API_KEY env)" + "Tool metadata API key (or ENGINE_API_KEY env; deprecated, only for tool-metadata source)" ) .option( "--tool-metadata-page-size ", @@ -2802,7 +2888,7 @@ program .option("--metadata-file ", "Path to metadata JSON file") .option( "--api-source ", - 'API source: "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata), or "mock" (default: auto-detect)' + 'API source: "public-catalog" (/v1/public/*), "list-tools" (/v1/tools), "tool-metadata" (/v1/tool_metadata, deprecated), or "mock" (default: auto-detect)' ) .option( "--list-tools-url ", @@ -2817,10 +2903,13 @@ program "List tools API page size", (value) => Number.parseInt(value, 10) ) - .option("--tool-metadata-url ", "Tool metadata API URL") + .option( + "--tool-metadata-url ", + "Engine API base URL (or ENGINE_API_URL env)" + ) .option( "--tool-metadata-key ", - "Tool metadata API key (or ENGINE_API_KEY env)" + "Tool metadata API key (or ENGINE_API_KEY env; deprecated, only for tool-metadata source)" ) .option( "--custom-sections ", diff --git a/toolkit-docs-generator/src/llm/secret-edit-generator.ts b/toolkit-docs-generator/src/llm/secret-edit-generator.ts index 06c2e52c8..7659ad61e 100644 --- a/toolkit-docs-generator/src/llm/secret-edit-generator.ts +++ b/toolkit-docs-generator/src/llm/secret-edit-generator.ts @@ -39,7 +39,7 @@ export interface SecretCoverageEditInput { readonly requireConfigLink: boolean; } -export interface ISecretEditGenerator { +export interface SecretEditGenerator { /** * Edit the provided content to remove all references to `removedSecrets` * while preserving every other sentence, bullet, table row, heading, and @@ -56,6 +56,9 @@ export interface ISecretEditGenerator { fillCoverageGaps: (input: SecretCoverageEditInput) => Promise; } +/** @deprecated Use {@link SecretEditGenerator} */ +export type ISecretEditGenerator = SecretEditGenerator; + const DEFAULT_SYSTEM_PROMPT = "You are a careful documentation editor for the Arcade MCP toolkit docs. " + "You make the smallest possible change that satisfies the request. " + @@ -146,7 +149,7 @@ const buildCoveragePrompt = (input: SecretCoverageEditInput): string => { ].join("\n"); }; -export class LlmSecretEditGenerator implements ISecretEditGenerator { +export class LlmSecretEditGenerator implements SecretEditGenerator { private readonly client: LlmClient; private readonly model: string; private readonly temperature: number | undefined; diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts index 39268d257..833d89407 100644 --- a/toolkit-docs-generator/src/merger/data-merger.ts +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -6,15 +6,15 @@ */ import { createHash } from "node:crypto"; -import type { ISecretEditGenerator } from "../llm/secret-edit-generator"; +import type { SecretEditGenerator } from "../llm/secret-edit-generator"; import { isApiSuffixedToolkitId, normalizeToolkitId, } from "../shared/toolkit-primitives"; import type { ICustomSectionsSource } from "../sources/interfaces"; import type { - IToolkitDataSource, ToolkitData, + ToolkitDataSource, } from "../sources/toolkit-data-source"; import type { CustomSections, @@ -48,7 +48,7 @@ import { // ============================================================================ export interface DataMergerConfig { - toolkitDataSource: IToolkitDataSource; + toolkitDataSource: ToolkitDataSource; customSectionsSource: ICustomSectionsSource; toolExampleGenerator?: ToolExampleGenerator; toolkitSummaryGenerator?: ToolkitSummaryGenerator; @@ -57,7 +57,7 @@ export interface DataMergerConfig { * coverage gaps in summary / documentation chunks. When omitted the * scanners still run and emit warnings, but no content is rewritten. */ - secretEditGenerator?: ISecretEditGenerator; + secretEditGenerator?: SecretEditGenerator; /** * When true, the secret-coherence step is disabled entirely — neither * the scan nor the LLM edit runs, and no warnings are emitted. Wired @@ -1079,11 +1079,11 @@ export const mergeToolkit = async ( * Data merger that combines all sources */ export class DataMerger { - private readonly toolkitDataSource: IToolkitDataSource; + private readonly toolkitDataSource: ToolkitDataSource; private readonly customSectionsSource: ICustomSectionsSource; private readonly toolExampleGenerator: ToolExampleGenerator | undefined; private readonly toolkitSummaryGenerator: ToolkitSummaryGenerator | undefined; - private readonly secretEditGenerator: ISecretEditGenerator | undefined; + private readonly secretEditGenerator: SecretEditGenerator | undefined; private readonly skipSecretCoherence: boolean; private readonly previousToolkits: | ReadonlyMap diff --git a/toolkit-docs-generator/src/sources/arcade-api.ts b/toolkit-docs-generator/src/sources/arcade-api.ts index f0c7dce9c..3f03020b3 100644 --- a/toolkit-docs-generator/src/sources/arcade-api.ts +++ b/toolkit-docs-generator/src/sources/arcade-api.ts @@ -16,7 +16,7 @@ import { parseArcadeErrorResponse, parseArcadeToolsResponse, } from "./arcade-api-types"; -import type { FetchOptions, IToolDataSource } from "./internal"; +import type { FetchOptions, ToolDataSource } from "./internal"; // ============================================================================ // Configuration @@ -153,7 +153,8 @@ const extractToolkitId = (qualifiedName: string): string => { // Arcade API Source Implementation // ============================================================================ -export class ArcadeApiSource implements IToolDataSource { +/** @deprecated Use the public catalog source ({@link createPublicCatalogApiSource}) instead. */ +export class ArcadeApiSource implements ToolDataSource { private readonly endpoint: string; private readonly apiKey: string; private readonly fetchFn: typeof fetch; @@ -327,7 +328,7 @@ export class ArcadeApiSource implements IToolDataSource { */ export const createArcadeApiSource = ( config: ArcadeApiSourceConfig -): IToolDataSource => new ArcadeApiSource(config); +): ToolDataSource => new ArcadeApiSource(config); /** * Create an Arcade API source with default production URL @@ -335,7 +336,7 @@ export const createArcadeApiSource = ( export const createProductionArcadeApiSource = ( apiKey: string, options?: Partial> -): IToolDataSource => +): ToolDataSource => new ArcadeApiSource({ baseUrl: DEFAULT_BASE_URL, apiKey, diff --git a/toolkit-docs-generator/src/sources/design-system-metadata.ts b/toolkit-docs-generator/src/sources/design-system-metadata.ts index e9f3e0864..6d46d2f5c 100644 --- a/toolkit-docs-generator/src/sources/design-system-metadata.ts +++ b/toolkit-docs-generator/src/sources/design-system-metadata.ts @@ -11,7 +11,7 @@ import { z } from "zod"; import { normalizeToolkitId } from "../shared/toolkit-primitives"; import type { ToolkitMetadata } from "../types/index"; import { ToolkitMetadataSchema } from "../types/index"; -import type { IMetadataSource } from "./internal"; +import type { MetadataSource } from "./internal"; // ============================================================================ // Types @@ -69,7 +69,7 @@ function withApiSuffix(label: string): string { // Source // ============================================================================ -export class DesignSystemMetadataSource implements IMetadataSource { +export class DesignSystemMetadataSource implements MetadataSource { private readonly toolkits: readonly ToolkitMetadata[]; private readonly indexByIdOrLabel: Map; @@ -122,11 +122,11 @@ export class DesignSystemMetadataSource implements IMetadataSource { export function createDesignSystemMetadataSourceFromToolkits( toolkits: readonly ToolkitMetadata[] -): IMetadataSource { +): MetadataSource { return new DesignSystemMetadataSource(toolkits); } -export async function createDesignSystemMetadataSource(): Promise { +export async function createDesignSystemMetadataSource(): Promise { const parsed: ToolkitMetadata[] = []; for (const raw of DESIGN_SYSTEM_TOOLKITS) { const dsParsed = DesignSystemToolkitSchema.safeParse(raw); diff --git a/toolkit-docs-generator/src/sources/engine-api.ts b/toolkit-docs-generator/src/sources/engine-api.ts index 5b6862691..3cb838660 100644 --- a/toolkit-docs-generator/src/sources/engine-api.ts +++ b/toolkit-docs-generator/src/sources/engine-api.ts @@ -1,5 +1,5 @@ import type { ToolDefinition } from "../types/index"; -import type { FetchOptions, IToolDataSource } from "./internal"; +import type { FetchOptions, ToolDataSource } from "./internal"; import { parseToolMetadataError, parseToolMetadataResponse, @@ -60,7 +60,8 @@ const parseJsonResponse = async ( } }; -export class EngineApiSource implements IToolDataSource { +/** @deprecated Use the public catalog source ({@link createPublicCatalogApiSource}) instead. */ +export class EngineApiSource implements ToolDataSource { private readonly endpoint: string; private readonly summaryEndpoint: string; private readonly apiKey: string; @@ -216,4 +217,4 @@ export class EngineApiSource implements IToolDataSource { export const createEngineApiSource = ( config: EngineApiSourceConfig -): IToolDataSource => new EngineApiSource(config); +): ToolDataSource => new EngineApiSource(config); diff --git a/toolkit-docs-generator/src/sources/in-memory.ts b/toolkit-docs-generator/src/sources/in-memory.ts index 1ea321b65..658636c18 100644 --- a/toolkit-docs-generator/src/sources/in-memory.ts +++ b/toolkit-docs-generator/src/sources/in-memory.ts @@ -12,11 +12,7 @@ import type { } from "../types/index"; import { normalizeId } from "../utils/fp"; import type { ICustomSectionsSource } from "./interfaces"; -import type { - FetchOptions, - IMetadataSource, - IToolDataSource, -} from "./internal"; +import type { FetchOptions, MetadataSource, ToolDataSource } from "./internal"; // ============================================================================ // In-Memory Tool Data Source @@ -37,7 +33,7 @@ import type { * const tools = await source.fetchToolsByToolkit('Github'); * ``` */ -export class InMemoryToolDataSource implements IToolDataSource { +export class InMemoryToolDataSource implements ToolDataSource { private readonly tools: readonly ToolDefinition[]; constructor(tools: readonly ToolDefinition[]) { @@ -104,7 +100,7 @@ export class InMemoryToolDataSource implements IToolDataSource { * const metadata = await source.getToolkitMetadata('Github'); * ``` */ -export class InMemoryMetadataSource implements IMetadataSource { +export class InMemoryMetadataSource implements MetadataSource { private readonly metadata: ReadonlyMap; constructor(toolkits: readonly ToolkitMetadata[]) { @@ -237,14 +233,14 @@ export class EmptyCustomSectionsSource implements ICustomSectionsSource { */ export const createInMemoryToolDataSource = ( tools: readonly ToolDefinition[] -): IToolDataSource => new InMemoryToolDataSource(tools); +): ToolDataSource => new InMemoryToolDataSource(tools); /** * Create an in-memory metadata source from test fixtures */ export const createInMemoryMetadataSource = ( toolkits: readonly ToolkitMetadata[] -): IMetadataSource => new InMemoryMetadataSource(toolkits); +): MetadataSource => new InMemoryMetadataSource(toolkits); /** * Create an in-memory custom sections source from test fixtures diff --git a/toolkit-docs-generator/src/sources/index.ts b/toolkit-docs-generator/src/sources/index.ts index 7b9c7046d..9785949f4 100644 --- a/toolkit-docs-generator/src/sources/index.ts +++ b/toolkit-docs-generator/src/sources/index.ts @@ -12,6 +12,8 @@ export * from "./markdown-curation"; export * from "./mock-engine-api"; export * from "./mock-metadata"; export * from "./oauth-provider-resolver"; +export * from "./public-catalog-api"; +export * from "./public-catalog-pagination"; export * from "./toolkit-data-source"; // Note: Design System source requires @arcadeai/design-system to be installed. diff --git a/toolkit-docs-generator/src/sources/internal.ts b/toolkit-docs-generator/src/sources/internal.ts index f628d9249..fbb836fc2 100644 --- a/toolkit-docs-generator/src/sources/internal.ts +++ b/toolkit-docs-generator/src/sources/internal.ts @@ -23,7 +23,7 @@ export interface FetchOptions { // Tool Data Source Interface (internal) // ============================================================================ -export interface IToolDataSource { +export interface ToolDataSource { /** Fetch tools for a specific toolkit */ readonly fetchToolsByToolkit: ( toolkitId: string @@ -36,11 +36,14 @@ export interface IToolDataSource { readonly isAvailable: () => Promise; } +/** @deprecated Use {@link ToolDataSource} */ +export type IToolDataSource = ToolDataSource; + // ============================================================================ // Metadata Source Interface (internal) // ============================================================================ -export interface IMetadataSource { +export interface MetadataSource { /** Get metadata for a specific toolkit */ readonly getToolkitMetadata: ( toolkitId: string @@ -50,3 +53,6 @@ export interface IMetadataSource { /** List all available toolkit IDs */ readonly listToolkitIds: () => Promise; } + +/** @deprecated Use {@link MetadataSource} */ +export type IMetadataSource = MetadataSource; diff --git a/toolkit-docs-generator/src/sources/mock-engine-api.ts b/toolkit-docs-generator/src/sources/mock-engine-api.ts index 00c958c59..dfe234e83 100644 --- a/toolkit-docs-generator/src/sources/mock-engine-api.ts +++ b/toolkit-docs-generator/src/sources/mock-engine-api.ts @@ -8,7 +8,7 @@ import { readFile } from "fs/promises"; import type { ToolDefinition } from "../types/index"; import { normalizeId } from "../utils/fp"; -import type { FetchOptions, IToolDataSource } from "./internal"; +import type { FetchOptions, ToolDataSource } from "./internal"; import { parseToolMetadataResponse } from "./tool-metadata-schema"; export interface MockEngineApiConfig { @@ -22,7 +22,7 @@ export interface MockEngineApiConfig { * Use this until the real Engine API endpoint is available. * The fixture format matches the expected API response schema. */ -export class MockEngineApiSource implements IToolDataSource { +export class MockEngineApiSource implements ToolDataSource { private readonly fixtureFilePath: string; private cachedData: ToolDefinition[] | null = null; @@ -99,4 +99,4 @@ export class MockEngineApiSource implements IToolDataSource { export const createMockEngineApiSource = ( fixtureFilePath: string -): IToolDataSource => new MockEngineApiSource({ fixtureFilePath }); +): ToolDataSource => new MockEngineApiSource({ fixtureFilePath }); diff --git a/toolkit-docs-generator/src/sources/mock-metadata.ts b/toolkit-docs-generator/src/sources/mock-metadata.ts index cdb88ddd0..6e7b07907 100644 --- a/toolkit-docs-generator/src/sources/mock-metadata.ts +++ b/toolkit-docs-generator/src/sources/mock-metadata.ts @@ -10,7 +10,7 @@ import { z } from "zod"; import type { ToolkitMetadata } from "../types/index"; import { ToolkitMetadataSchema } from "../types/index"; import { normalizeId } from "../utils/fp"; -import type { IMetadataSource } from "./internal"; +import type { MetadataSource } from "./internal"; // ============================================================================ // File Schema @@ -32,7 +32,7 @@ export interface MockMetadataConfig { /** * Mock implementation of IMetadataSource that loads from JSON fixtures */ -export class MockMetadataSource implements IMetadataSource { +export class MockMetadataSource implements MetadataSource { private readonly fixtureFilePath: string; private cachedData: MetadataFile | null = null; private normalizedIndex: Map | null = null; @@ -111,4 +111,4 @@ export class MockMetadataSource implements IMetadataSource { export const createMockMetadataSource = ( fixtureFilePath: string -): IMetadataSource => new MockMetadataSource({ fixtureFilePath }); +): MetadataSource => new MockMetadataSource({ fixtureFilePath }); diff --git a/toolkit-docs-generator/src/sources/public-catalog-api.ts b/toolkit-docs-generator/src/sources/public-catalog-api.ts new file mode 100644 index 000000000..27cf8f89e --- /dev/null +++ b/toolkit-docs-generator/src/sources/public-catalog-api.ts @@ -0,0 +1,144 @@ +import type { ToolDefinition } from "../types/index"; +import type { FetchOptions, ToolDataSource } from "./internal"; +import { fetchAllPages } from "./public-catalog-pagination"; +import { + type PublicCatalogToolkit, + parsePublicToolsResponse, + transformPublicToolItem, +} from "./public-catalog-schema"; + +export interface PublicCatalogApiSourceConfig { + /** Base URL for Engine (e.g., https://api.arcade.dev) */ + readonly baseUrl: string; + /** Optional fetch implementation for testing */ + readonly fetchFn?: typeof fetch; + /** Page size for toolkit catalog pagination */ + readonly catalogPageSize?: number; + /** Page size for the flat tools read */ + readonly toolsPageSize?: number; +} + +const DEFAULT_CATALOG_PAGE_SIZE = 100; +const DEFAULT_TOOLS_PAGE_SIZE = 25_000; + +const normalizeBaseUrl = (baseUrl: string): string => + baseUrl.replace(/\/+$/, ""); + +const buildEndpointUrl = (baseUrl: string, path: string): string => { + const normalized = normalizeBaseUrl(baseUrl); + if (normalized.endsWith("/v1")) { + return `${normalized}/${path}`; + } + return `${normalized}/v1/${path}`; +}; + +type CatalogSnapshot = { + toolkits: readonly PublicCatalogToolkit[]; + tools: readonly ToolDefinition[]; +}; + +export class PublicCatalogApiSource implements ToolDataSource { + private readonly catalogEndpoint: string; + private readonly toolsEndpoint: string; + private readonly fetchFn: typeof fetch; + private readonly catalogPageSize: number; + private readonly toolsPageSize: number; + private snapshot: Promise | undefined; + + constructor(config: PublicCatalogApiSourceConfig) { + this.catalogEndpoint = buildEndpointUrl( + config.baseUrl, + "public/tool_catalog" + ); + this.toolsEndpoint = buildEndpointUrl(config.baseUrl, "public/tools"); + this.fetchFn = config.fetchFn ?? fetch; + this.catalogPageSize = config.catalogPageSize ?? DEFAULT_CATALOG_PAGE_SIZE; + this.toolsPageSize = config.toolsPageSize ?? DEFAULT_TOOLS_PAGE_SIZE; + } + + private async loadSnapshot(): Promise { + const [catalogItems, toolItems] = await Promise.all([ + fetchAllPages( + this.catalogEndpoint, + this.fetchFn, + this.catalogPageSize + ), + fetchAllPages(this.toolsEndpoint, this.fetchFn, this.toolsPageSize), + ]); + + const parsedTools = parsePublicToolsResponse(toolItems); + const requirementsByToolkit = new Map( + catalogItems.map((toolkit) => [toolkit.name, toolkit.requirements]) + ); + + const tools = parsedTools.map((tool) => { + const toolkitName = tool.toolkit.name; + return transformPublicToolItem( + tool, + requirementsByToolkit.get(toolkitName) ?? null + ); + }); + + return { + toolkits: catalogItems, + tools, + }; + } + + private getSnapshot(): Promise { + this.snapshot ??= this.loadSnapshot(); + return this.snapshot; + } + + async fetchToolsByToolkit( + toolkitId: string + ): Promise { + const { tools } = await this.getSnapshot(); + return tools.filter( + (tool) => tool.qualifiedName.split(".")[0] === toolkitId + ); + } + + async fetchAllTools( + options?: FetchOptions + ): Promise { + const { tools } = await this.getSnapshot(); + + return tools.filter((tool) => { + const toolkitId = tool.qualifiedName.split(".")[0]; + if (options?.toolkitId && toolkitId !== options.toolkitId) { + return false; + } + + if (options?.version) { + const toolVersion = tool.fullyQualifiedName.split("@")[1]; + if (toolVersion !== options.version) { + return false; + } + } + + if (options?.providerId) { + if (tool.auth?.providerId !== options.providerId) { + return false; + } + } + + return true; + }); + } + + async isAvailable(): Promise { + try { + const response = await this.fetchFn( + `${this.catalogEndpoint}?limit=1&offset=0` + ); + return response.ok; + } catch { + return false; + } + } +} + +export const createPublicCatalogApiSource = ( + config: PublicCatalogApiSourceConfig +): ToolDataSource => new PublicCatalogApiSource(config); diff --git a/toolkit-docs-generator/src/sources/public-catalog-pagination.ts b/toolkit-docs-generator/src/sources/public-catalog-pagination.ts new file mode 100644 index 000000000..a4f0eb5ab --- /dev/null +++ b/toolkit-docs-generator/src/sources/public-catalog-pagination.ts @@ -0,0 +1,48 @@ +/** + * Paginated reads for Engine public catalog endpoints. + * + * Upstream defaults `limit` to 100, which silently truncates large catalogs. + * Loop on `total_count` and refuse short reads so a partial fetch cannot look + * like "toolkits were deleted". + */ +export const fetchAllPages = async ( + url: string, + fetchFn: typeof fetch = fetch, + pageSize = 100 +): Promise => { + const items: T[] = []; + let total = Number.POSITIVE_INFINITY; + + while (items.length < total) { + const separator = url.includes("?") ? "&" : "?"; + const paged = `${url}${separator}limit=${pageSize}&offset=${items.length}`; + const response = await fetchFn(paged); + + if (!response.ok) { + throw new Error( + `Public catalog API error ${response.status} from ${paged}` + ); + } + + const body = (await response.json()) as { + items?: T[]; + total_count?: number; + }; + const page = body.items ?? []; + total = body.total_count ?? page.length; + + if (page.length === 0) { + break; + } + + items.push(...page); + } + + if (items.length < total) { + throw new Error( + `Public catalog read ${items.length} of ${total} from ${url} — refusing a partial catalog` + ); + } + + return items; +}; diff --git a/toolkit-docs-generator/src/sources/public-catalog-schema.ts b/toolkit-docs-generator/src/sources/public-catalog-schema.ts new file mode 100644 index 000000000..7c50591cd --- /dev/null +++ b/toolkit-docs-generator/src/sources/public-catalog-schema.ts @@ -0,0 +1,149 @@ +import { z } from "zod"; +import type { ToolAuth, ToolDefinition } from "../types/index"; +import { + ToolMetadataItemSchema, + transformToolMetadataItem, +} from "./tool-metadata-schema"; + +const PublicCatalogAuthorizationItemSchema = z.object({ + provider_id: z.string().nullable().optional(), + provider_type: z.string().nullable().optional(), + scopes: z.array(z.string()).optional(), +}); + +const PublicCatalogRequirementsSchema = z + .object({ + authorization: z + .object({ + items: z + .record(z.string(), PublicCatalogAuthorizationItemSchema) + .nullable() + .optional(), + }) + .nullable() + .optional(), + secrets: z + .object({ + items: z.record(z.string(), z.unknown()).nullable().optional(), + }) + .nullable() + .optional(), + }) + .nullable() + .optional(); + +export const PublicCatalogToolkitSchema = z.object({ + name: z.string(), + description: z.string(), + version: z.string(), + tool_count: z.number(), + requirements: PublicCatalogRequirementsSchema, +}); + +export const PublicCatalogToolkitResponseSchema = z.object({ + items: z.array(PublicCatalogToolkitSchema), + limit: z.number().optional(), + offset: z.number().optional(), + page_count: z.number().optional(), + total_count: z.number(), +}); + +export const PublicCatalogToolResponseSchema = z.object({ + items: z.array(ToolMetadataItemSchema), + limit: z.number().optional(), + offset: z.number().optional(), + page_count: z.number().optional(), + total_count: z.number(), +}); + +export type PublicCatalogToolkit = z.infer; +export type PublicCatalogRequirements = z.infer< + typeof PublicCatalogRequirementsSchema +>; + +const DEFAULT_OAUTH_PROVIDER_TYPE = "oauth2"; + +export const extractToolkitRequirements = ( + requirements: PublicCatalogRequirements | null | undefined +): { auth: ToolAuth | null; secrets: string[] } => { + const authItems = requirements?.authorization?.items ?? {}; + const authEntries = Object.values(authItems); + const secrets = Object.keys(requirements?.secrets?.items ?? {}); + + if (authEntries.length === 0) { + return { auth: null, secrets }; + } + + const providerId = + authEntries.find((entry) => entry.provider_id)?.provider_id ?? null; + const providerType = + authEntries.find((entry) => entry.provider_type)?.provider_type ?? + DEFAULT_OAUTH_PROVIDER_TYPE; + const scopes = [ + ...new Set(authEntries.flatMap((entry) => entry.scopes ?? [])), + ]; + + return { + auth: { + providerId, + providerType, + scopes, + }, + secrets, + }; +}; + +export const transformPublicToolItem = ( + apiTool: z.infer, + toolkitRequirements: PublicCatalogRequirements | null | undefined +): ToolDefinition => { + const { auth, secrets } = extractToolkitRequirements(toolkitRequirements); + + return transformToolMetadataItem({ + ...apiTool, + requirements: { + authorization: auth + ? [ + { + provider_id: auth.providerId, + provider_type: auth.providerType, + scopes: auth.scopes, + }, + ] + : null, + secrets: secrets.length > 0 ? secrets.map((key) => ({ key })) : null, + }, + }); +}; + +export const groupToolsByToolkit = ( + items: readonly T[] +): Map => { + const byToolkit = new Map(); + + for (const item of items) { + const toolkitName = item.toolkit?.name; + if (!toolkitName) { + continue; + } + + const existing = byToolkit.get(toolkitName); + if (existing) { + existing.push(item); + } else { + byToolkit.set(toolkitName, [item]); + } + } + + return byToolkit; +}; + +export const parsePublicCatalogResponse = ( + payload: unknown +): PublicCatalogToolkit[] => + PublicCatalogToolkitResponseSchema.parse(payload).items; + +export const parsePublicToolsResponse = ( + items: unknown[] +): z.infer[] => + items.map((item) => ToolMetadataItemSchema.parse(item)); diff --git a/toolkit-docs-generator/src/sources/tool-metadata-schema.ts b/toolkit-docs-generator/src/sources/tool-metadata-schema.ts index 5ae406fe8..9f7fc0205 100644 --- a/toolkit-docs-generator/src/sources/tool-metadata-schema.ts +++ b/toolkit-docs-generator/src/sources/tool-metadata-schema.ts @@ -79,7 +79,7 @@ const ToolItemMetadataSchema = z .optional() .nullable(); -const ToolMetadataItemSchema = z.object({ +export const ToolMetadataItemSchema = z.object({ fully_qualified_name: z.string(), qualified_name: z.string(), name: z.string(), diff --git a/toolkit-docs-generator/src/sources/toolkit-data-source.ts b/toolkit-docs-generator/src/sources/toolkit-data-source.ts index 7c7ac3a84..64a0b1644 100644 --- a/toolkit-docs-generator/src/sources/toolkit-data-source.ts +++ b/toolkit-docs-generator/src/sources/toolkit-data-source.ts @@ -18,9 +18,13 @@ import { createEngineApiSource, type EngineApiSourceConfig, } from "./engine-api"; -import type { IMetadataSource, IToolDataSource } from "./internal"; +import type { MetadataSource, ToolDataSource } from "./internal"; import { createMockEngineApiSource } from "./mock-engine-api"; import { createMockMetadataSource } from "./mock-metadata"; +import { + createPublicCatalogApiSource, + type PublicCatalogApiSourceConfig, +} from "./public-catalog-api"; // ============================================================================ // Unified Toolkit Data Interface @@ -48,10 +52,10 @@ export interface ToolkitData { * 2. Future: Use a single unified source when Engine API includes metadata * * Implementations: - * - CombinedToolkitDataSource: Combines IToolDataSource + IMetadataSource + * - CombinedToolkitDataSource: Combines ToolDataSource + MetadataSource * - UnifiedToolkitDataSource: Single source (future implementation) */ -export interface IToolkitDataSource { +export interface ToolkitDataSource { /** * Fetch combined data for a specific toolkit * @param toolkitId - The toolkit identifier (e.g., "Github", "Slack") @@ -75,6 +79,9 @@ export interface IToolkitDataSource { readonly isAvailable: () => Promise; } +/** @deprecated Use {@link ToolkitDataSource} */ +export type IToolkitDataSource = ToolkitDataSource; + /** * Reuse one all-toolkit snapshot for the lifetime of a generation run. * @@ -82,8 +89,8 @@ export interface IToolkitDataSource { * instead of issuing independent API reads that can disagree mid-run. */ export const createCachedToolkitDataSource = ( - source: IToolkitDataSource -): IToolkitDataSource => { + source: ToolkitDataSource +): ToolkitDataSource => { let allToolkitsSnapshot: | Promise> | undefined; @@ -108,24 +115,24 @@ export const createCachedToolkitDataSource = ( */ export interface CombinedToolkitDataSourceConfig { /** Source for tool definitions */ - readonly toolSource: IToolDataSource; + readonly toolSource: ToolDataSource; /** Source for toolkit metadata */ - readonly metadataSource: IMetadataSource; + readonly metadataSource: MetadataSource; } /** * Combined implementation that merges separate tool and metadata sources * * This is the current implementation that combines: - * - Engine API (via IToolDataSource) - * - Design System (via IMetadataSource) + * - Engine API (via ToolDataSource) + * - Design System (via MetadataSource) * * In the future, this can be replaced with UnifiedToolkitDataSource * when Engine API includes metadata. */ -export class CombinedToolkitDataSource implements IToolkitDataSource { - private readonly toolSource: IToolDataSource; - private readonly metadataSource: IMetadataSource; +export class CombinedToolkitDataSource implements ToolkitDataSource { + private readonly toolSource: ToolDataSource; + private readonly metadataSource: MetadataSource; constructor(config: CombinedToolkitDataSourceConfig) { this.toolSource = config.toolSource; @@ -259,7 +266,26 @@ export class CombinedToolkitDataSource implements IToolkitDataSource { */ export const createCombinedToolkitDataSource = ( config: CombinedToolkitDataSourceConfig -): IToolkitDataSource => new CombinedToolkitDataSource(config); +): ToolkitDataSource => new CombinedToolkitDataSource(config); + +// ============================================================================ +// Public Catalog Toolkit Data Source +// ============================================================================ + +export interface PublicCatalogToolkitDataSourceConfig { + /** Public catalog API configuration */ + readonly publicCatalog: PublicCatalogApiSourceConfig; + /** Source for toolkit metadata */ + readonly metadataSource: MetadataSource; +} + +export const createPublicCatalogToolkitDataSource = ( + config: PublicCatalogToolkitDataSourceConfig +): ToolkitDataSource => + createCombinedToolkitDataSource({ + toolSource: createPublicCatalogApiSource(config.publicCatalog), + metadataSource: config.metadataSource, + }); // ============================================================================ // Engine Toolkit Data Source @@ -269,12 +295,13 @@ export interface EngineToolkitDataSourceConfig { /** Engine API configuration */ readonly engine: EngineApiSourceConfig; /** Source for toolkit metadata */ - readonly metadataSource: IMetadataSource; + readonly metadataSource: MetadataSource; } +/** @deprecated Use {@link createPublicCatalogToolkitDataSource} instead. */ export const createEngineToolkitDataSource = ( config: EngineToolkitDataSourceConfig -): IToolkitDataSource => +): ToolkitDataSource => createCombinedToolkitDataSource({ toolSource: createEngineApiSource(config.engine), metadataSource: config.metadataSource, @@ -288,15 +315,13 @@ export interface ArcadeToolkitDataSourceConfig { /** Arcade API configuration */ readonly arcade: ArcadeApiSourceConfig; /** Source for toolkit metadata */ - readonly metadataSource: IMetadataSource; + readonly metadataSource: MetadataSource; } -/** - * Create a toolkit data source using the Arcade Production API (/v1/tools) - */ +/** @deprecated Use {@link createPublicCatalogToolkitDataSource} instead. */ export const createArcadeToolkitDataSource = ( config: ArcadeToolkitDataSourceConfig -): IToolkitDataSource => +): ToolkitDataSource => createCombinedToolkitDataSource({ toolSource: createArcadeApiSource(config.arcade), metadataSource: config.metadataSource, @@ -322,7 +347,7 @@ export interface MockToolkitDataSourceConfig { */ export const createMockToolkitDataSource = ( config: MockToolkitDataSourceConfig -): IToolkitDataSource => { +): ToolkitDataSource => { const toolFixturePath = join(config.dataDir, "engine-api-response.json"); const metadataFixturePath = join(config.dataDir, "metadata.json"); diff --git a/toolkit-docs-generator/tests/cli/api-source.test.ts b/toolkit-docs-generator/tests/cli/api-source.test.ts index 18237cafb..664dbe382 100644 --- a/toolkit-docs-generator/tests/cli/api-source.test.ts +++ b/toolkit-docs-generator/tests/cli/api-source.test.ts @@ -27,6 +27,13 @@ describe("resolveApiSource", () => { resetEnv(); }); + it("returns public-catalog when explicitly requested", () => { + expect(resolveApiSource({ apiSource: "public-catalog" })).toBe( + "public-catalog" + ); + expect(resolveApiSource({ apiSource: "public" })).toBe("public-catalog"); + }); + it("returns list-tools only when explicitly requested", () => { expect(resolveApiSource({ apiSource: "list-tools" })).toBe("list-tools"); }); @@ -41,6 +48,12 @@ describe("resolveApiSource", () => { ); }); + it("auto-selects public-catalog when only the Engine URL is set", () => { + process.env.ENGINE_API_URL = "https://api.arcade.dev"; + + expect(resolveApiSource({})).toBe("public-catalog"); + }); + it("auto-selects tool-metadata when Engine credentials exist", () => { process.env.ENGINE_API_KEY = "test-key"; process.env.ENGINE_API_URL = "https://api.arcade.dev"; diff --git a/toolkit-docs-generator/tests/sources/public-catalog-api.test.ts b/toolkit-docs-generator/tests/sources/public-catalog-api.test.ts new file mode 100644 index 000000000..09677d427 --- /dev/null +++ b/toolkit-docs-generator/tests/sources/public-catalog-api.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from "vitest"; +import { PublicCatalogApiSource } from "../../src/sources/public-catalog-api"; +import { fetchAllPages } from "../../src/sources/public-catalog-pagination"; +import { + extractToolkitRequirements, + groupToolsByToolkit, + transformPublicToolItem, +} from "../../src/sources/public-catalog-schema"; + +const githubCatalogEntry = { + name: "Github", + description: "Arcade.dev LLM tools for Github", + version: "4.1.1", + tool_count: 2, + requirements: { + authorization: { + items: { + github: { + provider_id: "arcade-github", + scopes: ["repo"], + }, + }, + }, + secrets: { + items: { + GITHUB_SERVER_URL: {}, + }, + }, + }, +}; + +const githubTool = { + fully_qualified_name: "Github.CreateIssue@4.1.1", + qualified_name: "Github.CreateIssue", + name: "CreateIssue", + description: "Create issue", + toolkit: { + name: "Github", + version: "4.1.1", + description: "GitHub toolkit", + }, + input: { parameters: [] }, + output: null, + metadata: { + behavior: { + operations: ["create"], + read_only: false, + destructive: false, + idempotent: false, + open_world: true, + }, + }, +}; + +describe("fetchAllPages", () => { + it("reads every page until total_count is satisfied", async () => { + const calls: string[] = []; + const items = Array.from({ length: 125 }, (_, index) => ({ + name: `Toolkit${index}`, + })); + const fetchFn = (async (input: string | URL | Request) => { + calls.push(input.toString()); + const url = new URL(input.toString()); + const offset = Number(url.searchParams.get("offset") ?? 0); + const limit = Number(url.searchParams.get("limit") ?? 100); + return new Response( + JSON.stringify({ + items: items.slice(offset, offset + limit), + total_count: items.length, + }), + { status: 200 } + ); + }) as typeof fetch; + + const result = await fetchAllPages<{ name: string }>( + "https://api.example/v1/public/tool_catalog", + fetchFn + ); + + expect(result).toHaveLength(125); + expect(calls).toHaveLength(2); + expect(calls[0]).toContain("offset=0"); + expect(calls[1]).toContain("offset=100"); + }); + + it("refuses a short read", async () => { + const fetchFn = (async (input: string | URL | Request) => { + const url = new URL(input.toString()); + const offset = Number(url.searchParams.get("offset") ?? 0); + + if (offset > 0) { + return new Response( + JSON.stringify({ + items: [], + total_count: 5, + }), + { status: 200 } + ); + } + + return new Response( + JSON.stringify({ + items: [{ name: "Github" }], + total_count: 5, + }), + { status: 200 } + ); + }) as typeof fetch; + + await expect( + fetchAllPages("https://api.example/v1/public/tools", fetchFn) + ).rejects.toThrow(/read 1 of 5/); + }); +}); + +describe("extractToolkitRequirements", () => { + it("maps catalog authorization and secrets onto tool auth fields", () => { + expect(extractToolkitRequirements(githubCatalogEntry.requirements)).toEqual( + { + auth: { + providerId: "arcade-github", + providerType: "oauth2", + scopes: ["repo"], + }, + secrets: ["GITHUB_SERVER_URL"], + } + ); + }); + + it("returns no auth when only secrets are required", () => { + expect( + extractToolkitRequirements({ + authorization: { items: {} }, + secrets: { items: { SERP_API_KEY: {} } }, + }) + ).toEqual({ + auth: null, + secrets: ["SERP_API_KEY"], + }); + }); +}); + +describe("transformPublicToolItem", () => { + it("fans toolkit requirements out to each tool", () => { + const tool = transformPublicToolItem( + githubTool, + githubCatalogEntry.requirements + ); + + expect(tool.auth).toEqual({ + providerId: "arcade-github", + providerType: "oauth2", + scopes: ["repo"], + }); + expect(tool.secrets).toEqual(["GITHUB_SERVER_URL"]); + expect(tool.metadata?.behavior.readOnly).toBe(false); + }); +}); + +describe("groupToolsByToolkit", () => { + it("groups tools by toolkit name", () => { + const grouped = groupToolsByToolkit([ + githubTool, + { + ...githubTool, + qualified_name: "Slack.SendMessage", + toolkit: { name: "Slack" }, + }, + ]); + + expect(grouped.get("Github")).toHaveLength(1); + expect(grouped.get("Slack")).toHaveLength(1); + }); +}); + +describe("PublicCatalogApiSource", () => { + it("loads catalog and tools once, then filters by toolkit", async () => { + let catalogCalls = 0; + let toolsCalls = 0; + + const fetchFn = (async (input: string | URL | Request) => { + const url = new URL(input.toString()); + + if (url.pathname.endsWith("/public/tool_catalog")) { + catalogCalls += 1; + return new Response( + JSON.stringify({ + items: [githubCatalogEntry], + total_count: 1, + }), + { status: 200 } + ); + } + + if (url.pathname.endsWith("/public/tools")) { + toolsCalls += 1; + return new Response( + JSON.stringify({ + items: [githubTool], + total_count: 1, + }), + { status: 200 } + ); + } + + return new Response("not found", { status: 404 }); + }) as typeof fetch; + + const source = new PublicCatalogApiSource({ + baseUrl: "https://api.example", + fetchFn, + }); + + const first = await source.fetchToolsByToolkit("Github"); + const second = await source.fetchAllTools({ toolkitId: "Github" }); + + expect(first).toHaveLength(1); + expect(second).toHaveLength(1); + expect(first[0]?.auth?.providerId).toBe("arcade-github"); + expect(catalogCalls).toBe(1); + expect(toolsCalls).toBe(1); + }); +}); diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts index bad7b93cf..e31845ef3 100644 --- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts +++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts @@ -29,9 +29,9 @@ test("porter workflow generates docs and opens a PR", () => { expect(workflowContents).toContain("--skip-unchanged"); expect(workflowContents).toContain("--preserve-last-known-good"); expect(workflowContents).toContain("--verbose"); - expect(workflowContents).toContain("--api-source tool-metadata"); + expect(workflowContents).toContain("--api-source public-catalog"); expect(workflowContents).toContain("--tool-metadata-url"); - expect(workflowContents).toContain("--tool-metadata-key"); + expect(workflowContents).not.toContain("--tool-metadata-key"); expect(workflowContents).toContain("--llm-provider anthropic"); expect(workflowContents).toContain("--llm-model"); expect(workflowContents).toContain("--llm-api-key");