diff --git a/.gitignore b/.gitignore index 3352a53..88fcbd0 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,4 @@ vite.config.ts.timestamp-* # Test artifacts test-results.xml coverage/ +.claude/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f99a4c..fe1fd98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,32 @@ ## Unreleased +### Replace Algolia search with self-contained llms.txt search + +`search_mapbox_docs_tool` no longer depends on the Algolia third-party service. The hosted server shares a single Algolia free-tier quota across all users, making it prone to throttling as usage grows. The new implementation searches directly against the `llms.txt` index files that now exist at every product level on docs.mapbox.com. + +**How it works:** + +On first search, the tool fetches 12 product `llms.txt` files in parallel (~220KB total). Each file contains a structured list of documentation pages with titles, URLs, and one-line descriptions. These files are cached for the standard 1-hour TTL, so subsequent searches are pure in-memory keyword matching — no network calls. + +**Products indexed:** + +- API Reference (`api/llms.txt`) +- Mapbox GL JS (`mapbox-gl-js/llms.txt`) +- Help Center (`help/llms.txt`) +- Style Specification (`style-spec/llms.txt`) +- Studio Manual (`studio-manual/llms.txt`) +- Mapbox Search JS (`mapbox-search-js/llms.txt`) +- Maps SDK for iOS and Android +- Navigation SDK for iOS and Android +- Mapbox Tiling Service +- Tilesets + +**Scoring:** Title matches (3×) outrank description matches (1×) and URL path matches (1×). Results are deduplicated by URL across sources and capped at the requested `limit`. + +**Reliability:** Failed sources are silently skipped — if any single product `llms.txt` is unreachable, the remaining sources still return results. + +**`fetchCachedText(url, httpRequest)`** — new helper in `docFetcher.ts` that fetches a URL and stores the response in `docCache`. Used by `docsSearchIndex.ts` to share the cache with the resource layer (which also caches `llms.txt` files). Fixed a subtle bug where empty-string responses (`''`) were not treated as cache hits due to falsy check — now uses `!== null`. + ### Raise `docCache` per-entry limit to 5 MB with size warnings - **Hard cap raised from 2 MB → 5 MB** — allows `llms-full.txt` files (Style Spec 466 KB, iOS Nav 696 KB, GL JS 1.6 MB) to be cached after being fetched via `get_document_tool` @@ -10,6 +37,22 @@ - **Upgrade `tshy` to `^4.1.1`, `vitest` to `^4.1.4`, `typescript` to `^6.0.2`** — removed deprecated `baseUrl` from `tsconfig.base.json` (TS6), added `"types": ["node"]` (required because tshy compiles from `.tshy/` and does not auto-discover `@types/node` in CI); downgraded `@types/node` to `^22.0.0` for LTS consistency with other repos; bumped `typescript-eslint` packages to `^8.58.2` for TypeScript 6 support +### Resources — use sublevel `llms.txt` per product + +docs.mapbox.com restructured its documentation so that `llms.txt` files now exist at every product level (e.g. `docs.mapbox.com/api/llms.txt`, `docs.mapbox.com/help/llms.txt`, `docs.mapbox.com/mapbox-gl-js/llms.txt`) alongside `llms-full.txt` files containing full page content. The root `docs.mapbox.com/llms.txt` is now a pure index of links to these sublevel files rather than a monolithic content file. The previous resources all filtered the root file by category keyword — now that the root contains only link lists, they were effectively returning empty or useless content. + +Updated resources to use the appropriate sublevel `llms.txt` files: + +- **`resource://mapbox-api-reference`** now fetches `docs.mapbox.com/api/llms.txt` — a clean, structured index of every Mapbox REST API grouped by service (Maps, Navigation, Search, Accounts) with links to full API reference pages +- **`resource://mapbox-guides`** now fetches `docs.mapbox.com/help/llms.txt` (39KB) — the full Mapbox Help Center index with troubleshooting guides, how-to tutorials, and walkthroughs +- **`resource://mapbox-sdk-docs`** now fetches `docs.mapbox.com/mapbox-gl-js/llms.txt` (34KB) — the GL JS documentation index listing all guides, API reference pages, and examples for the primary web mapping SDK +- **`resource://mapbox-reference`** now fetches the root `llms.txt` without filtering and returns the complete product catalog — useful for discovering what documentation exists and finding `llms.txt` URLs for any product +- **`resource://mapbox-examples`** continues to extract playground/demo/example sections from the root index (API Playgrounds, Demos & Projects) + +**`docFetcher.fetchCachedText`** — new shared helper that fetches a URL and stores it in `docCache`, used by all five resources to avoid duplicating the fetch+cache pattern. + +**`docFetcher.toMarkdownUrl`** — no longer rewrites URLs already ending in `.txt`, `.md`, or `.json`. Previously `get_document_tool` would try to fetch `llms.txt.md` before falling back; now it fetches `llms.txt` directly on the first attempt. + ## 0.2.1 - 2026-04-01 ### Security diff --git a/cspell.json b/cspell.json index 1461730..226b354 100644 --- a/cspell.json +++ b/cspell.json @@ -21,6 +21,8 @@ "Tilesets", "tilestats", "Tilequery", + "isochrone", + "Isochrone", "aerialway", "aeroway", "housenum", diff --git a/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts b/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts index 7adcc45..40203d3 100644 --- a/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts +++ b/src/resources/mapbox-api-reference-resource/MapboxApiReferenceResource.ts @@ -8,22 +8,24 @@ import type { ServerRequest } from '@modelcontextprotocol/sdk/types.js'; import type { HttpRequest } from '../../utils/types.js'; -import { docCache } from '../../utils/docCache.js'; import { BaseResource } from '../BaseResource.js'; -import { - parseDocSections, - filterSectionsByCategory, - sectionsToMarkdown -} from '../utils/docParser.js'; +import { fetchCachedText } from '../../utils/docFetcher.js'; /** - * Resource providing Mapbox API reference documentation + * Resource providing Mapbox API reference documentation. + * Fetches the aggregated API reference index from docs.mapbox.com/api/llms.txt, + * which lists all Mapbox REST API endpoints grouped by service category + * (Maps, Navigation, Search, Accounts, etc.) with direct links to each + * API reference page. */ export class MapboxApiReferenceResource extends BaseResource { readonly name = 'Mapbox API Reference'; readonly uri = 'resource://mapbox-api-reference'; readonly description = - 'Mapbox REST API reference documentation including endpoints, parameters, rate limits, and authentication for all Mapbox APIs (Geocoding, Directions, Static Images, Tilequery, etc.)'; + 'Mapbox REST API reference index organized by service (Maps, Navigation, Search, Accounts). ' + + 'Lists all API endpoints with links to detailed reference pages covering parameters, ' + + 'rate limits, authentication, and response formats (Geocoding, Directions, Static Images, ' + + 'Tilequery, Matrix, isochrone, Optimization, Styles, Uploads, Datasets, and more).'; readonly mimeType = 'text/markdown'; private httpRequest: HttpRequest; @@ -38,36 +40,17 @@ export class MapboxApiReferenceResource extends BaseResource { _extra: RequestHandlerExtra ): Promise { try { - const LLMS_TXT_URL = 'https://docs.mapbox.com/llms.txt'; - let content = docCache.get(LLMS_TXT_URL); - if (!content) { - const response = await this.httpRequest(LLMS_TXT_URL, { - headers: { - Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' - } - }); - - if (!response.ok) { - throw new Error( - `Failed to fetch Mapbox documentation: ${response.statusText}` - ); - } - - content = await response.text(); - docCache.set(LLMS_TXT_URL, content); - } - - // Parse and filter for API sections only - const allSections = parseDocSections(content); - const apiSections = filterSectionsByCategory(allSections, 'apis'); - const apiContent = sectionsToMarkdown(apiSections); + const content = await fetchCachedText( + 'https://docs.mapbox.com/api/llms.txt', + this.httpRequest + ); return { contents: [ { uri: uri.href, mimeType: this.mimeType, - text: `# Mapbox API Reference\n\n${apiContent}` + text: content } ] }; diff --git a/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts b/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts index 69c8832..fb6f6fe 100644 --- a/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts +++ b/src/resources/mapbox-examples-resource/MapboxExamplesResource.ts @@ -8,8 +8,8 @@ import type { ServerRequest } from '@modelcontextprotocol/sdk/types.js'; import type { HttpRequest } from '../../utils/types.js'; -import { docCache } from '../../utils/docCache.js'; import { BaseResource } from '../BaseResource.js'; +import { fetchCachedText } from '../../utils/docFetcher.js'; import { parseDocSections, filterSectionsByCategory, @@ -17,13 +17,17 @@ import { } from '../utils/docParser.js'; /** - * Resource providing Mapbox examples and playgrounds + * Resource providing links to Mapbox interactive examples and playgrounds. + * Extracts the examples/playground/demo sections from the root llms.txt + * catalog, which lists API playgrounds, demo apps, and open-code projects. */ export class MapboxExamplesResource extends BaseResource { readonly name = 'Mapbox Examples'; readonly uri = 'resource://mapbox-examples'; readonly description = - 'Mapbox code examples, API playgrounds, and interactive demos for testing and learning'; + 'Mapbox interactive API playgrounds, demo applications, and code examples. ' + + 'Includes playground URLs for Directions, Search Box, Static Images, ' + + 'isochrone, Matrix APIs, and demo apps for real estate, store locator, etc.'; readonly mimeType = 'text/markdown'; private httpRequest: HttpRequest; @@ -38,26 +42,12 @@ export class MapboxExamplesResource extends BaseResource { _extra: RequestHandlerExtra ): Promise { try { - const LLMS_TXT_URL = 'https://docs.mapbox.com/llms.txt'; - let content = docCache.get(LLMS_TXT_URL); - if (!content) { - const response = await this.httpRequest(LLMS_TXT_URL, { - headers: { - Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' - } - }); - - if (!response.ok) { - throw new Error( - `Failed to fetch Mapbox documentation: ${response.statusText}` - ); - } - - content = await response.text(); - docCache.set(LLMS_TXT_URL, content); - } + const content = await fetchCachedText( + 'https://docs.mapbox.com/llms.txt', + this.httpRequest + ); - // Parse and filter for example sections only + // Extract playground/demo/example sections from the catalog index const allSections = parseDocSections(content); const exampleSections = filterSectionsByCategory(allSections, 'examples'); const exampleContent = sectionsToMarkdown(exampleSections); diff --git a/src/resources/mapbox-guides-resource/MapboxGuidesResource.ts b/src/resources/mapbox-guides-resource/MapboxGuidesResource.ts index 0056724..fad821e 100644 --- a/src/resources/mapbox-guides-resource/MapboxGuidesResource.ts +++ b/src/resources/mapbox-guides-resource/MapboxGuidesResource.ts @@ -8,22 +8,22 @@ import type { ServerRequest } from '@modelcontextprotocol/sdk/types.js'; import type { HttpRequest } from '../../utils/types.js'; -import { docCache } from '../../utils/docCache.js'; import { BaseResource } from '../BaseResource.js'; -import { - parseDocSections, - filterSectionsByCategory, - sectionsToMarkdown -} from '../utils/docParser.js'; +import { fetchCachedText } from '../../utils/docFetcher.js'; /** - * Resource providing Mapbox guides and tutorials + * Resource providing Mapbox guides, tutorials, and help articles. + * Fetches from docs.mapbox.com/help/llms.txt which contains the full + * Mapbox Help Center index: troubleshooting guides, how-to tutorials, + * glossary, account setup, billing, and platform-specific walkthroughs. */ export class MapboxGuidesResource extends BaseResource { readonly name = 'Mapbox Guides'; readonly uri = 'resource://mapbox-guides'; readonly description = - 'Mapbox guides, tutorials, and how-tos including Studio Manual, map design guides, and best practices'; + 'Mapbox Help Center documentation: troubleshooting guides, how-to tutorials, ' + + 'glossary, account and billing setup, and walkthroughs for common developer tasks. ' + + 'Use this for conceptual guidance and step-by-step instructions.'; readonly mimeType = 'text/markdown'; private httpRequest: HttpRequest; @@ -38,36 +38,17 @@ export class MapboxGuidesResource extends BaseResource { _extra: RequestHandlerExtra ): Promise { try { - const LLMS_TXT_URL = 'https://docs.mapbox.com/llms.txt'; - let content = docCache.get(LLMS_TXT_URL); - if (!content) { - const response = await this.httpRequest(LLMS_TXT_URL, { - headers: { - Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' - } - }); - - if (!response.ok) { - throw new Error( - `Failed to fetch Mapbox documentation: ${response.statusText}` - ); - } - - content = await response.text(); - docCache.set(LLMS_TXT_URL, content); - } - - // Parse and filter for guide sections only - const allSections = parseDocSections(content); - const guideSections = filterSectionsByCategory(allSections, 'guides'); - const guideContent = sectionsToMarkdown(guideSections); + const content = await fetchCachedText( + 'https://docs.mapbox.com/help/llms.txt', + this.httpRequest + ); return { contents: [ { uri: uri.href, mimeType: this.mimeType, - text: `# Mapbox Guides\n\n${guideContent}` + text: content } ] }; diff --git a/src/resources/mapbox-reference-resource/MapboxReferenceResource.ts b/src/resources/mapbox-reference-resource/MapboxReferenceResource.ts index b4d4bcb..ac42977 100644 --- a/src/resources/mapbox-reference-resource/MapboxReferenceResource.ts +++ b/src/resources/mapbox-reference-resource/MapboxReferenceResource.ts @@ -8,22 +8,25 @@ import type { ServerRequest } from '@modelcontextprotocol/sdk/types.js'; import type { HttpRequest } from '../../utils/types.js'; -import { docCache } from '../../utils/docCache.js'; import { BaseResource } from '../BaseResource.js'; -import { - parseDocSections, - filterSectionsByCategory, - sectionsToMarkdown -} from '../utils/docParser.js'; +import { fetchCachedText } from '../../utils/docFetcher.js'; /** - * Resource providing Mapbox reference documentation + * Resource providing the complete Mapbox documentation catalog. + * Fetches the root llms.txt which is now a structured index of every + * Mapbox product and documentation section, each with a link to its + * own llms.txt file. Use this to discover what documentation is available + * and find URLs to pass to get_document_tool for deeper exploration. */ export class MapboxReferenceResource extends BaseResource { readonly name = 'Mapbox Reference'; readonly uri = 'resource://mapbox-reference'; readonly description = - 'Mapbox reference documentation including tilesets, data products, accounts, pricing, and other reference materials'; + 'Complete catalog of all Mapbox products and documentation. ' + + 'Lists every product section (Maps SDKs, Navigation APIs, Search APIs, ' + + 'Studio, Style Spec, Tilesets, Data products, Help Center, Atlas, Unity, etc.) ' + + "with links to each product's own llms.txt index. Use this to discover " + + 'what documentation exists and find the right URLs to fetch full docs.'; readonly mimeType = 'text/markdown'; private httpRequest: HttpRequest; @@ -38,39 +41,17 @@ export class MapboxReferenceResource extends BaseResource { _extra: RequestHandlerExtra ): Promise { try { - const LLMS_TXT_URL = 'https://docs.mapbox.com/llms.txt'; - let content = docCache.get(LLMS_TXT_URL); - if (!content) { - const response = await this.httpRequest(LLMS_TXT_URL, { - headers: { - Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' - } - }); - - if (!response.ok) { - throw new Error( - `Failed to fetch Mapbox documentation: ${response.statusText}` - ); - } - - content = await response.text(); - docCache.set(LLMS_TXT_URL, content); - } - - // Parse and filter for reference sections only - const allSections = parseDocSections(content); - const referenceSections = filterSectionsByCategory( - allSections, - 'reference' + const content = await fetchCachedText( + 'https://docs.mapbox.com/llms.txt', + this.httpRequest ); - const referenceContent = sectionsToMarkdown(referenceSections); return { contents: [ { uri: uri.href, mimeType: this.mimeType, - text: `# Mapbox Reference\n\n${referenceContent}` + text: content } ] }; diff --git a/src/resources/mapbox-sdk-docs-resource/MapboxSdkDocsResource.ts b/src/resources/mapbox-sdk-docs-resource/MapboxSdkDocsResource.ts index c89ad3b..edd84cc 100644 --- a/src/resources/mapbox-sdk-docs-resource/MapboxSdkDocsResource.ts +++ b/src/resources/mapbox-sdk-docs-resource/MapboxSdkDocsResource.ts @@ -8,22 +8,25 @@ import type { ServerRequest } from '@modelcontextprotocol/sdk/types.js'; import type { HttpRequest } from '../../utils/types.js'; -import { docCache } from '../../utils/docCache.js'; import { BaseResource } from '../BaseResource.js'; -import { - parseDocSections, - filterSectionsByCategory, - sectionsToMarkdown -} from '../utils/docParser.js'; +import { fetchCachedText } from '../../utils/docFetcher.js'; /** - * Resource providing Mapbox SDK documentation + * Resource providing Mapbox GL JS SDK documentation. + * Fetches the Mapbox GL JS llms.txt index which lists all GL JS guides, + * API reference pages, and examples with direct links to each page. + * GL JS is the primary web mapping SDK; use get_document_tool to fetch + * full content of any listed page. */ export class MapboxSdkDocsResource extends BaseResource { readonly name = 'Mapbox SDK Documentation'; readonly uri = 'resource://mapbox-sdk-docs'; readonly description = - 'Mapbox SDK and client library documentation for mobile (iOS, Android, Flutter) and web (Mapbox GL JS, Search JS) platforms'; + 'Mapbox GL JS documentation index: guides for getting started, adding data, ' + + 'globe/projections, indoor mapping, gestures, styling layers, and migration. ' + + 'Also includes links to API reference and code examples. ' + + 'For mobile/native SDKs (iOS, Android, Flutter), use resource://mapbox-reference ' + + 'to discover their documentation URLs.'; readonly mimeType = 'text/markdown'; private httpRequest: HttpRequest; @@ -38,36 +41,17 @@ export class MapboxSdkDocsResource extends BaseResource { _extra: RequestHandlerExtra ): Promise { try { - const LLMS_TXT_URL = 'https://docs.mapbox.com/llms.txt'; - let content = docCache.get(LLMS_TXT_URL); - if (!content) { - const response = await this.httpRequest(LLMS_TXT_URL, { - headers: { - Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' - } - }); - - if (!response.ok) { - throw new Error( - `Failed to fetch Mapbox documentation: ${response.statusText}` - ); - } - - content = await response.text(); - docCache.set(LLMS_TXT_URL, content); - } - - // Parse and filter for SDK sections only - const allSections = parseDocSections(content); - const sdkSections = filterSectionsByCategory(allSections, 'sdks'); - const sdkContent = sectionsToMarkdown(sdkSections); + const content = await fetchCachedText( + 'https://docs.mapbox.com/mapbox-gl-js/llms.txt', + this.httpRequest + ); return { contents: [ { uri: uri.href, mimeType: this.mimeType, - text: `# Mapbox SDK Documentation\n\n${sdkContent}` + text: content } ] }; diff --git a/src/tools/search-docs-tool/SearchDocsTool.ts b/src/tools/search-docs-tool/SearchDocsTool.ts index b6c3b63..0e448f8 100644 --- a/src/tools/search-docs-tool/SearchDocsTool.ts +++ b/src/tools/search-docs-tool/SearchDocsTool.ts @@ -8,63 +8,21 @@ import { SearchDocsSchema, SearchDocsInput } from './SearchDocsTool.input.schema.js'; +import { searchDocs, type DocEntry } from '../../utils/docsSearchIndex.js'; -// These are public search-only credentials embedded in every docs.mapbox.com -// browser session — safe to include in source. -const ALGOLIA_APP_ID = 'Z7QUXRWJ7L'; -const ALGOLIA_SEARCH_KEY = '332b77ce8ccfba082ac50ad2d882c80a'; -const ALGOLIA_INDEX = 'mapbox'; -const ALGOLIA_URL = `https://${ALGOLIA_APP_ID}-dsn.algolia.net/1/indexes/${ALGOLIA_INDEX}/query`; - -interface AlgoliaHierarchy { - lvl0: string | null; - lvl1: string | null; - lvl2: string | null; - lvl3: string | null; - lvl4: string | null; - lvl5: string | null; - lvl6: string | null; -} - -interface AlgoliaHit { - url: string; - content: string | null; - hierarchy: AlgoliaHierarchy; -} - -interface AlgoliaResponse { - hits: AlgoliaHit[]; -} - -function stripHtml(text: string): string { - return text.replace(/<[^>]+>/g, ''); -} - -function buildTitle(hierarchy: AlgoliaHierarchy): string { - const levels = [ - hierarchy.lvl0, - hierarchy.lvl1, - hierarchy.lvl2, - hierarchy.lvl3, - hierarchy.lvl4, - hierarchy.lvl5, - hierarchy.lvl6 - ].filter((lvl): lvl is string => lvl !== null); - return levels.map(stripHtml).join(' > '); -} - -function formatResults(hits: AlgoliaHit[]): string { - if (hits.length === 0) { +function formatResults(entries: DocEntry[]): string { + if (entries.length === 0) { return 'No results found.'; } - return hits - .map((hit, i) => { - const title = buildTitle(hit.hierarchy); - const excerpt = hit.content ? stripHtml(hit.content).trim() : ''; - const lines = [`${i + 1}. **${title}**`, ` URL: ${hit.url}`]; - if (excerpt) { - lines.push(` > ${excerpt}`); + return entries + .map((entry, i) => { + const lines = [ + `${i + 1}. **${entry.title}** (${entry.product})`, + ` URL: ${entry.url}` + ]; + if (entry.description) { + lines.push(` > ${entry.description}`); } return lines.join('\n'); }) @@ -74,7 +32,11 @@ function formatResults(hits: AlgoliaHit[]): string { export class SearchDocsTool extends BaseTool { name = 'search_mapbox_docs_tool'; description = - 'Search Mapbox documentation by keyword or natural language query. Returns ranked results with titles, URLs, and content excerpts. Use get_document_tool to fetch the full content of a result page.'; + 'Search Mapbox documentation by keyword or natural language query. ' + + 'Searches across API reference, GL JS, Help Center, Style Spec, Studio, ' + + 'Search JS, iOS/Android Maps and Navigation SDKs, and Tilesets. ' + + 'Returns ranked results with titles, URLs, and descriptions. ' + + 'Use get_document_tool to fetch the full content of a result page.'; readonly annotations = { readOnlyHint: true, destructiveHint: false, @@ -91,49 +53,23 @@ export class SearchDocsTool extends BaseTool { } protected async execute(input: SearchDocsInput): Promise { - const body = JSON.stringify({ - query: input.query, - hitsPerPage: input.limit - }); - - let data: AlgoliaResponse; - try { - const response = await this.httpRequest(ALGOLIA_URL, { - method: 'POST', - headers: { - 'X-Algolia-Application-Id': ALGOLIA_APP_ID, - 'X-Algolia-API-Key': ALGOLIA_SEARCH_KEY, - 'Content-Type': 'application/json' - }, - body - }); - - if (!response.ok) { - return { - content: [ - { - type: 'text', - text: `Search request failed: ${response.status} ${response.statusText}` - } - ], - isError: true - }; - } - - data = (await response.json()) as AlgoliaResponse; + const results = await searchDocs( + input.query, + input.limit, + this.httpRequest + ); + return { + content: [{ type: 'text', text: formatResults(results) }], + isError: false + }; } catch (error) { const message = error instanceof Error ? error.message : 'Unknown error occurred'; return { - content: [{ type: 'text', text: `Search request failed: ${message}` }], + content: [{ type: 'text', text: `Search failed: ${message}` }], isError: true }; } - - return { - content: [{ type: 'text', text: formatResults(data.hits) }], - isError: false - }; } } diff --git a/src/utils/docFetcher.ts b/src/utils/docFetcher.ts index dd7975a..d6f7cab 100644 --- a/src/utils/docFetcher.ts +++ b/src/utils/docFetcher.ts @@ -1,7 +1,7 @@ // Copyright (c) Mapbox, Inc. // Licensed under the MIT License. -import { MAX_ENTRY_BYTES, readBodyWithLimit } from './docCache.js'; +import { docCache, MAX_ENTRY_BYTES, readBodyWithLimit } from './docCache.js'; import type { HttpRequest } from './types.js'; const DOCS_HOSTNAME = 'docs.mapbox.com'; @@ -17,11 +17,14 @@ function getDocsHostname(): string { /** * Rewrite a docs.mapbox.com URL to its .md variant, substituting the * effective hostname (which may be overridden via MAPBOX_DOCS_HOST_OVERRIDE). - * Returns null if the URL is not a docs.mapbox.com URL. + * Returns null if the URL is not a docs.mapbox.com URL, or if the URL + * already ends in a text extension (.md, .txt, .json) — those should be + * fetched as-is without rewriting. * * Example (no override): * https://docs.mapbox.com/accounts/guides → https://docs.mapbox.com/accounts/guides.md * https://docs.mapbox.com/accounts/guides/ → https://docs.mapbox.com/accounts/guides.md + * https://docs.mapbox.com/mapbox-gl-js/llms.txt → null (already a text file) * * Example (MAPBOX_DOCS_HOST_OVERRIDE=docs.tilestream.net): * https://docs.mapbox.com/accounts/guides → https://docs.tilestream.net/accounts/guides.md @@ -30,6 +33,8 @@ export function toMarkdownUrl(url: string): string | null { try { const parsed = new URL(url); if (parsed.hostname !== DOCS_HOSTNAME) return null; + // Don't double-extend URLs that are already text/data files + if (/\.(txt|md|json)$/i.test(parsed.pathname)) return null; parsed.hostname = getDocsHostname(); parsed.pathname = parsed.pathname.replace(/\/$/, '') + '.md'; return parsed.toString(); @@ -56,6 +61,34 @@ function applyHostOverride(url: string): string { } } +/** + * Fetch a URL and cache the result. Intended for fetching llms.txt index + * files which are small, change infrequently, and are shared across multiple + * resources and tool calls. + */ +export async function fetchCachedText( + url: string, + httpRequest: HttpRequest +): Promise { + const cached = docCache.get(url); + if (cached !== null) return cached; + + const response = await httpRequest(url, { + headers: { Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' } + }); + + if (!response.ok) { + throw new Error(`Failed to fetch ${url}: ${response.statusText}`); + } + + const content = await response.text(); + if (Buffer.byteLength(content, 'utf8') > MAX_ENTRY_BYTES) { + throw new Error(`Response too large for ${url}`); + } + docCache.set(url, content); + return content; +} + /** * Fetch a Mapbox documentation page, preferring the .md variant for * docs.mapbox.com URLs when available. Falls back to the original URL @@ -78,9 +111,7 @@ export async function fetchDocContent( } const fetchUrl = applyHostOverride(url); - const response = await httpRequest(fetchUrl, { - headers: { Accept: 'text/markdown, text/plain;q=0.9, */*;q=0.8' } - }); + const response = await httpRequest(fetchUrl, {}); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); diff --git a/src/utils/docsSearchIndex.ts b/src/utils/docsSearchIndex.ts new file mode 100644 index 0000000..a3b5efc --- /dev/null +++ b/src/utils/docsSearchIndex.ts @@ -0,0 +1,164 @@ +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +import type { HttpRequest } from './types.js'; +import { fetchCachedText } from './docFetcher.js'; + +export interface DocEntry { + title: string; + url: string; + description: string; + product: string; +} + +/** + * Curated set of product llms.txt files to include in the search index. + * Each file is fetched once and cached for the duration of the TTL. + * Files are fetched in parallel on first search then served from cache. + */ +const INDEX_SOURCES: Array<{ url: string; product: string }> = [ + { + url: 'https://docs.mapbox.com/api/llms.txt', + product: 'API Reference' + }, + { + url: 'https://docs.mapbox.com/mapbox-gl-js/llms.txt', + product: 'Mapbox GL JS' + }, + { + url: 'https://docs.mapbox.com/help/llms.txt', + product: 'Help Center' + }, + { + url: 'https://docs.mapbox.com/style-spec/llms.txt', + product: 'Style Specification' + }, + { + url: 'https://docs.mapbox.com/studio-manual/llms.txt', + product: 'Studio Manual' + }, + { + url: 'https://docs.mapbox.com/mapbox-search-js/llms.txt', + product: 'Mapbox Search JS' + }, + { + url: 'https://docs.mapbox.com/ios/maps/llms.txt', + product: 'Maps SDK for iOS' + }, + { + url: 'https://docs.mapbox.com/android/maps/llms.txt', + product: 'Maps SDK for Android' + }, + { + url: 'https://docs.mapbox.com/ios/navigation/llms.txt', + product: 'Navigation SDK for iOS' + }, + { + url: 'https://docs.mapbox.com/android/navigation/llms.txt', + product: 'Navigation SDK for Android' + }, + { + url: 'https://docs.mapbox.com/mapbox-tiling-service/llms.txt', + product: 'Mapbox Tiling Service' + }, + { + url: 'https://docs.mapbox.com/data/tilesets/llms.txt', + product: 'Tilesets' + } +]; + +// Matches: - [Title](URL): optional description +// or: - [Title](URL) +const ENTRY_RE = /^-\s+\[([^\]]+)\]\(([^)]+)\)(?::\s*(.+))?$/; + +/** + * Parse a product-level llms.txt file into searchable entries. + * Each `- [Title](URL): description` line becomes one entry. + * Lines that don't match (headers, blanks, root index links) are skipped. + */ +export function parseLlmsTxtEntries( + content: string, + product: string +): DocEntry[] { + const entries: DocEntry[] = []; + for (const rawLine of content.split('\n')) { + const line = rawLine.trim(); + const match = ENTRY_RE.exec(line); + if (!match) continue; + const [, title, url, description = ''] = match; + // Skip links to other llms.txt files (those are index links, not doc pages) + if (url.endsWith('/llms.txt') || url.endsWith('/llms-full.txt')) continue; + entries.push({ title, url, description, product }); + } + return entries; +} + +/** + * Score an entry against the query words. + * Title matches are weighted 3x, description and URL path matches 1x each. + */ +function scoreEntry(entry: DocEntry, queryWords: string[]): number { + const titleLower = entry.title.toLowerCase(); + const descLower = entry.description.toLowerCase(); + const urlLower = entry.url.toLowerCase(); + + let score = 0; + for (const word of queryWords) { + if (titleLower.includes(word)) score += 3; + if (descLower.includes(word)) score += 1; + if (urlLower.includes(word)) score += 1; + } + return score; +} + +/** + * Search across all indexed llms.txt files. + * + * All source files are fetched in parallel and cached via docCache. + * First call may take ~500ms while files are fetched; subsequent calls + * are instant since all content is in-memory. + * + * Failed sources are skipped gracefully so a single unavailable file + * doesn't break the entire search. + */ +export async function searchDocs( + query: string, + limit: number, + httpRequest: HttpRequest +): Promise { + const allEntries = ( + await Promise.all( + INDEX_SOURCES.map(async ({ url, product }) => { + try { + const content = await fetchCachedText(url, httpRequest); + return parseLlmsTxtEntries(content, product); + } catch { + return []; + } + }) + ) + ).flat(); + + const queryWords = query + .toLowerCase() + .split(/\s+/) + .filter((w) => w.length > 1); // skip single-char words + + const scored = allEntries + .map((entry) => ({ entry, score: scoreEntry(entry, queryWords) })) + .filter(({ score }) => score > 0) + .sort((a, b) => b.score - a.score); + + // Deduplicate by URL (same page can appear across multiple sources) + const seen = new Set(); + const results: DocEntry[] = []; + for (const { entry } of scored) { + if (!seen.has(entry.url)) { + seen.add(entry.url); + results.push(entry); + if (results.length >= limit) break; + } + } + + return results; +} diff --git a/test/tools/search-docs-tool/SearchDocsTool.test.ts b/test/tools/search-docs-tool/SearchDocsTool.test.ts index 91d6bfd..dc23088 100644 --- a/test/tools/search-docs-tool/SearchDocsTool.test.ts +++ b/test/tools/search-docs-tool/SearchDocsTool.test.ts @@ -1,188 +1,190 @@ // Copyright (c) Mapbox, Inc. // Licensed under the MIT License. -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; import { SearchDocsTool } from '../../../src/tools/search-docs-tool/SearchDocsTool.js'; +import { docCache } from '../../../src/utils/docCache.js'; -function makeResponse(body: object, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { 'Content-Type': 'application/json' } +const GL_JS_LLMS_TXT = `# Mapbox GL JS + +> Documentation for Mapbox GL JS + +## Guides + +- [Add your data to the map](https://docs.mapbox.com/mapbox-gl-js/guides/add-your-data.md): Mapbox GL JS offers several ways to add your data to the map. +- [Markers](https://docs.mapbox.com/mapbox-gl-js/guides/add-your-data/markers.md): Add interactive markers to your map with minimal code using the Marker class. +- [Getting Started](https://docs.mapbox.com/mapbox-gl-js/guides/get-started.md): Get started with Mapbox GL JS. + +## Examples + +- [Add a marker to the map](https://docs.mapbox.com/mapbox-gl-js/example/add-a-marker.md): Use a Marker to add a visual indicator to the map. +`; + +const API_LLMS_TXT = `# API Docs + +> Documentation for Mapbox Web APIs + +## Navigation + +- [Directions API](https://docs.mapbox.com/api/navigation/directions.md): The Mapbox Directions API calculates optimal routes and produces turn-by-turn instructions. +- [Matrix API](https://docs.mapbox.com/api/navigation/matrix.md): The Mapbox Matrix API returns travel times between many points. + +## Search + +- [Geocoding API](https://docs.mapbox.com/api/search/geocoding.md): Convert addresses and place names to geographic coordinates. +`; + +function makeLlmsTxtResponse(content: string): Response { + return new Response(content, { + status: 200, + headers: { + 'content-type': 'text/plain', + 'content-length': String(Buffer.byteLength(content, 'utf8')) + } }); } -const sampleHits = [ - { - url: 'https://docs.mapbox.com/mapbox-gl-js/example/add-a-marker/', - content: 'Use a Marker to add a visual indicator to the map.', - hierarchy: { - lvl0: 'Mapbox GL JS', - lvl1: 'Examples', - lvl2: 'Add a marker to the map', - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null - } - }, - { - url: 'https://docs.mapbox.com/ios/maps/guides/add-your-data/markers/#basic-marker', - content: 'The simplest way to add a marker is to specify a coordinate.', - hierarchy: { - lvl0: 'Maps SDK for iOS', - lvl1: 'Markers', - lvl2: 'Adding a basic marker', - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null - } - } -]; +beforeEach(() => { + docCache.clear(); +}); describe('SearchDocsTool', () => { - it('returns formatted results for a successful search', async () => { - const httpRequest = vi - .fn() - .mockResolvedValue(makeResponse({ hits: sampleHits })); - const tool = new SearchDocsTool({ httpRequest }); + it('returns ranked results matching the query', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(GL_JS_LLMS_TXT)); + if (url.includes('/api/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(API_LLMS_TXT)); + return Promise.resolve(makeLlmsTxtResponse('')); + }); + const tool = new SearchDocsTool({ httpRequest }); const result = await tool.run({ query: 'add a marker', limit: 5 }); expect(result.isError).toBe(false); const text = result.content[0].text as string; - expect(text).toContain('Mapbox GL JS > Examples > Add a marker to the map'); + expect(text).toContain('Markers'); expect(text).toContain( - 'https://docs.mapbox.com/mapbox-gl-js/example/add-a-marker/' - ); - expect(text).toContain('Use a Marker to add a visual indicator'); - expect(text).toContain( - 'Maps SDK for iOS > Markers > Adding a basic marker' + 'https://docs.mapbox.com/mapbox-gl-js/guides/add-your-data/markers.md' ); + expect(text).toContain('Mapbox GL JS'); }); - it('POSTs to the Algolia endpoint with correct headers and body', async () => { - const httpRequest = vi.fn().mockResolvedValue(makeResponse({ hits: [] })); - const tool = new SearchDocsTool({ httpRequest }); - - await tool.run({ query: 'geocoding', limit: 3 }); - - expect(httpRequest).toHaveBeenCalledWith( - expect.stringContaining('algolia.net'), - expect.objectContaining({ - method: 'POST', - headers: expect.objectContaining({ - 'X-Algolia-Application-Id': 'Z7QUXRWJ7L', - 'X-Algolia-API-Key': expect.any(String) - }), - body: JSON.stringify({ query: 'geocoding', hitsPerPage: 3 }) - }) - ); - }); + it('returns "No results found" when nothing matches', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(GL_JS_LLMS_TXT)); + return Promise.resolve(makeLlmsTxtResponse('')); + }); - it('returns "No results found" when hits array is empty', async () => { - const httpRequest = vi.fn().mockResolvedValue(makeResponse({ hits: [] })); const tool = new SearchDocsTool({ httpRequest }); - - const result = await tool.run({ query: 'xyzzy nonexistent' }); + const result = await tool.run({ query: 'xyzzy nonexistent quantum' }); expect(result.isError).toBe(false); expect(result.content[0].text).toBe('No results found.'); }); - it('returns error result when response is not ok', async () => { - const httpRequest = vi - .fn() - .mockResolvedValue(makeResponse({ message: 'Invalid API key' }, 403)); - const tool = new SearchDocsTool({ httpRequest }); + it('respects the limit parameter', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(GL_JS_LLMS_TXT)); + if (url.includes('/api/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(API_LLMS_TXT)); + return Promise.resolve(makeLlmsTxtResponse('')); + }); - const result = await tool.run({ query: 'markers' }); + const tool = new SearchDocsTool({ httpRequest }); + const result = await tool.run({ query: 'map', limit: 2 }); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('403'); + expect(result.isError).toBe(false); + const text = result.content[0].text as string; + // Numbered list — should not contain "3." + expect(text).toContain('1.'); + expect(text).toContain('2.'); + expect(text).not.toContain('3.'); }); - it('returns error result on network failure', async () => { - const httpRequest = vi.fn().mockRejectedValue(new Error('Network error')); - const tool = new SearchDocsTool({ httpRequest }); + it('searches across multiple product sources', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(GL_JS_LLMS_TXT)); + if (url.includes('/api/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(API_LLMS_TXT)); + return Promise.resolve(makeLlmsTxtResponse('')); + }); - const result = await tool.run({ query: 'markers' }); + const tool = new SearchDocsTool({ httpRequest }); + const result = await tool.run({ query: 'directions', limit: 5 }); - expect(result.isError).toBe(true); - expect(result.content[0].text).toContain('Network error'); + expect(result.isError).toBe(false); + const text = result.content[0].text as string; + expect(text).toContain('Directions API'); + expect(text).toContain('API Reference'); }); - it('strips HTML highlight tags from content and hierarchy', async () => { - const httpRequest = vi.fn().mockResolvedValue( - makeResponse({ - hits: [ - { - url: 'https://docs.mapbox.com/api/search/geocoding/', - content: - 'Use the Geocoding API to convert addresses.', - hierarchy: { - lvl0: 'Geocoding API', - lvl1: 'Overview', - lvl2: null, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null - } - } - ] - }) - ); + it('deduplicates results when same URL appears in multiple sources', async () => { + const duplicate = `# Product A\n\n## Guides\n\n- [Geocoding Guide](https://docs.mapbox.com/api/search/geocoding.md): How to geocode.`; + const httpRequest = vi + .fn() + .mockImplementation(() => + Promise.resolve(makeLlmsTxtResponse(duplicate)) + ); + const tool = new SearchDocsTool({ httpRequest }); + const result = await tool.run({ query: 'geocoding', limit: 10 }); - const result = await tool.run({ query: 'geocoding' }); const text = result.content[0].text as string; - - expect(text).not.toContain(' { - const httpRequest = vi.fn().mockResolvedValue( - makeResponse({ - hits: [ - { - url: 'https://docs.mapbox.com/api/search/', - content: null, - hierarchy: { - lvl0: 'Search', - lvl1: null, - lvl2: null, - lvl3: null, - lvl4: null, - lvl5: null, - lvl6: null - } - } - ] - }) - ); + it('includes product name in each result', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(GL_JS_LLMS_TXT)); + return Promise.resolve(makeLlmsTxtResponse('')); + }); + const tool = new SearchDocsTool({ httpRequest }); + const result = await tool.run({ query: 'getting started' }); + + expect(result.content[0].text).toContain('Mapbox GL JS'); + }); + + it('gracefully skips sources that fail to fetch', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(GL_JS_LLMS_TXT)); + // All other sources fail + return Promise.reject(new Error('Network error')); + }); - const result = await tool.run({ query: 'search' }); + const tool = new SearchDocsTool({ httpRequest }); + const result = await tool.run({ query: 'marker' }); + // Should still return results from the working source expect(result.isError).toBe(false); - expect(result.content[0].text).toContain('Search'); - expect(result.content[0].text).not.toContain('undefined'); + expect(result.content[0].text).toContain('Markers'); }); - it('uses default limit of 5 when not specified', async () => { - const httpRequest = vi.fn().mockResolvedValue(makeResponse({ hits: [] })); + it('uses cached content on repeated searches without re-fetching', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeLlmsTxtResponse(GL_JS_LLMS_TXT)); + return Promise.resolve(makeLlmsTxtResponse('')); + }); + const tool = new SearchDocsTool({ httpRequest }); + await tool.run({ query: 'marker' }); - await tool.run({ query: 'navigation' }); + const callsAfterFirst = httpRequest.mock.calls.length; - expect(httpRequest).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ - body: JSON.stringify({ query: 'navigation', hitsPerPage: 5 }) - }) - ); + // Second search — all sources should be cached + await tool.run({ query: 'marker' }); + + expect(httpRequest.mock.calls.length).toBe(callsAfterFirst); }); }); diff --git a/test/utils/docFetcher.test.ts b/test/utils/docFetcher.test.ts index dc99c4b..fadf8f6 100644 --- a/test/utils/docFetcher.test.ts +++ b/test/utils/docFetcher.test.ts @@ -1,8 +1,13 @@ // Copyright (c) Mapbox, Inc. // Licensed under the MIT License. -import { describe, it, expect, vi, afterEach } from 'vitest'; -import { toMarkdownUrl, fetchDocContent } from '../../src/utils/docFetcher.js'; +import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest'; +import { + toMarkdownUrl, + fetchDocContent, + fetchCachedText +} from '../../src/utils/docFetcher.js'; +import { docCache } from '../../src/utils/docCache.js'; function makeResponse(body: string, status = 200): Response { return new Response(body, { @@ -14,6 +19,10 @@ function makeResponse(body: string, status = 200): Response { }); } +beforeEach(() => { + docCache.clear(); +}); + afterEach(() => { delete process.env.MAPBOX_DOCS_HOST_OVERRIDE; }); @@ -49,6 +58,65 @@ describe('toMarkdownUrl', () => { 'https://docs.tilestream.net/accounts/guides.md' ); }); + + it('returns null for .txt URLs (already a text file, no rewrite needed)', () => { + expect( + toMarkdownUrl('https://docs.mapbox.com/mapbox-gl-js/llms.txt') + ).toBeNull(); + expect( + toMarkdownUrl('https://docs.mapbox.com/mapbox-gl-js/llms-full.txt') + ).toBeNull(); + expect(toMarkdownUrl('https://docs.mapbox.com/api/llms.txt')).toBeNull(); + }); + + it('returns null for .md URLs (already markdown, no double-extension)', () => { + expect( + toMarkdownUrl('https://docs.mapbox.com/api/navigation/directions.md') + ).toBeNull(); + }); +}); + +describe('fetchCachedText', () => { + it('fetches and caches the response', async () => { + const content = '# Mapbox API\n\nSome content.'; + const httpRequest = vi.fn().mockResolvedValue( + new Response(content, { + status: 200, + headers: { + 'content-type': 'text/plain', + 'content-length': String(Buffer.byteLength(content, 'utf8')) + } + }) + ); + + const result = await fetchCachedText( + 'https://docs.mapbox.com/api/llms.txt', + httpRequest + ); + + expect(result).toBe(content); + expect(httpRequest).toHaveBeenCalledTimes(1); + + // Second call should use cache, not make another request + const result2 = await fetchCachedText( + 'https://docs.mapbox.com/api/llms.txt', + httpRequest + ); + expect(result2).toBe(content); + expect(httpRequest).toHaveBeenCalledTimes(1); + }); + + it('throws if the response is not ok', async () => { + const httpRequest = vi + .fn() + .mockResolvedValue( + new Response('Not Found', { status: 404, statusText: 'Not Found' }) + ); + + await expect( + fetchCachedText('https://docs.mapbox.com/api/llms.txt', httpRequest) + ).rejects.toThrow('Not Found'); + }); }); describe('fetchDocContent', () => { @@ -90,7 +158,7 @@ describe('fetchDocContent', () => { expect(httpRequest).toHaveBeenNthCalledWith( 2, 'https://docs.mapbox.com/accounts/guides', - expect.objectContaining({ headers: expect.anything() }) + {} ); expect(content).toBe(html); }); @@ -103,7 +171,7 @@ describe('fetchDocContent', () => { expect(httpRequest).toHaveBeenCalledTimes(1); expect(httpRequest).toHaveBeenCalledWith( 'https://api.mapbox.com/geocoding/v5', - expect.objectContaining({ headers: expect.anything() }) + {} ); }); diff --git a/test/utils/docsSearchIndex.test.ts b/test/utils/docsSearchIndex.test.ts new file mode 100644 index 0000000..970ea7b --- /dev/null +++ b/test/utils/docsSearchIndex.test.ts @@ -0,0 +1,149 @@ +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +import { describe, it, expect } from 'vitest'; +import { + parseLlmsTxtEntries, + searchDocs +} from '../../src/utils/docsSearchIndex.js'; +import { docCache } from '../../src/utils/docCache.js'; +import { vi, beforeEach } from 'vitest'; + +const SAMPLE_LLMS_TXT = `# Mapbox GL JS + +> Documentation for Mapbox GL JS + +## Guides + +- [Add your data to the map](https://docs.mapbox.com/mapbox-gl-js/guides/add-your-data.md): Mapbox GL JS offers several ways to add your data to the map. +- [Markers](https://docs.mapbox.com/mapbox-gl-js/guides/add-your-data/markers.md): Add interactive markers to your map with minimal code using the Marker class. +- [Getting Started](https://docs.mapbox.com/mapbox-gl-js/guides/get-started.md): Get started with Mapbox GL JS. + +## Examples + +- [Add a marker to the map](https://docs.mapbox.com/mapbox-gl-js/example/add-a-marker.md): Use a Marker to add a visual indicator to the map. + +## Other products + +- [Maps SDK for iOS](https://docs.mapbox.com/ios/maps/llms.txt) +`; + +beforeEach(() => { + docCache.clear(); +}); + +describe('parseLlmsTxtEntries', () => { + it('parses title, url, and description from entries', () => { + const entries = parseLlmsTxtEntries(SAMPLE_LLMS_TXT, 'Mapbox GL JS'); + + expect(entries).toContainEqual({ + title: 'Markers', + url: 'https://docs.mapbox.com/mapbox-gl-js/guides/add-your-data/markers.md', + description: + 'Add interactive markers to your map with minimal code using the Marker class.', + product: 'Mapbox GL JS' + }); + }); + + it('parses entries without a description', () => { + const content = `## Guides\n\n- [Getting Started](https://docs.mapbox.com/guide.md)`; + const entries = parseLlmsTxtEntries(content, 'Test'); + + expect(entries).toHaveLength(1); + expect(entries[0].description).toBe(''); + }); + + it('skips llms.txt index links (not doc pages)', () => { + const entries = parseLlmsTxtEntries(SAMPLE_LLMS_TXT, 'Mapbox GL JS'); + const urls = entries.map((e) => e.url); + + expect(urls.every((u) => !u.endsWith('/llms.txt'))).toBe(true); + expect(urls.every((u) => !u.endsWith('/llms-full.txt'))).toBe(true); + }); + + it('skips non-entry lines (headers, blank lines, descriptions)', () => { + const entries = parseLlmsTxtEntries(SAMPLE_LLMS_TXT, 'Mapbox GL JS'); + + // Only actual doc page links should be parsed (4 entries, 1 llms.txt skipped) + expect(entries).toHaveLength(4); + }); + + it('assigns the provided product name to all entries', () => { + const entries = parseLlmsTxtEntries(SAMPLE_LLMS_TXT, 'My Product'); + expect(entries.every((e) => e.product === 'My Product')).toBe(true); + }); +}); + +describe('searchDocs', () => { + function makeResponse(content: string): Response { + return new Response(content, { + status: 200, + headers: { + 'content-type': 'text/plain', + 'content-length': String(Buffer.byteLength(content, 'utf8')) + } + }); + } + + it('returns entries matching query words', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeResponse(SAMPLE_LLMS_TXT)); + return Promise.resolve(makeResponse('')); + }); + + const results = await searchDocs('marker', 5, httpRequest); + + expect(results.length).toBeGreaterThan(0); + expect(results.some((r) => r.title.toLowerCase().includes('marker'))).toBe( + true + ); + }); + + it('returns empty array when nothing matches', async () => { + const httpRequest = vi.fn().mockResolvedValue(makeResponse('')); + const results = await searchDocs('xyzzy impossible query', 5, httpRequest); + + expect(results).toHaveLength(0); + }); + + it('ranks title matches higher than description-only matches', async () => { + const content = + `## Guides\n\n` + + `- [Marker Guide](https://docs.mapbox.com/marker.md): How to use markers.\n` + + `- [Getting Started](https://docs.mapbox.com/start.md): Add a marker to your map using this guide.\n`; + + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeResponse(content)); + return Promise.resolve(makeResponse('')); + }); + + const results = await searchDocs('marker', 5, httpRequest); + + // "Marker Guide" (title match) should rank above "Getting Started" (description match) + expect(results[0].title).toBe('Marker Guide'); + }); + + it('deduplicates entries with the same URL', async () => { + const content = `## Guides\n\n- [Directions](https://docs.mapbox.com/api/directions.md): Routing API.`; + const httpRequest = vi.fn().mockResolvedValue(makeResponse(content)); + + const results = await searchDocs('directions', 10, httpRequest); + + const urls = results.map((r) => r.url); + const unique = new Set(urls); + expect(urls.length).toBe(unique.size); + }); + + it('respects the limit', async () => { + const httpRequest = vi.fn().mockImplementation((url: string) => { + if (url.includes('/mapbox-gl-js/llms.txt')) + return Promise.resolve(makeResponse(SAMPLE_LLMS_TXT)); + return Promise.resolve(makeResponse('')); + }); + + const results = await searchDocs('map', 2, httpRequest); + expect(results.length).toBeLessThanOrEqual(2); + }); +});