From 747e9357c7f166ac0721399ba9c211de2e2a122a Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Tue, 5 May 2026 09:19:26 -1000 Subject: [PATCH 1/2] feat: improve agent-friendly docs score (AFDocs quick wins) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add llms.txt directive to all HTML pages (visually-hidden div in header) and all generated .md files (blockquote at top) - Add Astro middleware for Accept: text/markdown content negotiation, using existing shouldServeMarkdown() helper - Fix llms.txt coverage: university/** → guides/** (directory was renamed), add community pages to Support customSet - Add MCP discovery file at .well-known/mcp.json pointing to Kapa MCP - Verify already appears early in — no repositioning needed Checks addressed: llms-txt-directive-html (FAIL → PASS) llms-txt-directive-md (FAIL → PASS) content-negotiation (FAIL → PASS) content-start-position (mitigated via Tasks 1+2) llms-txt-coverage (80% → improved) MCP Server Discoverable (FAIL → PASS) Co-Authored-By: Oz --- astro.config.mjs | 17 ++++++---- public/.well-known/mcp.json | 6 ++++ src/components/CustomHeader.astro | 4 +++ src/integrations/docs-markdown-integration.js | 5 ++- src/middleware.ts | 32 +++++++++++++++++++ 5 files changed, 56 insertions(+), 8 deletions(-) create mode 100644 public/.well-known/mcp.json create mode 100644 src/middleware.ts diff --git a/astro.config.mjs b/astro.config.mjs index f66ae03..f330868 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -139,9 +139,9 @@ export default defineConfig({ // widely consumed by AI agents. starlightLlmsTxt({ projectName: 'Warp', - // Excludes open-source-licenses from llms-full.txt and llms-small.txt. - // The file is ~25k lines and causes a stack overflow in hast-util-to-text. - // llms-custom sets exclude it separately via explicit path enumeration below. + // Excludes pages from llms-small.txt that cause a stack overflow + // in hast-util-to-text due to their size. Note: the `exclude` + // option only applies to llms-small.txt, not llms-full.txt. exclude: ['support-and-community/community/open-source-licenses'], description: 'Documentation for Warp, the agentic development environment, and Oz, Warp\'s programmable agent for running and coordinating agents at scale.', @@ -153,10 +153,13 @@ export default defineConfig({ { label: 'Getting Started', description: 'Installation, quickstart, and migration guides.', paths: ['getting-started/**'] }, { label: 'Knowledge and Collaboration', description: 'Warp Drive, teams, and the Admin Panel.', paths: ['knowledge-and-collaboration/**'] }, { label: 'Reference', description: 'CLI and API reference.', paths: ['reference/**'] }, - // Excludes support-and-community/community/ — open-source-licenses.mdx is ~25k - // lines and causes a stack overflow in hast-util-to-text during llms-txt generation. - { label: 'Support', description: 'Troubleshooting, billing, and privacy.', paths: ['support-and-community/index', 'support-and-community/plans-and-billing/**', 'support-and-community/privacy-and-security/**', 'support-and-community/troubleshooting-and-support/**'] }, - { label: 'Guides (Warp University)', description: 'Task-oriented walkthroughs.', paths: ['university/**'] }, + // Includes all support-and-community/ pages except open-source-licenses.mdx + // (excluded globally above — ~25k lines causes a stack overflow in hast-util-to-text). + { label: 'Support', description: 'Troubleshooting, billing, and privacy.', paths: ['support-and-community/index', 'support-and-community/plans-and-billing/**', 'support-and-community/privacy-and-security/**', 'support-and-community/troubleshooting-and-support/**', 'support-and-community/community/contributing', 'support-and-community/community/open-source-partnership', 'support-and-community/community/refer-a-friend'] }, + { label: 'Guides', description: 'Task-oriented walkthroughs and tutorials.', paths: ['guides/**'] }, + // Changelog excluded — the single page (4k lines) causes a stack overflow + // in hast-util-to-text during llms-full.txt generation. The page is still + // available at /changelog/ and indexed by the sitemap. ], }), ], diff --git a/public/.well-known/mcp.json b/public/.well-known/mcp.json new file mode 100644 index 0000000..c371a07 --- /dev/null +++ b/public/.well-known/mcp.json @@ -0,0 +1,6 @@ +{ + "name": "Warp Documentation", + "description": "Search and retrieve Warp documentation — the agentic development environment and Oz, Warp's programmable agent platform.", + "url": "https://warp.mcp.kapa.ai/sse", + "transport": "sse" +} diff --git a/src/components/CustomHeader.astro b/src/components/CustomHeader.astro index 59eac2e..276b204 100644 --- a/src/components/CustomHeader.astro +++ b/src/components/CustomHeader.astro @@ -18,6 +18,10 @@ import ThemeSelect from 'virtual:starlight/components/ThemeSelect'; import WarpTopicNav from './WarpTopicNav.astro'; --- +
diff --git a/src/integrations/docs-markdown-integration.js b/src/integrations/docs-markdown-integration.js index 81ce540..76c43d7 100644 --- a/src/integrations/docs-markdown-integration.js +++ b/src/integrations/docs-markdown-integration.js @@ -88,7 +88,10 @@ function convertHtmlToMarkdown(html) { const clone = /** @type {HTMLElement} */ (contentRoot.cloneNode(true)); sanitizeRoot(clone); const markdownBody = turndown.turndown(clone.innerHTML).trim(); - const sections = [`# ${normalizeWhitespace(title)}`]; + const llmsDirective = + '> For the complete documentation index, see [llms.txt](/llms.txt).\n' + + '> Markdown versions of each page are available by appending .md to any URL.'; + const sections = [llmsDirective, `# ${normalizeWhitespace(title)}`]; if (description) { sections.push(description); diff --git a/src/middleware.ts b/src/middleware.ts new file mode 100644 index 0000000..5375916 --- /dev/null +++ b/src/middleware.ts @@ -0,0 +1,32 @@ +import { defineMiddleware } from 'astro:middleware'; +import { + shouldServeMarkdown, + isEligibleDocHtmlPath, + getMarkdownPathFromHtmlPath, +} from './lib/docs-markdown.js'; + +/** + * Content negotiation middleware. + * + * When an agent sends `Accept: text/markdown` (or is identified by user-agent), + * rewrite the request to the pre-rendered `.md` variant of the page. This lets + * agents like Claude Code, Cursor, and OpenCode get clean markdown without + * needing to discover the `.md` URL convention first. + * + * The `shouldServeMarkdown` helper handles both explicit Accept-header + * negotiation and user-agent fallback detection (see `src/lib/docs-markdown.js`). + */ +export const onRequest = defineMiddleware(async (context, next) => { + const { request, url } = context; + + if (!isEligibleDocHtmlPath(url.pathname)) { + return next(); + } + + if (!shouldServeMarkdown(request)) { + return next(); + } + + const mdPath = getMarkdownPathFromHtmlPath(url.pathname); + return context.rewrite(mdPath); +}); From d93f0a1b275d73b2d502ed03f7f5a54835c1b98e Mon Sep 17 00:00:00 2001 From: Rachael Rose Renk <91027132+rachaelrenk@users.noreply.github.com> Date: Tue, 5 May 2026 12:24:07 -1000 Subject: [PATCH 2/2] fix: use correct Kapa MCP base URL (remove /sse path) The Kapa MCP server uses OAuth authentication at the base URL, not an SSE endpoint at /sse. Remove the incorrect path and transport field. Co-Authored-By: Oz --- public/.well-known/mcp.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/.well-known/mcp.json b/public/.well-known/mcp.json index c371a07..a522985 100644 --- a/public/.well-known/mcp.json +++ b/public/.well-known/mcp.json @@ -1,6 +1,5 @@ { "name": "Warp Documentation", "description": "Search and retrieve Warp documentation — the agentic development environment and Oz, Warp's programmable agent platform.", - "url": "https://warp.mcp.kapa.ai/sse", - "transport": "sse" + "url": "https://warp.mcp.kapa.ai" }