From bfc3ee46e3c494daf4148fb259e8e53a5ce1943d Mon Sep 17 00:00:00 2001 From: Edgars Date: Mon, 24 Aug 2026 19:26:13 +0100 Subject: [PATCH 1/2] fix(docs-mcp): detect true mirror duplicates --- docs-mcp/OPERATIONS.md | 7 ++- docs-mcp/smoke-test.mjs | 59 ++++++++++++++++++++++--- docs-mcp/smoke-test.test.mjs | 86 +++++++++++++++++++++++++++++++++++- 3 files changed, 144 insertions(+), 8 deletions(-) diff --git a/docs-mcp/OPERATIONS.md b/docs-mcp/OPERATIONS.md index b6491477..0d80d418 100644 --- a/docs-mcp/OPERATIONS.md +++ b/docs-mcp/OPERATIONS.md @@ -49,8 +49,11 @@ and allows the two serving replicas to run on different nodes. The previous EBS PVC is intentionally retained, unmounted, for short-term rollback evidence; it is not part of the active data path. -Rendered pages and their `.md` mirrors contain the same content. The scraper -excludes `.md` URLs so each canonical page is indexed once. +Rendered pages and their `.md` mirrors contain the same content. Direct `.md` +links are excluded so each canonical page is indexed once. The v3 scraper may +still use a Markdown representation discovered through `llms.txt`; the canary +therefore rejects an HTML/Markdown pair for the same page, not a lone Markdown +source URL. ## Health checks diff --git a/docs-mcp/smoke-test.mjs b/docs-mcp/smoke-test.mjs index 423eed13..9bb51b59 100644 --- a/docs-mcp/smoke-test.mjs +++ b/docs-mcp/smoke-test.mjs @@ -235,6 +235,51 @@ function validateTools(payload, expectedTool) { return tools.map((tool) => tool.name); } +function markdownMirrorIdentity(rawUrl) { + try { + const url = new URL(rawUrl); + let pathname = url.pathname; + + if (/\/index(?:\.html)?\.md$/i.test(pathname)) { + pathname = pathname.replace(/\/index(?:\.html)?\.md$/i, ""); + } else if (/\.html\.md$/i.test(pathname)) { + pathname = pathname.replace(/\.md$/i, ""); + } else if (/\.md$/i.test(pathname)) { + pathname = pathname.replace(/\.md$/i, ""); + } + + pathname = pathname.replace(/\/+$/, "") || "/"; + return `${url.origin}${pathname}`; + } catch { + return rawUrl + .replace(/(?:\/index(?:\.html)?)?\.md(?:[?#].*)?$/i, "") + .replace(/\/+$/, ""); + } +} + +function findMarkdownMirrorDuplicate(text) { + const resultUrls = [...text.matchAll(/^Result \d+:\s+(\S+)\s*$/gim)].map( + (match) => match[1], + ); + const seen = new Map(); + + for (const resultUrl of resultUrls) { + const identity = markdownMirrorIdentity(resultUrl); + const previousUrl = seen.get(identity); + if ( + previousUrl && + previousUrl !== resultUrl && + (/\.md(?:[?#]|$)/i.test(previousUrl) || + /\.md(?:[?#]|$)/i.test(resultUrl)) + ) { + return [previousUrl, resultUrl]; + } + seen.set(identity, resultUrl); + } + + return null; +} + function validateSearch(payload, search) { if (payload.result?.isError) { throw new Error("search_docs returned isError=true"); @@ -256,11 +301,15 @@ function validateSearch(payload, search) { `search_docs did not contain expected text ${JSON.stringify(search.expectedText)}`, ); } - if ( - search.rejectMarkdownMirror && - /Result \d+:\s+\S+\.md(?:[?#]\S*)?(?:\r?\n|$)/i.test(text) - ) { - throw new Error("search_docs returned a duplicate .md mirror result"); + const markdownMirrorDuplicate = search.rejectMarkdownMirror + ? findMarkdownMirrorDuplicate(text) + : null; + if (markdownMirrorDuplicate) { + throw new Error( + `search_docs returned duplicate HTML/Markdown mirror results: ${markdownMirrorDuplicate.join( + " and ", + )}`, + ); } if ( search.rejectUndefinedMetadata && diff --git a/docs-mcp/smoke-test.test.mjs b/docs-mcp/smoke-test.test.mjs index eb13a9e2..1c6045c2 100644 --- a/docs-mcp/smoke-test.test.mjs +++ b/docs-mcp/smoke-test.test.mjs @@ -184,7 +184,91 @@ test("rejects duplicate Markdown mirror search results", async () => { await assert.rejects( runSmokeTest({ endpoint: `${baseUrl}/mcp`, timeoutMs: 2_000, search }), - /duplicate \.md mirror result/, + /duplicate HTML\/Markdown mirror results/, + ); +}); + +test("allows a Markdown source when no HTML mirror is also returned", async () => { + const baseUrl = await listen(async (request, response) => { + const payload = await readJson(request); + if (payload.id === undefined) { + response.writeHead(202).end(); + return; + } + + response.writeHead(200, { "Content-Type": "application/json" }); + response.end( + JSON.stringify({ + jsonrpc: "2.0", + id: payload.id, + result: resultFor( + payload, + "Result 1: https://docs.genlayer.com/equivalence-principle/index.html.md\nEquivalence Principle", + ), + }), + ); + }); + + const result = await runSmokeTest({ + endpoint: `${baseUrl}/mcp`, + timeoutMs: 2_000, + search, + }); + + assert.equal(result.searchValidated, true); +}); + +test("rejects a canonical URL paired with the v3 Markdown variant", async () => { + const baseUrl = await listen(async (request, response) => { + const payload = await readJson(request); + if (payload.id === undefined) { + response.writeHead(202).end(); + return; + } + + response.writeHead(200, { "Content-Type": "application/json" }); + response.end( + JSON.stringify({ + jsonrpc: "2.0", + id: payload.id, + result: resultFor( + payload, + "Result 1: https://docs.genlayer.com/equivalence-principle\nEquivalence Principle\nResult 2: https://docs.genlayer.com/equivalence-principle/index.html.md\nEquivalence Principle", + ), + }), + ); + }); + + await assert.rejects( + runSmokeTest({ endpoint: `${baseUrl}/mcp`, timeoutMs: 2_000, search }), + /duplicate HTML\/Markdown mirror results/, + ); +}); + +test("rejects a root URL paired with index.md", async () => { + const baseUrl = await listen(async (request, response) => { + const payload = await readJson(request); + if (payload.id === undefined) { + response.writeHead(202).end(); + return; + } + + response.writeHead(200, { "Content-Type": "application/json" }); + response.end( + JSON.stringify({ + jsonrpc: "2.0", + id: payload.id, + result: resultFor( + payload, + "Result 1: https://docs.genlayer.com/\nEquivalence Principle\nResult 2: https://docs.genlayer.com/index.md\nEquivalence Principle", + ), + }), + ); + }); + + await assert.rejects( + runSmokeTest({ endpoint: baseUrl + "/mcp", timeoutMs: 2_000, search }), + /duplicate HTML\/Markdown mirror results/, ); }); From 9eaa334cfa4b3f4d8090230394dc1e76a1d69dec Mon Sep 17 00:00:00 2001 From: Edgars Date: Mon, 24 Aug 2026 19:43:37 +0100 Subject: [PATCH 2/2] fix(docs-mcp): force canonical HTML indexing --- .github/workflows/docs-mcp.yml | 4 +-- docs-mcp/OPERATIONS.md | 9 +++--- docs-mcp/entrypoint.sh | 1 + docs-mcp/entrypoint.test.mjs | 59 ++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 docs-mcp/entrypoint.test.mjs diff --git a/.github/workflows/docs-mcp.yml b/.github/workflows/docs-mcp.yml index 317fcee3..d7f1b763 100644 --- a/.github/workflows/docs-mcp.yml +++ b/.github/workflows/docs-mcp.yml @@ -31,8 +31,8 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Test MCP handshake canary - run: node --test docs-mcp/smoke-test.test.mjs + - name: Test MCP canaries + run: node --test docs-mcp/*.test.mjs - name: Log in to GitHub Container Registry if: github.event_name == 'push' diff --git a/docs-mcp/OPERATIONS.md b/docs-mcp/OPERATIONS.md index 0d80d418..823db02e 100644 --- a/docs-mcp/OPERATIONS.md +++ b/docs-mcp/OPERATIONS.md @@ -49,10 +49,11 @@ and allows the two serving replicas to run on different nodes. The previous EBS PVC is intentionally retained, unmounted, for short-term rollback evidence; it is not part of the active data path. -Rendered pages and their `.md` mirrors contain the same content. Direct `.md` -links are excluded so each canonical page is indexed once. The v3 scraper may -still use a Markdown representation discovered through `llms.txt`; the canary -therefore rejects an HTML/Markdown pair for the same page, not a lone Markdown +Rendered pages and their `.md` mirrors contain the same content. The docs +scrape explicitly requests HTML and excludes direct `.md` links so each +canonical page is indexed once. Without the explicit `Accept` header, the v3 +scraper can store a Markdown final URL after its pre-fetch exclusion check. The +canary rejects an HTML/Markdown pair for the same page, not a lone Markdown source URL. ## Health checks diff --git a/docs-mcp/entrypoint.sh b/docs-mcp/entrypoint.sh index f2c7ae40..a46bdb63 100644 --- a/docs-mcp/entrypoint.sh +++ b/docs-mcp/entrypoint.sh @@ -23,6 +23,7 @@ index_docs() { --max-pages "$MAX_PAGES" \ --max-depth "$MAX_DEPTH" \ --max-concurrency "$MAX_CONCURRENCY" \ + --header 'Accept: text/html' \ --exclude-pattern '/full-documentation\.txt/' \ --exclude-pattern '/\.md(?:\?.*)?$/' \ --exclude-pattern '/developers/intelligent-contracts/?$/' \ diff --git a/docs-mcp/entrypoint.test.mjs b/docs-mcp/entrypoint.test.mjs new file mode 100644 index 00000000..dcb908c1 --- /dev/null +++ b/docs-mcp/entrypoint.test.mjs @@ -0,0 +1,59 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { + chmod, + mkdtemp, + mkdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import test from "node:test"; + +const execFileAsync = promisify(execFile); +const entrypoint = join(dirname(fileURLToPath(import.meta.url)), "entrypoint.sh"); + +test("indexes docs as HTML through the image-owned entrypoint", async (t) => { + const root = await mkdtemp(join(tmpdir(), "docs-mcp-entrypoint-")); + const bin = join(root, "bin"); + const store = join(root, "data"); + const callsFile = join(root, "calls.log"); + const fakeServer = join(bin, "docs-mcp-server"); + t.after(() => rm(root, { recursive: true, force: true })); + + await mkdir(bin); + await writeFile( + fakeServer, + `#!/bin/sh +for arg in "$@"; do + printf '[%s]' "$arg" >> "$CALLS_FILE" +done +printf '\n' >> "$CALLS_FILE" +`, + ); + await chmod(fakeServer, 0o755); + + await execFileAsync("/bin/sh", [entrypoint], { + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + CALLS_FILE: callsFile, + MODE: "index", + STORE_PATH: store, + DOCS_URL: "https://docs.example.test", + SDK_URL: "https://sdk.example.test/main/", + }, + }); + + const calls = (await readFile(callsFile, "utf8")).trim().split("\n"); + assert.equal(calls.length, 4); + assert.match(calls[0], /^\[scrape\]\[genlayer-docs\]/); + assert.match(calls[0], /\[--header\]\[Accept: text\/html\]/); + assert.match(calls[0], /\[--exclude-pattern\]\[\/\\\.md/); + assert.match(calls[1], /^\[scrape\]\[genlayer-sdk\]/); + assert.doesNotMatch(calls[1], /\[--header\]/); +});