fix(docs-mcp): detect true mirror duplicates

.github/workflows/docs-mcp.yml

@@ -31,8 +31,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Test MCP handshake canary
-        run: node --test docs-mcp/smoke-test.test.mjs
+      - name: Test MCP canaries
+        run: node --test docs-mcp/*.test.mjs
 
       - name: Log in to GitHub Container Registry
         if: github.event_name == 'push'

docs-mcp/OPERATIONS.md

@@ -49,8 +49,12 @@ and allows the two serving replicas to run on different nodes. The previous
 EBS PVC is intentionally retained, unmounted, for short-term rollback evidence;
 it is not part of the active data path.
 
-Rendered pages and their `.md` mirrors contain the same content. The scraper
-excludes `.md` URLs so each canonical page is indexed once.
+Rendered pages and their `.md` mirrors contain the same content. The docs
+scrape explicitly requests HTML and excludes direct `.md` links so each
+canonical page is indexed once. Without the explicit `Accept` header, the v3
+scraper can store a Markdown final URL after its pre-fetch exclusion check. The
+canary rejects an HTML/Markdown pair for the same page, not a lone Markdown
+source URL.
 
 ## Health checks
 

docs-mcp/entrypoint.sh

@@ -23,6 +23,7 @@ index_docs() {
     --max-pages "$MAX_PAGES" \
     --max-depth "$MAX_DEPTH" \
     --max-concurrency "$MAX_CONCURRENCY" \
+    --header 'Accept: text/html' \
     --exclude-pattern '/full-documentation\.txt/' \
     --exclude-pattern '/\.md(?:\?.*)?$/' \
     --exclude-pattern '/developers/intelligent-contracts/?$/' \

docs-mcp/entrypoint.test.mjs

@@ -0,0 +1,59 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import {
+  chmod,
+  mkdtemp,
+  mkdir,
+  readFile,
+  rm,
+  writeFile,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import test from "node:test";
+
+const execFileAsync = promisify(execFile);
+const entrypoint = join(dirname(fileURLToPath(import.meta.url)), "entrypoint.sh");
+
+test("indexes docs as HTML through the image-owned entrypoint", async (t) => {
+  const root = await mkdtemp(join(tmpdir(), "docs-mcp-entrypoint-"));
+  const bin = join(root, "bin");
+  const store = join(root, "data");
+  const callsFile = join(root, "calls.log");
+  const fakeServer = join(bin, "docs-mcp-server");
+  t.after(() => rm(root, { recursive: true, force: true }));
+
+  await mkdir(bin);
+  await writeFile(
+    fakeServer,
+    `#!/bin/sh
+for arg in "$@"; do
+  printf '[%s]' "$arg" >> "$CALLS_FILE"
+done
+printf '\n' >> "$CALLS_FILE"
+`,
+  );
+  await chmod(fakeServer, 0o755);
+
+  await execFileAsync("/bin/sh", [entrypoint], {
+    env: {
+      ...process.env,
+      PATH: `${bin}:${process.env.PATH}`,
+      CALLS_FILE: callsFile,
+      MODE: "index",
+      STORE_PATH: store,
+      DOCS_URL: "https://docs.example.test",
+      SDK_URL: "https://sdk.example.test/main/",
+    },
+  });
+
+  const calls = (await readFile(callsFile, "utf8")).trim().split("\n");
+  assert.equal(calls.length, 4);
+  assert.match(calls[0], /^\[scrape\]\[genlayer-docs\]/);
+  assert.match(calls[0], /\[--header\]\[Accept: text\/html\]/);
+  assert.match(calls[0], /\[--exclude-pattern\]\[\/\\\.md/);
+  assert.match(calls[1], /^\[scrape\]\[genlayer-sdk\]/);
+  assert.doesNotMatch(calls[1], /\[--header\]/);
+});

docs-mcp/smoke-test.mjs

@@ -235,6 +235,51 @@ function validateTools(payload, expectedTool) {
   return tools.map((tool) => tool.name);
 }
 
+function markdownMirrorIdentity(rawUrl) {
+  try {
+    const url = new URL(rawUrl);
+    let pathname = url.pathname;
+
+    if (/\/index(?:\.html)?\.md$/i.test(pathname)) {
+      pathname = pathname.replace(/\/index(?:\.html)?\.md$/i, "");
+    } else if (/\.html\.md$/i.test(pathname)) {
+      pathname = pathname.replace(/\.md$/i, "");
+    } else if (/\.md$/i.test(pathname)) {
+      pathname = pathname.replace(/\.md$/i, "");
+    }
+
+    pathname = pathname.replace(/\/+$/, "") || "/";
+    return `${url.origin}${pathname}`;
+  } catch {
+    return rawUrl
+      .replace(/(?:\/index(?:\.html)?)?\.md(?:[?#].*)?$/i, "")
+      .replace(/\/+$/, "");
+  }
+}
+
+function findMarkdownMirrorDuplicate(text) {
+  const resultUrls = [...text.matchAll(/^Result \d+:\s+(\S+)\s*$/gim)].map(
+    (match) => match[1],
+  );
+  const seen = new Map();
+
+  for (const resultUrl of resultUrls) {
+    const identity = markdownMirrorIdentity(resultUrl);
+    const previousUrl = seen.get(identity);
+    if (
+      previousUrl &&
+      previousUrl !== resultUrl &&
+      (/\.md(?:[?#]|$)/i.test(previousUrl) ||
+        /\.md(?:[?#]|$)/i.test(resultUrl))
+    ) {
+      return [previousUrl, resultUrl];
+    }
+    seen.set(identity, resultUrl);
+  }
+
+  return null;
+}
+
 function validateSearch(payload, search) {
   if (payload.result?.isError) {
     throw new Error("search_docs returned isError=true");
@@ -256,11 +301,15 @@ function validateSearch(payload, search) {
       `search_docs did not contain expected text ${JSON.stringify(search.expectedText)}`,
     );
   }
-  if (
-    search.rejectMarkdownMirror &&
-    /Result \d+:\s+\S+\.md(?:[?#]\S*)?(?:\r?\n|$)/i.test(text)
-  ) {
-    throw new Error("search_docs returned a duplicate .md mirror result");
+  const markdownMirrorDuplicate = search.rejectMarkdownMirror
+    ? findMarkdownMirrorDuplicate(text)
+    : null;
+  if (markdownMirrorDuplicate) {
+    throw new Error(
+      `search_docs returned duplicate HTML/Markdown mirror results: ${markdownMirrorDuplicate.join(
+        " and ",
+      )}`,
+    );
   }
   if (
     search.rejectUndefinedMetadata &&

docs-mcp/smoke-test.test.mjs

@@ -184,7 +184,91 @@ test("rejects duplicate Markdown mirror search results", async () => {
 
   await assert.rejects(
     runSmokeTest({ endpoint: `${baseUrl}/mcp`, timeoutMs: 2_000, search }),
-    /duplicate \.md mirror result/,
+    /duplicate HTML\/Markdown mirror results/,
+  );
+});
+
+test("allows a Markdown source when no HTML mirror is also returned", async () => {
+  const baseUrl = await listen(async (request, response) => {
+    const payload = await readJson(request);
+    if (payload.id === undefined) {
+      response.writeHead(202).end();
+      return;
+    }
+
+    response.writeHead(200, { "Content-Type": "application/json" });
+    response.end(
+      JSON.stringify({
+        jsonrpc: "2.0",
+        id: payload.id,
+        result: resultFor(
+          payload,
+          "Result 1: https://docs.genlayer.com/equivalence-principle/index.html.md\nEquivalence Principle",
+        ),
+      }),
+    );
+  });
+
+  const result = await runSmokeTest({
+    endpoint: `${baseUrl}/mcp`,
+    timeoutMs: 2_000,
+    search,
+  });
+
+  assert.equal(result.searchValidated, true);
+});
+
+test("rejects a canonical URL paired with the v3 Markdown variant", async () => {
+  const baseUrl = await listen(async (request, response) => {
+    const payload = await readJson(request);
+    if (payload.id === undefined) {
+      response.writeHead(202).end();
+      return;
+    }
+
+    response.writeHead(200, { "Content-Type": "application/json" });
+    response.end(
+      JSON.stringify({
+        jsonrpc: "2.0",
+        id: payload.id,
+        result: resultFor(
+          payload,
+          "Result 1: https://docs.genlayer.com/equivalence-principle\nEquivalence Principle\nResult 2: https://docs.genlayer.com/equivalence-principle/index.html.md\nEquivalence Principle",
+        ),
+      }),
+    );
+  });
+
+  await assert.rejects(
+    runSmokeTest({ endpoint: `${baseUrl}/mcp`, timeoutMs: 2_000, search }),
+    /duplicate HTML\/Markdown mirror results/,
+  );
+});
+
+test("rejects a root URL paired with index.md", async () => {
+  const baseUrl = await listen(async (request, response) => {
+    const payload = await readJson(request);
+    if (payload.id === undefined) {
+      response.writeHead(202).end();
+      return;
+    }
+
+    response.writeHead(200, { "Content-Type": "application/json" });
+    response.end(
+      JSON.stringify({
+        jsonrpc: "2.0",
+        id: payload.id,
+        result: resultFor(
+          payload,
+          "Result 1: https://docs.genlayer.com/\nEquivalence Principle\nResult 2: https://docs.genlayer.com/index.md\nEquivalence Principle",
+        ),
+      }),
+    );
+  });
+
+  await assert.rejects(
+    runSmokeTest({ endpoint: baseUrl + "/mcp", timeoutMs: 2_000, search }),
+    /duplicate HTML\/Markdown mirror results/,
   );
 });