From f177dfd311e63ad7e2babcb3f07332dc5f97b0dd Mon Sep 17 00:00:00 2001 From: Ayesh Rodrigo Date: Tue, 27 Oct 2026 10:33:51 +1300 Subject: [PATCH] Links in the page HTML use the served address, and lab tool links resolve from the site root Content links written into each page's HTML had no trailing slash, so a crawler followed a redirect for each one. Links to the lab tool files ("lab/lab") resolved against the page's own path and ended in a 404; they now point to /lab/ and carry rel=nofollow. Co-Authored-By: Claude Opus 5.5 --- packages/engine/src/components/prose/elements.jsx | 4 ++++ scripts/prerender.mjs | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/engine/src/components/prose/elements.jsx b/packages/engine/src/components/prose/elements.jsx index 860a90c..21a7a07 100644 --- a/packages/engine/src/components/prose/elements.jsx +++ b/packages/engine/src/components/prose/elements.jsx @@ -30,5 +30,9 @@ export function Anchor({ href = '', onClick, ...props }) { }; return ; } + if (href.startsWith('lab/')) { + // Files of the lab tools live at the site root, whatever page links to them. + return ; + } return ; } diff --git a/scripts/prerender.mjs b/scripts/prerender.mjs index d30f305..260320a 100644 --- a/scripts/prerender.mjs +++ b/scripts/prerender.mjs @@ -40,7 +40,14 @@ const TEXT_TAGS = new Set(['p', 'h2', 'h3', 'h4', 'ul', 'ol', 'li', 'strong', 'e /** Content links: "#/ch02/x" is a page of the same program, "#heading" a heading on this page. */ function link(href, programId) { if (typeof href !== 'string') return null; - if (href.startsWith('#/')) return `${base}${programId ? `${programId}/` : ''}${href.slice(2)}`.replace(/\/\/+/g, '/'); + if (href.startsWith('#/')) { + // The address the host serves, with its trailing slash, so a crawler never follows a redirect. + const [route, heading] = href.slice(2).split('#'); + const path = `${base}${programId ? `${programId}/` : ''}${route}`.replace(/\/\/+/g, '/').replace(/\/?$/, '/'); + return heading ? `${path}#${heading}` : path; + } + // Files of the lab tools ("lab/lab"): from the site root, whatever page links to them. + if (href.startsWith('lab/')) return `${base}${href}`; if (href.startsWith('#') || /^https:\/\//.test(href) || href.startsWith('mailto:')) return href; return null; } @@ -63,7 +70,10 @@ function renderNode(node, ctx) { const tag = node.tag; if (tag === 'a') { const href = link(node.attrs?.href, ctx.programId); - return href ? `${render(node.c, ctx)}` : render(node.c, ctx); + if (!href) return render(node.c, ctx); + // Lab tool files are for the lab command, not for search engines (robots.txt keeps them out). + const rel = href.startsWith(`${base}lab/`) ? ' rel="nofollow"' : ''; + return `${render(node.c, ctx)}`; } if (tag === 'pre') return `
${escape(text(node.c))}
`; if (!TEXT_TAGS.has(tag)) return render(node.c, ctx);