From a7c57547fa028555e08656514dcc2c9f4df1c7c1 Mon Sep 17 00:00:00 2001 From: Alex Patterson Date: Thu, 22 Oct 2026 13:06:16 -0400 Subject: [PATCH 1/3] fix(site): allow /api/og/ and social card crawlers in robots.txt --- apps/site/src/pages/robots.txt.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/site/src/pages/robots.txt.ts b/apps/site/src/pages/robots.txt.ts index f161a269..d8444616 100644 --- a/apps/site/src/pages/robots.txt.ts +++ b/apps/site/src/pages/robots.txt.ts @@ -22,6 +22,7 @@ export const GET: APIRoute = async ({ locals }) => { ? `User-Agent: * Content-Signal: ai-train=no, search=yes, ai-input=yes Allow: / +Allow: /api/og/ Allow: /api/search Allow: /api/mcp Allow: /.well-known/ @@ -33,6 +34,15 @@ Allow: /pricing.md Disallow: /api/ Disallow: /dashboard/ +User-Agent: Twitterbot +Allow: / + +User-Agent: facebookexternalhit +Allow: / + +User-Agent: LinkedInBot +Allow: / + User-Agent: GPTBot Allow: / From c84d03b57a0f8694dcef596eb54d6178345e5af8 Mon Sep 17 00:00:00 2001 From: Alex Patterson Date: Thu, 22 Oct 2026 13:18:09 -0400 Subject: [PATCH 2/3] fix(site): remove Disallow /api/ so all social, search, and AI bots can crawl OG cards and APIs --- apps/site/src/pages/robots.txt.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/apps/site/src/pages/robots.txt.ts b/apps/site/src/pages/robots.txt.ts index d8444616..72835f69 100644 --- a/apps/site/src/pages/robots.txt.ts +++ b/apps/site/src/pages/robots.txt.ts @@ -20,7 +20,7 @@ export const GET: APIRoute = async ({ locals }) => { const body = isProduction ? `User-Agent: * -Content-Signal: ai-train=no, search=yes, ai-input=yes +Content-Signal: ai-train=yes, search=yes, ai-input=yes Allow: / Allow: /api/og/ Allow: /api/search @@ -31,12 +31,21 @@ Allow: /llms-full.txt Allow: /auth.md Allow: /AGENTS.md Allow: /pricing.md -Disallow: /api/ Disallow: /dashboard/ +Disallow: /guest-portal/ User-Agent: Twitterbot Allow: / +User-Agent: Redditbot +Allow: / + +User-Agent: Discordbot +Allow: / + +User-Agent: Slackbot +Allow: / + User-Agent: facebookexternalhit Allow: / From 94934f1df02313ce2aca765ad5b520a9083a40af Mon Sep 17 00:00:00 2001 From: Alex Patterson Date: Thu, 22 Oct 2026 13:21:25 -0400 Subject: [PATCH 3/3] test(site): update baseline robots.txt contract for open social & AI crawling --- apps/site/baseline/robots.txt | 23 +++++++++++++++++-- apps/site/tests/tier1-features.test.mjs | 5 ++-- .../tests/tier4-baseline-contract.test.mjs | 4 ++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/apps/site/baseline/robots.txt b/apps/site/baseline/robots.txt index cc5da2d8..601c2456 100644 --- a/apps/site/baseline/robots.txt +++ b/apps/site/baseline/robots.txt @@ -1,6 +1,7 @@ User-Agent: * -Content-Signal: ai-train=no, search=yes, ai-input=yes +Content-Signal: ai-train=yes, search=yes, ai-input=yes Allow: / +Allow: /api/og/ Allow: /api/search Allow: /api/mcp Allow: /.well-known/ @@ -9,8 +10,26 @@ Allow: /llms-full.txt Allow: /auth.md Allow: /AGENTS.md Allow: /pricing.md -Disallow: /api/ Disallow: /dashboard/ +Disallow: /guest-portal/ + +User-Agent: Twitterbot +Allow: / + +User-Agent: Redditbot +Allow: / + +User-Agent: Discordbot +Allow: / + +User-Agent: Slackbot +Allow: / + +User-Agent: facebookexternalhit +Allow: / + +User-Agent: LinkedInBot +Allow: / User-Agent: GPTBot Allow: / diff --git a/apps/site/tests/tier1-features.test.mjs b/apps/site/tests/tier1-features.test.mjs index a04b5cd6..47ab84ec 100644 --- a/apps/site/tests/tier1-features.test.mjs +++ b/apps/site/tests/tier1-features.test.mjs @@ -653,15 +653,16 @@ describe("Tier 1 - Feature Coverage", () => { ); }); - it("4.2 Production robots allows root and disallows /api/ and /dashboard/", async () => { + it("4.2 Production robots allows root and /api/og/ and disallows /dashboard/ and /guest-portal/", async () => { const locals = { siteUrl: new URL("https://codingcat.dev") }; const res = await robotsModule.GET({ locals }); const body = await res.text(); assert.ok(body.includes("User-Agent: *")); assert.ok(body.includes("Allow: /")); - assert.ok(body.includes("Disallow: /api/")); + assert.ok(body.includes("Allow: /api/og/")); assert.ok(body.includes("Disallow: /dashboard/")); + assert.ok(body.includes("Disallow: /guest-portal/")); assert.ok(body.includes("Host: https://codingcat.dev")); assert.ok(body.includes("Sitemap: https://codingcat.dev/sitemap.xml")); }); diff --git a/apps/site/tests/tier4-baseline-contract.test.mjs b/apps/site/tests/tier4-baseline-contract.test.mjs index 69e0a7fa..2d98ab3e 100644 --- a/apps/site/tests/tier4-baseline-contract.test.mjs +++ b/apps/site/tests/tier4-baseline-contract.test.mjs @@ -320,10 +320,10 @@ describe("Tier 4 - Real-World Application & Baseline Contract", () => { const baselineRobots = loadBaseline("robots.txt"); const robotsModule = loadModule("src/pages/robots.txt.ts"); - it("4.1 Baseline robots contains User-Agent: * block with /api/ and /dashboard/ disallowed", () => { + it("4.1 Baseline robots contains User-Agent: * block with /api/og/ allowed and /dashboard/ disallowed", () => { assert.ok(baselineRobots.includes("User-Agent: *")); assert.ok(baselineRobots.includes("Allow: /")); - assert.ok(baselineRobots.includes("Disallow: /api/")); + assert.ok(baselineRobots.includes("Allow: /api/og/")); assert.ok(baselineRobots.includes("Disallow: /dashboard/")); });