[codex] fix site crawler checks

package.json

@@ -19,6 +19,7 @@
     "lint": "oxlint . --config oxlint.json",
     "format": "oxfmt --write .",
     "format:check": "oxfmt --check .",
+    "website:smoke": "node scripts/website-smoke.mjs",
     "test": "vitest run",
     "pack:smoke": "node scripts/package-smoke.mjs"
   },

scripts/website-smoke.mjs

@@ -0,0 +1,109 @@
+import { readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+
+const root = process.cwd();
+const website = join(root, "website");
+const failures = [];
+
+function fail(message) {
+  failures.push(message);
+}
+
+async function mustRead(relativePath) {
+  try {
+    return await readFile(join(root, relativePath), "utf8");
+  } catch {
+    fail(`missing ${relativePath}`);
+    return "";
+  }
+}
+
+function stripTags(value) {
+  return value
+    .replace(/<br\s*\/?>/giu, " ")
+    .replace(/<[^>]+>/gu, "")
+    .replace(/\s+/gu, " ")
+    .trim();
+}
+
+function extractFirst(html, pattern, label) {
+  const match = html.match(pattern);
+  if (!match) {
+    fail(`missing ${label}`);
+    return "";
+  }
+  return match[1] || "";
+}
+
+const html = await mustRead("website/index.html");
+const robots = await mustRead("website/robots.txt");
+const sitemap = await mustRead("website/sitemap.xml");
+const headers = await mustRead("website/_headers");
+
+const title = stripTags(extractFirst(html, /<title>([\s\S]*?)<\/title>/iu, "title"));
+if (title !== "Clawpatch — Automated Code Review") {
+  fail(`unexpected title: ${title}`);
+}
+
+const description = html.match(/<meta\s+name="description"\s+content="([^"]+)"/iu)?.[1] || "";
+if (!description.includes("Automated code review that lands fixes")) {
+  fail("meta description does not contain the product promise");
+}
+
+const h1 = stripTags(extractFirst(html, /<h1>([\s\S]*?)<\/h1>/iu, "h1"));
+if (h1 !== "Code review with explicit fixes") {
+  fail(`unexpected h1 text: ${h1}`);
+}
+
+const ids = new Set([...html.matchAll(/\sid="([^"]+)"/giu)].map((match) => match[1]));
+const anchorLinks = [...html.matchAll(/href="#([^"]+)"/giu)].map((match) => match[1]);
+for (const id of anchorLinks) {
+  if (!ids.has(id)) fail(`missing anchor target: #${id}`);
+}
+
+if (!robots.includes("Sitemap: https://clawpatch.ai/sitemap.xml")) {
+  fail("robots.txt missing sitemap reference");
+}
+
+if (!sitemap.includes("<loc>https://clawpatch.ai/</loc>")) {
+  fail("sitemap.xml missing canonical homepage loc");
+}
+
+for (const expectedHeader of [
+  "Strict-Transport-Security",
+  "X-Content-Type-Options",
+  "X-Frame-Options",
+  "Referrer-Policy",
+  "Permissions-Policy",
+  "Content-Security-Policy",
+]) {
+  if (!headers.includes(expectedHeader)) {
+    fail(`_headers missing ${expectedHeader}`);
+  }
+}
+
+const socialCard = await readFile(join(website, "social-card.png"));
+if (socialCard.toString("ascii", 1, 4) !== "PNG") {
+  fail("social-card.png is not a PNG");
+} else {
+  const width = socialCard.readUInt32BE(16);
+  const height = socialCard.readUInt32BE(20);
+  if (width !== 1200 || height !== 630) {
+    fail(`social-card.png dimensions are ${width}x${height}, expected 1200x630`);
+  }
+}
+
+for (const file of ["website/favicon.svg", "website/CNAME", "website/.nojekyll"]) {
+  try {
+    await stat(join(root, file));
+  } catch {
+    fail(`missing ${file}`);
+  }
+}
+
+if (failures.length) {
+  console.error(failures.join("\n"));
+  process.exit(1);
+}
+
+console.log("Website smoke checks passed.");

src/exec.test.ts

@@ -92,7 +92,7 @@ describe("runCommandArgs", () => {
           "import { writeFileSync } from 'node:fs';",
           "process.on('SIGTERM', () => {});",
           "process.send?.('ready');",
-          `setTimeout(() => writeFileSync(${JSON.stringify(marker)}, 'alive'), 2500);`,
+          `setTimeout(() => writeFileSync(${JSON.stringify(marker)}, 'alive'), 4500);`,
           "setInterval(() => {}, 1000);",
         ].join("\n"),
         "utf8",
@@ -111,7 +111,7 @@ describe("runCommandArgs", () => {
       );
 
       const result = await runCommandArgs(process.execPath, [parentScript], dir, undefined, {
-        timeoutMs: 1000,
+        timeoutMs: 3000,
       });
       await new Promise((resolve) => setTimeout(resolve, 1200));
 

website/README.md

@@ -8,6 +8,9 @@ Files:
 - `favicon.svg`: browser icon
 - `social-card.svg`: link preview card
 - `social-card.png`: raster link preview card for Open Graph/Twitter
+- `robots.txt`: crawler policy with sitemap reference
+- `sitemap.xml`: canonical single-page sitemap
+- `_headers`: static security headers for hosts that support header files
 
 Preview:
 

website/_headers

@@ -0,0 +1,7 @@
+/*
+  Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
+  X-Content-Type-Options: nosniff
+  X-Frame-Options: DENY
+  Referrer-Policy: strict-origin-when-cross-origin
+  Permissions-Policy: camera=(), microphone=(), geolocation=(), payment=(), usb=()
+  Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src https://fonts.gstatic.com; img-src 'self' data:; connect-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; object-src 'none'

website/index.html

@@ -1057,7 +1057,7 @@ <h2>Reference</h2>
       <main>
         <header class="home-hero">
           <p class="eyebrow">Automated Code Review · Explicit Fixes</p>
-          <h1>Code review with<br />explicit fixes</h1>
+          <h1>Code review with <br />explicit fixes</h1>
           <p class="lede">
             Clawpatch maps codebases into semantic feature slices, reviews them for bugs and quality
             issues, and records explicit fix attempts with validation.

website/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://clawpatch.ai/sitemap.xml

website/sitemap.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://clawpatch.ai/</loc>
+    <lastmod>2026-05-20</lastmod>
+    <changefreq>weekly</changefreq>
+    <priority>1.0</priority>
+  </url>
+</urlset>