Blog: a new post is announced to IndexNow in all ten languages, once, from getpluck.io only

apps/web/lib/indexnow.test.ts

@@ -0,0 +1,23 @@
+import { afterEach, expect, test, vi } from "vitest";
+import { POSTS } from "./blog";
+import { announceNewPosts, INDEXNOW_KEY } from "./indexnow";
+
+afterEach(() => vi.unstubAllGlobals());
+
+test("a new post is announced once, in every language, and only by getpluck.io", async () => {
+  const sent = vi.fn(async (_url: string, _init: RequestInit) => new Response(null, { status: 202 }));
+  vi.stubGlobal("fetch", sent);
+  const post = POSTS[0]!;
+  const day = new Date(`${post.date}T12:00:00Z`);
+
+  expect(await announceNewPosts(day, "https://pluck.example.org")).toBe(0); // a self-hosted copy stays quiet
+  expect(await announceNewPosts(new Date(day.getTime() + 10 * 86_400_000), "https://getpluck.io")).toBe(0); // old news
+  const count = await announceNewPosts(day, "https://getpluck.io");
+  expect(count % 10).toBe(0);
+  const body = JSON.parse(String(sent.mock.calls[0]![1].body)) as { host: string; key: string; urlList: string[] };
+  expect(body).toMatchObject({ host: "getpluck.io", key: INDEXNOW_KEY });
+  expect(body.urlList).toContain(`https://getpluck.io/blog/${post.slug}`);
+  expect(body.urlList).toContain(`https://getpluck.io/zh-hans/blog/${post.slug}`);
+  expect(await announceNewPosts(day, "https://getpluck.io")).toBe(0); // claimed: never twice
+  expect(sent).toHaveBeenCalledTimes(1);
+});

apps/web/lib/indexnow.ts

@@ -0,0 +1,33 @@
+import { claimOnce } from "@pluck/db";
+import { LOCALES } from "@pluck/i18n";
+import { POSTS } from "./blog";
+import { log } from "./log";
+import { ORIGIN } from "./origin";
+import { localePath } from "./siteUrls";
+
+// IndexNow (indexnow.org): a new post is announced to Bing (and Yandex, Seznam, Naver) in all ten languages as soon
+// as it is out, instead of waiting for a crawl. Only getpluck.io announces; the protocol checks the key against the
+// file of the same name at the site's root (public/<key>.txt).
+export const INDEXNOW_KEY = "2a2940bd0ceed62a1fd5c0a15b0b5836";
+const HOSTED = "https://getpluck.io";
+// A post counts as new for three days from its date, and its claim lasts seven (meta's expiry), so it goes once.
+const NEW_FOR_MS = 3 * 86_400_000;
+
+export async function announceNewPosts(now = new Date(), origin = ORIGIN): Promise<number> {
+  if (origin !== HOSTED) return 0;
+  const since = new Date(now.getTime() - NEW_FOR_MS).toISOString().slice(0, 10);
+  const urlList: string[] = [];
+  for (const p of POSTS) {
+    if (p.date >= since && (await claimOnce(`indexnow:${p.slug}`))) urlList.push(...LOCALES.map((l) => origin + localePath(l, `/blog/${p.slug}`)));
+  }
+  if (!urlList.length) return 0;
+  // A failed announcement is not retried: the sitemap still lists the post, so it is only found later.
+  const res = await fetch("https://api.indexnow.org/indexnow", {
+    method: "POST",
+    headers: { "content-type": "application/json; charset=utf-8" },
+    body: JSON.stringify({ host: new URL(origin).host, key: INDEXNOW_KEY, keyLocation: `${origin}/${INDEXNOW_KEY}.txt`, urlList }),
+    signal: AbortSignal.timeout(5_000),
+  }).catch((e: unknown) => (log("warn", "indexnow failed", { error: e }), null));
+  if (res && !res.ok) log("warn", "indexnow refused", { status: res.status });
+  return urlList.length;
+}

apps/web/lib/schedule.ts

@@ -1,6 +1,7 @@
 import { claimOnce, narrowAiGrants, runDueExpiry } from "@pluck/db";
 import { FIRST_PARTY_REDIRECTS } from "./auth";
 import { sortBackfill } from "./capture";
+import { announceNewPosts } from "./indexnow";
 import { pollInbound, purgeBinNow, queueDueDigests } from "./library";
 import { log } from "./log";
 
@@ -19,5 +20,7 @@ export async function runScheduled(now = new Date()): Promise<{ purged: number;
   if (digests) log("info", "digests queued", { digests });
   const inbound = await pollInbound();
   const sorted = await sortBackfill(); // until every Item read before Previews has one; then a single lookup
+  const announced = await announceNewPosts(now);
+  if (announced) log("info", "new posts announced to IndexNow", { urls: announced });
   return { purged, expired: expiry.archived, digests, inbound, sorted };
 }

apps/web/public/2a2940bd0ceed62a1fd5c0a15b0b5836.txt

@@ -0,0 +1 @@
+2a2940bd0ceed62a1fd5c0a15b0b5836

docs/i18n.md

@@ -48,5 +48,5 @@ Portuguese of any region reads Brazilian Portuguese. Chinese reads Simplified on
 
 ## The blog
 
-Every post is in every language: `apps/web/content/blog/<locale>/<slug>.md`, with the same slug (the URL stays English in every language). The front matter holds the title, the description (the search result snippet) and the keywords, which go only into the structured data. A new post lands in all ten languages at once, and `apps/web/lib/siteUrls.test.ts` fails until it does. Translate a post the way the site is translated: the glossary, the tone rules, and any UI it names quoted from that language's catalog. Link to site pages unprefixed (`/open-source`); the post adds its language.
+Every post is in every language: `apps/web/content/blog/<locale>/<slug>.md`, with the same slug (the URL stays English in every language). The front matter holds the title, the description (the search result snippet) and the keywords, which go only into the structured data. A new post lands in all ten languages at once, and `apps/web/lib/siteUrls.test.ts` fails until it does. Translate a post the way the site is translated: the glossary, the tone rules, and any UI it names quoted from that language's catalog. Link to site pages unprefixed (`/open-source`); the post adds its language. The post's `date` in `apps/web/lib/blog.ts` is the day it goes out: for three days from then, getpluck.io announces it to IndexNow in all ten languages, once (`apps/web/lib/indexnow.ts`).