From 6f31ad60a8bb857212256c91217a6fe03c591aa5 Mon Sep 17 00:00:00 2001 From: NiallJoeMaher Date: Mon, 26 Oct 2026 22:57:32 +0000 Subject: [PATCH] Blog: a new post is announced to IndexNow in all ten languages, once, from getpluck.io only Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Per4f8yL5A7nL9frZVDJj7 --- apps/web/lib/indexnow.test.ts | 23 +++++++++++++ apps/web/lib/indexnow.ts | 33 +++++++++++++++++++ apps/web/lib/schedule.ts | 3 ++ .../2a2940bd0ceed62a1fd5c0a15b0b5836.txt | 1 + docs/i18n.md | 2 +- 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 apps/web/lib/indexnow.test.ts create mode 100644 apps/web/lib/indexnow.ts create mode 100644 apps/web/public/2a2940bd0ceed62a1fd5c0a15b0b5836.txt diff --git a/apps/web/lib/indexnow.test.ts b/apps/web/lib/indexnow.test.ts new file mode 100644 index 0000000..ef8165d --- /dev/null +++ b/apps/web/lib/indexnow.test.ts @@ -0,0 +1,23 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { POSTS } from "./blog"; +import { announceNewPosts, INDEXNOW_KEY } from "./indexnow"; + +afterEach(() => vi.unstubAllGlobals()); + +test("a new post is announced once, in every language, and only by getpluck.io", async () => { + const sent = vi.fn(async (_url: string, _init: RequestInit) => new Response(null, { status: 202 })); + vi.stubGlobal("fetch", sent); + const post = POSTS[0]!; + const day = new Date(`${post.date}T12:00:00Z`); + + expect(await announceNewPosts(day, "https://pluck.example.org")).toBe(0); // a self-hosted copy stays quiet + expect(await announceNewPosts(new Date(day.getTime() + 10 * 86_400_000), "https://getpluck.io")).toBe(0); // old news + const count = await announceNewPosts(day, "https://getpluck.io"); + expect(count % 10).toBe(0); + const body = JSON.parse(String(sent.mock.calls[0]![1].body)) as { host: string; key: string; urlList: string[] }; + expect(body).toMatchObject({ host: "getpluck.io", key: INDEXNOW_KEY }); + expect(body.urlList).toContain(`https://getpluck.io/blog/${post.slug}`); + expect(body.urlList).toContain(`https://getpluck.io/zh-hans/blog/${post.slug}`); + expect(await announceNewPosts(day, "https://getpluck.io")).toBe(0); // claimed: never twice + expect(sent).toHaveBeenCalledTimes(1); +}); diff --git a/apps/web/lib/indexnow.ts b/apps/web/lib/indexnow.ts new file mode 100644 index 0000000..3d27e11 --- /dev/null +++ b/apps/web/lib/indexnow.ts @@ -0,0 +1,33 @@ +import { claimOnce } from "@pluck/db"; +import { LOCALES } from "@pluck/i18n"; +import { POSTS } from "./blog"; +import { log } from "./log"; +import { ORIGIN } from "./origin"; +import { localePath } from "./siteUrls"; + +// IndexNow (indexnow.org): a new post is announced to Bing (and Yandex, Seznam, Naver) in all ten languages as soon +// as it is out, instead of waiting for a crawl. Only getpluck.io announces; the protocol checks the key against the +// file of the same name at the site's root (public/.txt). +export const INDEXNOW_KEY = "2a2940bd0ceed62a1fd5c0a15b0b5836"; +const HOSTED = "https://getpluck.io"; +// A post counts as new for three days from its date, and its claim lasts seven (meta's expiry), so it goes once. +const NEW_FOR_MS = 3 * 86_400_000; + +export async function announceNewPosts(now = new Date(), origin = ORIGIN): Promise { + if (origin !== HOSTED) return 0; + const since = new Date(now.getTime() - NEW_FOR_MS).toISOString().slice(0, 10); + const urlList: string[] = []; + for (const p of POSTS) { + if (p.date >= since && (await claimOnce(`indexnow:${p.slug}`))) urlList.push(...LOCALES.map((l) => origin + localePath(l, `/blog/${p.slug}`))); + } + if (!urlList.length) return 0; + // A failed announcement is not retried: the sitemap still lists the post, so it is only found later. + const res = await fetch("https://api.indexnow.org/indexnow", { + method: "POST", + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify({ host: new URL(origin).host, key: INDEXNOW_KEY, keyLocation: `${origin}/${INDEXNOW_KEY}.txt`, urlList }), + signal: AbortSignal.timeout(5_000), + }).catch((e: unknown) => (log("warn", "indexnow failed", { error: e }), null)); + if (res && !res.ok) log("warn", "indexnow refused", { status: res.status }); + return urlList.length; +} diff --git a/apps/web/lib/schedule.ts b/apps/web/lib/schedule.ts index 9b1e538..1168cef 100644 --- a/apps/web/lib/schedule.ts +++ b/apps/web/lib/schedule.ts @@ -1,6 +1,7 @@ import { claimOnce, narrowAiGrants, runDueExpiry } from "@pluck/db"; import { FIRST_PARTY_REDIRECTS } from "./auth"; import { sortBackfill } from "./capture"; +import { announceNewPosts } from "./indexnow"; import { pollInbound, purgeBinNow, queueDueDigests } from "./library"; import { log } from "./log"; @@ -19,5 +20,7 @@ export async function runScheduled(now = new Date()): Promise<{ purged: number; if (digests) log("info", "digests queued", { digests }); const inbound = await pollInbound(); const sorted = await sortBackfill(); // until every Item read before Previews has one; then a single lookup + const announced = await announceNewPosts(now); + if (announced) log("info", "new posts announced to IndexNow", { urls: announced }); return { purged, expired: expiry.archived, digests, inbound, sorted }; } diff --git a/apps/web/public/2a2940bd0ceed62a1fd5c0a15b0b5836.txt b/apps/web/public/2a2940bd0ceed62a1fd5c0a15b0b5836.txt new file mode 100644 index 0000000..726eba6 --- /dev/null +++ b/apps/web/public/2a2940bd0ceed62a1fd5c0a15b0b5836.txt @@ -0,0 +1 @@ +2a2940bd0ceed62a1fd5c0a15b0b5836 \ No newline at end of file diff --git a/docs/i18n.md b/docs/i18n.md index 2b31eb6..8c77e28 100644 --- a/docs/i18n.md +++ b/docs/i18n.md @@ -48,5 +48,5 @@ Portuguese of any region reads Brazilian Portuguese. Chinese reads Simplified on ## The blog -Every post is in every language: `apps/web/content/blog//.md`, with the same slug (the URL stays English in every language). The front matter holds the title, the description (the search result snippet) and the keywords, which go only into the structured data. A new post lands in all ten languages at once, and `apps/web/lib/siteUrls.test.ts` fails until it does. Translate a post the way the site is translated: the glossary, the tone rules, and any UI it names quoted from that language's catalog. Link to site pages unprefixed (`/open-source`); the post adds its language. +Every post is in every language: `apps/web/content/blog//.md`, with the same slug (the URL stays English in every language). The front matter holds the title, the description (the search result snippet) and the keywords, which go only into the structured data. A new post lands in all ten languages at once, and `apps/web/lib/siteUrls.test.ts` fails until it does. Translate a post the way the site is translated: the glossary, the tone rules, and any UI it names quoted from that language's catalog. Link to site pages unprefixed (`/open-source`); the post adds its language. The post's `date` in `apps/web/lib/blog.ts` is the day it goes out: for three days from then, getpluck.io announces it to IndexNow in all ten languages, once (`apps/web/lib/indexnow.ts`).