feat(seo): IndexNow submissions for changed pages

.github/workflows/indexnow-submit.yml

@@ -0,0 +1,162 @@
+name: "SEO: IndexNow submit"
+run-name: "IndexNow: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }} URLs"
+
+# Tells the IndexNow engines (Bing, Yandex, Seznam, Naver, Yep) which pages
+# changed, instead of waiting for their next crawl. Google does not take part;
+# it keeps reading the sitemap.
+#
+# On every push to main that touches plots/, the diff is mapped to page URLs:
+#   plots/<spec>/specification.*                  → https://anyplot.ai/<spec>
+#   plots/<spec>/metadata/<lang>/<lib>.yaml        → https://anyplot.ai/<spec>/<lang>/<lib>
+#   plots/<spec>/implementations/<lang>/<lib>.<ext> → same page
+# A deleted implementation is submitted too — IndexNow is "this URL changed",
+# which covers removals.
+#
+# The key is public by design: it only proves the submitter controls the host,
+# and the engines verify it by fetching https://anyplot.ai/<key>.txt. The same
+# key appears in three places — app/public/<key>.txt (the file itself), the
+# exact-match `location` in app/nginx.conf (so a crawler UA is not proxied to
+# /seo-proxy and a 404), and INDEXNOW_KEY below. Keep all three in sync when
+# rotating it.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'plots/**'
+  workflow_dispatch:
+    inputs:
+      scope:
+        description: "'changed' submits the URLs touched by the latest commit on main; 'sitemap' submits every URL in the live sitemap (initial load, or after a long outage)"
+        required: false
+        default: changed
+        type: choice
+        options: [changed, sitemap]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: indexnow-submit
+  cancel-in-progress: false
+
+env:
+  HOST: anyplot.ai
+  INDEXNOW_KEY: anyplot-indexnow-ab738f04ea92446a
+  SCOPE: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }}
+
+jobs:
+  submit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          # HEAD~1 is the previous main for a squash merge, which is how every
+          # pipeline PR lands; a rare multi-commit push submits its last commit.
+          fetch-depth: 2
+
+      - name: Collect URLs
+        id: urls
+        env:
+          # Number of commits in the push; a squash merge is exactly one.
+          PUSH_COMMITS: ${{ github.event_name == 'push' && toJSON(github.event.commits) || '[]' }}
+        run: |
+          set -euo pipefail
+          # The full list comes from the checkout, not from the live sitemap:
+          # every spec directory is a hub page and every metadata file an
+          # implementation page (the same rule the sitemap follows), plus the
+          # static pages. No fetch means no Cloudflare edge or bot management
+          # between a GitHub runner and the list, and the shallow checkout
+          # still carries the complete tree.
+          full_list() {
+            for p in / /plots /specs /libraries /map /palette /about /mcp /legal /stats; do
+              echo "https://${HOST}${p}"
+            done
+            git ls-tree -d --name-only HEAD plots/ | awk -F/ '{ print "https://'"$HOST"'/" $2 }'
+            git ls-tree -r --name-only HEAD plots/ | awk -F/ '
+              $3 == "metadata" && NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }'
+          }
+          commits=$(jq 'length' <<<"$PUSH_COMMITS")
+          if [ "$SCOPE" = "sitemap" ]; then
+            full_list | sort -u > urls.txt
+          elif [ "$commits" -gt 1 ]; then
+            # A multi-commit push (rare on main) is not what fetch-depth 2 can
+            # diff; submitting everything is cheap and always correct.
+            echo "::notice::push carries ${commits} commits; submitting the full list instead of a diff"
+            full_list | sort -u > urls.txt
+          else
+            # A changed specification touches every page of that spec: the
+            # implementation pages render the spec's title and description
+            # too. A changed implementation or metadata file touches its own
+            # page and the hub that lists it.
+            impl_pages() {
+              git ls-tree -r --name-only HEAD "plots/$1/metadata/" | awk -F/ '
+                NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }'
+            }
+            git diff --name-only HEAD~1 HEAD -- plots/ | while IFS=/ read -r top spec third fourth fifth rest; do
+              [ "$top" = plots ] && [ -n "$spec" ] && [ -n "$third" ] || continue
+              case "$third" in
+                specification.*)
+                  echo "https://${HOST}/${spec}"
+                  impl_pages "$spec" ;;
+                metadata|implementations)
+                  [ -n "$fifth" ] && [ -z "$rest" ] || continue
+                  echo "https://${HOST}/${spec}"
+                  echo "https://${HOST}/${spec}/${fourth}/${fifth%.*}" ;;
+              esac
+            done | sort -u > urls.txt
+          fi
+          n=$(wc -l < urls.txt)
+          echo "count=$n" >> "$GITHUB_OUTPUT"
+          echo "::notice::${n} URL(s) to submit (scope: ${SCOPE})"
+          head -20 urls.txt
+
+      - name: Submit to IndexNow
+        if: steps.urls.outputs.count != '0'
+        run: |
+          set -euo pipefail
+          # The engines validate a submission by fetching the key file. It is
+          # served by the app deploy, which a rollout or a key rotation may
+          # still have in flight — wait for it (up to ~8 min), but submit
+          # either way: a runner behind Cloudflare's bot management may see a
+          # 403 that Bing's own fetch does not, and IndexNow verifies itself.
+          for i in $(seq 1 16); do
+            if curl -fsS --max-time 15 -o /dev/null "https://${HOST}/${INDEXNOW_KEY}.txt"; then
+              echo "::notice::key file reachable"; break
+            fi
+            if [ "$i" -eq 16 ]; then
+              echo "::warning::key file not confirmed reachable after 8 min; submitting anyway"
+            else
+              sleep 30
+            fi
+          done
+          # 10,000 URLs per request is the protocol limit; the full sitemap is
+          # under that today (4.6k) but the split keeps this future-proof.
+          split -l 10000 -d urls.txt batch_
+          for f in batch_*; do
+            body=$(jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \
+              --arg loc "https://${HOST}/${INDEXNOW_KEY}.txt" \
+              --rawfile list "$f" \
+              '{host: $host, key: $key, keyLocation: $loc,
+                urlList: ($list | split("\n") | map(select(length > 0)))}')
+            # Bounded: a slow or flaky api.indexnow.org must not burn the job
+            # timeout; three retries cover a transient error, and a transport
+            # error after them yields code 000 for the branch below instead of
+            # aborting under `set -e`.
+            code=$(curl -sS -o response.txt -w '%{http_code}' \
+              --max-time 30 --retry 3 --retry-delay 5 --retry-all-errors \
+              -X POST "https://api.indexnow.org/indexnow" \
+              -H "Content-Type: application/json; charset=utf-8" \
+              --data "$body") || code="000"
+            n=$(grep -c . "$f")
+            case "$code" in
+              200|202) echo "::notice::IndexNow accepted ${n} URL(s) (HTTP ${code})" ;;
+              # 4xx is a protocol or key problem on our side; make it visible.
+              4*) echo "::error::IndexNow rejected ${n} URL(s) (HTTP ${code}): $(head -c 300 response.txt 2>/dev/null)"; exit 1 ;;
+              # 5xx / no response: their side. Every later push resubmits its
+              # own URLs and `scope=sitemap` covers a longer gap, so warn.
+              *) echo "::warning::IndexNow unavailable (HTTP ${code}) for ${n} URL(s); resubmit with scope=sitemap if it persists" ;;
+            esac
+          done

CHANGELOG.md

@@ -26,6 +26,19 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
 
 ## [Unreleased]
 
+### Added
+
+- **IndexNow: changed pages are pushed to Bing, Yandex, Seznam, Naver and Yep instead of
+  waiting for a crawl** — Bing Webmaster Tools' first recommendation for the site. A public
+  key file (`app/public/<key>.txt`, served by an explicit nginx `location` so crawler UAs
+  are not proxied away from it) proves control of the host, and the new
+  `.github/workflows/indexnow-submit.yml` maps every push to `main` that touches `plots/`
+  onto the affected `/{spec}` and `/{spec}/{language}/{library}` URLs and POSTs them to
+  `api.indexnow.org` (10,000 per request; a deleted implementation is submitted too).
+  `workflow_dispatch` with `scope=sitemap` submits the whole live sitemap for the initial
+  load. Google does not take part and keeps reading the sitemap. The protocol is free.
+  (#11202)
+
 ### Fixed
 
 - **Infrastructure failures no longer spend a pair's generation budget** — the

app/nginx.conf

@@ -258,6 +258,14 @@ server {
         try_files $uri =404;
     }
 
+    # IndexNow key file (docs/reference/seo.md "IndexNow"). Bing verifies a
+    # submission by fetching it with a crawler UA, which the map above would
+    # otherwise send to /seo-proxy/<key>.txt and a 404. Same key as
+    # .github/workflows/indexnow-submit.yml; rotate both together.
+    location = /anyplot-indexnow-ab738f04ea92446a.txt {
+        try_files $uri =404;
+    }
+
     # llms.txt has no registered .well-known name, but agents guess this path
     # (docs/reference/seo.md "Discoverability") — answer with the file's
     # canonical location instead of the SPA shell, which soft-404'd it with

app/public/anyplot-indexnow-ab738f04ea92446a.txt

@@ -0,0 +1 @@
+anyplot-indexnow-ab738f04ea92446a

docs/reference/seo.md

@@ -499,6 +499,23 @@ onto the seo-proxy path, and `.github/workflows/bot-serving-check.yml` guards it
 daily against the Cloud Run origin (origin, not edge — so it reports on the
 nginx map no matter what the zone policy is, and will never catch edge drift).
 
+## IndexNow
+
+[IndexNow](https://www.indexnow.org/) is the free, open push protocol shared by
+Bing, Yandex, Seznam, Naver and Yep: instead of waiting for a crawl, the site
+posts the URLs that changed. Google does not take part and keeps reading the
+sitemap. Three pieces, kept in sync when the key is rotated:
+
+| Piece | Where | Purpose |
+|-------|-------|---------|
+| Key file | `app/public/<key>.txt` (served by nginx to every client, bots included — an explicit `location =` like `robots.txt`) | Proves the submitter controls `anyplot.ai`; the engines fetch it on every submission. The key is public by design. |
+| Submission workflow | `.github/workflows/indexnow-submit.yml` | On every push to `main` that touches `plots/`, maps the diff to page URLs (`/{spec}` and `/{spec}/{language}/{library}`) and POSTs them to `https://api.indexnow.org/indexnow`, 10,000 per request. A deleted implementation is submitted too — the protocol means "this URL changed". |
+| Manual full load | `gh workflow run indexnow-submit.yml -f scope=sitemap` | Submits every URL of the live sitemap; used once at rollout and after a long outage of the workflow. |
+
+The engines answer `200` or `202` for an accepted batch; `4xx` means a key or
+payload problem and fails the run so it is visible. Bing Webmaster Tools shows
+the received submissions under *IndexNow*.
+
 ## Discoverability for assistants
 
 Two session protocols of external assistants against the sister project

docs/workflows/overview.md

@@ -172,6 +172,7 @@ Located in `.github/workflows/`:
 | `report-validate.yml` | Validates user-submitted issue reports |
 | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main |
 | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges |
+| `indexnow-submit.yml` | Pushes changed page URLs to IndexNow (Bing, Yandex, Seznam, Naver, Yep) on every push to main that touches `plots/`; `workflow_dispatch` with `scope=sitemap` submits the whole sitemap |
 | `codeql.yml` | CodeQL scanning (actions, JavaScript/TypeScript, Python) on pushes to main, PRs and a weekly cron; `plots/**` is excluded from triggers and analysis, so pipeline PRs never start a scan |
 | `ci-lint.yml` | Ruff lint check on PRs |
 | `ci-tests.yml` | Unit + integration tests on PRs |