From 9b3353707227e240f95c7a0831c731a94d742102 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:37:34 +0200 Subject: [PATCH 1/5] =?UTF-8?q?feat(seo):=20IndexNow=20=E2=80=94=20push=20?= =?UTF-8?q?changed=20page=20URLs=20to=20Bing,=20Yandex,=20Seznam,=20Naver,?= =?UTF-8?q?=20Yep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bing Webmaster Tools' first recommendation for the site. A public key file (app/public/.txt, served by an explicit nginx location so crawler UAs are not proxied to /seo-proxy and a 404) proves control of the host, and a new workflow maps every push to main that touches plots/ onto the affected /{spec} and /{spec}/{language}/{library} URLs and POSTs them to api.indexnow.org. workflow_dispatch with scope=sitemap submits the whole live sitemap for the initial load. Google does not take part. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SrKzcwZBnref1sWYtdXynu --- .github/workflows/indexnow-submit.yml | 104 ++++++++++++++++++ CHANGELOG.md | 12 ++ app/nginx.conf | 8 ++ .../anyplot-indexnow-ab738f04ea92446a.txt | 1 + docs/reference/seo.md | 17 +++ docs/workflows/overview.md | 1 + 6 files changed, 143 insertions(+) create mode 100644 .github/workflows/indexnow-submit.yml create mode 100644 app/public/anyplot-indexnow-ab738f04ea92446a.txt diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml new file mode 100644 index 00000000000..e04b7cd1366 --- /dev/null +++ b/.github/workflows/indexnow-submit.yml @@ -0,0 +1,104 @@ +name: "SEO: IndexNow submit" +run-name: "IndexNow: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }} URLs" + +# Tells the IndexNow engines (Bing, Yandex, Seznam, Naver, Yep) which pages +# changed, instead of waiting for their next crawl. Google does not take part; +# it keeps reading the sitemap. +# +# On every push to main that touches plots/, the diff is mapped to page URLs: +# plots//specification.* → https://anyplot.ai/ +# plots//metadata//.yaml → https://anyplot.ai/// +# plots//implementations//. → same page +# A deleted implementation is submitted too — IndexNow is "this URL changed", +# which covers removals. +# +# The key is public by design: it only proves the submitter controls the host, +# and the engines verify it by fetching https://anyplot.ai/.txt. The same +# key lives in app/public/.txt (what humans and non-bot clients get) and +# in api/routers/seo.py (what the nginx bot map proxies crawlers to). Keep the +# three in sync when rotating it. + +on: + push: + branches: [main] + paths: + - 'plots/**' + workflow_dispatch: + inputs: + scope: + description: "'changed' submits the URLs touched by the latest commit on main; 'sitemap' submits every URL in the live sitemap (initial load, or after a long outage)" + required: false + default: changed + type: choice + options: [changed, sitemap] + +permissions: + contents: read + +concurrency: + group: indexnow-submit + cancel-in-progress: false + +env: + HOST: anyplot.ai + INDEXNOW_KEY: anyplot-indexnow-ab738f04ea92446a + SCOPE: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }} + +jobs: + submit: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # HEAD~1 is the previous main for a squash merge, which is how every + # pipeline PR lands; a rare multi-commit push submits its last commit. + fetch-depth: 2 + + - name: Collect URLs + id: urls + run: | + set -euo pipefail + if [ "$SCOPE" = "sitemap" ]; then + curl -sfS "https://${HOST}/sitemap.xml" \ + | grep -oE '[^<]+' | sed -E 's###g' | sort -u > urls.txt + else + git diff --name-only HEAD~1 HEAD -- plots/ | awk -F/ ' + $1 != "plots" || NF < 3 { next } + $3 ~ /^specification\./ { print "https://'"$HOST"'/" $2; next } + ($3 == "metadata" || $3 == "implementations") && NF == 5 { + lib = $5; sub(/\.[^.]+$/, "", lib) + print "https://'"$HOST"'/" $2 + print "https://'"$HOST"'/" $2 "/" $4 "/" lib + }' | sort -u > urls.txt + fi + n=$(wc -l < urls.txt) + echo "count=$n" >> "$GITHUB_OUTPUT" + echo "::notice::${n} URL(s) to submit (scope: ${SCOPE})" + head -20 urls.txt + + - name: Submit to IndexNow + if: steps.urls.outputs.count != '0' + run: | + set -euo pipefail + # 10,000 URLs per request is the protocol limit; the full sitemap is + # under that today (4.6k) but the split keeps this future-proof. + split -l 10000 -d urls.txt batch_ + for f in batch_*; do + body=$(jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \ + --arg loc "https://${HOST}/${INDEXNOW_KEY}.txt" \ + --rawfile list "$f" \ + '{host: $host, key: $key, keyLocation: $loc, + urlList: ($list | split("\n") | map(select(length > 0)))}') + code=$(curl -sS -o response.txt -w '%{http_code}' \ + -X POST "https://api.indexnow.org/indexnow" \ + -H "Content-Type: application/json; charset=utf-8" \ + --data "$body") + n=$(grep -c . "$f") + case "$code" in + 200|202) echo "::notice::IndexNow accepted ${n} URL(s) (HTTP ${code})" ;; + # 4xx is a protocol or key problem on our side; make it visible. + *) echo "::error::IndexNow returned HTTP ${code} for ${n} URL(s): $(head -c 300 response.txt)"; exit 1 ;; + esac + done diff --git a/CHANGELOG.md b/CHANGELOG.md index a24df73e936..73b060b8301 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,18 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ## [Unreleased] +### Added + +- **IndexNow: changed pages are pushed to Bing, Yandex, Seznam, Naver and Yep instead of + waiting for a crawl** — Bing Webmaster Tools' first recommendation for the site. A public + key file (`app/public/.txt`, served by an explicit nginx `location` so crawler UAs + are not proxied away from it) proves control of the host, and the new + `.github/workflows/indexnow-submit.yml` maps every push to `main` that touches `plots/` + onto the affected `/{spec}` and `/{spec}/{language}/{library}` URLs and POSTs them to + `api.indexnow.org` (10,000 per request; a deleted implementation is submitted too). + `workflow_dispatch` with `scope=sitemap` submits the whole live sitemap for the initial + load. Google does not take part and keeps reading the sitemap. The protocol is free. + ### Fixed - **Infrastructure failures no longer spend a pair's generation budget** — the diff --git a/app/nginx.conf b/app/nginx.conf index 001e0352e4d..05aba16f267 100644 --- a/app/nginx.conf +++ b/app/nginx.conf @@ -258,6 +258,14 @@ server { try_files $uri =404; } + # IndexNow key file (docs/reference/seo.md "IndexNow"). Bing verifies a + # submission by fetching it with a crawler UA, which the map above would + # otherwise send to /seo-proxy/.txt and a 404. Same key as + # .github/workflows/indexnow-submit.yml; rotate both together. + location = /anyplot-indexnow-ab738f04ea92446a.txt { + try_files $uri =404; + } + # llms.txt has no registered .well-known name, but agents guess this path # (docs/reference/seo.md "Discoverability") — answer with the file's # canonical location instead of the SPA shell, which soft-404'd it with diff --git a/app/public/anyplot-indexnow-ab738f04ea92446a.txt b/app/public/anyplot-indexnow-ab738f04ea92446a.txt new file mode 100644 index 00000000000..3ceb768403d --- /dev/null +++ b/app/public/anyplot-indexnow-ab738f04ea92446a.txt @@ -0,0 +1 @@ +anyplot-indexnow-ab738f04ea92446a diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 51ae5f04b4a..c318b2973da 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -499,6 +499,23 @@ onto the seo-proxy path, and `.github/workflows/bot-serving-check.yml` guards it daily against the Cloud Run origin (origin, not edge — so it reports on the nginx map no matter what the zone policy is, and will never catch edge drift). +## IndexNow + +[IndexNow](https://www.indexnow.org/) is the free, open push protocol shared by +Bing, Yandex, Seznam, Naver and Yep: instead of waiting for a crawl, the site +posts the URLs that changed. Google does not take part and keeps reading the +sitemap. Three pieces, kept in sync when the key is rotated: + +| Piece | Where | Purpose | +|-------|-------|---------| +| Key file | `app/public/.txt` (served by nginx to every client, bots included — an explicit `location =` like `robots.txt`) | Proves the submitter controls `anyplot.ai`; the engines fetch it on every submission. The key is public by design. | +| Submission workflow | `.github/workflows/indexnow-submit.yml` | On every push to `main` that touches `plots/`, maps the diff to page URLs (`/{spec}` and `/{spec}/{language}/{library}`) and POSTs them to `https://api.indexnow.org/indexnow`, 10,000 per request. A deleted implementation is submitted too — the protocol means "this URL changed". | +| Manual full load | `gh workflow run indexnow-submit.yml -f scope=sitemap` | Submits every URL of the live sitemap; used once at rollout and after a long outage of the workflow. | + +The engines answer `200` or `202` for an accepted batch; `4xx` means a key or +payload problem and fails the run so it is visible. Bing Webmaster Tools shows +the received submissions under *IndexNow*. + ## Discoverability for assistants Two session protocols of external assistants against the sister project diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index 37ca87f2b20..7bde8754bfe 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -172,6 +172,7 @@ Located in `.github/workflows/`: | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main | | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges | +| `indexnow-submit.yml` | Pushes changed page URLs to IndexNow (Bing, Yandex, Seznam, Naver, Yep) on every push to main that touches `plots/`; `workflow_dispatch` with `scope=sitemap` submits the whole sitemap | | `codeql.yml` | CodeQL scanning (actions, JavaScript/TypeScript, Python) on pushes to main, PRs and a weekly cron; `plots/**` is excluded from triggers and analysis, so pipeline PRs never start a scan | | `ci-lint.yml` | Ruff lint check on PRs | | `ci-tests.yml` | Unit + integration tests on PRs | From 3107757227929bc6afd5faeb5127ebc73ac4c578 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:42:38 +0200 Subject: [PATCH 2/5] docs(workflow): name the three places the IndexNow key lives Review feedback: the header pointed at api/routers/seo.py, which does not carry the key; the nginx exact-match location does. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SrKzcwZBnref1sWYtdXynu --- .github/workflows/indexnow-submit.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml index e04b7cd1366..08e18d51635 100644 --- a/.github/workflows/indexnow-submit.yml +++ b/.github/workflows/indexnow-submit.yml @@ -14,9 +14,10 @@ run-name: "IndexNow: ${{ github.event_name == 'workflow_dispatch' && inputs.scop # # The key is public by design: it only proves the submitter controls the host, # and the engines verify it by fetching https://anyplot.ai/.txt. The same -# key lives in app/public/.txt (what humans and non-bot clients get) and -# in api/routers/seo.py (what the nginx bot map proxies crawlers to). Keep the -# three in sync when rotating it. +# key appears in three places — app/public/.txt (the file itself), the +# exact-match `location` in app/nginx.conf (so a crawler UA is not proxied to +# /seo-proxy and a 404), and INDEXNOW_KEY below. Keep all three in sync when +# rotating it. on: push: From a4ae1b61ae157160a21d174a67e2cc825b7ec907 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:47:29 +0200 Subject: [PATCH 3/5] ci(indexnow): bound the submission curl with a timeout and retries Mirrors the sister repo's review: a slow or flaky api.indexnow.org must not burn the job timeout, and three retries cover a transient error. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SrKzcwZBnref1sWYtdXynu --- .github/workflows/indexnow-submit.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml index 08e18d51635..75932e761e9 100644 --- a/.github/workflows/indexnow-submit.yml +++ b/.github/workflows/indexnow-submit.yml @@ -92,7 +92,10 @@ jobs: --rawfile list "$f" \ '{host: $host, key: $key, keyLocation: $loc, urlList: ($list | split("\n") | map(select(length > 0)))}') + # Bounded: a slow or flaky api.indexnow.org must not burn the job + # timeout; three retries cover a transient error. code=$(curl -sS -o response.txt -w '%{http_code}' \ + --max-time 30 --retry 3 --retry-delay 5 --retry-all-errors \ -X POST "https://api.indexnow.org/indexnow" \ -H "Content-Type: application/json; charset=utf-8" \ --data "$body") From 950c1c7395c88e0b5b548690d75115f1fa8a1e9c Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:54:42 +0200 Subject: [PATCH 4/5] ci(indexnow): derive the full URL list from the checkout, handle multi-commit pushes Review feedback: fetching the live sitemap puts the Cloudflare edge between a GitHub runner and the list; the checkout carries the same information (spec directories, metadata files, the static pages). A push with more than one commit falls back to the full list instead of a diff that fetch-depth 2 cannot resolve. Transport errors after the curl retries warn instead of failing the run; 4xx still fails. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SrKzcwZBnref1sWYtdXynu --- .github/workflows/indexnow-submit.yml | 37 +++++++++++++++++++++++---- CHANGELOG.md | 1 + 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml index 75932e761e9..99a0f827245 100644 --- a/.github/workflows/indexnow-submit.yml +++ b/.github/workflows/indexnow-submit.yml @@ -59,11 +59,33 @@ jobs: - name: Collect URLs id: urls + env: + # Number of commits in the push; a squash merge is exactly one. + PUSH_COMMITS: ${{ github.event_name == 'push' && toJSON(github.event.commits) || '[]' }} run: | set -euo pipefail + # The full list comes from the checkout, not from the live sitemap: + # every spec directory is a hub page and every metadata file an + # implementation page (the same rule the sitemap follows), plus the + # static pages. No fetch means no Cloudflare edge or bot management + # between a GitHub runner and the list, and the shallow checkout + # still carries the complete tree. + full_list() { + for p in / /plots /specs /libraries /map /palette /about /mcp /legal /stats; do + echo "https://${HOST}${p}" + done + git ls-tree -d --name-only HEAD plots/ | awk -F/ '{ print "https://'"$HOST"'/" $2 }' + git ls-tree -r --name-only HEAD plots/ | awk -F/ ' + $3 == "metadata" && NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }' + } + commits=$(jq 'length' <<<"$PUSH_COMMITS") if [ "$SCOPE" = "sitemap" ]; then - curl -sfS "https://${HOST}/sitemap.xml" \ - | grep -oE '[^<]+' | sed -E 's###g' | sort -u > urls.txt + full_list | sort -u > urls.txt + elif [ "$commits" -gt 1 ]; then + # A multi-commit push (rare on main) is not what fetch-depth 2 can + # diff; submitting everything is cheap and always correct. + echo "::notice::push carries ${commits} commits; submitting the full list instead of a diff" + full_list | sort -u > urls.txt else git diff --name-only HEAD~1 HEAD -- plots/ | awk -F/ ' $1 != "plots" || NF < 3 { next } @@ -93,16 +115,21 @@ jobs: '{host: $host, key: $key, keyLocation: $loc, urlList: ($list | split("\n") | map(select(length > 0)))}') # Bounded: a slow or flaky api.indexnow.org must not burn the job - # timeout; three retries cover a transient error. + # timeout; three retries cover a transient error, and a transport + # error after them yields code 000 for the branch below instead of + # aborting under `set -e`. code=$(curl -sS -o response.txt -w '%{http_code}' \ --max-time 30 --retry 3 --retry-delay 5 --retry-all-errors \ -X POST "https://api.indexnow.org/indexnow" \ -H "Content-Type: application/json; charset=utf-8" \ - --data "$body") + --data "$body") || code="000" n=$(grep -c . "$f") case "$code" in 200|202) echo "::notice::IndexNow accepted ${n} URL(s) (HTTP ${code})" ;; # 4xx is a protocol or key problem on our side; make it visible. - *) echo "::error::IndexNow returned HTTP ${code} for ${n} URL(s): $(head -c 300 response.txt)"; exit 1 ;; + 4*) echo "::error::IndexNow rejected ${n} URL(s) (HTTP ${code}): $(head -c 300 response.txt 2>/dev/null)"; exit 1 ;; + # 5xx / no response: their side. Every later push resubmits its + # own URLs and `scope=sitemap` covers a longer gap, so warn. + *) echo "::warning::IndexNow unavailable (HTTP ${code}) for ${n} URL(s); resubmit with scope=sitemap if it persists" ;; esac done diff --git a/CHANGELOG.md b/CHANGELOG.md index 73b060b8301..527364ac02c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ aggregate instead: an italic *Catalog* line at the end of the version section an `api.indexnow.org` (10,000 per request; a deleted implementation is submitted too). `workflow_dispatch` with `scope=sitemap` submits the whole live sitemap for the initial load. Google does not take part and keeps reading the sitemap. The protocol is free. + (#11202) ### Fixed From acbd5c4a12823d0fb706b49739e2bee4c8f15e0e Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:01:04 +0200 Subject: [PATCH 5/5] ci(indexnow): a spec change submits its implementation pages; wait for the key file Review feedback: implementation pages render the spec's title and description, so a changed specification touches all of them, not only the hub. The submit step now also waits up to 8 min for the key file the app deploy serves (a rollout or key rotation), submitting either way because IndexNow verifies the key itself and a runner may be 403'd at the edge. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SrKzcwZBnref1sWYtdXynu --- .github/workflows/indexnow-submit.yml | 43 ++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml index 99a0f827245..83ed76988ab 100644 --- a/.github/workflows/indexnow-submit.yml +++ b/.github/workflows/indexnow-submit.yml @@ -87,14 +87,26 @@ jobs: echo "::notice::push carries ${commits} commits; submitting the full list instead of a diff" full_list | sort -u > urls.txt else - git diff --name-only HEAD~1 HEAD -- plots/ | awk -F/ ' - $1 != "plots" || NF < 3 { next } - $3 ~ /^specification\./ { print "https://'"$HOST"'/" $2; next } - ($3 == "metadata" || $3 == "implementations") && NF == 5 { - lib = $5; sub(/\.[^.]+$/, "", lib) - print "https://'"$HOST"'/" $2 - print "https://'"$HOST"'/" $2 "/" $4 "/" lib - }' | sort -u > urls.txt + # A changed specification touches every page of that spec: the + # implementation pages render the spec's title and description + # too. A changed implementation or metadata file touches its own + # page and the hub that lists it. + impl_pages() { + git ls-tree -r --name-only HEAD "plots/$1/metadata/" | awk -F/ ' + NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }' + } + git diff --name-only HEAD~1 HEAD -- plots/ | while IFS=/ read -r top spec third fourth fifth rest; do + [ "$top" = plots ] && [ -n "$spec" ] && [ -n "$third" ] || continue + case "$third" in + specification.*) + echo "https://${HOST}/${spec}" + impl_pages "$spec" ;; + metadata|implementations) + [ -n "$fifth" ] && [ -z "$rest" ] || continue + echo "https://${HOST}/${spec}" + echo "https://${HOST}/${spec}/${fourth}/${fifth%.*}" ;; + esac + done | sort -u > urls.txt fi n=$(wc -l < urls.txt) echo "count=$n" >> "$GITHUB_OUTPUT" @@ -105,6 +117,21 @@ jobs: if: steps.urls.outputs.count != '0' run: | set -euo pipefail + # The engines validate a submission by fetching the key file. It is + # served by the app deploy, which a rollout or a key rotation may + # still have in flight — wait for it (up to ~8 min), but submit + # either way: a runner behind Cloudflare's bot management may see a + # 403 that Bing's own fetch does not, and IndexNow verifies itself. + for i in $(seq 1 16); do + if curl -fsS --max-time 15 -o /dev/null "https://${HOST}/${INDEXNOW_KEY}.txt"; then + echo "::notice::key file reachable"; break + fi + if [ "$i" -eq 16 ]; then + echo "::warning::key file not confirmed reachable after 8 min; submitting anyway" + else + sleep 30 + fi + done # 10,000 URLs per request is the protocol limit; the full sitemap is # under that today (4.6k) but the split keeps this future-proof. split -l 10000 -d urls.txt batch_