diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml new file mode 100644 index 00000000000..83ed76988ab --- /dev/null +++ b/.github/workflows/indexnow-submit.yml @@ -0,0 +1,162 @@ +name: "SEO: IndexNow submit" +run-name: "IndexNow: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }} URLs" + +# Tells the IndexNow engines (Bing, Yandex, Seznam, Naver, Yep) which pages +# changed, instead of waiting for their next crawl. Google does not take part; +# it keeps reading the sitemap. +# +# On every push to main that touches plots/, the diff is mapped to page URLs: +# plots//specification.* → https://anyplot.ai/ +# plots//metadata//.yaml → https://anyplot.ai/// +# plots//implementations//. → same page +# A deleted implementation is submitted too — IndexNow is "this URL changed", +# which covers removals. +# +# The key is public by design: it only proves the submitter controls the host, +# and the engines verify it by fetching https://anyplot.ai/.txt. The same +# key appears in three places — app/public/.txt (the file itself), the +# exact-match `location` in app/nginx.conf (so a crawler UA is not proxied to +# /seo-proxy and a 404), and INDEXNOW_KEY below. Keep all three in sync when +# rotating it. + +on: + push: + branches: [main] + paths: + - 'plots/**' + workflow_dispatch: + inputs: + scope: + description: "'changed' submits the URLs touched by the latest commit on main; 'sitemap' submits every URL in the live sitemap (initial load, or after a long outage)" + required: false + default: changed + type: choice + options: [changed, sitemap] + +permissions: + contents: read + +concurrency: + group: indexnow-submit + cancel-in-progress: false + +env: + HOST: anyplot.ai + INDEXNOW_KEY: anyplot-indexnow-ab738f04ea92446a + SCOPE: ${{ github.event_name == 'workflow_dispatch' && inputs.scope || 'changed' }} + +jobs: + submit: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # HEAD~1 is the previous main for a squash merge, which is how every + # pipeline PR lands; a rare multi-commit push submits its last commit. + fetch-depth: 2 + + - name: Collect URLs + id: urls + env: + # Number of commits in the push; a squash merge is exactly one. + PUSH_COMMITS: ${{ github.event_name == 'push' && toJSON(github.event.commits) || '[]' }} + run: | + set -euo pipefail + # The full list comes from the checkout, not from the live sitemap: + # every spec directory is a hub page and every metadata file an + # implementation page (the same rule the sitemap follows), plus the + # static pages. No fetch means no Cloudflare edge or bot management + # between a GitHub runner and the list, and the shallow checkout + # still carries the complete tree. + full_list() { + for p in / /plots /specs /libraries /map /palette /about /mcp /legal /stats; do + echo "https://${HOST}${p}" + done + git ls-tree -d --name-only HEAD plots/ | awk -F/ '{ print "https://'"$HOST"'/" $2 }' + git ls-tree -r --name-only HEAD plots/ | awk -F/ ' + $3 == "metadata" && NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }' + } + commits=$(jq 'length' <<<"$PUSH_COMMITS") + if [ "$SCOPE" = "sitemap" ]; then + full_list | sort -u > urls.txt + elif [ "$commits" -gt 1 ]; then + # A multi-commit push (rare on main) is not what fetch-depth 2 can + # diff; submitting everything is cheap and always correct. + echo "::notice::push carries ${commits} commits; submitting the full list instead of a diff" + full_list | sort -u > urls.txt + else + # A changed specification touches every page of that spec: the + # implementation pages render the spec's title and description + # too. A changed implementation or metadata file touches its own + # page and the hub that lists it. + impl_pages() { + git ls-tree -r --name-only HEAD "plots/$1/metadata/" | awk -F/ ' + NF == 5 { lib = $5; sub(/\.[^.]+$/, "", lib); print "https://'"$HOST"'/" $2 "/" $4 "/" lib }' + } + git diff --name-only HEAD~1 HEAD -- plots/ | while IFS=/ read -r top spec third fourth fifth rest; do + [ "$top" = plots ] && [ -n "$spec" ] && [ -n "$third" ] || continue + case "$third" in + specification.*) + echo "https://${HOST}/${spec}" + impl_pages "$spec" ;; + metadata|implementations) + [ -n "$fifth" ] && [ -z "$rest" ] || continue + echo "https://${HOST}/${spec}" + echo "https://${HOST}/${spec}/${fourth}/${fifth%.*}" ;; + esac + done | sort -u > urls.txt + fi + n=$(wc -l < urls.txt) + echo "count=$n" >> "$GITHUB_OUTPUT" + echo "::notice::${n} URL(s) to submit (scope: ${SCOPE})" + head -20 urls.txt + + - name: Submit to IndexNow + if: steps.urls.outputs.count != '0' + run: | + set -euo pipefail + # The engines validate a submission by fetching the key file. It is + # served by the app deploy, which a rollout or a key rotation may + # still have in flight — wait for it (up to ~8 min), but submit + # either way: a runner behind Cloudflare's bot management may see a + # 403 that Bing's own fetch does not, and IndexNow verifies itself. + for i in $(seq 1 16); do + if curl -fsS --max-time 15 -o /dev/null "https://${HOST}/${INDEXNOW_KEY}.txt"; then + echo "::notice::key file reachable"; break + fi + if [ "$i" -eq 16 ]; then + echo "::warning::key file not confirmed reachable after 8 min; submitting anyway" + else + sleep 30 + fi + done + # 10,000 URLs per request is the protocol limit; the full sitemap is + # under that today (4.6k) but the split keeps this future-proof. + split -l 10000 -d urls.txt batch_ + for f in batch_*; do + body=$(jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \ + --arg loc "https://${HOST}/${INDEXNOW_KEY}.txt" \ + --rawfile list "$f" \ + '{host: $host, key: $key, keyLocation: $loc, + urlList: ($list | split("\n") | map(select(length > 0)))}') + # Bounded: a slow or flaky api.indexnow.org must not burn the job + # timeout; three retries cover a transient error, and a transport + # error after them yields code 000 for the branch below instead of + # aborting under `set -e`. + code=$(curl -sS -o response.txt -w '%{http_code}' \ + --max-time 30 --retry 3 --retry-delay 5 --retry-all-errors \ + -X POST "https://api.indexnow.org/indexnow" \ + -H "Content-Type: application/json; charset=utf-8" \ + --data "$body") || code="000" + n=$(grep -c . "$f") + case "$code" in + 200|202) echo "::notice::IndexNow accepted ${n} URL(s) (HTTP ${code})" ;; + # 4xx is a protocol or key problem on our side; make it visible. + 4*) echo "::error::IndexNow rejected ${n} URL(s) (HTTP ${code}): $(head -c 300 response.txt 2>/dev/null)"; exit 1 ;; + # 5xx / no response: their side. Every later push resubmits its + # own URLs and `scope=sitemap` covers a longer gap, so warn. + *) echo "::warning::IndexNow unavailable (HTTP ${code}) for ${n} URL(s); resubmit with scope=sitemap if it persists" ;; + esac + done diff --git a/CHANGELOG.md b/CHANGELOG.md index a24df73e936..527364ac02c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,19 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ## [Unreleased] +### Added + +- **IndexNow: changed pages are pushed to Bing, Yandex, Seznam, Naver and Yep instead of + waiting for a crawl** — Bing Webmaster Tools' first recommendation for the site. A public + key file (`app/public/.txt`, served by an explicit nginx `location` so crawler UAs + are not proxied away from it) proves control of the host, and the new + `.github/workflows/indexnow-submit.yml` maps every push to `main` that touches `plots/` + onto the affected `/{spec}` and `/{spec}/{language}/{library}` URLs and POSTs them to + `api.indexnow.org` (10,000 per request; a deleted implementation is submitted too). + `workflow_dispatch` with `scope=sitemap` submits the whole live sitemap for the initial + load. Google does not take part and keeps reading the sitemap. The protocol is free. + (#11202) + ### Fixed - **Infrastructure failures no longer spend a pair's generation budget** — the diff --git a/app/nginx.conf b/app/nginx.conf index 001e0352e4d..05aba16f267 100644 --- a/app/nginx.conf +++ b/app/nginx.conf @@ -258,6 +258,14 @@ server { try_files $uri =404; } + # IndexNow key file (docs/reference/seo.md "IndexNow"). Bing verifies a + # submission by fetching it with a crawler UA, which the map above would + # otherwise send to /seo-proxy/.txt and a 404. Same key as + # .github/workflows/indexnow-submit.yml; rotate both together. + location = /anyplot-indexnow-ab738f04ea92446a.txt { + try_files $uri =404; + } + # llms.txt has no registered .well-known name, but agents guess this path # (docs/reference/seo.md "Discoverability") — answer with the file's # canonical location instead of the SPA shell, which soft-404'd it with diff --git a/app/public/anyplot-indexnow-ab738f04ea92446a.txt b/app/public/anyplot-indexnow-ab738f04ea92446a.txt new file mode 100644 index 00000000000..3ceb768403d --- /dev/null +++ b/app/public/anyplot-indexnow-ab738f04ea92446a.txt @@ -0,0 +1 @@ +anyplot-indexnow-ab738f04ea92446a diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 51ae5f04b4a..c318b2973da 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -499,6 +499,23 @@ onto the seo-proxy path, and `.github/workflows/bot-serving-check.yml` guards it daily against the Cloud Run origin (origin, not edge — so it reports on the nginx map no matter what the zone policy is, and will never catch edge drift). +## IndexNow + +[IndexNow](https://www.indexnow.org/) is the free, open push protocol shared by +Bing, Yandex, Seznam, Naver and Yep: instead of waiting for a crawl, the site +posts the URLs that changed. Google does not take part and keeps reading the +sitemap. Three pieces, kept in sync when the key is rotated: + +| Piece | Where | Purpose | +|-------|-------|---------| +| Key file | `app/public/.txt` (served by nginx to every client, bots included — an explicit `location =` like `robots.txt`) | Proves the submitter controls `anyplot.ai`; the engines fetch it on every submission. The key is public by design. | +| Submission workflow | `.github/workflows/indexnow-submit.yml` | On every push to `main` that touches `plots/`, maps the diff to page URLs (`/{spec}` and `/{spec}/{language}/{library}`) and POSTs them to `https://api.indexnow.org/indexnow`, 10,000 per request. A deleted implementation is submitted too — the protocol means "this URL changed". | +| Manual full load | `gh workflow run indexnow-submit.yml -f scope=sitemap` | Submits every URL of the live sitemap; used once at rollout and after a long outage of the workflow. | + +The engines answer `200` or `202` for an accepted batch; `4xx` means a key or +payload problem and fails the run so it is visible. Bing Webmaster Tools shows +the received submissions under *IndexNow*. + ## Discoverability for assistants Two session protocols of external assistants against the sister project diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index 37ca87f2b20..7bde8754bfe 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -172,6 +172,7 @@ Located in `.github/workflows/`: | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main | | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges | +| `indexnow-submit.yml` | Pushes changed page URLs to IndexNow (Bing, Yandex, Seznam, Naver, Yep) on every push to main that touches `plots/`; `workflow_dispatch` with `scope=sitemap` submits the whole sitemap | | `codeql.yml` | CodeQL scanning (actions, JavaScript/TypeScript, Python) on pushes to main, PRs and a weekly cron; `plots/**` is excluded from triggers and analysis, so pipeline PRs never start a scan | | `ci-lint.yml` | Ruff lint check on PRs | | `ci-tests.yml` | Unit + integration tests on PRs |