diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml
index 83ed76988a..7a7b12160e 100644
--- a/.github/workflows/indexnow-submit.yml
+++ b/.github/workflows/indexnow-submit.yml
@@ -24,6 +24,9 @@ on:
branches: [main]
paths:
- 'plots/**'
+ # A change to this workflow exercises itself with the full list; a
+ # fix that never runs until the next plot merge is a fix nobody saw.
+ - '.github/workflows/indexnow-submit.yml'
workflow_dispatch:
inputs:
scope:
@@ -48,7 +51,10 @@ env:
jobs:
submit:
runs-on: ubuntu-latest
- timeout-minutes: 10
+ # Worst case: ~12 min waiting for the key file (16 probes, 15 s timeout,
+ # 30 s apart) + the 10 min verification deadline + one last request with
+ # its retries (up to ~4 min), with room to spare.
+ timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -81,6 +87,11 @@ jobs:
commits=$(jq 'length' <<<"$PUSH_COMMITS")
if [ "$SCOPE" = "sitemap" ]; then
full_list | sort -u > urls.txt
+ elif [ "$commits" -eq 1 ] && git diff --name-only HEAD~1 HEAD -- .github/workflows/indexnow-submit.yml | grep -q .; then
+ # The workflow itself changed (the push.paths entry above): run
+ # the real thing end to end rather than an empty diff.
+ echo "::notice::workflow file changed; submitting the full list"
+ full_list | sort -u > urls.txt
elif [ "$commits" -gt 1 ]; then
# A multi-commit push (rare on main) is not what fetch-depth 2 can
# diff; submitting everything is cheap and always correct.
@@ -120,14 +131,22 @@ jobs:
# The engines validate a submission by fetching the key file. It is
# served by the app deploy, which a rollout or a key rotation may
# still have in flight — wait for it (up to ~8 min), but submit
- # either way: a runner behind Cloudflare's bot management may see a
- # 403 that Bing's own fetch does not, and IndexNow verifies itself.
+ # either way: this probe is a courtesy, IndexNow's own key fetch
+ # below is the authoritative check.
for i in $(seq 1 16); do
- if curl -fsS --max-time 15 -o /dev/null "https://${HOST}/${INDEXNOW_KEY}.txt"; then
- echo "::notice::key file reachable"; break
- fi
+ # On a transport failure curl still prints 000 for %{http_code};
+ # the fallback is assigned afterwards so nothing is appended.
+ status=$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' \
+ "https://${HOST}/${INDEXNOW_KEY}.txt") || status=000
+ case "$status" in
+ 200) echo "::notice::key file reachable"; break ;;
+ # Cloudflare's bot management answers 403 before the origin is
+ # asked, so this says nothing about the deploy either way, and
+ # waiting cannot change it. Bing's fetch is not subject to it.
+ 403) echo "::notice::key file answers 403 to this runner (edge bot management), which cannot confirm the deploy; skipping the wait, IndexNow verifies the key itself"; break ;;
+ esac
if [ "$i" -eq 16 ]; then
- echo "::warning::key file not confirmed reachable after 8 min; submitting anyway"
+ echo "::warning::key file not confirmed reachable after 8 min (last status ${status}); submitting anyway"
else
sleep 30
fi
@@ -136,24 +155,48 @@ jobs:
# under that today (4.6k) but the split keeps this future-proof.
split -l 10000 -d urls.txt batch_
for f in batch_*; do
- body=$(jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \
+ # The body goes through a file: a full-sitemap batch is ~250 KB,
+ # and a single command-line argument is capped at 128 KB on Linux
+ # — the first sitemap-scope run (33665718870) failed to exec curl
+ # at all and reported HTTP 000 for 4,593 URLs.
+ jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \
--arg loc "https://${HOST}/${INDEXNOW_KEY}.txt" \
--rawfile list "$f" \
'{host: $host, key: $key, keyLocation: $loc,
- urlList: ($list | split("\n") | map(select(length > 0)))}')
- # Bounded: a slow or flaky api.indexnow.org must not burn the job
- # timeout; three retries cover a transient error, and a transport
- # error after them yields code 000 for the branch below instead of
- # aborting under `set -e`.
- code=$(curl -sS -o response.txt -w '%{http_code}' \
- --max-time 30 --retry 3 --retry-delay 5 --retry-all-errors \
- -X POST "https://api.indexnow.org/indexnow" \
- -H "Content-Type: application/json; charset=utf-8" \
- --data "$body") || code="000"
+ urlList: ($list | split("\n") | map(select(length > 0)))}' > body.json
n=$(grep -c . "$f")
+ # First use of a key (rollout or rotation): IndexNow verifies the
+ # key file asynchronously and answers 403 SiteVerificationNotCompleted
+ # until it has — seen minutes after the file went live. Retry that
+ # case once a minute until a ten-minute deadline has passed (an
+ # elapsed deadline, so slow responses cannot stretch it); if it
+ # persists, the run FAILS, so a `changed` run keeps its URL set and
+ # can be re-run — a green warning would have dropped those URLs for
+ # good, because later pushes submit only their own diffs.
+ deadline=$(( $(date +%s) + 600 ))
+ while :; do
+ # Bounded: a slow or flaky api.indexnow.org must not burn the job
+ # timeout; three retries cover a transient error, and a transport
+ # error after them yields code 000 for the branch below instead of
+ # aborting under `set -e`.
+ code=$(curl -sS -o response.txt -w '%{http_code}' \
+ --max-time 60 --retry 3 --retry-delay 5 --retry-all-errors \
+ -X POST "https://api.indexnow.org/indexnow" \
+ -H "Content-Type: application/json; charset=utf-8" \
+ --data @body.json) || code="000"
+ if [ "$code" = 403 ] && grep -q SiteVerificationNotCompleted response.txt 2>/dev/null \
+ && [ "$(date +%s)" -lt "$deadline" ]; then
+ echo "::notice::IndexNow has not finished verifying the key file yet; retrying in 60 s"
+ sleep 60
+ continue
+ fi
+ break
+ done
case "$code" in
200|202) echo "::notice::IndexNow accepted ${n} URL(s) (HTTP ${code})" ;;
- # 4xx is a protocol or key problem on our side; make it visible.
+ # 4xx is a protocol or key problem on our side (or the key still
+ # unverified after ten minutes); make it visible and keep the run
+ # re-runnable.
4*) echo "::error::IndexNow rejected ${n} URL(s) (HTTP ${code}): $(head -c 300 response.txt 2>/dev/null)"; exit 1 ;;
# 5xx / no response: their side. Every later push resubmits its
# own URLs and `scope=sitemap` covers a longer gap, so warn.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 53c76205a1..7d2675e6f2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,17 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
### Fixed
+- **The IndexNow workflow no longer waits eight minutes behind an edge 403** — its
+ key-file readiness loop treated every non-200 as "not deployed yet"; a GitHub runner
+ that Cloudflare's bot management answers with 403 would have slept the full budget on
+ every run. A 403 now ends the wait without claiming anything about the deploy (the
+ edge answers before the origin is asked, so it is inconclusive either way; IndexNow's
+ own key fetch is the authoritative check), and the final warning names the last
+ status seen. The submission body goes
+ through a file (a full-sitemap batch is ~260 KB, above the 128 KB command-line
+ argument cap that made the first sitemap-scope run report HTTP 000), and a first-use
+ `403 SiteVerificationNotCompleted` from IndexNow is retried for up to ten minutes
+ before the run fails, so a `changed` run keeps its URL set re-runnable. (#11204)
- **Implementation pages no longer share one meta description per spec** — the SEO
proxy reused the spec description verbatim as the `` and OG
description of every implementation page, so up to 15 library pages and their hub
diff --git a/docs/reference/seo.md b/docs/reference/seo.md
index 99b1e90d3e..63427d5d16 100644
--- a/docs/reference/seo.md
+++ b/docs/reference/seo.md
@@ -517,11 +517,17 @@ sitemap. Three pieces, kept in sync when the key is rotated:
| Piece | Where | Purpose |
|-------|-------|---------|
| Key file | `app/public/.txt` (served by nginx to every client, bots included — an explicit `location =` like `robots.txt`) | Proves the submitter controls `anyplot.ai`; the engines fetch it on every submission. The key is public by design. |
-| Submission workflow | `.github/workflows/indexnow-submit.yml` | On every push to `main` that touches `plots/`, maps the diff to page URLs (`/{spec}` and `/{spec}/{language}/{library}`) and POSTs them to `https://api.indexnow.org/indexnow`, 10,000 per request. A deleted implementation is submitted too — the protocol means "this URL changed". |
+| Submission workflow | `.github/workflows/indexnow-submit.yml` | On every push to `main` that touches `plots/`, maps the diff to page URLs (`/{spec}` and `/{spec}/{language}/{library}`; a changed specification covers the hub and all of its implementation pages) and POSTs them to `https://api.indexnow.org/indexnow`, 10,000 per request. A deleted implementation is submitted too — the protocol means "this URL changed". A push that changes the workflow file itself, or one carrying more than one commit, submits the full list instead of a diff. |
| Manual full load | `gh workflow run indexnow-submit.yml -f scope=sitemap` | Submits every URL of the live sitemap; used once at rollout and after a long outage of the workflow. |
The engines answer `200` or `202` for an accepted batch; `4xx` means a key or
-payload problem and fails the run so it is visible. Bing Webmaster Tools shows
+payload problem and fails the run so it is visible. One `4xx` is transient by
+nature: after a rollout or a key rotation IndexNow verifies the key file
+asynchronously and answers `403 SiteVerificationNotCompleted` for a while, so
+the workflow retries that case once a minute until a ten-minute deadline has
+passed (elapsed time, so slow responses cannot stretch it) and then fails — a
+`changed` run must not report success without its URLs accepted, because later
+pushes submit only their own diffs. Bing Webmaster Tools shows
the received submissions under *IndexNow*.
## Discoverability for assistants
diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md
index 7bde8754bf..04c57d679e 100644
--- a/docs/workflows/overview.md
+++ b/docs/workflows/overview.md
@@ -172,7 +172,7 @@ Located in `.github/workflows/`:
| `report-validate.yml` | Validates user-submitted issue reports |
| `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main |
| `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges |
-| `indexnow-submit.yml` | Pushes changed page URLs to IndexNow (Bing, Yandex, Seznam, Naver, Yep) on every push to main that touches `plots/`; `workflow_dispatch` with `scope=sitemap` submits the whole sitemap |
+| `indexnow-submit.yml` | Pushes changed page URLs to IndexNow (Bing, Yandex, Seznam, Naver, Yep) on every push to main that touches `plots/`; a push that changes the workflow file itself submits the full list, as does `workflow_dispatch` with `scope=sitemap` |
| `codeql.yml` | CodeQL scanning (actions, JavaScript/TypeScript, Python) on pushes to main, PRs and a weekly cron; `plots/**` is excluded from triggers and analysis, so pipeline PRs never start a scan |
| `ci-lint.yml` | Ruff lint check on PRs |
| `ci-tests.yml` | Unit + integration tests on PRs |