diff --git a/.github/workflows/indexnow-submit.yml b/.github/workflows/indexnow-submit.yml index 83ed76988a..7a7b12160e 100644 --- a/.github/workflows/indexnow-submit.yml +++ b/.github/workflows/indexnow-submit.yml @@ -24,6 +24,9 @@ on: branches: [main] paths: - 'plots/**' + # A change to this workflow exercises itself with the full list; a + # fix that never runs until the next plot merge is a fix nobody saw. + - '.github/workflows/indexnow-submit.yml' workflow_dispatch: inputs: scope: @@ -48,7 +51,10 @@ env: jobs: submit: runs-on: ubuntu-latest - timeout-minutes: 10 + # Worst case: ~12 min waiting for the key file (16 probes, 15 s timeout, + # 30 s apart) + the 10 min verification deadline + one last request with + # its retries (up to ~4 min), with room to spare. + timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -81,6 +87,11 @@ jobs: commits=$(jq 'length' <<<"$PUSH_COMMITS") if [ "$SCOPE" = "sitemap" ]; then full_list | sort -u > urls.txt + elif [ "$commits" -eq 1 ] && git diff --name-only HEAD~1 HEAD -- .github/workflows/indexnow-submit.yml | grep -q .; then + # The workflow itself changed (the push.paths entry above): run + # the real thing end to end rather than an empty diff. + echo "::notice::workflow file changed; submitting the full list" + full_list | sort -u > urls.txt elif [ "$commits" -gt 1 ]; then # A multi-commit push (rare on main) is not what fetch-depth 2 can # diff; submitting everything is cheap and always correct. @@ -120,14 +131,22 @@ jobs: # The engines validate a submission by fetching the key file. It is # served by the app deploy, which a rollout or a key rotation may # still have in flight — wait for it (up to ~8 min), but submit - # either way: a runner behind Cloudflare's bot management may see a - # 403 that Bing's own fetch does not, and IndexNow verifies itself. + # either way: this probe is a courtesy, IndexNow's own key fetch + # below is the authoritative check. for i in $(seq 1 16); do - if curl -fsS --max-time 15 -o /dev/null "https://${HOST}/${INDEXNOW_KEY}.txt"; then - echo "::notice::key file reachable"; break - fi + # On a transport failure curl still prints 000 for %{http_code}; + # the fallback is assigned afterwards so nothing is appended. + status=$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' \ + "https://${HOST}/${INDEXNOW_KEY}.txt") || status=000 + case "$status" in + 200) echo "::notice::key file reachable"; break ;; + # Cloudflare's bot management answers 403 before the origin is + # asked, so this says nothing about the deploy either way, and + # waiting cannot change it. Bing's fetch is not subject to it. + 403) echo "::notice::key file answers 403 to this runner (edge bot management), which cannot confirm the deploy; skipping the wait, IndexNow verifies the key itself"; break ;; + esac if [ "$i" -eq 16 ]; then - echo "::warning::key file not confirmed reachable after 8 min; submitting anyway" + echo "::warning::key file not confirmed reachable after 8 min (last status ${status}); submitting anyway" else sleep 30 fi @@ -136,24 +155,48 @@ jobs: # under that today (4.6k) but the split keeps this future-proof. split -l 10000 -d urls.txt batch_ for f in batch_*; do - body=$(jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \ + # The body goes through a file: a full-sitemap batch is ~250 KB, + # and a single command-line argument is capped at 128 KB on Linux + # — the first sitemap-scope run (33665718870) failed to exec curl + # at all and reported HTTP 000 for 4,593 URLs. + jq -n --arg host "$HOST" --arg key "$INDEXNOW_KEY" \ --arg loc "https://${HOST}/${INDEXNOW_KEY}.txt" \ --rawfile list "$f" \ '{host: $host, key: $key, keyLocation: $loc, - urlList: ($list | split("\n") | map(select(length > 0)))}') - # Bounded: a slow or flaky api.indexnow.org must not burn the job - # timeout; three retries cover a transient error, and a transport - # error after them yields code 000 for the branch below instead of - # aborting under `set -e`. - code=$(curl -sS -o response.txt -w '%{http_code}' \ - --max-time 30 --retry 3 --retry-delay 5 --retry-all-errors \ - -X POST "https://api.indexnow.org/indexnow" \ - -H "Content-Type: application/json; charset=utf-8" \ - --data "$body") || code="000" + urlList: ($list | split("\n") | map(select(length > 0)))}' > body.json n=$(grep -c . "$f") + # First use of a key (rollout or rotation): IndexNow verifies the + # key file asynchronously and answers 403 SiteVerificationNotCompleted + # until it has — seen minutes after the file went live. Retry that + # case once a minute until a ten-minute deadline has passed (an + # elapsed deadline, so slow responses cannot stretch it); if it + # persists, the run FAILS, so a `changed` run keeps its URL set and + # can be re-run — a green warning would have dropped those URLs for + # good, because later pushes submit only their own diffs. + deadline=$(( $(date +%s) + 600 )) + while :; do + # Bounded: a slow or flaky api.indexnow.org must not burn the job + # timeout; three retries cover a transient error, and a transport + # error after them yields code 000 for the branch below instead of + # aborting under `set -e`. + code=$(curl -sS -o response.txt -w '%{http_code}' \ + --max-time 60 --retry 3 --retry-delay 5 --retry-all-errors \ + -X POST "https://api.indexnow.org/indexnow" \ + -H "Content-Type: application/json; charset=utf-8" \ + --data @body.json) || code="000" + if [ "$code" = 403 ] && grep -q SiteVerificationNotCompleted response.txt 2>/dev/null \ + && [ "$(date +%s)" -lt "$deadline" ]; then + echo "::notice::IndexNow has not finished verifying the key file yet; retrying in 60 s" + sleep 60 + continue + fi + break + done case "$code" in 200|202) echo "::notice::IndexNow accepted ${n} URL(s) (HTTP ${code})" ;; - # 4xx is a protocol or key problem on our side; make it visible. + # 4xx is a protocol or key problem on our side (or the key still + # unverified after ten minutes); make it visible and keep the run + # re-runnable. 4*) echo "::error::IndexNow rejected ${n} URL(s) (HTTP ${code}): $(head -c 300 response.txt 2>/dev/null)"; exit 1 ;; # 5xx / no response: their side. Every later push resubmits its # own URLs and `scope=sitemap` covers a longer gap, so warn. diff --git a/CHANGELOG.md b/CHANGELOG.md index 53c76205a1..7d2675e6f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,17 @@ aggregate instead: an italic *Catalog* line at the end of the version section an ### Fixed +- **The IndexNow workflow no longer waits eight minutes behind an edge 403** — its + key-file readiness loop treated every non-200 as "not deployed yet"; a GitHub runner + that Cloudflare's bot management answers with 403 would have slept the full budget on + every run. A 403 now ends the wait without claiming anything about the deploy (the + edge answers before the origin is asked, so it is inconclusive either way; IndexNow's + own key fetch is the authoritative check), and the final warning names the last + status seen. The submission body goes + through a file (a full-sitemap batch is ~260 KB, above the 128 KB command-line + argument cap that made the first sitemap-scope run report HTTP 000), and a first-use + `403 SiteVerificationNotCompleted` from IndexNow is retried for up to ten minutes + before the run fails, so a `changed` run keeps its URL set re-runnable. (#11204) - **Implementation pages no longer share one meta description per spec** — the SEO proxy reused the spec description verbatim as the `` and OG description of every implementation page, so up to 15 library pages and their hub diff --git a/docs/reference/seo.md b/docs/reference/seo.md index 99b1e90d3e..63427d5d16 100644 --- a/docs/reference/seo.md +++ b/docs/reference/seo.md @@ -517,11 +517,17 @@ sitemap. Three pieces, kept in sync when the key is rotated: | Piece | Where | Purpose | |-------|-------|---------| | Key file | `app/public/.txt` (served by nginx to every client, bots included — an explicit `location =` like `robots.txt`) | Proves the submitter controls `anyplot.ai`; the engines fetch it on every submission. The key is public by design. | -| Submission workflow | `.github/workflows/indexnow-submit.yml` | On every push to `main` that touches `plots/`, maps the diff to page URLs (`/{spec}` and `/{spec}/{language}/{library}`) and POSTs them to `https://api.indexnow.org/indexnow`, 10,000 per request. A deleted implementation is submitted too — the protocol means "this URL changed". | +| Submission workflow | `.github/workflows/indexnow-submit.yml` | On every push to `main` that touches `plots/`, maps the diff to page URLs (`/{spec}` and `/{spec}/{language}/{library}`; a changed specification covers the hub and all of its implementation pages) and POSTs them to `https://api.indexnow.org/indexnow`, 10,000 per request. A deleted implementation is submitted too — the protocol means "this URL changed". A push that changes the workflow file itself, or one carrying more than one commit, submits the full list instead of a diff. | | Manual full load | `gh workflow run indexnow-submit.yml -f scope=sitemap` | Submits every URL of the live sitemap; used once at rollout and after a long outage of the workflow. | The engines answer `200` or `202` for an accepted batch; `4xx` means a key or -payload problem and fails the run so it is visible. Bing Webmaster Tools shows +payload problem and fails the run so it is visible. One `4xx` is transient by +nature: after a rollout or a key rotation IndexNow verifies the key file +asynchronously and answers `403 SiteVerificationNotCompleted` for a while, so +the workflow retries that case once a minute until a ten-minute deadline has +passed (elapsed time, so slow responses cannot stretch it) and then fails — a +`changed` run must not report success without its URLs accepted, because later +pushes submit only their own diffs. Bing Webmaster Tools shows the received submissions under *IndexNow*. ## Discoverability for assistants diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index 7bde8754bf..04c57d679e 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -172,7 +172,7 @@ Located in `.github/workflows/`: | `report-validate.yml` | Validates user-submitted issue reports | | `sync-postgres.yml` | Syncs `plots/` filesystem state to PostgreSQL on push to main | | `sync-labels.yml` | Auto-syncs spec/impl labels after manual PR merges | -| `indexnow-submit.yml` | Pushes changed page URLs to IndexNow (Bing, Yandex, Seznam, Naver, Yep) on every push to main that touches `plots/`; `workflow_dispatch` with `scope=sitemap` submits the whole sitemap | +| `indexnow-submit.yml` | Pushes changed page URLs to IndexNow (Bing, Yandex, Seznam, Naver, Yep) on every push to main that touches `plots/`; a push that changes the workflow file itself submits the full list, as does `workflow_dispatch` with `scope=sitemap` | | `codeql.yml` | CodeQL scanning (actions, JavaScript/TypeScript, Python) on pushes to main, PRs and a weekly cron; `plots/**` is excluded from triggers and analysis, so pipeline PRs never start a scan | | `ci-lint.yml` | Ruff lint check on PRs | | `ci-tests.yml` | Unit + integration tests on PRs |