MarkusNeusinger · MarkusNeusinger · Sep 2, 2026 · Sep 2, 2026 · Sep 2, 2026 · Sep 2, 2026
diff --git a/.github/workflows/impl-generate.yml b/.github/workflows/impl-generate.yml
@@ -874,13 +874,15 @@ jobs:
       # Upload to GCS Staging
       # ========================================================================
       - name: Authenticate to GCP
+        id: gcp_auth
         if: steps.pr.outputs.pr_exists == 'true'
         uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093  # v3
         with:
           project_id: anyplot
           workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
 
       - name: Set up Cloud SDK
+        id: gcloud
         if: steps.pr.outputs.pr_exists == 'true'
         uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db  # v3
 
@@ -980,6 +982,7 @@ jobs:
           gh issue comment "$ISSUE" --body "$BODY"
 
       - name: Trigger review workflow
+        id: review_dispatch
         if: steps.pr.outputs.pr_exists == 'true'
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -1021,9 +1024,57 @@ jobs:
           # well under an hour) with room for a stalled tail, without letting
           # last month's failures veto today's retries.
           CAMPAIGN_WINDOW_H: '12'
+          # Step outcomes, to tell an infrastructure failure (provider incident,
+          # cloud auth, GitHub API) from the agent's own — only the latter counts
+          # toward the 3-attempt cap. Only the retry step is inspected: it runs
+          # solely when the first Claude step failed, so its failure means both
+          # runs died (the first step has continue-on-error and never fails the
+          # job by itself).
+          CLAUDE_RETRY_OUTCOME: ${{ steps.claude_retry.outcome }}
+          CLAUDE_EXEC_FILE: ${{ steps.claude_retry.outputs.execution_file }}
+          GCP_AUTH_OUTCOME: ${{ steps.gcp_auth.outcome }}
+          GCLOUD_OUTCOME: ${{ steps.gcloud.outcome }}
+          GCS_OUTCOME: ${{ steps.gcs.outcome }}
+          PR_OUTCOME: ${{ steps.pr.outcome }}
+          REVIEW_DISPATCH_OUTCOME: ${{ steps.review_dispatch.outcome }}
+          # Infrastructure failures are retried without spending the pair's
+          # budget, but not forever: after this many in the window the pair is
+          # parked WITHOUT `impl:<lib>:failed` (it is not a capability verdict)
+          # and left for the next dispatch.
+          INFRA_WINDOW_CAP: '5'
         run: |
           echo "::notice::Handling generation failure for $LIBRARY/$SPEC_ID"
 
+          # Classify the failure. The retry cap exists to stop re-running a pair
+          # the model cannot solve; a provider incident, a cloud token refresh
+          # or a GitHub 5xx says nothing about the pair. During the Claude
+          # outage of 2026-09-02 (03:15–03:45 UTC) every run failed with
+          # `is_error:true` + "Internal error", 27 pairs burned all three
+          # attempts in twenty minutes, and for the next twelve hours each
+          # re-dispatch ran without any auto-retry because the markers counted.
+          #
+          # Infrastructure = both Claude runs ended in the action's own error
+          # path with a provider-side signature (the execution log is checked,
+          # so an agent that gave up — max turns, refused — still counts), or
+          # the GCP auth / Cloud SDK / GCS upload / PR creation / review
+          # dispatch step failed. "Implementation file not found" and a missing
+          # theme render are the agent's own and keep counting.
+          INFRA_CAUSE=""
+          if [ "${CLAUDE_RETRY_OUTCOME}" = "failure" ]; then
+            if [ -n "${CLAUDE_EXEC_FILE}" ] && [ -f "${CLAUDE_EXEC_FILE}" ] \
+               && grep -qiE 'Internal error|overloaded|rate.?limit|too many requests|ECONNRESET|ETIMEDOUT|"status": *5[0-9][0-9]' "${CLAUDE_EXEC_FILE}"; then
+              SNIPPET=$(grep -oiE 'Internal error[^"]{0,60}|overloaded[^"]{0,40}|rate.?limit[^"]{0,40}|too many requests|ECONNRESET|ETIMEDOUT|"status": *5[0-9][0-9]' "${CLAUDE_EXEC_FILE}" | head -1)
+              INFRA_CAUSE="Claude Code action error: ${SNIPPET:-provider-side error, see the execution log}"
+            fi
+          elif [ "${GCP_AUTH_OUTCOME}" = "failure" ] || [ "${GCLOUD_OUTCOME}" = "failure" ] || [ "${GCS_OUTCOME}" = "failure" ]; then
+            INFRA_CAUSE="Google Cloud auth/upload step failed"
+          elif [ "${PR_OUTCOME}" = "failure" ] || [ "${REVIEW_DISPATCH_OUTCOME}" = "failure" ]; then
+            INFRA_CAUSE="GitHub API step failed (PR creation or review dispatch)"
+          fi
+          if [ -n "${INFRA_CAUSE}" ]; then
+            echo "::notice::Classified as infrastructure failure: ${INFRA_CAUSE}"
+          fi
+
           # Count previous failures via hidden marker comments (more reliable than workflow runs).
           # Paginate so the marker is found even on issues with >30 comments
           # (which is common because all 15 library impls land on the same issue).
@@ -1053,15 +1104,90 @@ jobs:
           # on the very next manual dispatch, and 87 pairs parked repo-wide.
           # A generation campaign is minutes long, so a window of hours is
           # generous while still letting stale history age out on its own.
+          #
+          # INFRASTRUCTURE-AWARE: a marker that also carries INFRA_TAG records a
+          # failure that was not the agent's (see the classification above).
+          # Those are counted separately: they never spend the 3-attempt budget
+          # and are bounded by INFRA_WINDOW_CAP instead.
           MARKER="<!-- impl-fail:${SPEC_ID}:${LIBRARY} -->"
+          INFRA_TAG="<!-- impl-fail-cause:infra -->"
           CAMPAIGN_CUTOFF=$(date -u -d "${CAMPAIGN_WINDOW_H} hours ago" +%Y-%m-%dT%H:%M:%SZ)
-          if FAILURE_COUNT=$(gh api --paginate "repos/${{ github.repository }}/issues/${ISSUE}/comments?per_page=100" \
-            --jq "[.[] | select(.body != null and (.body | contains(\"$MARKER\")) and .created_at > \"$CAMPAIGN_CUTOFF\")] | length" \
-            | awk '{ sum += $1 } END { print sum + 0 }'); then
-            echo "::notice::Previous failures for ${LIBRARY}/${SPEC_ID} since ${CAMPAIGN_CUTOFF}: $FAILURE_COUNT"
+          # One paginated call, two numbers per page ("<genuine> <infra>"), summed by awk.
+          if COUNTS=$(gh api --paginate "repos/${{ github.repository }}/issues/${ISSUE}/comments?per_page=100" \
+            --jq "[.[] | select(.body != null and (.body | contains(\"$MARKER\")) and .created_at > \"$CAMPAIGN_CUTOFF\")] | \"\(map(select(.body | contains(\"$INFRA_TAG\") | not)) | length) \(map(select(.body | contains(\"$INFRA_TAG\"))) | length)\"" \
+            | awk '{ g += $1; i += $2 } END { print (g + 0) " " (i + 0) }'); then
+            FAILURE_COUNT=${COUNTS% *}
+            INFRA_COUNT=${COUNTS#* }
+            echo "::notice::Previous failures for ${LIBRARY}/${SPEC_ID} since ${CAMPAIGN_CUTOFF}: $FAILURE_COUNT (plus $INFRA_COUNT infrastructure failures, not counted)"
           else
             echo "::warning::Failure-count API call failed — failing closed (treating retry cap as reached, no auto-retry)"
             FAILURE_COUNT=999
+            INFRA_COUNT=999
+          fi
+
+          # Retry dispatch, shared by both branches below. Forwards
+          # `change_request` so cross-library divergence hints from daily-regen
+          # pre-flight survive the retry — otherwise the first attempt has the
+          # hint but the retry doesn't, defeating the audit. CHANGE_REQUEST is
+          # read from env to keep raw quotes/$/backticks inside the hint from
+          # breaking shell parsing.
+          dispatch_retry() {
+            gh workflow run impl-generate.yml \
+              -f specification_id="${SPEC_ID}" \
+              -f library="${LIBRARY}" \
+              -f issue_number="${ISSUE}" \
+              -f model="${MODEL}" \
+              -f change_request="${CHANGE_REQUEST}"
+            if [ -n "${CHANGE_REQUEST}" ]; then
+              echo "::notice::Triggered automatic retry for ${LIBRARY}/${SPEC_ID} ($1, model=${MODEL}, change_request=present)"
+            else
+              echo "::notice::Triggered automatic retry for ${LIBRARY}/${SPEC_ID} ($1, model=${MODEL}, change_request=none)"
+            fi
+          }
+
+          if [ -n "${INFRA_CAUSE}" ]; then
+            INFRA_ATTEMPT=$((INFRA_COUNT + 1))
+            if [ "${INFRA_COUNT}" -ge $((INFRA_WINDOW_CAP - 1)) ]; then
+              # Parked, not failed: nothing here says the pair is impossible.
+              # No `impl:<lib>:failed`, so the watchdog does not treat it as a
+              # capability verdict either; the next backfill dispatch picks it
+              # up with a clean budget once the markers age out.
+              echo "::warning::Parking $LIBRARY/$SPEC_ID after ${INFRA_ATTEMPT} infrastructure failures in the last ${CAMPAIGN_WINDOW_H}h (cap: ${INFRA_WINDOW_CAP}) — not marked failed, no auto-retry"
+              # One label per call: a name `gh` cannot resolve fails a whole
+              # comma-separated edit and would leave both labels in place.
+              for stale in "generate:${LIBRARY}" "impl:${LIBRARY}:pending"; do
+                gh issue edit "$ISSUE" --remove-label "$stale" 2>/dev/null || true
+              done
+              gh issue comment "$ISSUE" --body "${MARKER}
+          ${INFRA_TAG}
+          ## :pause_button: ${LIBRARY} Paused (infrastructure failures)
+
+          The **${LIBRARY}** implementation for \`${SPEC_ID}\` hit ${INFRA_ATTEMPT} infrastructure failures in the last ${CAMPAIGN_WINDOW_H}h (cap: ${INFRA_WINDOW_CAP}) and is paused. This is **not** a capability verdict — the pair keeps its 3-attempt budget.
+
+          **Last cause:** ${INFRA_CAUSE}
+
+          To retry once the incident is over:
+          \`\`\`
+          gh workflow run impl-generate.yml -f specification_id=${SPEC_ID} -f library=${LIBRARY} -f issue_number=${ISSUE} -f model=${MODEL}
+          \`\`\`
+
+          ---
+          :robot: *[impl-generate](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*"
+            else
+              gh issue comment "$ISSUE" --body "${MARKER}
+          ${INFRA_TAG}
+          ## :cloud: ${LIBRARY} Infrastructure Failure (${INFRA_ATTEMPT}/${INFRA_WINDOW_CAP} in window)
+
+          **Cause:** ${INFRA_CAUSE}
+
+          Not counted toward the 3-attempt cap. Automatically retrying...
+
+          ---
+          :robot: *[impl-generate](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})*"
+              gh issue edit "$ISSUE" --remove-label "generate:${LIBRARY}" 2>/dev/null || true
+              dispatch_retry "infrastructure retry ${INFRA_ATTEMPT}/${INFRA_WINDOW_CAP}"
+            fi
+            exit 0
           fi
 
           # FAILURE_COUNT counts marker comments BEFORE this run.
@@ -1089,10 +1215,13 @@ jobs:
             gh label create "impl:${LIBRARY}:failed" --color "d73a4a" \
               --description "${LIBRARY} implementation failed" 2>/dev/null || true
 
-            # Remove stale labels and add failed
-            gh issue edit "$ISSUE" \
-              --remove-label "generate:${LIBRARY},impl:${LIBRARY}:pending" \
-              --add-label "impl:${LIBRARY}:failed" 2>/dev/null || true
+            # Add failed first, then drop the stale labels one per call: a
+            # name `gh` cannot resolve fails a whole comma-separated edit, and
+            # the old single call could leave the pair marked pending forever.
+            gh issue edit "$ISSUE" --add-label "impl:${LIBRARY}:failed" 2>/dev/null || true
+            for stale in "generate:${LIBRARY}" "impl:${LIBRARY}:pending"; do
+              gh issue edit "$ISSUE" --remove-label "$stale" 2>/dev/null || true
+            done
 
             # Post final failure comment with marker
             gh issue comment "$ISSUE" --body "${MARKER}
@@ -1123,22 +1252,5 @@ jobs:
             # Clean up generate label before retry
             gh issue edit "$ISSUE" --remove-label "generate:${LIBRARY}" 2>/dev/null || true
 
-            # Automatic retry via workflow_dispatch.
-            # Forward `change_request` so cross-library divergence hints from
-            # daily-regen pre-flight survive the retry — otherwise the first
-            # attempt has the hint but the retry doesn't, defeating the audit.
-            # CHANGE_REQUEST is read from env to keep raw quotes/$/backticks
-            # inside the hint from breaking shell parsing.
-            gh workflow run impl-generate.yml \
-              -f specification_id="${SPEC_ID}" \
-              -f library="${LIBRARY}" \
-              -f issue_number="${ISSUE}" \
-              -f model="${MODEL}" \
-              -f change_request="${CHANGE_REQUEST}"
-
-            if [ -n "${CHANGE_REQUEST}" ]; then
-              echo "::notice::Triggered automatic retry for ${LIBRARY}/${SPEC_ID} (attempt $((ATTEMPT + 1)), model=${MODEL}, change_request=present)"
-            else
-              echo "::notice::Triggered automatic retry for ${LIBRARY}/${SPEC_ID} (attempt $((ATTEMPT + 1)), model=${MODEL}, change_request=none)"
-            fi
+            dispatch_retry "attempt $((ATTEMPT + 1))"
           fi
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,19 @@ aggregate instead: an italic *Catalog* line at the end of the version section an
 
 ### Fixed
 
+- **Infrastructure failures no longer spend a pair's generation budget** — the
+  3-attempt cap in `impl-generate.yml` counted every failed run alike, so the Claude
+  outage of 2026-09-02 (03:15–03:45 UTC, every run ending in `is_error:true` with an
+  "Internal error") burned all three attempts of 27 pairs in twenty minutes, and for the
+  rest of the 12-hour window each re-dispatch ran without any auto-retry. The failure
+  handler now classifies the run from the step outcomes: both Claude runs dying with a
+  provider-side signature in the execution log, a Google Cloud auth/SDK/upload failure,
+  or a GitHub API failure at PR creation or review dispatch is an infrastructure failure.
+  Those are recorded with an extra `<!-- impl-fail-cause:infra -->` tag, excluded from the
+  genuine count, and retried on a separate cap of 5 per window, after which the pair is
+  paused without `impl:<lib>:failed` (an incident is not a capability verdict). "Agent
+  reports success but writes no file" and a missing theme render still count as before.
+  (#11199)
 - **A merged implementation now clears its `impl:<lib>:failed` label and the watchdog's
   retry marker** — impl-merge removed the stale labels in one comma-separated
   `gh issue edit`, and `gh` rejects the whole call when any name in the list is not a

diff --git a/agentic/docs/project-guide.md b/agentic/docs/project-guide.md
@@ -789,7 +789,7 @@ uv run python -m automation.scripts.label_manager list
 - **`generate:all`** - Trigger all 15 libraries via bulk-generate
 - **`impl:{library}:pending`** - Generation in progress
 - **`impl:{library}:done`** - Implementation merged to main
-- **`impl:{library}:failed`** - Max retries exhausted (4 repair attempts)
+- **`impl:{library}:failed`** - Set by impl-generate after three failed generation attempts for the pair within a 12-hour window, or by impl-review when the PR still scores below 50 after four repair attempts; infrastructure failures (provider incident, cloud auth, GitHub API) are retried on their own cap of 5 and never set it
 
 ### PR Labels (set by workflows)
 

diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md
@@ -99,7 +99,7 @@ impl-review.yml
 | `generate:{library}` | Trigger generation for library | User |
 | `impl:{library}:pending` | Generation in progress | Workflow |
 | `impl:{library}:done` | Implementation merged to main | Workflow |
-| `impl:{library}:failed` | Max retries exhausted | Workflow |
+| `impl:{library}:failed` | Set by `impl-generate.yml` after three failed generation attempts for the pair within a 12-hour window, or by `impl-review.yml` when the PR still scores below 50 after four repair attempts. Infrastructure failures (provider incident, cloud auth, GitHub API) are retried on a separate cap of 5 and never set this label; a pair paused that way carries no label. | Workflow |
 
 ### PR labels (on Pull Requests)