From 5a0122b351fe1b2634d766703d9d66020f50a4e6 Mon Sep 17 00:00:00 2001 From: ayushtr-aws Date: Thu, 9 Jul 2026 13:54:13 -0400 Subject: [PATCH 1/4] fix(ci): remove broken 'Ensure stack torn down' safety net from integ workflow (#566) The if: always() backstop ran raw aws cloudformation delete-stack / wait stack-delete-complete as the GitHub OIDC role, which has no cloudformation:DeleteStack or DescribeStacks permission. Both calls AccessDenied on every run; the un-guarded wait exits 255 and fails the job even when the integ test passed and integ-runner already destroyed the stack (e.g. run 28892156232: SUCCESS in 291s, then teardown-step failure). integ-runner remains the teardown path (cdk destroy on success and failure via the CDK bootstrap roles). Stranded-stack reclaim for cancelled/crashed runs is owned by #400 (non-blocking cleanup) and #72 (scheduled sweep). --- .github/workflows/integ.yml | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/.github/workflows/integ.yml b/.github/workflows/integ.yml index 0586d9a2..345711dd 100644 --- a/.github/workflows/integ.yml +++ b/.github/workflows/integ.yml @@ -217,31 +217,15 @@ jobs: - name: Install dependencies run: yarn install --immutable + # Teardown: integ-runner forces `cdk destroy` on success and failure (via + # the CDK bootstrap roles, which hold the CloudFormation permissions). No + # raw-CLI safety net here: the GitHub OIDC role has no + # cloudformation:DeleteStack/DescribeStacks, so such a step can only fail + # (#566). If a cancelled/crashed run strands `backgroundagent-integ`, + # reclaim is the stranded-stack cleanup owned by #400/#72. - name: Run integ tests (deploy → assert → destroy) run: mise //cdk:integ - # Safety net: integ-runner forces teardown on success and failure, but if - # the run is cancelled or crashes mid-deploy the stack can be stranded in - # the shared account. Delete it directly via CloudFormation so we never - # leak billable resources. - # - # NOTE: `cdk destroy backgroundagent-integ` would NOT work here — it - # synthesizes the main app (src/main.ts), which does not contain the integ - # stack, so it exits 0 having deleted nothing. Target the stack by its - # literal CloudFormation name instead. delete-stack is idempotent (no-op if - # already gone), so `|| true` only guards transient API errors. - - name: Ensure stack torn down - if: always() - env: - AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} - AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} - run: | - set -euo pipefail - aws cloudformation delete-stack --stack-name backgroundagent-integ || true - # No `|| true` on the wait: a DELETE_FAILED must surface loudly so we - # never silently leak billable resources in the shared account. - aws cloudformation wait stack-delete-complete --stack-name backgroundagent-integ - # Post the final integ-smoke status back to the PR head so the check flips from # pending to success/failure. Skipped for workflow_dispatch (no PR to gate). report: From 7dd94a5290048a87fbb494b9783ee47820587a9a Mon Sep 17 00:00:00 2001 From: ayushtr-aws Date: Thu, 9 Jul 2026 14:08:07 -0400 Subject: [PATCH 2/4] feat(ci): map integ stack name to commit sha (int-) Replace the single hardcoded backgroundagent-integ stack name with a run-unique int- derived from the resolved head SHA, threaded into integ-runner via INTEG_STACK_NAME. The test falls back to a stable backgroundagent-integ-local name so the local mise //cdk:integ path is unchanged. A stranded stack from a cancelled/crashed run no longer collides with or head-of-line blocks a later run under one fixed name; the int- prefix marks it ephemeral for the scheduled sweep (#72). Concurrency-comment updated: run-unique names make single-concurrency a cost bound, not a correctness requirement. Partial pull-forward of #400's ephemeral-name ask, scoped to naming. --- .github/workflows/integ.yml | 25 ++++++++++++++++++++----- cdk/test/integ/integ.task-api-smoke.ts | 8 +++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integ.yml b/.github/workflows/integ.yml index 345711dd..726725d2 100644 --- a/.github/workflows/integ.yml +++ b/.github/workflows/integ.yml @@ -29,8 +29,10 @@ on: types: [completed] workflow_dispatch: {} -# Only one integ run at a time against the shared account — overlapping deploys -# would collide on the single hardcoded `backgroundagent-integ` stack name. +# Only one integ run at a time against the shared account. Stack names are now +# run-unique (`int-`, see the integ job) so this is no longer a correctness +# requirement — it bounds shared-account cost/parallelism rather than preventing +# a name collision. concurrency: group: cdk-integ cancel-in-progress: false @@ -221,10 +223,23 @@ jobs: # the CDK bootstrap roles, which hold the CloudFormation permissions). No # raw-CLI safety net here: the GitHub OIDC role has no # cloudformation:DeleteStack/DescribeStacks, so such a step can only fail - # (#566). If a cancelled/crashed run strands `backgroundagent-integ`, - # reclaim is the stranded-stack cleanup owned by #400/#72. + # (#566). If a cancelled/crashed run strands the stack, reclaim is the + # stranded-stack cleanup owned by #400/#72 — and the run-unique name below + # means a stranded stack never blocks a later run. + # + # INTEG_STACK_NAME maps the stack to the commit under test (`int-`), + # replacing the single fixed `backgroundagent-integ` name. The `int-` + # prefix marks it ephemeral for #72's sweep. HEAD_SHA is the resolved PR + # head (or dispatched ref); the substring is already lowercase-hex so it + # needs no further sanitizing to be CloudFormation-valid. - name: Run integ tests (deploy → assert → destroy) - run: mise //cdk:integ + env: + HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + run: | + set -euo pipefail + export INTEG_STACK_NAME="int-${HEAD_SHA:0:12}" + echo "Integ stack name: $INTEG_STACK_NAME" + mise //cdk:integ # Post the final integ-smoke status back to the PR head so the check flips from # pending to success/failure. Skipped for workflow_dispatch (no PR to gate). diff --git a/cdk/test/integ/integ.task-api-smoke.ts b/cdk/test/integ/integ.task-api-smoke.ts index 5775de59..fefdd745 100644 --- a/cdk/test/integ/integ.task-api-smoke.ts +++ b/cdk/test/integ/integ.task-api-smoke.ts @@ -89,8 +89,14 @@ class TaskApiSmokeStack extends Stack { } } +// Stack name is run-unique in CI (`int-`, set by integ.yml) so a +// stranded stack from a cancelled/crashed run never collides with — or blocks — +// a later run under one fixed name. Falls back to a stable local name so the +// `mise //cdk:integ` dev path is unchanged. The `int-` prefix keeps it +// letter-led and the SHA is lowercase hex, so the name is CloudFormation-valid. const app = new App(); -const stack = new TaskApiSmokeStack(app, 'backgroundagent-integ'); +const stackName = process.env.INTEG_STACK_NAME || 'backgroundagent-integ-local'; +const stack = new TaskApiSmokeStack(app, stackName); const integ = new IntegTest(app, 'TaskApiSmoke', { testCases: [stack], From bb54b609c950d1f6270d04e7ac5cfae413ee92b7 Mon Sep 17 00:00:00 2001 From: ayushtr-aws Date: Thu, 9 Jul 2026 14:11:35 -0400 Subject: [PATCH 3/4] feat(ci): add run-number to integ stack name (int--) Same-commit re-runs (re-run, or dispatch + workflow_run on one SHA) now get distinct stack names, matching #400's proposed int-{short-sha}-{run} scheme. --- .github/workflows/integ.yml | 15 +++++++++------ cdk/test/integ/integ.task-api-smoke.ts | 11 ++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integ.yml b/.github/workflows/integ.yml index 726725d2..559e1f43 100644 --- a/.github/workflows/integ.yml +++ b/.github/workflows/integ.yml @@ -227,17 +227,20 @@ jobs: # stranded-stack cleanup owned by #400/#72 — and the run-unique name below # means a stranded stack never blocks a later run. # - # INTEG_STACK_NAME maps the stack to the commit under test (`int-`), - # replacing the single fixed `backgroundagent-integ` name. The `int-` - # prefix marks it ephemeral for #72's sweep. HEAD_SHA is the resolved PR - # head (or dispatched ref); the substring is already lowercase-hex so it - # needs no further sanitizing to be CloudFormation-valid. + # INTEG_STACK_NAME maps the stack to the commit under test plus the run + # number (`int--`), replacing the single fixed + # `backgroundagent-integ` name. The run number makes same-commit re-runs + # distinct; the `int-` prefix marks it ephemeral for #72's sweep. HEAD_SHA + # is the resolved PR head (or dispatched ref) and RUN_NUMBER is a positive + # integer, so both are already lowercase-alphanumeric — no sanitizing + # needed to stay CloudFormation-valid. - name: Run integ tests (deploy → assert → destroy) env: HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + RUN_NUMBER: ${{ github.run_number }} run: | set -euo pipefail - export INTEG_STACK_NAME="int-${HEAD_SHA:0:12}" + export INTEG_STACK_NAME="int-${HEAD_SHA:0:12}-${RUN_NUMBER}" echo "Integ stack name: $INTEG_STACK_NAME" mise //cdk:integ diff --git a/cdk/test/integ/integ.task-api-smoke.ts b/cdk/test/integ/integ.task-api-smoke.ts index fefdd745..c5e78bc3 100644 --- a/cdk/test/integ/integ.task-api-smoke.ts +++ b/cdk/test/integ/integ.task-api-smoke.ts @@ -89,11 +89,12 @@ class TaskApiSmokeStack extends Stack { } } -// Stack name is run-unique in CI (`int-`, set by integ.yml) so a -// stranded stack from a cancelled/crashed run never collides with — or blocks — -// a later run under one fixed name. Falls back to a stable local name so the -// `mise //cdk:integ` dev path is unchanged. The `int-` prefix keeps it -// letter-led and the SHA is lowercase hex, so the name is CloudFormation-valid. +// Stack name is run-unique in CI (`int--`, set by +// integ.yml) so a stranded stack from a cancelled/crashed run never collides +// with — or blocks — a later run under one fixed name. Falls back to a stable +// local name so the `mise //cdk:integ` dev path is unchanged. The `int-` prefix +// keeps it letter-led and the rest is lowercase-alphanumeric + hyphens, so the +// name is CloudFormation-valid. const app = new App(); const stackName = process.env.INTEG_STACK_NAME || 'backgroundagent-integ-local'; const stack = new TaskApiSmokeStack(app, stackName); From 3b5024c8ecc5f30feb6df6adfcd102d9c05876db Mon Sep 17 00:00:00 2001 From: ayushtr-aws Date: Thu, 9 Jul 2026 14:35:53 -0400 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Scott Schreckengaust --- .github/workflows/integ.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integ.yml b/.github/workflows/integ.yml index 559e1f43..692b1010 100644 --- a/.github/workflows/integ.yml +++ b/.github/workflows/integ.yml @@ -242,7 +242,13 @@ jobs: set -euo pipefail export INTEG_STACK_NAME="int-${HEAD_SHA:0:12}-${RUN_NUMBER}" echo "Integ stack name: $INTEG_STACK_NAME" - mise //cdk:integ + mise //cdk:integ -- --all --require-approval never \ + -c stackName="integ-${{ github.run_id }}-${{ github.run_attempt }}" \ + -c github:sha="$(git rev-parse HEAD)" \ + -c github:ref="$(git branch --show-current || echo "detached")" \ + -c github:actor="${{ github.triggering_actor }}" \ + -c github:repository="$(gh repo view --json nameWithOwner --jq .nameWithOwner)" \ + -c github:clean="$([ -z "$(git status --porcelain)" ] && echo true || echo false)" # Post the final integ-smoke status back to the PR head so the check flips from # pending to success/failure. Skipped for workflow_dispatch (no PR to gate).