fix(pr-envs): list the SAM stack in the close reminder and retry blocked deploys by re-run (ENG-3872)

argocd-pr-env-deploy/action.yml

@@ -479,7 +479,7 @@ runs:
         # anchor PR so the cause is visible without digging through the log.
         if [ "${#unresolved_links[@]}" -gt 0 ]; then
           links="$(printf '%s, ' "${unresolved_links[@]}")"; links="${links%, }"
-          comment="$(printf '**PR-env deploy blocked.**\n\nThese `Deploys:` link(s) could not be resolved, so the deploy was refused: %s\n\nA linked PR resolves to no `merge_commit_sha` when it is missing, closed, or unmergeable (a conflicted/`dirty` PR has none). Rather than silently deploy that repo from its default branch, this job fails.\n\nFix: resolve the conflicts (or correct the link), then re-run the deploy by pushing a commit or toggling the `deploy` label.' "$links")"
+          comment="$(printf '**PR-env deploy blocked.**\n\nThese `Deploys:` link(s) could not be resolved, so the deploy was refused: %s\n\nA linked PR resolves to no `merge_commit_sha` when it is missing, closed, or in conflict with its base branch. Rather than silently deploy that repo from its default branch, this job fails.\n\nTo deploy:\n1. Fix each link above. For a PR in conflict, merge or rebase its base branch into it, push, and wait for its pipeline to finish. For a closed or wrong link, correct or remove that line in the description of this PR.\n2. Re-run the failed jobs from the Checks tab of this PR. The job reads this description and every linked PR again. Pushing a commit here also redeploys, from that new commit.\n\nDo not remove the `deploy` label to retry. Removing it deletes this env, and adding it back creates a new env with an empty database.' "$links")"
           post_blocking_comment "$comment"
           echo "::error::Refusing to deploy: unresolvable Deploys link(s): ${links}"
           exit 1

scripts/prune_pr_environments.py

@@ -32,7 +32,8 @@
 This customer env is gone, but its data is not. Delete by hand when the engagement is over:
 - Files: the `customer-envs/{host_base}/` prefix in the bucket named by `filesBucket` in argocd-envs `parent/values.yaml`
 - Cowork workspaces: the `pr-{slug}-{number}` directory on the cowork EFS
-- Postgres: the retained PersistentVolume for `data-postgres-postgresql-0` in namespace `pr-{slug}-{number}` (`kubectl get pv | grep pr-{slug}-{number}`) and its AWS Backup recovery points
+- Postgres: the retained PersistentVolume for `data-postgres-postgresql-0` in namespace `pr-{slug}-{number}` (`kubectl get pv | grep pr-{slug}-{number}`)
+- SAM stack: the `anton-services-pr-{slug}-{number}` CloudFormation stack and its DynamoDB tables, if it still exists
 """
 
 

tests/test_argocd_pr_env_deploy.py

@@ -0,0 +1,47 @@
+"""Contract tests for the blocked-deploy comment in `argocd-pr-env-deploy/action.yml`.
+
+When a `Deploys:` link cannot be resolved, the action refuses the deploy and
+comments on the PR. That comment is where the author learns how to retry, so
+it has to name a retry that keeps the env: removing the `deploy` label deletes
+the env and its database. The comment is rendered with the action's own
+`printf` line, run through bash, so the test reads the text authors see.
+"""
+
+import subprocess
+from pathlib import Path
+
+import yaml
+
+ACTION_PATH = Path(__file__).resolve().parents[1] / "argocd-pr-env-deploy" / "action.yml"
+ACTION = yaml.safe_load(ACTION_PATH.read_text(encoding="utf-8"))
+RUN = "\n".join(step.get("run", "") for step in ACTION["runs"]["steps"])
+UNRESOLVED_BRANCH = 'if [ "${#unresolved_links[@]}" -gt 0 ]; then'
+
+
+def unresolved_links_comment(links: str) -> str:
+    """Render the comment the action posts for unresolvable `Deploys:` links."""
+    branch = RUN.split(UNRESOLVED_BRANCH, 1)[1]
+    line = next(line.strip() for line in branch.splitlines() if line.strip().startswith('comment="$(printf'))
+    script = f'links="$1"\n{line}\nprintf "%s" "$comment"'
+    return subprocess.run(["bash", "-c", script, "_", links], capture_output=True, text=True, check=True).stdout
+
+
+class TestUnresolvedLinkComment:
+    def test_names_every_refused_link(self):
+        comment = unresolved_links_comment("mindsdb/api#5, mindsdb/web#9")
+        assert comment.startswith("**PR-env deploy blocked.**")
+        assert "the deploy was refused: mindsdb/api#5, mindsdb/web#9\n" in comment
+
+    def test_tells_the_author_how_to_fix_a_conflicted_link(self):
+        comment = unresolved_links_comment("mindsdb/api#5")
+        assert "in conflict with its base branch" in comment
+        assert "\n1. Fix each link above. For a PR in conflict, merge or rebase its base branch into it" in comment
+
+    def test_retries_by_re_running_the_failed_jobs(self):
+        comment = unresolved_links_comment("mindsdb/api#5")
+        assert "\n2. Re-run the failed jobs from the Checks tab of this PR." in comment
+
+    def test_warns_against_retrying_with_the_deploy_label(self):
+        comment = unresolved_links_comment("mindsdb/api#5")
+        assert "toggling the `deploy` label" not in comment
+        assert "Do not remove the `deploy` label to retry. Removing it deletes this env" in comment

tests/test_prune_pr_environments.py

@@ -141,6 +141,23 @@ def test_posts_once_per_closed_customer_pr(self):
         assert "`customer-envs/acme/`" in body
         assert "pr-auth-5" in body
 
+    def test_reminder_names_the_envs_sam_stack(self):
+        # The stack may already be gone when the reminder posts, so the line
+        # says to delete it only if it is still there.
+        gh = FakeGh(envs={}, prs={}, closed={"mindsdb/web_app": [{"number": 7, "labels": ["deploy", "customer:acme"]}]}, comments={})
+        prune.customer_reminders("mindsdb/web_app", gh, dry_run=False)
+        sam_line = next(line for line in gh.posted_bodies()[0].splitlines() if line.startswith("- SAM stack:"))
+        assert "`anton-services-pr-web-app-7`" in sam_line
+        assert sam_line.endswith("if it still exists")
+
+    def test_reminder_asks_for_nothing_the_backup_vault_refuses(self):
+        # The backup vault is locked, so no one can delete its recovery points.
+        gh = FakeGh(envs={}, prs={}, closed={"mindsdb/web_app": [{"number": 7, "labels": ["deploy", "customer:acme"]}]}, comments={})
+        prune.customer_reminders("mindsdb/web_app", gh, dry_run=False)
+        postgres_line = next(line for line in gh.posted_bodies()[0].splitlines() if line.startswith("- Postgres:"))
+        assert "recovery points" not in postgres_line
+        assert postgres_line.endswith("(`kubectl get pv | grep pr-web-app-7`)")
+
     def test_customer_pr_that_never_deployed_is_not_reminded(self):
         gh = FakeGh(envs={}, prs={}, closed={"mindsdb/auth": [{"number": 5, "labels": ["customer:acme"]}]}, comments={})
         assert prune.customer_reminders("mindsdb/auth", gh, dry_run=False) is True