mindsdb-devops · Oct 30, 2026 · Oct 30, 2026 · Oct 30, 2026 · Oct 30, 2026 · Oct 30, 2026
@@ -145,6 +145,7 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
           PR: ${{ github.event.pull_request.number }}
+          PR_LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }}
           ENV_NAME: ${{ inputs.env-name }}
           LINKS: ${{ inputs.links }}
           STANDARD: ${{ inputs.standard-links }}
@@ -203,13 +204,18 @@ jobs:
             # is identical for every anchor repo.
             ALL="${LINKS:-[]}"
             if [ "$STANDARD" = "true" ]; then
-              ALL="$(jq -cn --arg e "$ENV_NAME" --arg d "$HOST_SUFFIX" --argjson extra "${LINKS:-[]}" '
-                [ {label: "Console",   url: "https://\($e).\($d)"},
-                  {label: "Auth API",        url: "https://auth-\($e).\($d)"},
-                  {label: "Cowork Web",   url: "https://cowork-\($e).\($d)"},
-                  {label: "Keycloak Admin",  url: "https://auth-\($e).\($d)/auth/admin/"},
-                  {label: "Inference API",   url: "https://api-\($e).\($d)/v1"},
-                  {label: "Cowork Server API",   url: "https://cowork-\($e).\($d)/api"} ]
+              # A `customer:<hostBase>` label moves the hostnames to <hostBase>;
+              # the namespace stays ${ENV_NAME}. Labels come from the event
+              # payload, so no extra API call can fail this redirect group.
+              HOST_BASE="$(jq -r '[.[] | select(startswith("customer:")) | ltrimstr("customer:")] | first // empty' <<< "${PR_LABELS:-[]}")"
+              HOST="${HOST_BASE:-$ENV_NAME}"
+              ALL="$(jq -cn --arg e "$ENV_NAME" --arg h "$HOST" --arg d "$HOST_SUFFIX" --argjson extra "${LINKS:-[]}" '
+                [ {label: "Console",   url: "https://\($h).\($d)"},
+                  {label: "Auth API",        url: "https://auth-\($h).\($d)"},
+                  {label: "Cowork Web",   url: "https://cowork-\($h).\($d)"},
+                  {label: "Keycloak Admin",  url: "https://auth-\($h).\($d)/auth/admin/"},
+                  {label: "Inference API",   url: "https://api-\($h).\($d)/v1"},
+                  {label: "Cowork Server API",   url: "https://cowork-\($h).\($d)/api"} ]
                 + $extra
                 + [ {label: "Namespace", url: "`\($e)`"} ]')"
             fi

@@ -46,7 +46,8 @@ jobs:
           permission-actions: read
           permission-administration: write
           permission-environments: write
-          permission-pull-requests: read
+          # The customer close reminder is a PR comment.
+          permission-pull-requests: write
 
       - name: Delete stale PR environments
         env:

@@ -4,6 +4,8 @@
 # generator filter — removing the label causes ArgoCD to delete the generated
 # Applications and prune the namespace.
 #
+# PRs carrying a `customer:` label are never stripped; they live until the PR closes.
+#
 # Called as a reusable workflow by per-repo scheduled wrappers. See e.g.
 #   auth/.github/workflows/stale-deploy.yml
 #   mindshub_frontend/.github/workflows/stale-deploy.yml
@@ -42,8 +44,8 @@ jobs:
 
           mapfile -t STALE_PRS < <(
             gh pr list --repo "${TARGET_REPO}" --label deploy --state open \
-              --json number,updatedAt,title \
-              --jq ".[] | select(.updatedAt < \"${THRESHOLD_ISO}\") | .number"
+              --json number,updatedAt,title,labels \
+              --jq ".[] | select(.updatedAt < \"${THRESHOLD_ISO}\") | select(any(.labels[]?.name; startswith(\"customer:\")) | not) | .number"
           )
 
           if [[ ${#STALE_PRS[@]} -eq 0 ]]; then

@@ -9,6 +9,8 @@
 #   - `Deploys: repo#staging` — no API lookup; pins that repo to the
 #     `staging` ECR image tag + `staging` git branch. Useful for testing
 #     the anchor PR against non-PR-branch versions of a sibling service.
+#   - `Deploys: repo#main`: same, pinned to the `production` ECR image tag
+#     + `main` git branch.
 #   - `Deploys: mindshub_services#N` sets `revisions.mindshub_services` only.
 #     That repo ships a SAM stack, not an image: its pr-env-stacks workflow
 #     reads the revision off the parent Application and deploys it, so there
@@ -192,8 +194,17 @@ runs:
         # remember ECR is unusable, and report every tag as present from
         # then on.
         ECR_CHECKS=on
+        ecr_repo_name() {
+          # anton's worker image is published under this name, not
+          # `mindsdb-anton`.
+          case "$1" in
+            anton) echo "minds-anton-scratchpad" ;;
+            *) echo "mindsdb-${1//_/-}" ;;
+          esac
+        }
         ecr_tag_exists() {
-          local ecr_repo="mindsdb-${1//_/-}" tag="$2" err
+          local ecr_repo tag="$2" err
+          ecr_repo="$(ecr_repo_name "$1")"
           [ "$ECR_CHECKS" = on ] || return 0
           if ! command -v aws >/dev/null 2>&1; then
             ECR_CHECKS=off
@@ -222,17 +233,19 @@ runs:
         # lint below both reuse it. Strip \r so CRLF bodies don't trip the
         # line-anchored lint pattern. Fail with a diagnosable message rather
         # than letting `set -e` kill the step mutely on an API blip.
-        PR_BODY="$(gh_api "/repos/$OWN_REPO/pulls/$PR_NUMBER" | jq -r '.body // ""' | tr -d '\r')" || {
+        PR_JSON="$(gh_api "/repos/$OWN_REPO/pulls/$PR_NUMBER")" && jq -e 'type == "object"' <<<"$PR_JSON" >/dev/null || {
           echo "::error::Could not fetch PR #${PR_NUMBER} body from the GitHub API (needed to resolve Deploys: links) — transient API failure or bad GH_TOKEN. Re-run the job."
           exit 1
         }
+        PR_BODY="$(jq -r '.body // ""' <<<"$PR_JSON" | tr -d '\r')"
+        LABEL_HOST="$(jq -r '[.labels[].name | select(startswith("customer:")) | ltrimstr("customer:")] | first // ""' <<<"$PR_JSON")"
 
         # Lint: `Deploys:`-looking lines that don't match the strict form
         # would otherwise be silently ignored and the env would quietly run
         # defaults — the classic offender is a pasted PR URL. Warn per line.
         while IFS= read -r line; do
-          if ! printf '%s\n' "$line" | grep -qE 'Deploys:[[:space:]]+[^#[:space:]]+#([0-9]+|staging)'; then
-            echo "::warning::Ignoring unrecognised Deploys line: '${line}'. Accepted forms: 'Deploys: repo#N', 'Deploys: org/repo#N', 'Deploys: repo#staging' (PR URLs are not parsed)."
+          if ! printf '%s\n' "$line" | grep -qE 'Deploys:[[:space:]]+[^#[:space:]]+#([0-9]+|staging|main)'; then
+            echo "::warning::Ignoring unrecognised Deploys line: '${line}'. Accepted forms: 'Deploys: repo#N', 'Deploys: org/repo#N', 'Deploys: repo#staging', 'Deploys: repo#main' (PR URLs are not parsed)."
           fi
         done < <(printf '%s\n' "$PR_BODY" | grep -iE '^[[:space:]]*deploys?:' || true)
 
@@ -270,7 +283,7 @@ runs:
         else
           OWN_TAG="development-${OWN_MERGE_SHA}"
           if ! ecr_tag_exists "$OWN_SHORT" "$OWN_TAG"; then
-            echo "::error::ECR image mindsdb-${OWN_SLUG}:${OWN_TAG} not found — did the build job in this run succeed?"
+            echo "::error::ECR image $(ecr_repo_name "$OWN_SHORT"):${OWN_TAG} not found. Did the build job in this run succeed?"
             exit 1
           fi
           set_args=(
@@ -306,10 +319,27 @@ runs:
               # after the loop, in the main shell where exit actually works.
               unresolved_links+=("$link")
               ;;
-            staging)
+            line)
+              # `repo#staging` / `repo#main`: the branch field carries which.
+              # Repos without an image take the chart branch alone.
+              if is_revision_only_repo "$repo"; then
+                set_args+=(--helm-set "revisions.${repo}=${branch}")
+                continue
+              fi
+              case "$branch" in
+                staging) tag=staging ;;
+                main)    tag=production ;;
+              esac
+              # Not every repo publishes both moving tags (cowork has no
+              # `production`), so check before pinning: an absent tag leaves
+              # the child in ImagePullBackOff until wait-timeout expires.
+              if [ "$ECR_CHECKS" = on ] && ! ecr_tag_exists "$repo" "$tag"; then
+                echo "::error::No ECR image $(ecr_repo_name "$repo"):${tag} for ${link}. That repo does not publish a '${tag}' tag; link a PR number or the other branch instead."
+                exit 1
+              fi
               set_args+=(
-                --helm-set "tags.${repo}=staging"
-                --helm-set "revisions.${repo}=staging"
+                --helm-set "tags.${repo}=${tag}"
+                --helm-set "revisions.${repo}=${branch}"
               )
               ;;
             revision-only)
@@ -356,7 +386,7 @@ runs:
               if [ "$ECR_CHECKS" = on ] && ecr_tag_exists "$repo" "$head_tag" && [ "$ECR_CHECKS" = on ]; then
                 tag="$head_tag"
               elif [ "$ECR_CHECKS" = on ] && ! ecr_tag_exists "$repo" "$merge_tag"; then
-                echo "::error::No ECR image for linked PR ${link}: neither ${head_tag} nor ${merge_tag} exists in mindsdb-${repo//_/-}. The linked PR's base moved after its last build regenerated the merge SHA; re-run the linked repo's pipeline to build a fresh image, then re-run this job."
+                echo "::error::No ECR image for linked PR ${link}: neither ${head_tag} nor ${merge_tag} exists in $(ecr_repo_name "$repo"). The linked PR's base moved after its last build regenerated the merge SHA; re-run the linked repo's pipeline to build a fresh image, then re-run this job."
                 exit 1
               fi
               set_args+=(
@@ -367,14 +397,25 @@ runs:
           esac
         done < <(
           printf '%s\n' "$PR_BODY" \
-            | grep -oE 'Deploys:[[:space:]]+[^#[:space:]]+#([0-9]+|staging)' \
+            | grep -oE 'Deploys:[[:space:]]+[^#[:space:]]+#([0-9]+|staging|main)' \
             | while read -r _kw ref; do
                 full="${ref%%#*}"; num="${ref##*#}"
                 case "$full" in
                   */*) ;;
                   *)   full="${OWN_REPO%%/*}/$full" ;;
                 esac
                 [ "$full" = "$OWN_REPO" ] && continue
+                # `#staging` / `#main`: reuse the linked repo's release
+                # pipeline outputs, moving ECR image tag + git branch. For
+                # argocd-envs the branch is the parent chart revision itself.
+                if [ "$num" = "staging" ] || [ "$num" = "main" ]; then
+                  if [ "${full##*/}" = "argocd-envs" ]; then
+                    echo "envs:${full}#${num}:${full##*/}:::${num}"
+                  else
+                    echo "line:${full}#${num}:${full##*/}:::${num}"
+                  fi
+                  continue
+                fi
                 # argocd-envs is the env's own parent chart, not a workload:
                 # no image to tag and no child Application to revision. Emit
                 # the head BRANCH so phase 2 can move the parent Application's
@@ -398,12 +439,6 @@ runs:
                   fi
                   continue
                 fi
-                # `#staging` — reuse the linked repo's staging pipeline
-                # outputs: `staging` ECR image tag + `staging` git branch.
-                if [ "$num" = "staging" ]; then
-                  echo "staging:${full}#staging:${full##*/}::"
-                  continue
-                fi
                 # Repos without an image: head SHA only, no ECR lookup.
                 if is_revision_only_repo "${full##*/}"; then
                   head_sha="$(gh_api "/repos/$full/pulls/$num" | jq -r '.head.sha // ""')" || true
@@ -430,17 +465,22 @@ runs:
               done
         )
 
-        # Fail closed on any unresolvable Deploys link, and comment on the
-        # anchor PR so the cause is visible without digging through the log.
-        if [ "${#unresolved_links[@]}" -gt 0 ]; then
-          links="$(printf '%s, ' "${unresolved_links[@]}")"; links="${links%, }"
-          comment="$(printf '**PR-env deploy blocked.**\n\nThese `Deploys:` link(s) could not be resolved, so the deploy was refused: %s\n\nA linked PR resolves to no `merge_commit_sha` when it is missing, closed, or unmergeable (a conflicted/`dirty` PR has none). Rather than silently deploy that repo from its default branch, this job fails.\n\nFix: resolve the conflicts (or correct the link), then re-run the deploy by pushing a commit or toggling the `deploy` label.' "$links")"
-          jq -n --arg b "$comment" '{body: $b}' \
+        # Post a comment on the anchor PR explaining a refused deploy.
+        post_blocking_comment() {
+          jq -n --arg b "$1" '{body: $b}' \
             | curl -sfX POST -H "Authorization: Bearer $GH_TOKEN" \
                  -H "Accept: application/vnd.github+json" \
                  "https://api.github.com/repos/$OWN_REPO/issues/$PR_NUMBER/comments" \
                  --data @- >/dev/null \
             || echo "::warning::could not post the blocking comment to ${OWN_REPO}#${PR_NUMBER} (does the gh-token allow writing issue comments?)" >&2
+        }
+
+        # Fail closed on any unresolvable Deploys link, and comment on the
+        # anchor PR so the cause is visible without digging through the log.
+        if [ "${#unresolved_links[@]}" -gt 0 ]; then
+          links="$(printf '%s, ' "${unresolved_links[@]}")"; links="${links%, }"
+          comment="$(printf '**PR-env deploy blocked.**\n\nThese `Deploys:` link(s) could not be resolved, so the deploy was refused: %s\n\nA linked PR resolves to no `merge_commit_sha` when it is missing, closed, or unmergeable (a conflicted/`dirty` PR has none). Rather than silently deploy that repo from its default branch, this job fails.\n\nFix: resolve the conflicts (or correct the link), then re-run the deploy by pushing a commit or toggling the `deploy` label.' "$links")"
+          post_blocking_comment "$comment"
           echo "::error::Refusing to deploy: unresolvable Deploys link(s): ${links}"
           exit 1
         fi
@@ -505,6 +545,37 @@ runs:
         # ARGOCD_AUTH_TOKEN env vars set above are picked up by every
         # subsequent argocd command directly.
 
+        # The `customer:` label is read only when the env is created. A parent
+        # that is absent or being deleted has nothing to compare: skip, and the
+        # `app set` retry below waits for the ApplicationSet to create it.
+        PARENT_JSON=""
+        get_errfile="$(mktemp)"
+        if ! PARENT_JSON="$("$ARGOCD_BIN" app get "$PARENT_APP" "${ARGOCD_FLAGS[@]}" -o json 2>"$get_errfile")"; then
+          PARENT_JSON=""
+          get_err="$(cat "$get_errfile")"
+          case "$get_err" in
+            *PermissionDenied*|*NotFound*|*"not found"*|*"permission denied"*|*"being deleted"*)
+              echo "Application ${PARENT_APP} not found or being deleted; skipping the customer label check."
+              ;;
+            *)
+              echo "argocd app get ${PARENT_APP} failed unexpectedly:" >&2
+              echo "$get_err" >&2
+              rm -f "$get_errfile"
+              exit 1
+              ;;
+          esac
+        fi
+        rm -f "$get_errfile"
+        if [ -n "$PARENT_JSON" ]; then
+          PARENT_HOST="$(jq -r '[.spec.source.helm.parameters[]? | select(.name=="hostBase") | .value] | first // ""' <<<"$PARENT_JSON")"
+          if [ "$LABEL_HOST" != "$PARENT_HOST" ]; then
+            comment="**PR-env deploy blocked.** This PR's \`customer:\` label does not match the environment that already exists (label: \`${LABEL_HOST}\`, env: \`${PARENT_HOST}\`). The label is read only when the env is created. Remove the \`deploy\` label, wait for the env to disappear, then add \`customer:<hostBase>\` first and \`deploy\` second."
+            post_blocking_comment "$comment"
+            echo "::error::Refusing to deploy: customer label '${LABEL_HOST}' does not match the existing env's hostBase '${PARENT_HOST}'."
+            exit 1
+          fi
+        fi
+
         # Retry `app set` directly until the parent App is set-able. Three
         # transient states are tolerated; anything else is fatal:
         #   1. App doesn't exist yet — ApplicationSet hasn't run its 60s

@@ -8,6 +8,11 @@
 
 Environments are listed before PRs, so an environment created mid-run is left
 for the next run. A repo whose PR list cannot be read is skipped entirely.
+
+A closed PR labelled ``deploy`` and ``customer:<hostBase>`` had a customer env
+whose data nothing deletes, so each one gets a single comment listing it (see
+``REMINDER``). Every closed PR is scanned, so a PR missed while the job was
+failing is still reminded once it recovers.
 """
 
 from __future__ import annotations
@@ -21,6 +26,14 @@
 Runner = Callable[[Sequence[str]], "subprocess.CompletedProcess[str]"]
 
 DEPLOY_LABEL = "deploy"
+CUSTOMER_PREFIX = "customer:"
+MARKER = "<!-- customer-env-close-reminder -->"
+REMINDER = MARKER + """
+This customer env is gone, but its data is not. Delete by hand when the engagement is over:
+- Files: the `customer-envs/{host_base}/` prefix in the bucket named by `filesBucket` in argocd-envs `parent/values.yaml`
+- Cowork workspaces: the `pr-{slug}-{number}` directory on the cowork EFS
+- Postgres: the retained PersistentVolume for `data-postgres-postgresql-0` in namespace `pr-{slug}-{number}` (`kubectl get pv | grep pr-{slug}-{number}`) and its AWS Backup recovery points
+"""
 
 
 def _run(argv: Sequence[str]) -> "subprocess.CompletedProcess[str]":
@@ -47,6 +60,41 @@ def _error(result: "subprocess.CompletedProcess[str]") -> str:
     return result.stderr.strip() or result.stdout.strip()
 
 
+def customer_reminders(repo: str, runner: Runner = _run, *, dry_run: bool) -> bool:
+    """Comment once on each closed PR in ``repo`` that had a customer env; False if a gh call failed."""
+    slug = repo.split("/", 1)[1].replace("_", "-")
+    closed = runner([
+        "gh", "api", "--paginate", f"repos/{repo}/pulls?state=closed&per_page=100",
+        "--jq", f'.[] | select(any(.labels[]; .name == "{DEPLOY_LABEL}")) | '
+                f'select(any(.labels[]; .name | startswith("{CUSTOMER_PREFIX}"))) | '
+                f'"\\(.number) \\([.labels[].name | select(startswith("{CUSTOMER_PREFIX}"))][0])"',
+    ])
+    if closed.returncode != 0:
+        print(f"::error title={repo}::could not list closed PRs: {_error(closed)}", flush=True)
+        return False
+    ok = True
+    for line in _lines(closed):
+        number, label = line.split(" ", 1)
+        comments = runner(["gh", "api", "--paginate", f"repos/{repo}/issues/{number}/comments", "--jq", ".[].body"])
+        if comments.returncode != 0:
+            print(f"::error title={repo}::could not read comments on #{number}: {_error(comments)}", flush=True)
+            ok = False
+            continue
+        if MARKER in comments.stdout:
+            continue
+        if dry_run:
+            print(f"  would remind on {repo}#{number}", flush=True)
+            continue
+        body = REMINDER.format(host_base=label.removeprefix(CUSTOMER_PREFIX), slug=slug, number=number)
+        res = runner(["gh", "api", "-X", "POST", f"repos/{repo}/issues/{number}/comments", "-f", f"body={body}"])
+        if res.returncode != 0:
+            print(f"::error title={repo}::could not comment on #{number}: {_error(res)}", flush=True)
+            ok = False
+        else:
+            print(f"  reminded {repo}#{number}", flush=True)
+    return ok
+
+
 def prune(repos: Sequence[str], *, dry_run: bool, runner: Runner = _run) -> int:
     failed = False
     for repo in repos:
@@ -75,6 +123,8 @@ def prune(repos: Sequence[str], *, dry_run: bool, runner: Runner = _run) -> int:
                 failed = True
             else:
                 print(f"  deleted {name}", flush=True)
+        if not customer_reminders(repo, runner, dry_run=dry_run):
+            failed = True
     return 1 if failed else 0