pipeline-cmd: add --retries for transient connection errors

README.md

@@ -67,7 +67,7 @@ HEROKU_API_KEY="op://Private/Heroku/credential" op run -- heroku-scripts apps my
 
 ```sh
 heroku-scripts apps <pipeline> <stage>
-heroku-scripts pipeline-cmd <pipeline> <stage> "<heroku command>" [--concurrency=N] [--no-stream] [-a] [--table|--csv]
+heroku-scripts pipeline-cmd <pipeline> <stage> "<heroku command>" [--concurrency=N] [--retries=N] [--no-stream] [-a] [--table|--csv]
 heroku-scripts pipeline-task <pipeline> <stage> <MixTask> [--concurrency=N]
 heroku-scripts promote <app> <to-team> <pipeline> [--dry-run] [--yes]
 ```
@@ -126,6 +126,26 @@ The output field is the app's raw combined heroku output, so it may span
 multiple lines and contain semicolons. Treat the stream as something to read
 or grep, not as strict CSV.
 
+### Retrying transient connection errors
+
+`ps:exec`-style commands occasionally fail with a transient connection error
+even though the dyno is up — Heroku's exec-manager rejects the credential
+handshake, the SSH tunnel drops mid-session, or keepalives time out. These
+show up more under parallel load and succeed on a plain re-run. Pass
+`--retries=N` to re-run an app up to N extra times (with increasing backoff)
+when its output matches one of those known-transient errors:
+
+```sh
+heroku-scripts pipeline-cmd my-pipe production 'ps:exec bin/rails runner "Some.task"' --retries=3
+```
+
+Genuine command failures — anything that doesn't match the known connection
+errors — are never retried, and a persistently failing app still emits its
+last error as its record. The default is `--retries=0` (unchanged behavior).
+
+One caveat: a mid-session drop can happen *after* the remote command started
+running, so only use `--retries` with commands that are safe to run twice.
+
 Run a mix task on every production app, four at a time:
 
 ```sh

bin/heroku-scripts

@@ -20,14 +20,19 @@ Commands:
   apps <pipeline> <stage>
       List apps in <pipeline>/<stage>, one per line.
 
-  pipeline-cmd <pipeline> <stage> "<heroku command>" [--concurrency=N] [--no-stream] [-a] [--table|--csv]
+  pipeline-cmd <pipeline> <stage> "<heroku command>" [--concurrency=N] [--retries=N] [--no-stream] [-a] [--table|--csv]
       Run the given heroku command against every app in <pipeline>/<stage>,
       in chunks of N (default 3). One record per app is emitted as each app
       finishes (so output appears progressively); --no-stream withholds output
       until every app finishes and emits it sorted by app name. Apps with empty
       output are skipped (a count is reported on stderr); pass -a/--all to
       include them.
 
+      --retries=N re-runs an app whose output matches a known *transient*
+      connection error (the ones ps:exec emits when Heroku's exec-manager or
+      the SSH tunnel hiccups), up to N extra attempts with increasing backoff.
+      Genuine command failures are never retried. Default is 0 (no retries).
+
       On a terminal the output is an aligned table (\`appname | output\`);
       when piped it is CSV (\`appname;output\`). Force either with --table or
       --csv. The output field is the app's raw combined heroku output and may
@@ -164,6 +169,39 @@ heroku_for_app() {
   fi
 }
 
+# Errors that mean "the connection flaked, the command itself may be fine".
+# These are the messages ps:exec-style commands emit when Heroku's exec-manager
+# rejects the credential PUT, the SSH tunnel drops mid-session, or keepalives
+# time out — all observed to succeed on a plain re-run. Kept deliberately
+# narrow: anything else is treated as a real failure and is never retried.
+TRANSIENT_CONNECTION_ERRORS='Could not connect to dyno|error connecting to the dyno|Connection to the dyno timed out|networking error'
+
+# Runs heroku_for_app, re-running it up to $3 extra times when the combined
+# output matches TRANSIENT_CONNECTION_ERRORS. The final attempt's output is
+# printed either way, so a persistent failure still surfaces as the app's
+# record. Backoff grows linearly (15s, 30s, ...); HEROKU_SCRIPTS_RETRY_DELAY
+# overrides the 15s unit (the tests set it to 0).
+#
+# NOTE for callers: retrying re-runs the heroku command on the dyno, and a
+# mid-session drop can happen *after* the remote command started. Only pass
+# --retries for idempotent commands.
+heroku_for_app_with_retries() {
+  local app="$1" cmd="$2" retries="$3"
+  local delay="${HEROKU_SCRIPTS_RETRY_DELAY:-15}"
+  local attempt=0 output
+  while true; do
+    output="$(heroku_for_app "$app" "$cmd" | strip_heroku_noise)"
+    if (( attempt >= retries )) \
+      || ! grep -qE "$TRANSIENT_CONNECTION_ERRORS" <<< "$output"; then
+      printf '%s' "$output"
+      return 0
+    fi
+    (( ++attempt ))
+    echo "pipeline-cmd: $app: transient connection error, retrying ($attempt/$retries)" >&2
+    sleep $(( attempt * delay ))
+  done
+}
+
 # Runs `heroku` with the given arguments passed as a real argv array — NOT a
 # string through eval — against a single app, with ambient noise stripped.
 #
@@ -247,7 +285,7 @@ emit_record() {
 }
 
 cmd_pipeline_cmd() {
-  [[ $# -ge 3 ]] || die "Usage: $SCRIPT_NAME pipeline-cmd <pipeline> <stage> \"<heroku command>\" [--concurrency=N] [--no-stream] [-a] [--table|--csv]"
+  [[ $# -ge 3 ]] || die "Usage: $SCRIPT_NAME pipeline-cmd <pipeline> <stage> \"<heroku command>\" [--concurrency=N] [--retries=N] [--no-stream] [-a] [--table|--csv]"
   local pipeline="$1" stage="$2" command="$3"
   shift 3
 
@@ -256,7 +294,7 @@ cmd_pipeline_cmd() {
   # emits everything sorted by app name at the end. Apps whose output is empty
   # are skipped by default; -a/--all includes them. Output format defaults to a
   # table on a terminal and CSV when piped; --table/--csv force one.
-  local stream=true include_empty=false format=auto
+  local stream=true include_empty=false format=auto retries=0
   local -a opts=()
   while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -265,11 +303,16 @@ cmd_pipeline_cmd() {
       -a|--all) include_empty=true;;
       --table) format=table;;
       --csv) format=csv;;
+      --retries=*) retries="${1#*=}";;
       *) opts+=("$1");;
     esac
     shift
   done
 
+  if [[ ! "$retries" =~ ^[0-9]+$ ]]; then
+    die "--retries must be a non-negative integer"
+  fi
+
   local concurrency=3
   if [[ ${#opts[@]} -gt 0 ]]; then
     concurrency="$(parse_concurrency "${opts[@]}")" || exit 1
@@ -331,7 +374,7 @@ cmd_pipeline_cmd() {
       # is captured into the output column rather than killing the subshell
       # before it can emit its record.
       set +e
-      output="$(heroku_for_app "$app" "$command" | strip_heroku_noise)"
+      output="$(heroku_for_app_with_retries "$app" "$command" "$retries")"
       if [[ -z "$output" && "$include_empty" != "true" ]]; then
         # Empty output: record the skip for the summary and emit no record.
         : > "$tmpdir/.skipped/$app"

test/heroku-scripts.bats

@@ -266,6 +266,88 @@ STUB
   [[ "$output" != *"op should not have been called"* ]]
 }
 
+# heroku stub that counts its invocations in ./stub-calls (tests cd into
+# $TESTDIR, and the script's workers inherit that cwd) and fails with a
+# transient connection error until the third call.
+_heroku_stub_flaky_connection() {
+  cat > "$TESTDIR/bin/heroku" <<'STUB'
+#!/usr/bin/env bash
+if [[ "$1" == "pipelines:info" ]]; then
+  printf '=== %s\napp-one        staging\n' "$2"; exit 0
+fi
+n=$(cat ./stub-calls 2>/dev/null || echo 0)
+n=$((n + 1))
+echo "$n" > ./stub-calls
+if [ "$n" -lt 3 ]; then
+  echo "Could not connect to dyno!"
+  exit 1
+fi
+echo "success-after-$n"
+STUB
+  chmod +x "$TESTDIR/bin/heroku"
+}
+
+@test "pipeline-cmd --retries re-runs transient connection errors until success" {
+  _heroku_stub_flaky_connection
+  HEROKU_SCRIPTS_RETRY_DELAY=0 \
+    run --separate-stderr "$SCRIPT" pipeline-cmd mypipe staging "ps:exec ls" --retries=2
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"app-one;success-after-3"* ]]
+  [[ "$stderr" == *"transient connection error, retrying (1/2)"* ]]
+  [[ "$stderr" == *"transient connection error, retrying (2/2)"* ]]
+}
+
+@test "pipeline-cmd without --retries keeps the single-attempt behavior" {
+  _heroku_stub_flaky_connection
+  run "$SCRIPT" pipeline-cmd mypipe staging "ps:exec ls"
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"app-one;Could not connect to dyno!"* ]]
+  [ "$(cat stub-calls)" = "1" ]
+}
+
+@test "pipeline-cmd --retries surfaces a persistent transient error after the last attempt" {
+  _heroku_stub_flaky_connection
+  HEROKU_SCRIPTS_RETRY_DELAY=0 \
+    run --separate-stderr "$SCRIPT" pipeline-cmd mypipe staging "ps:exec ls" --retries=1
+  [ "$status" -eq 0 ]
+  # Two attempts (initial + 1 retry), both flaky, so the error is the record.
+  [[ "$output" == *"app-one;Could not connect to dyno!"* ]]
+  [ "$(cat stub-calls)" = "2" ]
+}
+
+# heroku stub that counts invocations and always fails with a NON-transient
+# error, so retries must not kick in.
+_heroku_stub_real_failure() {
+  cat > "$TESTDIR/bin/heroku" <<'STUB'
+#!/usr/bin/env bash
+if [[ "$1" == "pipelines:info" ]]; then
+  printf '=== %s\napp-one        staging\n' "$2"; exit 0
+fi
+n=$(cat ./stub-calls 2>/dev/null || echo 0)
+n=$((n + 1))
+echo "$n" > ./stub-calls
+echo "bash: some-remote-cmd: command not found"
+exit 127
+STUB
+  chmod +x "$TESTDIR/bin/heroku"
+}
+
+@test "pipeline-cmd --retries never retries a genuine command failure" {
+  _heroku_stub_real_failure
+  HEROKU_SCRIPTS_RETRY_DELAY=0 \
+    run --separate-stderr "$SCRIPT" pipeline-cmd mypipe staging "ps:exec some-remote-cmd" --retries=3
+  [ "$status" -eq 0 ]
+  [[ "$output" == *"app-one;bash: some-remote-cmd: command not found"* ]]
+  [ "$(cat stub-calls)" = "1" ]
+  [[ "$stderr" != *"retrying"* ]]
+}
+
+@test "pipeline-cmd rejects a non-numeric retries value" {
+  run "$SCRIPT" pipeline-cmd mypipe staging "config" --retries=abc
+  [ "$status" -eq 1 ]
+  [[ "$output" == *"non-negative integer"* ]]
+}
+
 @test "HEROKU_SCRIPTS_OP_REF set but op missing fails clearly" {
   # Restricted PATH: the heroku stub + coreutils, but no `op` anywhere.
   PATH="$TESTDIR/bin:/usr/bin:/bin:/usr/sbin:/sbin" \