From 12956d3d76df0177cfbb07c629e1b853c323f1fc Mon Sep 17 00:00:00 2001 From: Maarten Jacobs Date: Mon, 10 Aug 2026 15:35:39 +0200 Subject: [PATCH] pipeline-cmd: add --retries for transient connection errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ps:exec-style commands intermittently fail even when the dyno is up: Heroku's exec-manager rejects the credential handshake, the SSH tunnel drops mid-session, or keepalives time out. All of these succeed on a plain re-run and show up more often under parallel load — exactly how pipeline-cmd runs heroku. Add an opt-in --retries=N flag that re-runs an app up to N extra times, with linearly increasing backoff, when its combined output matches one of those known-transient error messages. The pattern is deliberately narrow so genuine command failures are never retried, and a persistently failing app still emits its last error as its record. Default is 0, so existing behavior is unchanged. HEROKU_SCRIPTS_RETRY_DELAY overrides the 15s backoff unit; the tests set it to 0 to stay fast. Co-Authored-By: Claude Fable 5 --- README.md | 22 ++++++++++- bin/heroku-scripts | 51 +++++++++++++++++++++++-- test/heroku-scripts.bats | 82 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0112db9..1939104 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ HEROKU_API_KEY="op://Private/Heroku/credential" op run -- heroku-scripts apps my ```sh heroku-scripts apps -heroku-scripts pipeline-cmd "" [--concurrency=N] [--no-stream] [-a] [--table|--csv] +heroku-scripts pipeline-cmd "" [--concurrency=N] [--retries=N] [--no-stream] [-a] [--table|--csv] heroku-scripts pipeline-task [--concurrency=N] heroku-scripts promote [--dry-run] [--yes] ``` @@ -126,6 +126,26 @@ The output field is the app's raw combined heroku output, so it may span multiple lines and contain semicolons. Treat the stream as something to read or grep, not as strict CSV. +### Retrying transient connection errors + +`ps:exec`-style commands occasionally fail with a transient connection error +even though the dyno is up — Heroku's exec-manager rejects the credential +handshake, the SSH tunnel drops mid-session, or keepalives time out. These +show up more under parallel load and succeed on a plain re-run. Pass +`--retries=N` to re-run an app up to N extra times (with increasing backoff) +when its output matches one of those known-transient errors: + +```sh +heroku-scripts pipeline-cmd my-pipe production 'ps:exec bin/rails runner "Some.task"' --retries=3 +``` + +Genuine command failures — anything that doesn't match the known connection +errors — are never retried, and a persistently failing app still emits its +last error as its record. The default is `--retries=0` (unchanged behavior). + +One caveat: a mid-session drop can happen *after* the remote command started +running, so only use `--retries` with commands that are safe to run twice. + Run a mix task on every production app, four at a time: ```sh diff --git a/bin/heroku-scripts b/bin/heroku-scripts index db33968..d9508ab 100755 --- a/bin/heroku-scripts +++ b/bin/heroku-scripts @@ -20,7 +20,7 @@ Commands: apps List apps in /, one per line. - pipeline-cmd "" [--concurrency=N] [--no-stream] [-a] [--table|--csv] + pipeline-cmd "" [--concurrency=N] [--retries=N] [--no-stream] [-a] [--table|--csv] Run the given heroku command against every app in /, in chunks of N (default 3). One record per app is emitted as each app finishes (so output appears progressively); --no-stream withholds output @@ -28,6 +28,11 @@ Commands: output are skipped (a count is reported on stderr); pass -a/--all to include them. + --retries=N re-runs an app whose output matches a known *transient* + connection error (the ones ps:exec emits when Heroku's exec-manager or + the SSH tunnel hiccups), up to N extra attempts with increasing backoff. + Genuine command failures are never retried. Default is 0 (no retries). + On a terminal the output is an aligned table (\`appname | output\`); when piped it is CSV (\`appname;output\`). Force either with --table or --csv. The output field is the app's raw combined heroku output and may @@ -164,6 +169,39 @@ heroku_for_app() { fi } +# Errors that mean "the connection flaked, the command itself may be fine". +# These are the messages ps:exec-style commands emit when Heroku's exec-manager +# rejects the credential PUT, the SSH tunnel drops mid-session, or keepalives +# time out — all observed to succeed on a plain re-run. Kept deliberately +# narrow: anything else is treated as a real failure and is never retried. +TRANSIENT_CONNECTION_ERRORS='Could not connect to dyno|error connecting to the dyno|Connection to the dyno timed out|networking error' + +# Runs heroku_for_app, re-running it up to $3 extra times when the combined +# output matches TRANSIENT_CONNECTION_ERRORS. The final attempt's output is +# printed either way, so a persistent failure still surfaces as the app's +# record. Backoff grows linearly (15s, 30s, ...); HEROKU_SCRIPTS_RETRY_DELAY +# overrides the 15s unit (the tests set it to 0). +# +# NOTE for callers: retrying re-runs the heroku command on the dyno, and a +# mid-session drop can happen *after* the remote command started. Only pass +# --retries for idempotent commands. +heroku_for_app_with_retries() { + local app="$1" cmd="$2" retries="$3" + local delay="${HEROKU_SCRIPTS_RETRY_DELAY:-15}" + local attempt=0 output + while true; do + output="$(heroku_for_app "$app" "$cmd" | strip_heroku_noise)" + if (( attempt >= retries )) \ + || ! grep -qE "$TRANSIENT_CONNECTION_ERRORS" <<< "$output"; then + printf '%s' "$output" + return 0 + fi + (( ++attempt )) + echo "pipeline-cmd: $app: transient connection error, retrying ($attempt/$retries)" >&2 + sleep $(( attempt * delay )) + done +} + # Runs `heroku` with the given arguments passed as a real argv array — NOT a # string through eval — against a single app, with ambient noise stripped. # @@ -247,7 +285,7 @@ emit_record() { } cmd_pipeline_cmd() { - [[ $# -ge 3 ]] || die "Usage: $SCRIPT_NAME pipeline-cmd \"\" [--concurrency=N] [--no-stream] [-a] [--table|--csv]" + [[ $# -ge 3 ]] || die "Usage: $SCRIPT_NAME pipeline-cmd \"\" [--concurrency=N] [--retries=N] [--no-stream] [-a] [--table|--csv]" local pipeline="$1" stage="$2" command="$3" shift 3 @@ -256,7 +294,7 @@ cmd_pipeline_cmd() { # emits everything sorted by app name at the end. Apps whose output is empty # are skipped by default; -a/--all includes them. Output format defaults to a # table on a terminal and CSV when piped; --table/--csv force one. - local stream=true include_empty=false format=auto + local stream=true include_empty=false format=auto retries=0 local -a opts=() while [[ $# -gt 0 ]]; do case "$1" in @@ -265,11 +303,16 @@ cmd_pipeline_cmd() { -a|--all) include_empty=true;; --table) format=table;; --csv) format=csv;; + --retries=*) retries="${1#*=}";; *) opts+=("$1");; esac shift done + if [[ ! "$retries" =~ ^[0-9]+$ ]]; then + die "--retries must be a non-negative integer" + fi + local concurrency=3 if [[ ${#opts[@]} -gt 0 ]]; then concurrency="$(parse_concurrency "${opts[@]}")" || exit 1 @@ -331,7 +374,7 @@ cmd_pipeline_cmd() { # is captured into the output column rather than killing the subshell # before it can emit its record. set +e - output="$(heroku_for_app "$app" "$command" | strip_heroku_noise)" + output="$(heroku_for_app_with_retries "$app" "$command" "$retries")" if [[ -z "$output" && "$include_empty" != "true" ]]; then # Empty output: record the skip for the summary and emit no record. : > "$tmpdir/.skipped/$app" diff --git a/test/heroku-scripts.bats b/test/heroku-scripts.bats index 7b54129..351abbd 100644 --- a/test/heroku-scripts.bats +++ b/test/heroku-scripts.bats @@ -266,6 +266,88 @@ STUB [[ "$output" != *"op should not have been called"* ]] } +# heroku stub that counts its invocations in ./stub-calls (tests cd into +# $TESTDIR, and the script's workers inherit that cwd) and fails with a +# transient connection error until the third call. +_heroku_stub_flaky_connection() { + cat > "$TESTDIR/bin/heroku" <<'STUB' +#!/usr/bin/env bash +if [[ "$1" == "pipelines:info" ]]; then + printf '=== %s\napp-one staging\n' "$2"; exit 0 +fi +n=$(cat ./stub-calls 2>/dev/null || echo 0) +n=$((n + 1)) +echo "$n" > ./stub-calls +if [ "$n" -lt 3 ]; then + echo "Could not connect to dyno!" + exit 1 +fi +echo "success-after-$n" +STUB + chmod +x "$TESTDIR/bin/heroku" +} + +@test "pipeline-cmd --retries re-runs transient connection errors until success" { + _heroku_stub_flaky_connection + HEROKU_SCRIPTS_RETRY_DELAY=0 \ + run --separate-stderr "$SCRIPT" pipeline-cmd mypipe staging "ps:exec ls" --retries=2 + [ "$status" -eq 0 ] + [[ "$output" == *"app-one;success-after-3"* ]] + [[ "$stderr" == *"transient connection error, retrying (1/2)"* ]] + [[ "$stderr" == *"transient connection error, retrying (2/2)"* ]] +} + +@test "pipeline-cmd without --retries keeps the single-attempt behavior" { + _heroku_stub_flaky_connection + run "$SCRIPT" pipeline-cmd mypipe staging "ps:exec ls" + [ "$status" -eq 0 ] + [[ "$output" == *"app-one;Could not connect to dyno!"* ]] + [ "$(cat stub-calls)" = "1" ] +} + +@test "pipeline-cmd --retries surfaces a persistent transient error after the last attempt" { + _heroku_stub_flaky_connection + HEROKU_SCRIPTS_RETRY_DELAY=0 \ + run --separate-stderr "$SCRIPT" pipeline-cmd mypipe staging "ps:exec ls" --retries=1 + [ "$status" -eq 0 ] + # Two attempts (initial + 1 retry), both flaky, so the error is the record. + [[ "$output" == *"app-one;Could not connect to dyno!"* ]] + [ "$(cat stub-calls)" = "2" ] +} + +# heroku stub that counts invocations and always fails with a NON-transient +# error, so retries must not kick in. +_heroku_stub_real_failure() { + cat > "$TESTDIR/bin/heroku" <<'STUB' +#!/usr/bin/env bash +if [[ "$1" == "pipelines:info" ]]; then + printf '=== %s\napp-one staging\n' "$2"; exit 0 +fi +n=$(cat ./stub-calls 2>/dev/null || echo 0) +n=$((n + 1)) +echo "$n" > ./stub-calls +echo "bash: some-remote-cmd: command not found" +exit 127 +STUB + chmod +x "$TESTDIR/bin/heroku" +} + +@test "pipeline-cmd --retries never retries a genuine command failure" { + _heroku_stub_real_failure + HEROKU_SCRIPTS_RETRY_DELAY=0 \ + run --separate-stderr "$SCRIPT" pipeline-cmd mypipe staging "ps:exec some-remote-cmd" --retries=3 + [ "$status" -eq 0 ] + [[ "$output" == *"app-one;bash: some-remote-cmd: command not found"* ]] + [ "$(cat stub-calls)" = "1" ] + [[ "$stderr" != *"retrying"* ]] +} + +@test "pipeline-cmd rejects a non-numeric retries value" { + run "$SCRIPT" pipeline-cmd mypipe staging "config" --retries=abc + [ "$status" -eq 1 ] + [[ "$output" == *"non-negative integer"* ]] +} + @test "HEROKU_SCRIPTS_OP_REF set but op missing fails clearly" { # Restricted PATH: the heroku stub + coreutils, but no `op` anywhere. PATH="$TESTDIR/bin:/usr/bin:/bin:/usr/sbin:/sbin" \