From 9e50ebf9398aa0f7c88bb8246b9d74e5a4281540 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 30 Apr 2026 14:46:41 -0400 Subject: [PATCH] fix: wrap in retry on outer call --- .github/actions/nix-build-retry/action.yml | 74 ++++++++++++++++++++++ .github/workflows/cli-release.yml | 12 ++-- .github/workflows/nix-build.yml | 30 +++++---- 3 files changed, 100 insertions(+), 16 deletions(-) create mode 100644 .github/actions/nix-build-retry/action.yml diff --git a/.github/actions/nix-build-retry/action.yml b/.github/actions/nix-build-retry/action.yml new file mode 100644 index 0000000000..24c5f60f94 --- /dev/null +++ b/.github/actions/nix-build-retry/action.yml @@ -0,0 +1,74 @@ +name: 'Nix build with retry' +description: | + Runs `nix build --accept-flake-config -L` and retries on transient network + errors (codeload.github.com 5xx, DNS, TLS, connection resets). Genuine build + failures exit immediately without retrying. +inputs: + attr: + description: 'The flake attribute to build, including the leading `.#`, e.g. `.#psql_17_cli_portable`' + required: true + extra-args: + description: 'Additional arguments inserted between `-L` and the attribute, e.g. `--system aarch64-linux`' + required: false + default: '' + attempts: + description: 'Maximum number of attempts' + required: false + default: '3' + initial-delay-seconds: + description: 'Backoff before the second attempt; doubled before each subsequent attempt' + required: false + default: '30' + +runs: + using: 'composite' + steps: + - name: nix build (with retry) + shell: bash + env: + ATTR: ${{ inputs.attr }} + EXTRA_ARGS: ${{ inputs.extra-args }} + ATTEMPTS: ${{ inputs.attempts }} + INITIAL_DELAY: ${{ inputs.initial-delay-seconds }} + run: | + set -uo pipefail + + log_file=$(mktemp) + trap 'rm -f "$log_file"' EXIT + + # Patterns that indicate a transient network/upstream failure rather + # than a real build error. Keep conservative — false positives waste + # CI minutes by re-running deterministic build failures. + transient_re='HTTP error 5[0-9][0-9]|error: unable to download|Failed to open archive|Connection (reset|refused|timed out)|Couldn'\''t resolve host|Could not resolve host|unexpected end[- ]of[- ]file|SSL connection|TLS connection|Resource temporarily unavailable|Temporary failure in name resolution' + + delay="$INITIAL_DELAY" + rc=0 + for attempt in $(seq 1 "$ATTEMPTS"); do + echo "::group::nix build attempt ${attempt}/${ATTEMPTS}: ${ATTR} ${EXTRA_ARGS}" + # Stream to console AND capture for post-hoc pattern matching. + # PIPESTATUS preserves nix's exit code through the tee. + nix build --accept-flake-config -L ${EXTRA_ARGS} "${ATTR}" 2>&1 | tee "$log_file" + rc=${PIPESTATUS[0]} + echo "::endgroup::" + + if [ "$rc" -eq 0 ]; then + exit 0 + fi + + if [ "$attempt" -ge "$ATTEMPTS" ]; then + echo "::error::nix build failed after ${attempt} attempt(s) (exit ${rc})" + exit "$rc" + fi + + if grep -qE "$transient_re" "$log_file"; then + echo "::warning::Transient error on attempt ${attempt}; sleeping ${delay}s then retrying" + sleep "$delay" + delay=$((delay * 2)) + else + echo "::error::nix build failed with non-transient error on attempt ${attempt} (exit ${rc}); not retrying" + exit "$rc" + fi + done + + # Defensive — loop should always exit via one of the branches above. + exit "$rc" diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index 59c77e1c78..f3fdd1d9e9 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -37,12 +37,16 @@ jobs: uses: ./.github/actions/nix-install-ephemeral - name: Run portability check - run: | - nix build .#checks.${{ matrix.system }}.psql_17_cli_portable --system ${{ matrix.system }} -L --accept-flake-config + uses: ./.github/actions/nix-build-retry + with: + attr: .#checks.${{ matrix.system }}.psql_17_cli_portable + extra-args: --system ${{ matrix.system }} - name: Build portable CLI bundle - run: | - nix build .#psql_17_cli_portable --system ${{ matrix.system }} -L --accept-flake-config + uses: ./.github/actions/nix-build-retry + with: + attr: .#psql_17_cli_portable + extra-args: --system ${{ matrix.system }} - name: Determine version id: version diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index d77f86b7c6..d1f1ea5d85 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -53,8 +53,9 @@ jobs: uses: ./.github/actions/nix-install-self-hosted - name: nix build if: ${{ matrix.attr != '' }} - shell: bash - run: nix build --accept-flake-config -L .#${{ matrix.attr }} + uses: ./.github/actions/nix-build-retry + with: + attr: .#${{ matrix.attr }} nix-build-checks-aarch64-linux: name: >- @@ -84,8 +85,9 @@ jobs: uses: ./.github/actions/nix-install-self-hosted - name: nix build if: ${{ matrix.attr != '' }} - shell: bash - run: nix build --accept-flake-config -L .#${{ matrix.attr }} + uses: ./.github/actions/nix-build-retry + with: + attr: .#${{ matrix.attr }} nix-build-packages-aarch64-darwin: name: >- @@ -107,8 +109,9 @@ jobs: uses: ./.github/actions/nix-install-self-hosted - name: nix build if: ${{ matrix.attr != '' }} - shell: bash - run: nix build --accept-flake-config -L .#${{ matrix.attr }} + uses: ./.github/actions/nix-build-retry + with: + attr: .#${{ matrix.attr }} nix-build-checks-aarch64-darwin: name: >- @@ -130,8 +133,9 @@ jobs: uses: ./.github/actions/nix-install-self-hosted - name: nix build if: ${{ matrix.attr != '' }} - shell: bash - run: nix build --accept-flake-config -L .#${{ matrix.attr }} + uses: ./.github/actions/nix-build-retry + with: + attr: .#${{ matrix.attr }} nix-build-packages-x86_64-linux: name: >- @@ -158,8 +162,9 @@ jobs: NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: nix build if: ${{ matrix.attr != '' }} - shell: bash - run: nix build --accept-flake-config -L .#${{ matrix.attr }} + uses: ./.github/actions/nix-build-retry + with: + attr: .#${{ matrix.attr }} nix-build-checks-x86_64-linux: name: >- @@ -186,8 +191,9 @@ jobs: NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - name: nix build if: ${{ matrix.attr != '' }} - shell: bash - run: nix build --accept-flake-config -L .#${{ matrix.attr }} + uses: ./.github/actions/nix-build-retry + with: + attr: .#${{ matrix.attr }} run-testinfra: needs: [nix-eval, nix-build-packages-aarch64-linux, nix-build-checks-aarch64-linux, nix-build-packages-aarch64-darwin, nix-build-checks-aarch64-darwin, nix-build-packages-x86_64-linux, nix-build-checks-x86_64-linux]