diff --git a/.github/scripts/retry-build.sh b/.github/scripts/retry-build.sh index 60fc2559a..e1c245075 100755 --- a/.github/scripts/retry-build.sh +++ b/.github/scripts/retry-build.sh @@ -1,6 +1,8 @@ #!/bin/bash # Provides retry_build(): 2-attempt loop. -# On failure of attempt 1, nukes the entire build directory before attempt 2. +# On failure of attempt 1, nukes the build directory before attempt 2, keeping +# build/venv: a compute node cannot reinstall it (no route to PyPI), and a +# failed reinstall is misread as a cluster-wide outage (#1813). # If RETRY_VALIDATE_CMD is set, runs it after a successful build; a non-zero # exit triggers the same nuke-and-retry, catching e.g. SIGILL from binaries # compiled on a different CPU architecture. @@ -12,6 +14,10 @@ # path without waiting on it; CI leaves it at the default. : "${MFC_BUILD_RETRY_DELAY:=30}" +nuke_build() { + find build -mindepth 1 -maxdepth 1 ! -name venv -exec rm -rf -- {} + 2>/dev/null || true +} + retry_build() { local max_attempts=2 local validate_cmd="${RETRY_VALIDATE_CMD:-}" @@ -23,8 +29,8 @@ retry_build() { if ! eval "$validate_cmd"; then echo "Post-build validation failed on attempt $attempt." if [ $attempt -lt $max_attempts ]; then - echo " Nuking build directory before retry..." - rm -rf build 2>/dev/null || true + echo " Clearing the build directory (keeping build/venv) before retry..." + nuke_build sleep 5 attempt=$((attempt + 1)) continue @@ -38,8 +44,8 @@ retry_build() { return 0 fi if [ $attempt -lt $max_attempts ]; then - echo " Build failed — nuking build directory before retry..." - rm -rf build 2>/dev/null || true + echo " Build failed — clearing the build directory (keeping build/venv) before retry..." + nuke_build sleep "$MFC_BUILD_RETRY_DELAY" else echo "Build failed after $max_attempts attempts."