From 2d1f453e3f6656d803a9f8f9284711f048fec462 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Thu, 3 Sep 2026 16:38:22 -0500
Subject: [PATCH 1/2] ci: keep build/venv when the build retry nukes the build
 directory

---
 .github/scripts/retry-build.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/retry-build.sh b/.github/scripts/retry-build.sh
index 60fc2559a..b5dc927ee 100755
--- a/.github/scripts/retry-build.sh
+++ b/.github/scripts/retry-build.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 # Provides retry_build(): 2-attempt loop.
-# On failure of attempt 1, nukes the entire build directory before attempt 2.
+# On failure of attempt 1, nukes the build directory before attempt 2, keeping
+# build/venv: a compute node cannot reinstall it (no route to PyPI), and a
+# failed reinstall is misread as a cluster-wide outage (#1813).
 # If RETRY_VALIDATE_CMD is set, runs it after a successful build; a non-zero
 # exit triggers the same nuke-and-retry, catching e.g. SIGILL from binaries
 # compiled on a different CPU architecture.
@@ -12,6 +14,10 @@
 # path without waiting on it; CI leaves it at the default.
 : "${MFC_BUILD_RETRY_DELAY:=30}"
 
+nuke_build() {
+    find build -mindepth 1 -maxdepth 1 ! -name venv -exec rm -rf {} + 2>/dev/null || true
+}
+
 retry_build() {
     local max_attempts=2
     local validate_cmd="${RETRY_VALIDATE_CMD:-}"
@@ -24,7 +30,7 @@ retry_build() {
                     echo "Post-build validation failed on attempt $attempt."
                     if [ $attempt -lt $max_attempts ]; then
                         echo "  Nuking build directory before retry..."
-                        rm -rf build 2>/dev/null || true
+                        nuke_build
                         sleep 5
                         attempt=$((attempt + 1))
                         continue
@@ -39,7 +45,7 @@ retry_build() {
         fi
         if [ $attempt -lt $max_attempts ]; then
             echo "  Build failed — nuking build directory before retry..."
-            rm -rf build 2>/dev/null || true
+            nuke_build
             sleep "$MFC_BUILD_RETRY_DELAY"
         else
             echo "Build failed after $max_attempts attempts."

From 608e137b654652c6a5a5353609823aab7f9351b7 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson@gmail.com>
Date: Thu, 3 Sep 2026 20:40:20 -0500
Subject: [PATCH 2/2] ci: say what the build retry clears, and pass -- to rm

---
 .github/scripts/retry-build.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/retry-build.sh b/.github/scripts/retry-build.sh
index b5dc927ee..e1c245075 100755
--- a/.github/scripts/retry-build.sh
+++ b/.github/scripts/retry-build.sh
@@ -15,7 +15,7 @@
 : "${MFC_BUILD_RETRY_DELAY:=30}"
 
 nuke_build() {
-    find build -mindepth 1 -maxdepth 1 ! -name venv -exec rm -rf {} + 2>/dev/null || true
+    find build -mindepth 1 -maxdepth 1 ! -name venv -exec rm -rf -- {} + 2>/dev/null || true
 }
 
 retry_build() {
@@ -29,7 +29,7 @@ retry_build() {
                 if ! eval "$validate_cmd"; then
                     echo "Post-build validation failed on attempt $attempt."
                     if [ $attempt -lt $max_attempts ]; then
-                        echo "  Nuking build directory before retry..."
+                        echo "  Clearing the build directory (keeping build/venv) before retry..."
                         nuke_build
                         sleep 5
                         attempt=$((attempt + 1))
@@ -44,7 +44,7 @@ retry_build() {
             return 0
         fi
         if [ $attempt -lt $max_attempts ]; then
-            echo "  Build failed — nuking build directory before retry..."
+            echo "  Build failed — clearing the build directory (keeping build/venv) before retry..."
             nuke_build
             sleep "$MFC_BUILD_RETRY_DELAY"
         else