From cffdc3bdba4c0761b9ff6fc00066034ade594598 Mon Sep 17 00:00:00 2001
From: Gabor Gevay <gabor@materialize.com>
Date: Fri, 21 Aug 2026 20:10:09 +0200
Subject: [PATCH] ci: self-heal cargo caches that kill build scripts with a
 bare ENOENT

A corrupted cached build-script binary fails with
"failed to run custom build command" plus a bare
"Error: No such file or directory (os error 2)", a signature
clear-corrupted-cargo-target-dir did not recognize, so the corruption
persisted across runs instead of being wiped and retried. On the
single-agent merge-skew queue this took the check down for every PR
until the agent was replaced by hand.

Require both halves of the signature, with the ENOENT as the exact bare
error line: the first half alone is any build-script bug, which a retry
cannot fix, and a build script that fails on a missing file with an
error context of its own reports the ENOENT in a "Caused by:" detail
line instead. Mirror the signature in
run_and_detect_retryable_build_failure, which the sync comments already
tie to the shell script.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FJqg3FMKRiGYJ4owEwVAdC
---
 bin/clear-corrupted-cargo-target-dir | 17 +++++++++++++++++
 misc/python/materialize/mzbuild.py   | 18 +++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/bin/clear-corrupted-cargo-target-dir b/bin/clear-corrupted-cargo-target-dir
index ae4bee910af63..0fcc9cdf25852 100755
--- a/bin/clear-corrupted-cargo-target-dir
+++ b/bin/clear-corrupted-cargo-target-dir
@@ -27,12 +27,29 @@ incremental_build_failure_pattern+="|signal: 11, SIGSEGV"
 
 registry_fetch_failure_pattern="unable to update registry"
 
+# A cached build-script binary poisoned by cargo cache corruption dies with a
+# bare ENOENT before doing any work. Both halves are required, and the error
+# must be the exact bare line: "failed to run custom build command" alone is
+# any build-script bug, which a retry cannot fix, and a build script that
+# fails on a missing file with an error context of its own produces the ENOENT
+# in a "Caused by:" detail line instead of the bare "Error:" line.
+custom_build_script_enoent() {
+  grep -q "failed to run custom build command" "$1" \
+    && grep -qE '^[[:space:]]*Error: No such file or directory \(os error 2\)[[:space:]]*$' "$1"
+}
+
 if grep -qE "$incremental_build_failure_pattern" "$1"; then
   echo "--- Detected incremental build failure, clearing cargo target directories"
   rm -rf target target-xcompile || true
   exit 199
 fi
 
+if custom_build_script_enoent "$1"; then
+  echo "--- Detected corrupted cached build script, clearing cargo target directories"
+  rm -rf target target-xcompile || true
+  exit 199
+fi
+
 # Nothing on disk is corrupted, so keep the target directories. Wiping them
 # would turn a network blip into a full rebuild of the workspace, here and in
 # every later job that shares the agent's target directory.
diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py
index 27264bc82909e..585f56f30fc38 100644
--- a/misc/python/materialize/mzbuild.py
+++ b/misc/python/materialize/mzbuild.py
@@ -142,7 +142,23 @@ def run_and_detect_retryable_build_failure(
             "unable to update registry",
         ]
         combined = stdout_contents + stderr_contents
-        if any(msg in combined for msg in incremental_build_failure_msgs):
+        # A cached build-script binary poisoned by cargo cache corruption dies
+        # with a bare ENOENT before doing any work. Both halves are required,
+        # and the error must be the exact bare line: "failed to run custom
+        # build command" alone is any build-script bug, which a retry cannot
+        # fix, and a build script that fails on a missing file with an error
+        # context of its own produces the ENOENT in a "Caused by:" detail line
+        # instead of the bare "Error:" line.
+        custom_build_script_enoent = (
+            "failed to run custom build command" in combined
+            and any(
+                line.strip() == "Error: No such file or directory (os error 2)"
+                for line in combined.splitlines()
+            )
+        )
+        if custom_build_script_enoent or any(
+            msg in combined for msg in incremental_build_failure_msgs
+        ):
             raise RustIncrementalBuildFailure()
         if any(msg in combined for msg in registry_fetch_failure_msgs):
             raise CargoRegistryFetchFailure()