ci: self-heal cargo caches that kill build scripts with a bare ENOENT

bin/clear-corrupted-cargo-target-dir

@@ -27,12 +27,29 @@ incremental_build_failure_pattern+="|signal: 11, SIGSEGV"
 
 registry_fetch_failure_pattern="unable to update registry"
 
+# A cached build-script binary poisoned by cargo cache corruption dies with a
+# bare ENOENT before doing any work. Both halves are required, and the error
+# must be the exact bare line: "failed to run custom build command" alone is
+# any build-script bug, which a retry cannot fix, and a build script that
+# fails on a missing file with an error context of its own produces the ENOENT
+# in a "Caused by:" detail line instead of the bare "Error:" line.
+custom_build_script_enoent() {
+  grep -q "failed to run custom build command" "$1" \
+    && grep -qE '^[[:space:]]*Error: No such file or directory \(os error 2\)[[:space:]]*$' "$1"
+}
+
 if grep -qE "$incremental_build_failure_pattern" "$1"; then
   echo "--- Detected incremental build failure, clearing cargo target directories"
   rm -rf target target-xcompile || true
   exit 199
 fi
 
+if custom_build_script_enoent "$1"; then
+  echo "--- Detected corrupted cached build script, clearing cargo target directories"
+  rm -rf target target-xcompile || true
+  exit 199
+fi
+
 # Nothing on disk is corrupted, so keep the target directories. Wiping them
 # would turn a network blip into a full rebuild of the workspace, here and in
 # every later job that shares the agent's target directory.

misc/python/materialize/mzbuild.py

@@ -142,7 +142,23 @@ def run_and_detect_retryable_build_failure(
             "unable to update registry",
         ]
         combined = stdout_contents + stderr_contents
-        if any(msg in combined for msg in incremental_build_failure_msgs):
+        # A cached build-script binary poisoned by cargo cache corruption dies
+        # with a bare ENOENT before doing any work. Both halves are required,
+        # and the error must be the exact bare line: "failed to run custom
+        # build command" alone is any build-script bug, which a retry cannot
+        # fix, and a build script that fails on a missing file with an error
+        # context of its own produces the ENOENT in a "Caused by:" detail line
+        # instead of the bare "Error:" line.
+        custom_build_script_enoent = (
+            "failed to run custom build command" in combined
+            and any(
+                line.strip() == "Error: No such file or directory (os error 2)"
+                for line in combined.splitlines()
+            )
+        )
+        if custom_build_script_enoent or any(
+            msg in combined for msg in incremental_build_failure_msgs
+        ):
             raise RustIncrementalBuildFailure()
         if any(msg in combined for msg in registry_fetch_failure_msgs):
             raise CargoRegistryFetchFailure()