diff --git a/bin/clear-corrupted-cargo-target-dir b/bin/clear-corrupted-cargo-target-dir index ae4bee910af63..0fcc9cdf25852 100755 --- a/bin/clear-corrupted-cargo-target-dir +++ b/bin/clear-corrupted-cargo-target-dir @@ -27,12 +27,29 @@ incremental_build_failure_pattern+="|signal: 11, SIGSEGV" registry_fetch_failure_pattern="unable to update registry" +# A cached build-script binary poisoned by cargo cache corruption dies with a +# bare ENOENT before doing any work. Both halves are required, and the error +# must be the exact bare line: "failed to run custom build command" alone is +# any build-script bug, which a retry cannot fix, and a build script that +# fails on a missing file with an error context of its own produces the ENOENT +# in a "Caused by:" detail line instead of the bare "Error:" line. +custom_build_script_enoent() { + grep -q "failed to run custom build command" "$1" \ + && grep -qE '^[[:space:]]*Error: No such file or directory \(os error 2\)[[:space:]]*$' "$1" +} + if grep -qE "$incremental_build_failure_pattern" "$1"; then echo "--- Detected incremental build failure, clearing cargo target directories" rm -rf target target-xcompile || true exit 199 fi +if custom_build_script_enoent "$1"; then + echo "--- Detected corrupted cached build script, clearing cargo target directories" + rm -rf target target-xcompile || true + exit 199 +fi + # Nothing on disk is corrupted, so keep the target directories. Wiping them # would turn a network blip into a full rebuild of the workspace, here and in # every later job that shares the agent's target directory. diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py index 27264bc82909e..585f56f30fc38 100644 --- a/misc/python/materialize/mzbuild.py +++ b/misc/python/materialize/mzbuild.py @@ -142,7 +142,23 @@ def run_and_detect_retryable_build_failure( "unable to update registry", ] combined = stdout_contents + stderr_contents - if any(msg in combined for msg in incremental_build_failure_msgs): + # A cached build-script binary poisoned by cargo cache corruption dies + # with a bare ENOENT before doing any work. Both halves are required, + # and the error must be the exact bare line: "failed to run custom + # build command" alone is any build-script bug, which a retry cannot + # fix, and a build script that fails on a missing file with an error + # context of its own produces the ENOENT in a "Caused by:" detail line + # instead of the bare "Error:" line. + custom_build_script_enoent = ( + "failed to run custom build command" in combined + and any( + line.strip() == "Error: No such file or directory (os error 2)" + for line in combined.splitlines() + ) + ) + if custom_build_script_enoent or any( + msg in combined for msg in incremental_build_failure_msgs + ): raise RustIncrementalBuildFailure() if any(msg in combined for msg in registry_fetch_failure_msgs): raise CargoRegistryFetchFailure()