From 80f7503bf6c99f8f7a608f0157e677b5e7258d60 Mon Sep 17 00:00:00 2001 From: Wynne Pirini Date: Thu, 19 Nov 2026 10:13:53 +1300 Subject: [PATCH] fix(ns8): tell a registry blip apart from a missing image in the manifest selftest The registry-backed selftest failed on a TCP reset while reading an arm64 config blob, and reported: Failed to add docker://ghcr.io/calnode/ns8-calnode:build-arm64 to ... If that tag does not exist in the registry, run build-images.sh with TARGETARCH=arm64 PUSH=1 first. The image existed. One connection dropped. That advice sends the reader to rebuild and push an artefact that is already published, which is real, pointless work, and the red build says nothing about the network. Both halves are fixed where the truth is, in push-manifest.sh, because that is the code that has buildah's actual error in hand. The workflow step only sees an exit code, so wrapping it there would have meant guessing from a bare status. RETRY. add_arch() retries `buildah manifest add` only when the error looks like a transport failure: connection reset or refused, i/o timeout, deadline exceeded, name resolution, no such host, unreachable network, TLS handshake, unexpected EOF, 502/503/504, bad gateway, service unavailable, gateway timeout, internal server error. Three attempts, 5s then 10s backoff, both overridable. The classifier defaults to "real failure". Anything unrecognised fails on the first attempt, immediately, with the existing message. That matters more than the retry: an unfamiliar buildah error must never become a loop that hides a broken release, and a genuine 404 must stay unmistakable. MESSAGE. On a transport failure the script now says the registry was unreachable, states explicitly that the tag was never reported absent, says nothing is wrong with the build, and says not to rebuild. On a real absence it keeps the old advice word for word, because there it is correct. The last lines of buildah's own output are printed in both cases so the classification can be judged. PROVEN with the real script and real buildah (docker run --privileged quay.io/buildah/stable), not by injecting a raise: happy path, registry, both arches exit 0, "Published ... (amd64 arm64)" missing, nonexistent repository exit 1, 2s, old advice, no retries missing, real repo, bad arch tag exit 1, 3s, old advice, no retries ("manifest unknown" from ghcr) unreachable, localhost:1 exit 1, 15s, 2 retry lines then "registry was unreachable", never the missing-tag advice hermetic selftest (SOURCE_PREFIX="") exit 0, unchanged Worth recording from the second case: a nonexistent ghcr repository answers 403, not 404, because the bearer-token request fails before any manifest lookup. It is classified as a real failure by default, which is the right outcome, and it is why the default matters. No workflow change, no pin change, no permission change, no reformatting. One file. --- packaging/ns8/push-manifest.sh | 65 ++++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 6 deletions(-) diff --git a/packaging/ns8/push-manifest.sh b/packaging/ns8/push-manifest.sh index 7c5ae8ea..bd80ddaa 100755 --- a/packaging/ns8/push-manifest.sh +++ b/packaging/ns8/push-manifest.sh @@ -47,6 +47,50 @@ if [ "${#arches[@]}" -eq 0 ]; then exit 1 fi +# A network blip and a missing image both come back as a failed `manifest add`, +# and telling them apart matters in both directions. Retrying a +# genuine 404 hides a build that never ran. Reporting a TCP reset as a missing +# tag sends the reader to rebuild and push an image that already exists, which +# is real pointless work: that is exactly what a reset during this script's CI +# selftest used to say. +# +# So: classify, retry only transport failures, and let the message name the case. +retry_attempts="${RETRY_ATTEMPTS:-3}" +retry_delay="${RETRY_DELAY:-5}" + +# Transport failures. Deliberately a specific list rather than a broad "error" +# match, because the fallback has to stay fail-fast: anything we do not recognise +# is a real failure, not something to retry until the job times out. +is_network_error() { + grep -qiE 'connection reset|connection refused|i/o timeout|context deadline exceeded|timed out|temporary failure in name resolution|no such host|network is unreachable|tls handshake|unexpected eof|(^|[^0-9])(502|503|504)([^0-9]|$)|bad gateway|service unavailable|gateway timeout|internal server error' +} + +# add_arch RETRIES the add, then records why it gave up in ADD_OUTPUT/ADD_KIND +# so the caller can print the right message. Returns 0 on success. +add_arch() { + local src="$1" attempt=1 delay="$retry_delay" output + while :; do + if output=$(buildah manifest add "${manifest}" "${src}" 2>&1); then + return 0 + fi + if printf '%s' "$output" | is_network_error; then + if [ "$attempt" -lt "$retry_attempts" ]; then + echo "Registry unreachable (attempt ${attempt}/${retry_attempts})," \ + "retrying in ${delay}s: ${src}" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + continue + fi + ADD_KIND=network + else + ADD_KIND=missing + fi + ADD_OUTPUT="$output" + return 1 + done +} + for imagetag in "${imagetags[@]}"; do manifest="${repobase}/${reponame}:${imagetag}" # Drop any stale manifest so a retry cannot inherit old entries. @@ -74,12 +118,21 @@ for imagetag in "${imagetags[@]}"; do # # Exactly two arguments: buildah 5.x takes only the list and the image, # with no instance-name argument. - if ! buildah manifest add "${manifest}" \ - "${source_prefix}${repobase}/${reponame}:build-${arch}"; then - echo "Failed to add ${source_prefix}${repobase}/${reponame}:build-${arch}" \ - "to ${manifest}." >&2 - echo "If that tag does not exist in the registry, run build-images.sh" \ - "with TARGETARCH=${arch} PUSH=1 first." >&2 + if ! add_arch "${source_prefix}${repobase}/${reponame}:build-${arch}"; then + src="${source_prefix}${repobase}/${reponame}:build-${arch}" + # Show what the tool actually said, then say which case it is. + printf '%s\n' "${ADD_OUTPUT}" | tail -n 4 | sed 's/^/ /' >&2 + if [ "${ADD_KIND}" = "network" ]; then + echo "The registry was unreachable while reading ${src}." >&2 + echo "This is a TRANSPORT failure, not a missing image: the tag was" >&2 + echo "never reported absent and nothing is wrong with the build. Do" >&2 + echo "not rebuild or push the image. Re-run the job; if it keeps" >&2 + echo "failing, check connectivity to the registry from this host." >&2 + else + echo "Failed to add ${src}" "to ${manifest}." >&2 + echo "If that tag does not exist in the registry, run build-images.sh" \ + "with TARGETARCH=${arch} PUSH=1 first." >&2 + fi buildah manifest rm "${manifest}" 2>/dev/null || true exit 1 fi