From 982d823d7a7836d64741d06dfaeaae7e988ba43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20Weberru=C3=9F?= Date: Fri, 16 Oct 2026 18:16:22 +0000 Subject: [PATCH] fix(configurator): expose failed staging droplet errors Report the exact failed build's linked droplet state and timestamps for Runner and API staging. Preserve privacy filtering and distinguish generic build failures from platform StagingTimeExpired. Exercise the actual workflow shell with isolated CF fixtures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/configurator-release.yml | 18 +++- .../deploy/cloud-foundry/README.md | 12 +++ .../infra/ci/image-staging.test.mjs | 83 ++++++++++++++++++- 3 files changed, 110 insertions(+), 3 deletions(-) diff --git a/.github/workflows/configurator-release.yml b/.github/workflows/configurator-release.yml index 6acbf22..5db0956 100644 --- a/.github/workflows/configurator-release.yml +++ b/.github/workflows/configurator-release.yml @@ -312,7 +312,14 @@ jobs: if: failure() && steps.stage_runner.outcome == 'failure' run: | app_id="$(cf app lzc-plan-template --guid)" - cf curl "/v3/builds?app_guids=$app_id&order_by=-created_at&per_page=1" | jq '.resources[] | {guid, state, error}' + build="$(cf curl "/v3/builds?app_guids=$app_id&order_by=-created_at&per_page=1" | jq -ce '.resources | if length == 1 then .[0] else error("Expected one latest build") end')" + printf '%s\n' "$build" | jq '{guid, state, error, created_at, updated_at, staging_disk_in_mb, staging_memory_in_mb}' + droplet_id="$(printf '%s\n' "$build" | jq -r '.droplet.guid // empty')" + if [ -n "$droplet_id" ]; then + cf curl "/v3/droplets/$droplet_id" | jq '{guid, state, error, created_at, updated_at}' + else + echo "Latest build has no associated droplet; no droplet diagnostics available." >&2 + fi cf logs lzc-plan-template --recent - name: CF login and target run: | @@ -352,7 +359,14 @@ jobs: if: failure() && steps.push.outcome == 'failure' run: | app_id="$(cf app landing-zone-configurator --guid)" - cf curl "/v3/builds?app_guids=$app_id&order_by=-created_at&per_page=1" | jq '.resources[] | {guid, state, error, staging_disk_in_mb, staging_memory_in_mb}' + build="$(cf curl "/v3/builds?app_guids=$app_id&order_by=-created_at&per_page=1" | jq -ce '.resources | if length == 1 then .[0] else error("Expected one latest build") end')" + printf '%s\n' "$build" | jq '{guid, state, error, created_at, updated_at, staging_disk_in_mb, staging_memory_in_mb}' + droplet_id="$(printf '%s\n' "$build" | jq -r '.droplet.guid // empty')" + if [ -n "$droplet_id" ]; then + cf curl "/v3/droplets/$droplet_id" | jq '{guid, state, error, created_at, updated_at}' + else + echo "Latest build has no associated droplet; no droplet diagnostics available." >&2 + fi cf logs landing-zone-configurator --recent | awk '/\[STG\/[0-9]+\]/ { print }' - name: CF network diagnostics timeout-minutes: 3 diff --git a/landing-zone-configurator/deploy/cloud-foundry/README.md b/landing-zone-configurator/deploy/cloud-foundry/README.md index 851f2e6..6c78713 100644 --- a/landing-zone-configurator/deploy/cloud-foundry/README.md +++ b/landing-zone-configurator/deploy/cloud-foundry/README.md @@ -79,6 +79,18 @@ the redundant image cache. Runner template and isolated job/task budgets remain 4096 MiB; the measured native Runner runtime budget is unchanged. If Runner staging fails, the release also reports the latest V3 build's state and error and recent Runner logs while still targeting the isolated Runner space. +Both Runner and Web/API diagnostics read the droplet linked to that failed build, +not the application's current (possibly still healthy) droplet, and report its +state, error and timestamps. A generic build `StagingError` can hide the more +specific droplet error `StagingTimeExpired`, even after the lifecycle exits zero +and logs a completed droplet upload. A preceding missing buildpack `config.yml` +message is not by itself proof of failure: the lifecycle reports it without +failing, and successful releases have emitted it too. Inspect both build and +droplet status before changing buildpack code or retrying. `CF_STAGING_TIMEOUT` +only controls the CLI's wait; it does not extend a platform-side timeout. +If the platform expires the staging result, retain the build/droplet identifiers +and timestamps for operator/provider investigation. Do not weaken image +verification, repoint protected source tags or add automatic deployment retries. This keeps lifecycle/cache-upload errors distinct from buildpack image verification. Failed web staging reports only staging (`STG`) log entries, never router/callback logs that could contain OAuth codes, plus V3 staging limits. diff --git a/landing-zone-configurator/infra/ci/image-staging.test.mjs b/landing-zone-configurator/infra/ci/image-staging.test.mjs index 0a6990d..1bc894e 100644 --- a/landing-zone-configurator/infra/ci/image-staging.test.mjs +++ b/landing-zone-configurator/infra/ci/image-staging.test.mjs @@ -254,7 +254,7 @@ test("release pins the final buildpack to the approved source and excludes image "/v3/builds?app_guids=$app_id&order_by=-created_at&per_page=1", ), ); - assert.ok(workflow.includes("jq '.resources[] | {guid, state, error}'")); + assert.ok(workflow.includes('cf curl "/v3/droplets/$droplet_id"')); assert.ok(pack.includes("--exclude='./accelerator/firewall-image.qcow2'")); assert.match( await readFile( @@ -304,3 +304,84 @@ test("release pins the final buildpack to the approved source and excludes image /buildpacks: \[binary_buildpack\]/, ); }); + +test("actual staging diagnostics report the failed build's droplet, not the current healthy droplet", async () => { + const workflow = await readFile( + resolve(repository, ".github/workflows/configurator-release.yml"), + "utf8", + ); + const work = await mkdtemp(resolve(tmpdir(), "lzc-staging-diagnostics-")); + try { + await writeFile( + resolve(work, "cf"), + `#!/usr/bin/env bash +set -euo pipefail +case "$1" in + app) printf '%s\\n' app-id ;; + curl) + case "$2" in + /v3/builds\\?*) printf '%s\\n' "$DIAGNOSTIC_BUILDS" ;; + /v3/droplets/failed-droplet) + if [ "\${DIAGNOSTIC_HTTP_FAILURE:-}" = true ]; then echo 'CF HTTP 403' >&2; exit 1; fi + printf '%s\\n' "$DIAGNOSTIC_DROPLET" ;; + *) echo 'Unexpected CF endpoint' >&2; exit 1 ;; + esac ;; + logs) printf '%s\\n' '[STG/0] OUT staging message' '[RTR/0] OUT private-router-code' ;; + *) exit 1 ;; +esac +`, + ); + await chmod(resolve(work, "cf"), 0o700); + const failed = { + guid: "failed-build", + state: "FAILED", + error: "StagingError - Staging error: staging failed", + droplet: { guid: "failed-droplet" }, + environment_variables: { SECRET: "private-build-value" }, + }; + for (const name of [ + "Runner staging failure diagnostics", + "Sanitized application staging failure diagnostics", + ]) { + const block = workflow + .split(`- name: ${name}\n`)[1] + ?.split("\n - ")[0]; + const command = block + ?.split(" run: |\n")[1] + ?.replace(/^ /gm, ""); + assert.ok(command); + const run = (builds, extra = {}) => + spawnSync("bash", ["-e", "-o", "pipefail", "-c", command], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${work}:${process.env.PATH}`, + DIAGNOSTIC_BUILDS: JSON.stringify({ resources: builds }), + DIAGNOSTIC_DROPLET: JSON.stringify({ + guid: "failed-droplet", + state: "FAILED", + error: "StagingTimeExpired", + execution_metadata: "private-droplet-value", + }), + ...extra, + }, + }); + const result = run([failed]); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /StagingTimeExpired/); + assert.doesNotMatch(result.stdout, /private-(build|droplet)-value/); + if (name.startsWith("Sanitized")) + assert.doesNotMatch(result.stdout, /private-router-code/); + const absent = run([{ ...failed, droplet: null }]); + assert.equal(absent.status, 0, absent.stderr); + assert.match(absent.stderr, /no associated droplet/); + assert.notEqual(run([]).status, 0); + assert.notEqual( + run([failed], { DIAGNOSTIC_HTTP_FAILURE: "true" }).status, + 0, + ); + } + } finally { + await rm(work, { recursive: true, force: true }); + } +});