From 0238c8896df7f55aebff4652cea2e6418828391b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 29 Sep 2026 02:43:28 +0400 Subject: [PATCH] Retry Cloud Run routing verification during propagation --- .../cloud-run-simulation-entry-smoke.sh | 61 +++++- .../workflows/simulation-deploy.reusable.yml | 4 +- changelog_entry.yaml | 1 + .../tests/test_deployment_assets.py | 203 +++++++++++++++++- 4 files changed, 258 insertions(+), 11 deletions(-) diff --git a/.github/scripts/cloud-run-simulation-entry-smoke.sh b/.github/scripts/cloud-run-simulation-entry-smoke.sh index d25d937bb..a83633c78 100755 --- a/.github/scripts/cloud-run-simulation-entry-smoke.sh +++ b/.github/scripts/cloud-run-simulation-entry-smoke.sh @@ -5,18 +5,38 @@ set -euo pipefail base_url="${1:?Simulation Entrypoint base URL is required}" base_url="${base_url%/}" expected_revision="${2:-}" +# Stable hostnames can briefly continue serving the revision that was active +# immediately before a traffic update. Supplying that revision as argument 3 +# enables bounded retries for only that documented transition state. +previous_revision="${3:-}" +routing_max_attempts="${SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS:-24}" +routing_retry_delay_seconds="${SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS:-5}" + +if ! [[ "${routing_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then + printf 'SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS must be a positive integer\n' >&2 + exit 2 +fi +if ! [[ "${routing_retry_delay_seconds}" =~ ^[0-9]+$ ]]; then + printf 'SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS must be a non-negative integer\n' >&2 + exit 2 +fi health_headers="$(mktemp)" health_body="$(mktemp)" trap 'rm -f "${health_headers}" "${health_body}"' EXIT -curl --fail --silent --show-error \ - --dump-header "${health_headers}" \ - --output "${health_body}" \ - "${base_url}/health" -jq -e '.status == "healthy"' "${health_body}" >/dev/null +attempt=1 +while true; do + curl --fail --silent --show-error \ + --dump-header "${health_headers}" \ + --output "${health_body}" \ + "${base_url}/health" + jq -e '.status == "healthy"' "${health_body}" >/dev/null + + if [ -z "${expected_revision}" ]; then + break + fi -if [ -n "${expected_revision}" ]; then actual_revision="$( awk ' tolower($1) == "x-policyengine-simulation-revision:" { @@ -26,12 +46,37 @@ if [ -n "${expected_revision}" ]; then ' "${health_headers}" | tail -n 1 )" - if [ "${actual_revision}" != "${expected_revision}" ]; then + + if [ "${actual_revision}" = "${expected_revision}" ]; then + break + fi + + if [ -z "${previous_revision}" ]; then printf 'Expected revision %s at %s, received %s\n' \ "${expected_revision}" "${base_url}" "${actual_revision:-no revision header}" >&2 exit 1 fi -fi + + if [ "${actual_revision}" != "${previous_revision}" ]; then + printf 'Unexpected revision at %s: expected %s or previous revision %s, received %s\n' \ + "${base_url}" "${expected_revision}" "${previous_revision}" \ + "${actual_revision:-no revision header}" >&2 + exit 1 + fi + + if [ "${attempt}" -ge "${routing_max_attempts}" ]; then + printf '%s did not serve revision %s after %s attempts; it still served previous revision %s\n' \ + "${base_url}" "${expected_revision}" "${routing_max_attempts}" \ + "${previous_revision}" >&2 + exit 1 + fi + + printf '%s still served previous revision %s; retrying target revision %s (%s/%s)\n' \ + "${base_url}" "${previous_revision}" "${expected_revision}" \ + "${attempt}" "${routing_max_attempts}" >&2 + sleep "${routing_retry_delay_seconds}" + attempt=$((attempt + 1)) +done curl --fail --silent --show-error "${base_url}/ready" | jq -e '.status == "ready"' >/dev/null diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 127435d4f..f859f173f 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -798,7 +798,7 @@ jobs: run: .github/scripts/set-cloud-run-simulation-entry-revision.sh - name: Verify stable service routing - run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" + run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}" - name: Verify stable service authentication working-directory: projects/policyengine-simulation-entry @@ -812,7 +812,7 @@ jobs: - name: Verify configured custom hostname if: ${{ env.CUSTOM_URL != '' }} - run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" + run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}" - name: Verify configured custom-hostname authentication if: ${{ env.CUSTOM_URL != '' }} diff --git a/changelog_entry.yaml b/changelog_entry.yaml index 4c283b4de..455d0317f 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -4,3 +4,4 @@ - Resolved PolicyEngine-managed regional datasets using bundled data versions instead of release-manifest commit hashes when loading from GCS - Added a Modal app-release bundle backfill for missing data artifact revisions - Ensured Stage 12-only deployments run authenticated candidate tests and verify every required deployment phase before reporting success + - Allowed Cloud Run stable endpoint checks to wait for documented traffic propagation before restoring the previous revision diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index 31a621330..f9f5f04e9 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -59,6 +59,199 @@ def test_deployment_scripts_have_valid_shell_syntax(): assert os.access(CLOUD_RUN_DEPLOY_SCRIPT, os.X_OK) +def _write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, Path]: + state_file = tmp_path / "health-request-count" + revisions_file = tmp_path / "health-revisions" + revisions_file.write_text("\n".join(revisions) + "\n", encoding="utf-8") + fake_curl = tmp_path / "curl" + fake_curl.write_text( + textwrap.dedent("""\ + #!/usr/bin/env bash + set -euo pipefail + + headers_file="" + body_file="" + url="" + while [ "$#" -gt 0 ]; do + case "$1" in + --dump-header|--output|--request|--header|--data) + option="$1" + value="$2" + shift 2 + case "${option}" in + --dump-header) headers_file="${value}" ;; + --output) body_file="${value}" ;; + esac + ;; + --fail|--silent|--show-error) + shift + ;; + *) + url="$1" + shift + ;; + esac + done + + case "${url}" in + */health) + count=0 + if [ -f "${FAKE_CURL_STATE}" ]; then + count="$(cat "${FAKE_CURL_STATE}")" + fi + line_number=$((count + 1)) + revision="$(sed -n "${line_number}p" "${FAKE_CURL_REVISIONS}")" + printf '%s' "${line_number}" > "${FAKE_CURL_STATE}" + printf 'HTTP/1.1 200 OK\r\nX-PolicyEngine-Simulation-Revision: %s\r\n\r\n' \ + "${revision}" > "${headers_file}" + printf '{"status":"healthy"}' > "${body_file}" + ;; + */ready) + printf '{"status":"ready"}' + ;; + */versions) + printf '{}' + ;; + */ping) + printf '{"incremented":2}' + ;; + *) + printf 'Unexpected fake curl URL: %s\n' "${url}" >&2 + exit 2 + ;; + esac + """), + encoding="utf-8", + ) + fake_curl.chmod(0o755) + return state_file, revisions_file + + +def _smoke_test_env( + tmp_path: Path, + state_file: Path, + revisions_file: Path, +) -> dict[str, str]: + return { + **os.environ, + "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}", + "FAKE_CURL_STATE": str(state_file), + "FAKE_CURL_REVISIONS": str(revisions_file), + "SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS": "3", + "SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS": "0", + } + + +def test_stable_smoke_retries_the_previous_revision_until_target_is_served( + tmp_path: Path, +): + old_revision = "policyengine-simulation-entry-00001-old" + target_revision = "policyengine-simulation-entry-00002-new" + state_file, revisions_file = _write_fake_smoke_curl( + tmp_path, + [old_revision, old_revision, target_revision], + ) + + subprocess.run( + [ + "bash", + SMOKE_SCRIPT, + "https://simulation.example.test", + target_revision, + old_revision, + ], + check=True, + capture_output=True, + text=True, + env=_smoke_test_env(tmp_path, state_file, revisions_file), + ) + + assert state_file.read_text(encoding="utf-8") == "3" + + +def test_stable_smoke_rejects_an_unexpected_revision_without_retrying(tmp_path: Path): + old_revision = "policyengine-simulation-entry-00001-old" + target_revision = "policyengine-simulation-entry-00002-new" + unexpected_revision = "policyengine-simulation-entry-00003-unexpected" + state_file, revisions_file = _write_fake_smoke_curl( + tmp_path, + [unexpected_revision, target_revision], + ) + + result = subprocess.run( + [ + "bash", + SMOKE_SCRIPT, + "https://simulation.example.test", + target_revision, + old_revision, + ], + check=False, + capture_output=True, + text=True, + env=_smoke_test_env(tmp_path, state_file, revisions_file), + ) + + assert result.returncode == 1 + assert "Unexpected revision" in result.stderr + assert state_file.read_text(encoding="utf-8") == "1" + + +def test_candidate_smoke_rejects_a_revision_mismatch_without_retrying(tmp_path: Path): + old_revision = "policyengine-simulation-entry-00001-old" + target_revision = "policyengine-simulation-entry-00002-new" + state_file, revisions_file = _write_fake_smoke_curl( + tmp_path, + [old_revision, target_revision], + ) + + result = subprocess.run( + [ + "bash", + SMOKE_SCRIPT, + "https://candidate.example.test", + target_revision, + ], + check=False, + capture_output=True, + text=True, + env=_smoke_test_env(tmp_path, state_file, revisions_file), + ) + + assert result.returncode == 1 + assert "Expected revision" in result.stderr + assert state_file.read_text(encoding="utf-8") == "1" + + +def test_stable_smoke_fails_when_the_previous_revision_does_not_converge( + tmp_path: Path, +): + old_revision = "policyengine-simulation-entry-00001-old" + target_revision = "policyengine-simulation-entry-00002-new" + state_file, revisions_file = _write_fake_smoke_curl( + tmp_path, + [old_revision, old_revision, old_revision], + ) + + result = subprocess.run( + [ + "bash", + SMOKE_SCRIPT, + "https://simulation.example.test", + target_revision, + old_revision, + ], + check=False, + capture_output=True, + text=True, + env=_smoke_test_env(tmp_path, state_file, revisions_file), + ) + + assert result.returncode == 1 + assert "did not serve revision" in result.stderr + assert state_file.read_text(encoding="utf-8") == "3" + + def test_stage12_deployment_uses_the_api_owned_schema_directly(): validation_script = STAGE12_VALIDATION_SCRIPT.read_text(encoding="utf-8") @@ -396,7 +589,15 @@ def test_full_stack_promotion_order_is_explicit(): ) assert "needs: [deploy_entrypoint, authenticated_test]" in reusable_workflow assert ( - 'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}"' + 'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"' + in reusable_workflow + ) + assert ( + 'cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"' + in reusable_workflow + ) + assert ( + 'cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}"' in reusable_workflow ) assert "failure() && steps.promote.outcome == 'success'" in reusable_workflow