Wait for Cloud Run routing convergence during promotion

.github/scripts/cloud-run-simulation-entry-smoke.sh

@@ -5,18 +5,38 @@ set -euo pipefail
 base_url="${1:?Simulation Entrypoint base URL is required}"
 base_url="${base_url%/}"
 expected_revision="${2:-}"
+# Stable hostnames can briefly continue serving the revision that was active
+# immediately before a traffic update. Supplying that revision as argument 3
+# enables bounded retries for only that documented transition state.
+previous_revision="${3:-}"
+routing_max_attempts="${SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS:-24}"
+routing_retry_delay_seconds="${SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS:-5}"
+
+if ! [[ "${routing_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then
+  printf 'SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS must be a positive integer\n' >&2
+  exit 2
+fi
+if ! [[ "${routing_retry_delay_seconds}" =~ ^[0-9]+$ ]]; then
+  printf 'SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS must be a non-negative integer\n' >&2
+  exit 2
+fi
 
 health_headers="$(mktemp)"
 health_body="$(mktemp)"
 trap 'rm -f "${health_headers}" "${health_body}"' EXIT
 
-curl --fail --silent --show-error \
-  --dump-header "${health_headers}" \
-  --output "${health_body}" \
-  "${base_url}/health"
-jq -e '.status == "healthy"' "${health_body}" >/dev/null
+attempt=1
+while true; do
+  curl --fail --silent --show-error \
+    --dump-header "${health_headers}" \
+    --output "${health_body}" \
+    "${base_url}/health"
+  jq -e '.status == "healthy"' "${health_body}" >/dev/null
+
+  if [ -z "${expected_revision}" ]; then
+    break
+  fi
 
-if [ -n "${expected_revision}" ]; then
   actual_revision="$(
     awk '
       tolower($1) == "x-policyengine-simulation-revision:" {
@@ -26,12 +46,37 @@ if [ -n "${expected_revision}" ]; then
     ' "${health_headers}" |
       tail -n 1
   )"
-  if [ "${actual_revision}" != "${expected_revision}" ]; then
+
+  if [ "${actual_revision}" = "${expected_revision}" ]; then
+    break
+  fi
+
+  if [ -z "${previous_revision}" ]; then
     printf 'Expected revision %s at %s, received %s\n' \
       "${expected_revision}" "${base_url}" "${actual_revision:-no revision header}" >&2
     exit 1
   fi
-fi
+
+  if [ "${actual_revision}" != "${previous_revision}" ]; then
+    printf 'Unexpected revision at %s: expected %s or previous revision %s, received %s\n' \
+      "${base_url}" "${expected_revision}" "${previous_revision}" \
+      "${actual_revision:-no revision header}" >&2
+    exit 1
+  fi
+
+  if [ "${attempt}" -ge "${routing_max_attempts}" ]; then
+    printf '%s did not serve revision %s after %s attempts; it still served previous revision %s\n' \
+      "${base_url}" "${expected_revision}" "${routing_max_attempts}" \
+      "${previous_revision}" >&2
+    exit 1
+  fi
+
+  printf '%s still served previous revision %s; retrying target revision %s (%s/%s)\n' \
+    "${base_url}" "${previous_revision}" "${expected_revision}" \
+    "${attempt}" "${routing_max_attempts}" >&2
+  sleep "${routing_retry_delay_seconds}"
+  attempt=$((attempt + 1))
+done
 
 curl --fail --silent --show-error "${base_url}/ready" |
   jq -e '.status == "ready"' >/dev/null

.github/workflows/simulation-deploy.reusable.yml

@@ -798,7 +798,7 @@ jobs:
         run: .github/scripts/set-cloud-run-simulation-entry-revision.sh
 
       - name: Verify stable service routing
-        run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}"
+        run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"
 
       - name: Verify stable service authentication
         working-directory: projects/policyengine-simulation-entry
@@ -812,7 +812,7 @@ jobs:
 
       - name: Verify configured custom hostname
         if: ${{ env.CUSTOM_URL != '' }}
-        run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}"
+        run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"
 
       - name: Verify configured custom-hostname authentication
         if: ${{ env.CUSTOM_URL != '' }}

changelog_entry.yaml

@@ -4,3 +4,4 @@
       - Resolved PolicyEngine-managed regional datasets using bundled data versions instead of release-manifest commit hashes when loading from GCS
       - Added a Modal app-release bundle backfill for missing data artifact revisions
       - Ensured Stage 12-only deployments run authenticated candidate tests and verify every required deployment phase before reporting success
+      - Allowed Cloud Run stable endpoint checks to wait for documented traffic propagation before restoring the previous revision

projects/policyengine-simulation-entry/tests/test_deployment_assets.py

@@ -59,6 +59,199 @@ def test_deployment_scripts_have_valid_shell_syntax():
     assert os.access(CLOUD_RUN_DEPLOY_SCRIPT, os.X_OK)
 
 
+def _write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, Path]:
+    state_file = tmp_path / "health-request-count"
+    revisions_file = tmp_path / "health-revisions"
+    revisions_file.write_text("\n".join(revisions) + "\n", encoding="utf-8")
+    fake_curl = tmp_path / "curl"
+    fake_curl.write_text(
+        textwrap.dedent("""\
+            #!/usr/bin/env bash
+            set -euo pipefail
+
+            headers_file=""
+            body_file=""
+            url=""
+            while [ "$#" -gt 0 ]; do
+              case "$1" in
+                --dump-header|--output|--request|--header|--data)
+                  option="$1"
+                  value="$2"
+                  shift 2
+                  case "${option}" in
+                    --dump-header) headers_file="${value}" ;;
+                    --output) body_file="${value}" ;;
+                  esac
+                  ;;
+                --fail|--silent|--show-error)
+                  shift
+                  ;;
+                *)
+                  url="$1"
+                  shift
+                  ;;
+              esac
+            done
+
+            case "${url}" in
+              */health)
+                count=0
+                if [ -f "${FAKE_CURL_STATE}" ]; then
+                  count="$(cat "${FAKE_CURL_STATE}")"
+                fi
+                line_number=$((count + 1))
+                revision="$(sed -n "${line_number}p" "${FAKE_CURL_REVISIONS}")"
+                printf '%s' "${line_number}" > "${FAKE_CURL_STATE}"
+                printf 'HTTP/1.1 200 OK\r\nX-PolicyEngine-Simulation-Revision: %s\r\n\r\n' \
+                  "${revision}" > "${headers_file}"
+                printf '{"status":"healthy"}' > "${body_file}"
+                ;;
+              */ready)
+                printf '{"status":"ready"}'
+                ;;
+              */versions)
+                printf '{}'
+                ;;
+              */ping)
+                printf '{"incremented":2}'
+                ;;
+              *)
+                printf 'Unexpected fake curl URL: %s\n' "${url}" >&2
+                exit 2
+                ;;
+            esac
+            """),
+        encoding="utf-8",
+    )
+    fake_curl.chmod(0o755)
+    return state_file, revisions_file
+
+
+def _smoke_test_env(
+    tmp_path: Path,
+    state_file: Path,
+    revisions_file: Path,
+) -> dict[str, str]:
+    return {
+        **os.environ,
+        "PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}",
+        "FAKE_CURL_STATE": str(state_file),
+        "FAKE_CURL_REVISIONS": str(revisions_file),
+        "SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS": "3",
+        "SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS": "0",
+    }
+
+
+def test_stable_smoke_retries_the_previous_revision_until_target_is_served(
+    tmp_path: Path,
+):
+    old_revision = "policyengine-simulation-entry-00001-old"
+    target_revision = "policyengine-simulation-entry-00002-new"
+    state_file, revisions_file = _write_fake_smoke_curl(
+        tmp_path,
+        [old_revision, old_revision, target_revision],
+    )
+
+    subprocess.run(
+        [
+            "bash",
+            SMOKE_SCRIPT,
+            "https://simulation.example.test",
+            target_revision,
+            old_revision,
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+        env=_smoke_test_env(tmp_path, state_file, revisions_file),
+    )
+
+    assert state_file.read_text(encoding="utf-8") == "3"
+
+
+def test_stable_smoke_rejects_an_unexpected_revision_without_retrying(tmp_path: Path):
+    old_revision = "policyengine-simulation-entry-00001-old"
+    target_revision = "policyengine-simulation-entry-00002-new"
+    unexpected_revision = "policyengine-simulation-entry-00003-unexpected"
+    state_file, revisions_file = _write_fake_smoke_curl(
+        tmp_path,
+        [unexpected_revision, target_revision],
+    )
+
+    result = subprocess.run(
+        [
+            "bash",
+            SMOKE_SCRIPT,
+            "https://simulation.example.test",
+            target_revision,
+            old_revision,
+        ],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=_smoke_test_env(tmp_path, state_file, revisions_file),
+    )
+
+    assert result.returncode == 1
+    assert "Unexpected revision" in result.stderr
+    assert state_file.read_text(encoding="utf-8") == "1"
+
+
+def test_candidate_smoke_rejects_a_revision_mismatch_without_retrying(tmp_path: Path):
+    old_revision = "policyengine-simulation-entry-00001-old"
+    target_revision = "policyengine-simulation-entry-00002-new"
+    state_file, revisions_file = _write_fake_smoke_curl(
+        tmp_path,
+        [old_revision, target_revision],
+    )
+
+    result = subprocess.run(
+        [
+            "bash",
+            SMOKE_SCRIPT,
+            "https://candidate.example.test",
+            target_revision,
+        ],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=_smoke_test_env(tmp_path, state_file, revisions_file),
+    )
+
+    assert result.returncode == 1
+    assert "Expected revision" in result.stderr
+    assert state_file.read_text(encoding="utf-8") == "1"
+
+
+def test_stable_smoke_fails_when_the_previous_revision_does_not_converge(
+    tmp_path: Path,
+):
+    old_revision = "policyengine-simulation-entry-00001-old"
+    target_revision = "policyengine-simulation-entry-00002-new"
+    state_file, revisions_file = _write_fake_smoke_curl(
+        tmp_path,
+        [old_revision, old_revision, old_revision],
+    )
+
+    result = subprocess.run(
+        [
+            "bash",
+            SMOKE_SCRIPT,
+            "https://simulation.example.test",
+            target_revision,
+            old_revision,
+        ],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=_smoke_test_env(tmp_path, state_file, revisions_file),
+    )
+
+    assert result.returncode == 1
+    assert "did not serve revision" in result.stderr
+    assert state_file.read_text(encoding="utf-8") == "3"
+
+
 def test_stage12_deployment_uses_the_api_owned_schema_directly():
     validation_script = STAGE12_VALIDATION_SCRIPT.read_text(encoding="utf-8")
 
@@ -396,7 +589,15 @@ def test_full_stack_promotion_order_is_explicit():
     )
     assert "needs: [deploy_entrypoint, authenticated_test]" in reusable_workflow
     assert (
-        'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}"'
+        'cloud-run-simulation-entry-smoke.sh "${STABLE_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"'
+        in reusable_workflow
+    )
+    assert (
+        'cloud-run-simulation-entry-smoke.sh "${CUSTOM_URL}" "${TARGET_REVISION}" "${PREVIOUS_REVISION}"'
+        in reusable_workflow
+    )
+    assert (
+        'cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}"'
         in reusable_workflow
     )
     assert "failure() && steps.promote.outcome == 'success'" in reusable_workflow