Configure and instrument simulation observability

.github/scripts/deploy-cloud-run-simulation-entry.sh

@@ -77,6 +77,10 @@ required_variables=(
   OLD_GATEWAY_AUTH_AUDIENCE_VALUE
   OLD_GATEWAY_AUTH_CLIENT_ID_VALUE
   OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME
+  OBSERVABILITY_SERVICE_NAMESPACE
+  OBSERVABILITY_TRACE_PROJECT_ID
+  OTEL_EXPORTER_OTLP_ENDPOINT
+  POLICYENGINE_OTEL_GOOGLE_AUDIENCE
   RUNNER_TEMP
 )
 for variable_name in "${required_variables[@]}"; do
@@ -148,6 +152,14 @@ jq -n '
     OLD_GATEWAY_AUTH_ISSUER: env.OLD_GATEWAY_AUTH_ISSUER_VALUE,
     OLD_GATEWAY_AUTH_AUDIENCE: env.OLD_GATEWAY_AUTH_AUDIENCE_VALUE,
     OLD_GATEWAY_AUTH_CLIENT_ID: env.OLD_GATEWAY_AUTH_CLIENT_ID_VALUE,
+    OBSERVABILITY_SERVICE_NAMESPACE: env.OBSERVABILITY_SERVICE_NAMESPACE,
+    OBSERVABILITY_TRACE_PROJECT_ID: env.OBSERVABILITY_TRACE_PROJECT_ID,
+    OTEL_EXPORTER_OTLP_ENDPOINT: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+    OTEL_EXPORTER_OTLP_PROTOCOL: "grpc",
+    OTEL_TRACES_EXPORTER: "otlp",
+    OTEL_METRICS_EXPORTER: "otlp",
+    OTEL_TRACES_SAMPLER_ARG: "1.0",
+    POLICYENGINE_OTEL_GOOGLE_AUDIENCE: env.POLICYENGINE_OTEL_GOOGLE_AUDIENCE,
     STAGE12_ENABLED: (
       if env.DEPLOY_STAGE12_V2 == "true" then env.STAGE12_ENABLED_VALUE else "0" end
     )

.github/scripts/modal-sync-secrets.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
 # Sync secrets from GitHub to Modal environment
 # Usage: ./modal-sync-secrets.sh <modal-environment> <gh-environment>
-# Required env vars: LOGFIRE_TOKEN, HF_TOKEN
-# Optional env vars: GCP_CREDENTIALS_JSON
+# Required env vars: HF_TOKEN, GCP_CREDENTIALS_JSON
 
 set -euo pipefail
 
@@ -24,6 +23,12 @@ if [ -z "${HF_TOKEN:-}" ]; then
   exit 1
 fi
 
+if [ -z "${GCP_CREDENTIALS_JSON:-}" ]; then
+  echo "GCP_CREDENTIALS_JSON is required to sync the shared artifact-store credential." >&2
+  echo "Add GCP_CREDENTIALS_JSON to the repository secrets." >&2
+  exit 1
+fi
+
 GATEWAY_AUTH_VARS=(
   GATEWAY_AUTH_ISSUER
   GATEWAY_AUTH_AUDIENCE
@@ -54,20 +59,14 @@ if truthy "${GATEWAY_AUTH_REQUIRED:-}" && [ ${#missing[@]} -gt 0 ]; then
   exit 1
 fi
 
-# Sync Logfire secret
-uv run modal secret create policyengine-logfire \
-  "LOGFIRE_TOKEN=${LOGFIRE_TOKEN:-}" \
-  "LOGFIRE_ENVIRONMENT=$GH_ENV" \
-  --env="$MODAL_ENV" \
-  --force || true
-
-# Sync GCP credentials if provided
-if [ -n "${GCP_CREDENTIALS_JSON:-}" ]; then
-  uv run modal secret create gcp-credentials \
-    "GOOGLE_APPLICATION_CREDENTIALS_JSON=$GCP_CREDENTIALS_JSON" \
-    --env="$MODAL_ENV" \
-    --force || true
-fi
+# The legacy executor explicitly reads this shared secret from Modal's main
+# environment in both staging and production. Synchronize that exact resource
+# during deployment and stop immediately if the update fails; runtime storage
+# and observability error handling remain independent of this deployment step.
+uv run modal secret create gcp-credentials \
+  "GOOGLE_APPLICATION_CREDENTIALS_JSON=$GCP_CREDENTIALS_JSON" \
+  --env="main" \
+  --force
 
 # Sync Hugging Face token for private certified datasets used during bundle
 # image build and worker runtime.

.github/workflows/simulation-deploy.reusable.yml

@@ -127,7 +127,6 @@ jobs:
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
-          LOGFIRE_TOKEN: ${{ secrets.LOGFIRE_TOKEN }}
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
           GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
@@ -222,6 +221,10 @@ jobs:
         env:
           IMAGE: ${{ steps.image.outputs.uri }}
           TAG: ${{ steps.metadata.outputs.tag }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
           DEPLOY_STAGE12_V2: ${{ inputs.deploy_stage12_v2 }}
           APP_ENVIRONMENT: ${{ inputs.deployment_environment }}
           MODAL_ENVIRONMENT: ${{ inputs.modal_environment }}
@@ -292,7 +295,7 @@ jobs:
   deploy_gateway:
     name: Deploy and unit test Modal gateway
     if: ${{ inputs.deploy_existing_stack }}
-    needs: prepare
+    needs: [prepare, deploy_executor]
     runs-on: ubuntu-latest
     environment: ${{ inputs.release_environment }}
     outputs:
@@ -315,6 +318,14 @@ jobs:
         env:
           MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
           MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
+          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
+          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
         run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/policyengine_simulation_gateway/app.py
 
       - name: Run gateway unit tests
@@ -383,6 +394,14 @@ jobs:
           POLICYENGINE_MANIFEST_DIGEST: ${{ steps.precompute.outputs.manifest_digest }}
           POLICYENGINE_ARTIFACT_BUCKET: ${{ vars.POLICYENGINE_ARTIFACT_BUCKET }}
           GCP_CREDENTIALS_JSON: ${{ secrets.GCP_CREDENTIALS_JSON }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
+          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
+          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
         run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/app.py
 
       - name: Run executor unit tests
@@ -543,6 +562,14 @@ jobs:
           STAGE12_EXPECT_US_VERSION: ${{ needs.prepare.outputs.us_version }}
           STAGE12_EXPECT_UK_VERSION: ${{ needs.prepare.outputs.uk_version }}
           STAGE12_ARTIFACT_BUCKET: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
+          OBSERVABILITY_SERVICE_NAMESPACE: ${{ vars.OBSERVABILITY_SERVICE_NAMESPACE }}
+          OBSERVABILITY_TRACE_PROJECT_ID: ${{ vars.OBSERVABILITY_TRACE_PROJECT_ID }}
+          OBSERVABILITY_LOGGING_PROJECT_ID: ${{ vars.OBSERVABILITY_LOGGING_PROJECT_ID }}
+          OBSERVABILITY_LOG_NAME: ${{ vars.OBSERVABILITY_LOG_NAME }}
+          OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER: ${{ vars.OBSERVABILITY_GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ vars.OBSERVABILITY_GOOGLE_SERVICE_ACCOUNT_EMAIL }}
+          OTEL_EXPORTER_OTLP_ENDPOINT: ${{ vars.OBSERVABILITY_OTLP_ENDPOINT }}
+          POLICYENGINE_OTEL_GOOGLE_AUDIENCE: ${{ vars.OBSERVABILITY_OTLP_GOOGLE_AUDIENCE }}
         run: uv run modal deploy --env="${{ inputs.modal_environment }}" src/modal/v2_app.py
 
       - name: Validate installed bundles, secrets, datasets, and calculations

AGENTS.md

@@ -14,6 +14,10 @@ Before opening, replacing, or sharing any pull request, read
 When adding, moving, or reviewing tests, read
 `docs/engineering/skills/testing.md`.
 
+Before changing correlation identifiers, telemetry transport, spans, stage
+names, logging, or observability failure handling, read
+`docs/engineering/skills/observability.md`.
+
 ## Development
 
 - Use Python 3.13 and `uv`.

docs/engineering/skills/README.md

@@ -13,3 +13,6 @@ Current skills:
   verification, and title conventions.
 - `testing.md`: test placement, dependency boundaries, and expected validation
   commands.
+- `observability.md`: identifier ownership, HTTP and Modal transport,
+  persistence, registered runtime stages, trace boundaries, and failure
+  isolation.

docs/engineering/skills/observability.md

@@ -0,0 +1,124 @@
+# Observability engineering rules
+
+Read this file before changing request correlation, asynchronous dispatch,
+logging, traces, metrics, or runtime stage names.
+
+## Identifier ownership
+
+Use each identifier for its defined scope:
+
+| Identifier | Scope | Created by | Durable |
+|---|---|---|---|
+| `request_id` | One HTTP request | HTTP instrumentation | No |
+| `observability_id` | One complete calculation or report | The first service that accepts the calculation submission | Yes |
+| `job_id` | One annual Modal invocation | Modal | Yes, as functional job state |
+| `batch_job_id` | One budget window Modal invocation | Modal | Yes, as functional batch state |
+| `evaluation_id` | One Stage 12 report | Stage 12 report construction | Yes, as functional report state |
+| `simulation_execution_id` | One Stage 12 baseline or reform simulation | Stage 12 coordinator | Yes, as functional simulation state |
+| `submission_claim_id` | One API v1 attempt to acquire ownership of a simulation submission | API v1 | Only where the simulation request is stored |
+
+`observability_id` is a canonical UUID string and has diagnostic meaning only.
+Do not use it for idempotency, authorization, database identity, routing, or
+business logic. Do not create one for health checks, version queries, invalid
+requests, missing jobs, or ordinary status requests.
+
+## HTTP transport
+
+Transport `observability_id` only in
+`X-PolicyEngine-Observability-Id`. A submission endpoint accepts a valid
+incoming value or creates a new value. It binds that value to the active
+observability runtime and returns it in the response header.
+
+A status endpoint reads functional state first. If that state has a persisted
+`observability_id`, bind and return it. The persisted value takes precedence
+over a value supplied on the status request. If an older record has no value,
+leave it absent. Never invent an identifier while polling.
+
+Do not read an `observability_id`, `run_id`, `request_id`, or `traceparent`
+from a simulation JSON body. The body telemetry model discards these fields.
+
+## Modal transport
+
+Pass captured observability context as the keyword-only
+`observability_context` argument to a Modal function:
+
+```python
+context = runtime.capture_context()
+call = function.spawn(payload, observability_context=context)
+```
+
+Keep `payload` limited to calculation inputs and functional metadata. Do not
+add `_observability_context` to it.
+
+At the receiving function, validate the separate argument with
+`normalize_observability_context` and pass the result to
+`runtime.operation(..., remote_context=context)`. The operation scope must
+cover the worker call. Child dispatches made inside that scope call
+`runtime.capture_context()` again and pass the resulting context through their
+own keyword-only argument.
+
+Old Modal function signatures are intentionally unsupported. Deploy the
+simulation API before API v1 and stop old workers during the deployment.
+
+## Persistence
+
+Persist `observability_id` beside the functional record used by later status
+requests:
+
+- annual runs: job metadata keyed by `job_id`;
+- budget windows: seed and current batch state keyed by `batch_job_id`;
+- Stage 12 reports: the comparison report row keyed by `evaluation_id`.
+
+Keep this field nullable so records created before this implementation remain
+readable. Do not create replacement identifiers for those records.
+
+## Runtime stages
+
+All span names for supported run configurations are defined in
+`policyengine_simulation_observability.stages`. Runtime code must import the
+appropriate `StagePlan` and call `plan.name(Stage.VALUE)`. Do not add span name
+string literals in services or workers.
+
+When adding a run configuration or runtime stage:
+
+1. add it to `RunConfiguration` or `Stage`;
+2. add it to the applicable plan in `RUN_STAGE_REGISTRY`;
+3. import the plan in runtime code;
+4. add focused tests for the new stage and identifier propagation.
+
+## Traces and polling
+
+HTTP instrumentation carries W3C trace context across synchronous service
+calls. `capture_context` and `remote_context` carry that context across Modal
+dispatch, so worker spans can continue the submission trace.
+
+A later status request is a new trace. Its spans and logs share the persisted
+`observability_id` with the original calculation. Queries that measure a full
+report must select all telemetry with that identifier and use registered stage
+names to break down elapsed time. Do not assume every status request belongs to
+the original trace tree.
+
+Stage 12 authoritative and shadow reports use the same `observability_id` as
+the production calculation that caused their dispatch. Their functional
+`evaluation_id` and simulation execution identifiers remain distinct.
+
+## Failure behavior
+
+Invalid observability configuration must fail during build or deployment
+validation. Once a service is serving application traffic, logging, tracing,
+metrics, context capture, context restoration, and export failures must not
+change a calculation result or HTTP status.
+
+Application boundary helpers must normalize untrusted context and treat
+malformed values as absent. Calls that bind context or emit a diagnostic event
+must rely on the package's nonthrowing runtime contract or use a local exception
+boundary where a package failure could otherwise escape into application code.
+
+Tests must cover:
+
+- creation at each submission boundary;
+- propagation through HTTP and Modal keyword arguments;
+- persistence and restoration during polling;
+- persisted values taking precedence over status request headers;
+- old records with null identifiers;
+- observability failures leaving application results unchanged.

docs/operations/api-v1-observability.md

@@ -0,0 +1,69 @@
+# API v1 simulation observability
+
+This document describes the correlation and trace path shared by API v1, the
+Cloud Run simulation entry service, the Modal gateway, and Modal workers.
+
+## Submission path
+
+```mermaid
+sequenceDiagram
+    participant API as API v1
+    participant Entry as Simulation entry
+    participant Gateway as Modal gateway
+    participant Worker as Modal worker
+    participant Store as Job or report state
+
+    API->>API: Create or reuse observability_id
+    API->>Entry: HTTP header and W3C trace context
+    Entry->>Gateway: Same header and trace context
+    Gateway->>Store: Persist observability_id with functional state
+    Gateway->>Worker: payload plus separate observability_context argument
+    Worker->>Worker: Restore context for worker operation
+    Gateway-->>Entry: Response header
+    Entry-->>API: Response header
+```
+
+When API v1 is absent, the simulation entry service creates the identifier. A
+direct call to the Modal gateway causes the gateway to create it. Every service
+preserves a valid value received from the service that accepted the report.
+
+## Poll path
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant Entry as Simulation entry
+    participant Gateway as Modal gateway
+    participant Store as Job or report state
+
+    Client->>Entry: Status request with functional identifier
+    Entry->>Gateway: Status request
+    Gateway->>Store: Read job, batch, or report state
+    Store-->>Gateway: Persisted observability_id
+    Gateway-->>Entry: Status plus response header
+    Entry-->>Client: Status plus response header
+```
+
+The stored identifier is authoritative during polling. A conflicting header on
+a status request is ignored. A missing or null stored value produces no
+observability response header.
+
+## Stage 12
+
+The simulation entry service dispatches the current production run and the
+Stage 12 report with the same `observability_id`. Stage 12 creates separate
+functional identifiers for the report and its baseline and reform simulations.
+The coordinator passes captured context to both simulation workers. When Stage
+12 becomes the authoritative runner, the same report identifier flow applies;
+only the selected functional result changes.
+
+## Query model
+
+Use `observability_id` to select logs and spans for one calculation or report.
+Use the stage names from `RUN_STAGE_REGISTRY` to measure individual operations.
+Use `job_id`, `batch_job_id`, `evaluation_id`, and
+`simulation_execution_id` to locate functional state.
+
+Submission work can form one distributed trace because HTTP and Modal dispatch
+carry W3C trace context. Status requests occur later and create separate traces
+that remain queryable through the same `observability_id`.

libs/policyengine-fastapi/pyproject.toml

@@ -10,14 +10,14 @@ requires-python = ">=3.13"
 dependencies = [
     "fastapi[standard] >=0.115.8,<0.116.0",
     "pyjwt >=2.10.1,<3.0.0",
-    "opentelemetry-sdk >=1.30.0,<2.0.0",
+    "opentelemetry-sdk >=1.44.0,<2.0.0",
     "sqlmodel >=0.0.22,<0.0.23",
     "python-json-logger >=3.2.1,<4.0.0",
-    "opentelemetry-instrumentation-logging >=0.51b0,<0.52",
+    "opentelemetry-instrumentation-logging >=0.65b0,<0.66",
     "opentelemetry-exporter-gcp-trace >=1.9.0,<2.0.0",
     "opentelemetry-exporter-gcp-monitoring >=1.9.0a0,<2.0.0",
-    "opentelemetry-instrumentation-fastapi >=0.51b0,<0.52",
-    "opentelemetry-instrumentation-sqlalchemy>=0.51b0",
+    "opentelemetry-instrumentation-fastapi >=0.65b0,<0.66",
+    "opentelemetry-instrumentation-sqlalchemy>=0.65b0,<0.66",
     "uvicorn>=0.35.0",
 ]
 

libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py

@@ -15,13 +15,12 @@
     SimulationCompositeTraceResponse as SimulationCompositeTraceResponse,
     SimulationLifecycleEvent as SimulationLifecycleEvent,
     SimulationRunSummary as SimulationRunSummary,
-    SimulationTelemetryEnvelope as SimulationTelemetryEnvelope,
     SimulationTimelineEntry as SimulationTimelineEntry,
     TracerArtifactManifest as TracerArtifactManifest,
     VersionStageMetricResponse as VersionStageMetricResponse,
 )
 from .correlation import (
-    generate_run_id as generate_run_id,
+    generate_observability_id as generate_observability_id,
     stable_config_hash as stable_config_hash,
 )
 from .emitters import (