Phase 28: Hosted GCE Data Platform — serving apps + IaC + SDK hosted surface (platform layer)

.github/parity-trigger-paths.json

@@ -15,7 +15,9 @@
     "packages/markets/src/mostlyright/markets/catalog/**/*.py",
     "packages/core/src/mostlyright/core/**/*.py",
     "packages/weather/src/mostlyright/weather/catalog/**/*.py",
-    "packages/weather/src/mostlyright/weather/_fetchers/**/*.py"
+    "packages/weather/src/mostlyright/weather/_fetchers/**/*.py",
+    "packages/weather/src/mostlyright/weather/satellite/__init__.py",
+    "packages/weather/src/mostlyright/weather/satellite/_hosted_client.py"
   ],
   "typescript_paths": [
     "packages-ts/core/src/**/*.ts",

.github/workflows/deploy-weather-serving.yml

@@ -0,0 +1,107 @@
+name: Deploy weather-serving (28-30)
+
+# Phase 28 (28-30 Task 2) — WIF-authenticated build+deploy for the SLIM weather
+# serving Cloud Run service (/satellite + /capabilities) in mr-serving/eu-west3.
+#
+# This is the per-service workflow the shared deploy.yml (28-00) defers to for
+# weather-serving. KEYLESS auth via Workload Identity Federation — no SA key
+# files anywhere. The Cloud Run service resource itself + its R2-read-only secret
+# wiring + the global request ceiling live in infra/ (cloud_run.tf
+# weather_serving + secrets.tf); this workflow only builds+pushes the image and
+# rolls a new revision onto the EXISTING service.
+#
+# H4 note (documented, enforced in infra + app): the single build-injected
+# MOSTLYRIGHT_API_KEY is a PUBLIC secret (ships in the MV3 extension). Revocation/
+# rotation path: rotate the `mostlyright-api-key` Secret Manager version, re-run
+# this workflow (the service reads `version = latest`, so a new revision picks up
+# the rotated key), and rebuild/re-publish the extension with the new key — the
+# OLD key is then rejected 401 by the auth middleware. CORS is NOT access control.
+#
+# Setup (repo/environment Variables, from `tofu -chdir=infra output`):
+#   WIF_PROVIDER       = <wif_provider_name>            (full resource name)
+#   DEPLOY_SA_SERVING  = deploy@mr-serving...            (serving deploy SA email)
+#   AR_HOST            = europe-west3-docker.pkg.dev     (reused Artifact Registry host)
+#   SERVING_PROJECT_ID = mr-serving                      (resolved serving project id)
+# No secrets required — WIF mints tokens at run time; runtime secrets are injected
+# by the Cloud Run service (Secret Manager), never by this workflow.
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: "Image tag to build + deploy (e.g. a git SHA or 'latest')."
+        required: true
+        default: "latest"
+        type: string
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  # Reused Artifact Registry (28-00): europe-west3-docker.pkg.dev/mostlyright-backend/mostlyright.
+  AR_HOST: ${{ vars.AR_HOST }}
+  AR_PROJECT: mostlyright-backend
+  AR_REPO: mostlyright
+  IMAGE_NAME: weather-serving
+  SERVICE: weather-serving
+  REGION: europe-west3
+
+jobs:
+  deploy:
+    name: Build + push slim image, roll weather-serving revision
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # Keyless federation — exchange the GitHub OIDC token for GCP creds using
+      # the serving deploy SA (mr-serving). No inline SA key.
+      - name: Authenticate to GCP (WIF, keyless)
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.WIF_PROVIDER }}
+          service_account: ${{ vars.DEPLOY_SA_SERVING }}
+
+      - name: Set up gcloud
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Configure Docker for Artifact Registry
+        run: gcloud auth configure-docker "${AR_HOST}" --quiet
+
+      - name: Build slim weather-serving image
+        run: |
+          IMAGE="${AR_HOST}/${AR_PROJECT}/${AR_REPO}/${IMAGE_NAME}:${{ inputs.image_tag }}"
+          echo "IMAGE=${IMAGE}" >> "$GITHUB_ENV"
+          # Build from the repo root so the Dockerfile can COPY packages/ + services/.
+          docker build \
+            -f deploy/weather/serving.Dockerfile \
+            -t "${IMAGE}" \
+            .
+
+      - name: Push image
+        run: docker push "${IMAGE}"
+
+      # Roll a new revision onto the EXISTING service (declared in infra/). We do
+      # NOT create/patch scaling, secrets, or env here — those are Terraform-owned
+      # (min=0, the R2 read-only token, MOSTLYRIGHT_API_KEY, GLOBAL_RPS_CEILING,
+      # and the max-instances cap that is the infra-layer global ceiling, H4).
+      # `--image` only swaps the container image on the current config.
+      - name: Deploy revision (image swap only; config is Terraform-owned)
+        run: |
+          gcloud run deploy "${SERVICE}" \
+            --project "${{ vars.SERVING_PROJECT_ID }}" \
+            --region "${REGION}" \
+            --image "${IMAGE}" \
+            --quiet
+
+      # Post-deploy smoke: min-instances 0 (idle-cheap) is preserved and the
+      # service is reachable. Auth/byte-identical/global-ceiling checks are the
+      # blocking human-verify gate (28-30 Task 3), not this smoke step.
+      - name: Verify min-instances 0 preserved
+        run: |
+          MIN=$(gcloud run services describe "${SERVICE}" \
+            --project "${{ vars.SERVING_PROJECT_ID }}" \
+            --region "${REGION}" \
+            --format="value(spec.template.metadata.annotations['autoscaling.knative.dev/minScale'])")
+          echo "min-instances = ${MIN:-0}"
+          test "${MIN:-0}" = "0" || { echo "expected min-instances 0 (idle-cheap)"; exit 1; }

.github/workflows/deploy.yml

@@ -0,0 +1,98 @@
+name: Deploy (hosted GCE platform)
+
+# Phase 28 (28-00) — WIF-authenticated deploy skeleton for the hosted data
+# platform (mr-earnings-ingest / mr-serving / mr-staging).
+#
+# KEYLESS auth via Workload Identity Federation (28-GCE-ARCHITECTURE §7): this
+# workflow authenticates to GCP with a short-lived OIDC token minted from the
+# GitHub Actions run — there is NO SA key file (no inline JSON key) anywhere,
+# and no SA key is stored in repo secrets. The WIF provider + per-project deploy SAs
+# are provisioned by the Terraform root in infra/ (28-00).
+#
+# The image-build/push + `gcloud run deploy` / Cloud Batch / MIG steps are
+# STUBBED here — later waves (W1 serving, W2 ingest/fleet) fill them in against
+# the existing Artifact Registry
+# (europe-west3-docker.pkg.dev/mostlyright-backend/mostlyright).
+#
+# Setup (one-time, after `tofu apply` in infra/):
+#   Set the following repo/environment variables (Settings -> Variables), read
+#   from `tofu -chdir=infra output`:
+#     WIF_PROVIDER          = <tofu output wif_provider_name>              (full resource name)
+#     DEPLOY_SA_SERVING     = <tofu output deploy_service_accounts.serving>
+#     DEPLOY_SA_INGEST      = <tofu output deploy_service_accounts.ingest>
+#     DEPLOY_SA_SATELLITE   = <tofu output deploy_service_account_satellite> (weather, EXISTING mostlyright-satellite, H1)
+#   No secrets required — WIF mints tokens at run time. The deploy SA is selected
+#   per target below (the weather backfill/incremental deploys use the satellite
+#   SA — H1: weather compute lives in the EXISTING mostlyright-satellite project).
+
+on:
+  # Manual, explicit deploys only for now (no auto-deploy on push/tag until the
+  # image jobs below are real). Later waves may add a tag trigger.
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Deploy target project"
+        required: true
+        default: "serving"
+        type: choice
+        options:
+          - serving
+          - ingest
+          - satellite
+          - staging
+
+# WIF requires id-token: write so the runner can mint the OIDC token that
+# google-github-actions/auth exchanges for a short-lived GCP access token.
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  deploy:
+    name: WIF auth + deploy (stub)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # Select the per-target deploy SA. The weather target uses the EXISTING
+      # mostlyright-satellite deploy SA (H1); ingest/serving use their own.
+      - name: Select deploy SA for target
+        id: sa
+        run: |
+          case "${{ inputs.target }}" in
+            serving)   echo "email=${{ vars.DEPLOY_SA_SERVING }}"   >> "$GITHUB_OUTPUT" ;;
+            ingest)    echo "email=${{ vars.DEPLOY_SA_INGEST }}"    >> "$GITHUB_OUTPUT" ;;
+            satellite) echo "email=${{ vars.DEPLOY_SA_SATELLITE }}" >> "$GITHUB_OUTPUT" ;;
+            staging)   echo "email=${{ vars.DEPLOY_SA_STAGING }}"   >> "$GITHUB_OUTPUT" ;;
+            *) echo "Unknown target ${{ inputs.target }}" >&2; exit 1 ;;
+          esac
+
+      # Keyless federation — exchange the GitHub OIDC token for GCP creds.
+      # workload_identity_provider is the full resource name emitted by
+      # `tofu -chdir=infra output wif_provider_name`. No inline SA key is used.
+      - name: Authenticate to GCP (WIF, keyless)
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: ${{ vars.WIF_PROVIDER }}
+          service_account: ${{ steps.sa.outputs.email }}
+
+      - name: Set up gcloud
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Verify auth (identity smoke test)
+        run: gcloud auth list --filter=status:ACTIVE --format="value(account)"
+
+      # ---------------------------------------------------------------------
+      # STUB — per-service deploy workflows fill these in (28-10/11/12/13/21/22/30):
+      #   serving   : earnings-serving + weather-serving → Cloud Run (eu-west3),
+      #               timeout 3600, SSE max-instances=1 + affinity deploy check (H2).
+      #   ingest    : capture Job + rolefact Job (eu-west3) + STT Cloud Run GPU L4
+      #               (us-central1, bounded concurrency ≤ L4 quota, H8).
+      #   satellite : weather backfill (Cloud Batch, us-central1) + incremental Job
+      #               (H1: EXISTING mostlyright-satellite project).
+      # All push to europe-west3-docker.pkg.dev/mostlyright-backend/mostlyright.
+      # ---------------------------------------------------------------------
+      - name: Build & deploy (placeholder)
+        run: |
+          echo "Deploy target: ${{ inputs.target }}"
+          echo "Image build + push + gcloud run deploy stubbed — filled in by W1/W2."

.github/workflows/test.yml

@@ -91,6 +91,10 @@ jobs:
           uv run ruff check .
           uv run ruff format --check .
 
+      - name: Hosted-call grep-gate (D-28.2 — default path stays hosted-call-free)
+        if: needs.changes.outputs.py == 'true'
+        run: uv run python scripts/check_no_hosted_calls.py
+
       - name: Run fast test suite (excludes @pytest.mark.live and @pytest.mark.polars)
         if: needs.changes.outputs.py == 'true'
         run: uv run pytest -m "not live and not polars" -q

.gitignore

@@ -86,3 +86,16 @@ docs/sphinx/api/
 # JSDoc/TSDoc on `packages-ts/*/src/`, plus the hand-written
 # `packages-ts/typedoc.json` config.
 docs-ts-build/
+
+# Phase 28 — Terraform/OpenTofu root (infra/). Secret/semi-sensitive values
+# (billing account ID) live in a GITIGNORED terraform.tfvars; only
+# terraform.tfvars.example (placeholders) is committed. State + provider
+# binaries + plan outputs are never committed.
+infra/terraform.tfvars
+infra/*.auto.tfvars
+**/.terraform/
+*.tfstate
+*.tfstate.*
+*.tfplan
+crash.log
+crash.*.log

CLAUDE.md

@@ -38,7 +38,7 @@ uv build                                   # build all three packages
 - **Never commit directly to main.** Always branch + PR.
 - **TDD mandatory.** Write tests first. RED → GREEN → REFACTOR. 80% coverage minimum.
 - **Pre-commit + pre-push hooks mandatory.** No `--no-verify`. Fix the underlying issue. Pre-commit runs fast checks (ruff, format, whitespace, YAML/TOML validation); pre-push runs `pytest -m "not live"`. Install both with `uv run pre-commit install && uv run pre-commit install --hook-type pre-push`.
-- **All API calls direct from SDK.** No `api.mostlyright.md`, no hosted-API client calls anywhere in `mostlyright.*`. Verified via grep on built wheels before publish.
+- **Default path calls public APIs direct from SDK; hosted is opt-in (Phase 28 carve-out, GATE #1 signed 2026-07-02).** The SDK **DEFAULT** path (`research()`, local `live`) makes **NO** hosted call and hits public APIs (AWC, IEM, GHCNh, NWS CLI, Kalshi) directly. The wheel grep-gate still runs `grep` on built wheels before publish to enforce that the **default/published-dist** path stays hosted-call-free — the gate is **amended (narrowed to the default path), not removed**. Hosted is reached **only via the opt-in seams** `delivery="hosted"` / `EARNINGS_HOSTED_URL` / `WEATHER_HOSTED_URL` + `MOSTLYRIGHT_API_KEY`; hosted rows are byte-identical to the local `live` path. The `services/` deploy code (uvicorn/ffmpeg/Chromium/whisper deploy deps) is **NON-published** and MUST NOT enter any PyPI dist. See [`.planning/phases/28-hosted-gce-data-platform/28-01-GATE-RECORD.md`](.planning/phases/28-hosted-gce-data-platform/28-01-GATE-RECORD.md).
 
 ## Dual-SDK Planning Rule
 
@@ -109,7 +109,7 @@ A local-first Python SDK for quants researching prediction-market weather contra
 
 ### Constraints
 
-- **Tech stack:** Python 3.11+. uv workspace. `httpx`, `pandas`, `pyarrow`, `filelock`, `jsonschema`, `hypothesis` (dev). No FastAPI, no Docker, no hosted infra in v0.1.
+- **Tech stack:** Python 3.11+. uv workspace. `httpx`, `pandas`, `pyarrow`, `filelock`, `jsonschema`, `hypothesis` (dev). No FastAPI, no Docker, no hosted infra in the **published SDK v0.1 default path**. **Phase 28 opt-in-hosted carve-out (GATE #1, 2026-07-02):** an opt-in hosted client + a served hosted API are now permitted, but only reached via `delivery="hosted"` / `EARNINGS_HOSTED_URL` / `WEATHER_HOSTED_URL` + `MOSTLYRIGHT_API_KEY`; the `services/` serving app (FastAPI/uvicorn + ffmpeg/Chromium/whisper deploy deps) is NON-published and MUST NOT enter any PyPI dist.
 - **Timeline:** 14 calendar days from Day 1. Phase A (parity lift) Days 1-4, Phase B (core+catalog) Days 5-14. v0.2 (MCP) is a later milestone.
 - **Execution model:** Two-lane parallel — Lane V (Vu) lifts from `monorepo-v0.14.1/`, Lane F (Founder) builds new code. Cross-review mandatory. Every PR runs the two-reviewer loop (Codex `high` + Python Architect) per [`.planning/REVIEW-DISCIPLINE.md`](.planning/REVIEW-DISCIPLINE.md) — applies to ALL branches, not just parity-critical paths.
 - **Testing discipline:** TDD mandatory (RED → GREEN → REFACTOR). Pre-commit hooks; no `--no-verify`. ≥90% branch coverage on `mostlyright.core`. 80% line coverage on `catalog/` and adapter wrappers. Lifted `_vendor/` code retains its monorepo coverage.
@@ -259,7 +259,7 @@ A local-first Python SDK for quants researching prediction-market weather contra
 | `filelock` for cache | ✓ Confirmed. Battle-tested; 3.29 brings Windows improvements (cheap floor bump). |
 | `jsonschema` for validation | ✓ Confirmed for v0.1. Reconsider Pydantic for v0.2 MCP work. |
 | `hypothesis` for property tests | ✓ Confirmed. Only mainstream choice in Python. |
-| No FastAPI, no Docker | ✓ Confirmed. Local-first SDK; no servers. |
+| No FastAPI, no Docker | ✓ Confirmed for the published SDK default path. **Amended Phase 28 (GATE #1, 2026-07-02):** an opt-in hosted serving API (`services/`, FastAPI) is allowed off the published dist; the SDK default stays local-first / hosted-call-free. |
 | MCP deferred to v0.2 | ✓ Confirmed. The `mcp` SDK at 1.27.1 is mature enough for v0.2; deferring avoids the FastMCP + Pydantic dep proliferation in v0.1. |
 ## Decisions to Consider Revisiting (Soft Flags)
 ## Sources

deploy/weather/serving.Dockerfile

@@ -0,0 +1,58 @@
+# Weather serving image — the hosted /satellite + /capabilities REST app (28-30).
+#
+# A SLIM serving image: it packages only `services/weather/` + the two SDK
+# packages it reads from (`mostlyrightmd`, `mostlyrightmd-weather` with the
+# `[satellite]` extra for boto3 + parquet), and runs uvicorn. NO audio toolchain,
+# NO backfill/compute toolchain (xarray/h5netcdf/s3fs full stack is not needed to
+# READ derived parquet — boto3 + pyarrow are, and they come with `[satellite]`).
+#
+# Read-only by construction: the container reads R2 with the READ-ONLY token
+# (r2-read-* + MOSTLYRIGHT_API_KEY injected from Secret Manager into the serving
+# SA env by the deploy layer). It never holds the write token / any ingest secret.
+#
+# Pinned digests are NOT used here (matching the repo's other stubs); the base is
+# the slim CPython image the SDK floors target (py3.11+). Cloud Run injects PORT.
+
+FROM python:3.12-slim AS base
+
+# Non-interactive, no .pyc, unbuffered logs (Cloud Run friendly).
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+WORKDIR /app
+
+# --- Dependency layer --------------------------------------------------------
+# Copy just the package sources needed to install the SDK distributions the
+# serving app imports (core + weather[satellite]) plus the serving runtime deps.
+# Copying pyproject/src first keeps the dep layer cached across app-code edits.
+COPY packages/core/ packages/core/
+COPY packages/weather/ packages/weather/
+
+# Install the two published distributions (editable-free, from the local tree)
+# with the [satellite] extra (boto3 + pyarrow: the R2 read + parquet parse), and
+# the serving runtime (FastAPI + uvicorn). FastAPI is a WORKSPACE dev/test dep
+# for the non-published service, so it is named explicitly here.
+RUN pip install \
+      ./packages/core \
+      "./packages/weather[satellite]" \
+      "fastapi>=0.115,<1" \
+      "uvicorn[standard]>=0.30"
+
+# --- App layer ---------------------------------------------------------------
+# The non-published serving app is imported as `services.weather.*` (matching the
+# repo-root conftest sys.path convention), so it is copied under /app/services.
+COPY services/weather/ services/weather/
+
+# Cloud Run sets $PORT (default 8080). The lazy `services.weather.app:app` factory
+# resolves MOSTLYRIGHT_API_KEY from the env and FAILS CLOSED if it is unset (the
+# public feed never serves keyless), so a misconfigured deploy crashes loud at
+# startup rather than serving unauthenticated.
+ENV PORT=8080
+EXPOSE 8080
+
+# One worker: the in-process per-key + global-ceiling limiters are per-process
+# (the Redis seam is deferred), and Cloud Run scales by adding instances under
+# the max-instances cap (the infrastructure-layer global ceiling, weather_serving.tf).
+CMD ["sh", "-c", "uvicorn services.weather.app:app --host 0.0.0.0 --port ${PORT} --workers 1"]