diff --git a/.github/parity-trigger-paths.json b/.github/parity-trigger-paths.json index ba5e8310..d24953fd 100644 --- a/.github/parity-trigger-paths.json +++ b/.github/parity-trigger-paths.json @@ -15,7 +15,9 @@ "packages/markets/src/mostlyright/markets/catalog/**/*.py", "packages/core/src/mostlyright/core/**/*.py", "packages/weather/src/mostlyright/weather/catalog/**/*.py", - "packages/weather/src/mostlyright/weather/_fetchers/**/*.py" + "packages/weather/src/mostlyright/weather/_fetchers/**/*.py", + "packages/weather/src/mostlyright/weather/satellite/__init__.py", + "packages/weather/src/mostlyright/weather/satellite/_hosted_client.py" ], "typescript_paths": [ "packages-ts/core/src/**/*.ts", diff --git a/.github/workflows/deploy-weather-serving.yml b/.github/workflows/deploy-weather-serving.yml new file mode 100644 index 00000000..a8b463f5 --- /dev/null +++ b/.github/workflows/deploy-weather-serving.yml @@ -0,0 +1,107 @@ +name: Deploy weather-serving (28-30) + +# Phase 28 (28-30 Task 2) — WIF-authenticated build+deploy for the SLIM weather +# serving Cloud Run service (/satellite + /capabilities) in mr-serving/eu-west3. +# +# This is the per-service workflow the shared deploy.yml (28-00) defers to for +# weather-serving. KEYLESS auth via Workload Identity Federation — no SA key +# files anywhere. The Cloud Run service resource itself + its R2-read-only secret +# wiring + the global request ceiling live in infra/ (cloud_run.tf +# weather_serving + secrets.tf); this workflow only builds+pushes the image and +# rolls a new revision onto the EXISTING service. +# +# H4 note (documented, enforced in infra + app): the single build-injected +# MOSTLYRIGHT_API_KEY is a PUBLIC secret (ships in the MV3 extension). Revocation/ +# rotation path: rotate the `mostlyright-api-key` Secret Manager version, re-run +# this workflow (the service reads `version = latest`, so a new revision picks up +# the rotated key), and rebuild/re-publish the extension with the new key — the +# OLD key is then rejected 401 by the auth middleware. CORS is NOT access control. +# +# Setup (repo/environment Variables, from `tofu -chdir=infra output`): +# WIF_PROVIDER = (full resource name) +# DEPLOY_SA_SERVING = deploy@mr-serving... (serving deploy SA email) +# AR_HOST = europe-west3-docker.pkg.dev (reused Artifact Registry host) +# SERVING_PROJECT_ID = mr-serving (resolved serving project id) +# No secrets required — WIF mints tokens at run time; runtime secrets are injected +# by the Cloud Run service (Secret Manager), never by this workflow. + +on: + workflow_dispatch: + inputs: + image_tag: + description: "Image tag to build + deploy (e.g. a git SHA or 'latest')." + required: true + default: "latest" + type: string + +permissions: + id-token: write + contents: read + +env: + # Reused Artifact Registry (28-00): europe-west3-docker.pkg.dev/mostlyright-backend/mostlyright. + AR_HOST: ${{ vars.AR_HOST }} + AR_PROJECT: mostlyright-backend + AR_REPO: mostlyright + IMAGE_NAME: weather-serving + SERVICE: weather-serving + REGION: europe-west3 + +jobs: + deploy: + name: Build + push slim image, roll weather-serving revision + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Keyless federation — exchange the GitHub OIDC token for GCP creds using + # the serving deploy SA (mr-serving). No inline SA key. + - name: Authenticate to GCP (WIF, keyless) + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.WIF_PROVIDER }} + service_account: ${{ vars.DEPLOY_SA_SERVING }} + + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Configure Docker for Artifact Registry + run: gcloud auth configure-docker "${AR_HOST}" --quiet + + - name: Build slim weather-serving image + run: | + IMAGE="${AR_HOST}/${AR_PROJECT}/${AR_REPO}/${IMAGE_NAME}:${{ inputs.image_tag }}" + echo "IMAGE=${IMAGE}" >> "$GITHUB_ENV" + # Build from the repo root so the Dockerfile can COPY packages/ + services/. + docker build \ + -f deploy/weather/serving.Dockerfile \ + -t "${IMAGE}" \ + . + + - name: Push image + run: docker push "${IMAGE}" + + # Roll a new revision onto the EXISTING service (declared in infra/). We do + # NOT create/patch scaling, secrets, or env here — those are Terraform-owned + # (min=0, the R2 read-only token, MOSTLYRIGHT_API_KEY, GLOBAL_RPS_CEILING, + # and the max-instances cap that is the infra-layer global ceiling, H4). + # `--image` only swaps the container image on the current config. + - name: Deploy revision (image swap only; config is Terraform-owned) + run: | + gcloud run deploy "${SERVICE}" \ + --project "${{ vars.SERVING_PROJECT_ID }}" \ + --region "${REGION}" \ + --image "${IMAGE}" \ + --quiet + + # Post-deploy smoke: min-instances 0 (idle-cheap) is preserved and the + # service is reachable. Auth/byte-identical/global-ceiling checks are the + # blocking human-verify gate (28-30 Task 3), not this smoke step. + - name: Verify min-instances 0 preserved + run: | + MIN=$(gcloud run services describe "${SERVICE}" \ + --project "${{ vars.SERVING_PROJECT_ID }}" \ + --region "${REGION}" \ + --format="value(spec.template.metadata.annotations['autoscaling.knative.dev/minScale'])") + echo "min-instances = ${MIN:-0}" + test "${MIN:-0}" = "0" || { echo "expected min-instances 0 (idle-cheap)"; exit 1; } diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..0105c03e --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,98 @@ +name: Deploy (hosted GCE platform) + +# Phase 28 (28-00) — WIF-authenticated deploy skeleton for the hosted data +# platform (mr-earnings-ingest / mr-serving / mr-staging). +# +# KEYLESS auth via Workload Identity Federation (28-GCE-ARCHITECTURE §7): this +# workflow authenticates to GCP with a short-lived OIDC token minted from the +# GitHub Actions run — there is NO SA key file (no inline JSON key) anywhere, +# and no SA key is stored in repo secrets. The WIF provider + per-project deploy SAs +# are provisioned by the Terraform root in infra/ (28-00). +# +# The image-build/push + `gcloud run deploy` / Cloud Batch / MIG steps are +# STUBBED here — later waves (W1 serving, W2 ingest/fleet) fill them in against +# the existing Artifact Registry +# (europe-west3-docker.pkg.dev/mostlyright-backend/mostlyright). +# +# Setup (one-time, after `tofu apply` in infra/): +# Set the following repo/environment variables (Settings -> Variables), read +# from `tofu -chdir=infra output`: +# WIF_PROVIDER = (full resource name) +# DEPLOY_SA_SERVING = +# DEPLOY_SA_INGEST = +# DEPLOY_SA_SATELLITE = (weather, EXISTING mostlyright-satellite, H1) +# No secrets required — WIF mints tokens at run time. The deploy SA is selected +# per target below (the weather backfill/incremental deploys use the satellite +# SA — H1: weather compute lives in the EXISTING mostlyright-satellite project). + +on: + # Manual, explicit deploys only for now (no auto-deploy on push/tag until the + # image jobs below are real). Later waves may add a tag trigger. + workflow_dispatch: + inputs: + target: + description: "Deploy target project" + required: true + default: "serving" + type: choice + options: + - serving + - ingest + - satellite + - staging + +# WIF requires id-token: write so the runner can mint the OIDC token that +# google-github-actions/auth exchanges for a short-lived GCP access token. +permissions: + id-token: write + contents: read + +jobs: + deploy: + name: WIF auth + deploy (stub) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Select the per-target deploy SA. The weather target uses the EXISTING + # mostlyright-satellite deploy SA (H1); ingest/serving use their own. + - name: Select deploy SA for target + id: sa + run: | + case "${{ inputs.target }}" in + serving) echo "email=${{ vars.DEPLOY_SA_SERVING }}" >> "$GITHUB_OUTPUT" ;; + ingest) echo "email=${{ vars.DEPLOY_SA_INGEST }}" >> "$GITHUB_OUTPUT" ;; + satellite) echo "email=${{ vars.DEPLOY_SA_SATELLITE }}" >> "$GITHUB_OUTPUT" ;; + staging) echo "email=${{ vars.DEPLOY_SA_STAGING }}" >> "$GITHUB_OUTPUT" ;; + *) echo "Unknown target ${{ inputs.target }}" >&2; exit 1 ;; + esac + + # Keyless federation — exchange the GitHub OIDC token for GCP creds. + # workload_identity_provider is the full resource name emitted by + # `tofu -chdir=infra output wif_provider_name`. No inline SA key is used. + - name: Authenticate to GCP (WIF, keyless) + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ vars.WIF_PROVIDER }} + service_account: ${{ steps.sa.outputs.email }} + + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v2 + + - name: Verify auth (identity smoke test) + run: gcloud auth list --filter=status:ACTIVE --format="value(account)" + + # --------------------------------------------------------------------- + # STUB — per-service deploy workflows fill these in (28-10/11/12/13/21/22/30): + # serving : earnings-serving + weather-serving → Cloud Run (eu-west3), + # timeout 3600, SSE max-instances=1 + affinity deploy check (H2). + # ingest : capture Job + rolefact Job (eu-west3) + STT Cloud Run GPU L4 + # (us-central1, bounded concurrency ≤ L4 quota, H8). + # satellite : weather backfill (Cloud Batch, us-central1) + incremental Job + # (H1: EXISTING mostlyright-satellite project). + # All push to europe-west3-docker.pkg.dev/mostlyright-backend/mostlyright. + # --------------------------------------------------------------------- + - name: Build & deploy (placeholder) + run: | + echo "Deploy target: ${{ inputs.target }}" + echo "Image build + push + gcloud run deploy stubbed — filled in by W1/W2." diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2f5d423..8dbe99cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,6 +91,10 @@ jobs: uv run ruff check . uv run ruff format --check . + - name: Hosted-call grep-gate (D-28.2 — default path stays hosted-call-free) + if: needs.changes.outputs.py == 'true' + run: uv run python scripts/check_no_hosted_calls.py + - name: Run fast test suite (excludes @pytest.mark.live and @pytest.mark.polars) if: needs.changes.outputs.py == 'true' run: uv run pytest -m "not live and not polars" -q diff --git a/.gitignore b/.gitignore index b1635c68..46e91774 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,16 @@ docs/sphinx/api/ # JSDoc/TSDoc on `packages-ts/*/src/`, plus the hand-written # `packages-ts/typedoc.json` config. docs-ts-build/ + +# Phase 28 — Terraform/OpenTofu root (infra/). Secret/semi-sensitive values +# (billing account ID) live in a GITIGNORED terraform.tfvars; only +# terraform.tfvars.example (placeholders) is committed. State + provider +# binaries + plan outputs are never committed. +infra/terraform.tfvars +infra/*.auto.tfvars +**/.terraform/ +*.tfstate +*.tfstate.* +*.tfplan +crash.log +crash.*.log diff --git a/CLAUDE.md b/CLAUDE.md index 0d8a2528..ecc34ef8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,7 +38,7 @@ uv build # build all three packages - **Never commit directly to main.** Always branch + PR. - **TDD mandatory.** Write tests first. RED → GREEN → REFACTOR. 80% coverage minimum. - **Pre-commit + pre-push hooks mandatory.** No `--no-verify`. Fix the underlying issue. Pre-commit runs fast checks (ruff, format, whitespace, YAML/TOML validation); pre-push runs `pytest -m "not live"`. Install both with `uv run pre-commit install && uv run pre-commit install --hook-type pre-push`. -- **All API calls direct from SDK.** No `api.mostlyright.md`, no hosted-API client calls anywhere in `mostlyright.*`. Verified via grep on built wheels before publish. +- **Default path calls public APIs direct from SDK; hosted is opt-in (Phase 28 carve-out, GATE #1 signed 2026-07-02).** The SDK **DEFAULT** path (`research()`, local `live`) makes **NO** hosted call and hits public APIs (AWC, IEM, GHCNh, NWS CLI, Kalshi) directly. The wheel grep-gate still runs `grep` on built wheels before publish to enforce that the **default/published-dist** path stays hosted-call-free — the gate is **amended (narrowed to the default path), not removed**. Hosted is reached **only via the opt-in seams** `delivery="hosted"` / `EARNINGS_HOSTED_URL` / `WEATHER_HOSTED_URL` + `MOSTLYRIGHT_API_KEY`; hosted rows are byte-identical to the local `live` path. The `services/` deploy code (uvicorn/ffmpeg/Chromium/whisper deploy deps) is **NON-published** and MUST NOT enter any PyPI dist. See [`.planning/phases/28-hosted-gce-data-platform/28-01-GATE-RECORD.md`](.planning/phases/28-hosted-gce-data-platform/28-01-GATE-RECORD.md). ## Dual-SDK Planning Rule @@ -109,7 +109,7 @@ A local-first Python SDK for quants researching prediction-market weather contra ### Constraints -- **Tech stack:** Python 3.11+. uv workspace. `httpx`, `pandas`, `pyarrow`, `filelock`, `jsonschema`, `hypothesis` (dev). No FastAPI, no Docker, no hosted infra in v0.1. +- **Tech stack:** Python 3.11+. uv workspace. `httpx`, `pandas`, `pyarrow`, `filelock`, `jsonschema`, `hypothesis` (dev). No FastAPI, no Docker, no hosted infra in the **published SDK v0.1 default path**. **Phase 28 opt-in-hosted carve-out (GATE #1, 2026-07-02):** an opt-in hosted client + a served hosted API are now permitted, but only reached via `delivery="hosted"` / `EARNINGS_HOSTED_URL` / `WEATHER_HOSTED_URL` + `MOSTLYRIGHT_API_KEY`; the `services/` serving app (FastAPI/uvicorn + ffmpeg/Chromium/whisper deploy deps) is NON-published and MUST NOT enter any PyPI dist. - **Timeline:** 14 calendar days from Day 1. Phase A (parity lift) Days 1-4, Phase B (core+catalog) Days 5-14. v0.2 (MCP) is a later milestone. - **Execution model:** Two-lane parallel — Lane V (Vu) lifts from `monorepo-v0.14.1/`, Lane F (Founder) builds new code. Cross-review mandatory. Every PR runs the two-reviewer loop (Codex `high` + Python Architect) per [`.planning/REVIEW-DISCIPLINE.md`](.planning/REVIEW-DISCIPLINE.md) — applies to ALL branches, not just parity-critical paths. - **Testing discipline:** TDD mandatory (RED → GREEN → REFACTOR). Pre-commit hooks; no `--no-verify`. ≥90% branch coverage on `mostlyright.core`. 80% line coverage on `catalog/` and adapter wrappers. Lifted `_vendor/` code retains its monorepo coverage. @@ -259,7 +259,7 @@ A local-first Python SDK for quants researching prediction-market weather contra | `filelock` for cache | ✓ Confirmed. Battle-tested; 3.29 brings Windows improvements (cheap floor bump). | | `jsonschema` for validation | ✓ Confirmed for v0.1. Reconsider Pydantic for v0.2 MCP work. | | `hypothesis` for property tests | ✓ Confirmed. Only mainstream choice in Python. | -| No FastAPI, no Docker | ✓ Confirmed. Local-first SDK; no servers. | +| No FastAPI, no Docker | ✓ Confirmed for the published SDK default path. **Amended Phase 28 (GATE #1, 2026-07-02):** an opt-in hosted serving API (`services/`, FastAPI) is allowed off the published dist; the SDK default stays local-first / hosted-call-free. | | MCP deferred to v0.2 | ✓ Confirmed. The `mcp` SDK at 1.27.1 is mature enough for v0.2; deferring avoids the FastMCP + Pydantic dep proliferation in v0.1. | ## Decisions to Consider Revisiting (Soft Flags) ## Sources diff --git a/deploy/weather/serving.Dockerfile b/deploy/weather/serving.Dockerfile new file mode 100644 index 00000000..98627f07 --- /dev/null +++ b/deploy/weather/serving.Dockerfile @@ -0,0 +1,58 @@ +# Weather serving image — the hosted /satellite + /capabilities REST app (28-30). +# +# A SLIM serving image: it packages only `services/weather/` + the two SDK +# packages it reads from (`mostlyrightmd`, `mostlyrightmd-weather` with the +# `[satellite]` extra for boto3 + parquet), and runs uvicorn. NO audio toolchain, +# NO backfill/compute toolchain (xarray/h5netcdf/s3fs full stack is not needed to +# READ derived parquet — boto3 + pyarrow are, and they come with `[satellite]`). +# +# Read-only by construction: the container reads R2 with the READ-ONLY token +# (r2-read-* + MOSTLYRIGHT_API_KEY injected from Secret Manager into the serving +# SA env by the deploy layer). It never holds the write token / any ingest secret. +# +# Pinned digests are NOT used here (matching the repo's other stubs); the base is +# the slim CPython image the SDK floors target (py3.11+). Cloud Run injects PORT. + +FROM python:3.12-slim AS base + +# Non-interactive, no .pyc, unbuffered logs (Cloud Run friendly). +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +# --- Dependency layer -------------------------------------------------------- +# Copy just the package sources needed to install the SDK distributions the +# serving app imports (core + weather[satellite]) plus the serving runtime deps. +# Copying pyproject/src first keeps the dep layer cached across app-code edits. +COPY packages/core/ packages/core/ +COPY packages/weather/ packages/weather/ + +# Install the two published distributions (editable-free, from the local tree) +# with the [satellite] extra (boto3 + pyarrow: the R2 read + parquet parse), and +# the serving runtime (FastAPI + uvicorn). FastAPI is a WORKSPACE dev/test dep +# for the non-published service, so it is named explicitly here. +RUN pip install \ + ./packages/core \ + "./packages/weather[satellite]" \ + "fastapi>=0.115,<1" \ + "uvicorn[standard]>=0.30" + +# --- App layer --------------------------------------------------------------- +# The non-published serving app is imported as `services.weather.*` (matching the +# repo-root conftest sys.path convention), so it is copied under /app/services. +COPY services/weather/ services/weather/ + +# Cloud Run sets $PORT (default 8080). The lazy `services.weather.app:app` factory +# resolves MOSTLYRIGHT_API_KEY from the env and FAILS CLOSED if it is unset (the +# public feed never serves keyless), so a misconfigured deploy crashes loud at +# startup rather than serving unauthenticated. +ENV PORT=8080 +EXPOSE 8080 + +# One worker: the in-process per-key + global-ceiling limiters are per-process +# (the Redis seam is deferred), and Cloud Run scales by adding instances under +# the max-instances cap (the infrastructure-layer global ceiling, weather_serving.tf). +CMD ["sh", "-c", "uvicorn services.weather.app:app --host 0.0.0.0 --port ${PORT} --workers 1"] diff --git a/docs/deploy-runbook.md b/docs/deploy-runbook.md new file mode 100644 index 00000000..5c0f38a5 --- /dev/null +++ b/docs/deploy-runbook.md @@ -0,0 +1,161 @@ +# Deploy runbook — hosted GCE data platform (Phase 28) + +Operator-side summary of **how the hosted platform ships**: the wave order, the +hard gates a human must clear, and the environment realities you will hit. This +is the narrative companion to the Terraform root in +[`../infra/README.md`](../infra/README.md) and the user-facing surface in +[`hosted-api.md`](hosted-api.md). + +> **Read `../infra/README.md` first for the bootstrap + `tofu apply` steps.** +> This runbook is the *order* and the *gates*, not the command reference. + +## Projects (reconciled reality) + +Five projects share billing account `Mostly Right Main` +(`011A98-02C05B-2E637A`). Phase 28 **reuses** the first three and **creates** the +serving + ingest projects; staging is quota-blocked. + +| Project | Role | Status | +|---|---|---| +| `mostlyright-backend` (661421560872) | Private ops / secrets home (Secret Manager + Artifact Registry `europe-west3-docker.pkg.dev/mostlyright-backend/mostlyright`). Serves no client data. | EXISTS — never modified | +| `mostlyright-satellite` (38183953819) | ALL weather compute — backfill fleet + daily incremental (`us-central1`). Gets a deploy SA added. | EXISTS — reused (H1) | +| `mr-earnings-ingest` (899892194978) | Audio island — scheduler, capture, STT-GPU, role/fact. Audio-only; no public ingress. | NEW | +| `mr-serving` (417910866339) | Internet-facing serving — earnings REST + SSE, weather REST. Read-only from R2. | NEW | +| `mr-staging` | One shared staging. | **Quota-blocked** — see below | + +**Weather compute reuses `mostlyright-satellite` (H1), not `mr-earnings-ingest`** +— this keeps the earnings-audio island audio-only (no NODD/EUMETSAT/anonymous-S3 +IAM leaks into it). + +## The 5-project billing cap (staging is gated off) + +The billing account hit its **default 5-project link cap** after linking ingest + +serving (5 linked: `mostlyright-backend`, `mostlyright-satellite`, +`steel-utility-495707-v9`, `mr-earnings-ingest`, `mr-serving`). `mr-staging` +could not be billing-linked — its `google_project` billing link returned a Cloud +Billing `QuotaFailure`. It is therefore **gated OFF** behind +`enable_staging = false`; ingest + serving are fully provisioned without it. + +**To finish staging (operator action):** + +1. Request a Cloud Billing project-quota increase for `Mostly Right Main`: + +2. Once granted, set `enable_staging = true` in `terraform.tfvars`. +3. `tofu -chdir=infra apply` — creates the staging project + its APIs, deploy SA, + WIF binding, and Artifact Registry reader binding. + +## Keyless CI (WIF) + +CI deploys via **Workload Identity Federation** — no SA key files, ever. The pool ++ provider are homed in `mr-serving`: + +- Pool / provider: `projects/417910866339/locations/global/workloadIdentityPools/github-actions` + → `.../providers/github-oidc`, repo-restricted (`assertion.repository == + "mostlyrightmd/mostlyright-sdk"`). +- Per-project deploy SAs: `deploy@mr-serving`, `deploy@mr-earnings-ingest`, and + (added this phase) `deploy@mostlyright-satellite` for weather. +- `deploy.yml` reads the repo vars `WIF_PROVIDER`, `DEPLOY_SA_SERVING`, + `DEPLOY_SA_INGEST` (already set). + +## Secrets (bindings only — no secret values are created) + +All secret **resources** already live in Secret Manager (`mostlyright-backend`): +`r2-account-id`, `r2-write-access-key-id`, `r2-write-secret-access-key`, +`r2-read-access-key-id`, `r2-read-secret-access-key`, `mostlyright-api-key`, +`eumetsat-consumer-key`, `eumetsat-consumer-secret`. Phase 28 adds **per-SA +`secretAccessor` bindings** only — an IAM-enforced R2 firewall: + +- **serving SA** → `r2-read-*` only (never write). +- **ingest SA + satellite SA** → `r2-write-*`. +- **eumetsat-\*** → satellite SA only (the keyed Meteosat backfill path, D-28.9). +- **mostlyright-api-key** → serving + ingest. + +## Wave order + +The build order (mirrors `28-GCE-ARCHITECTURE.md` §8 / `28-CONTEXT.md` §6). Each +wave depends on the prior; the two operator gates block everything downstream of +them. + +| Wave | What ships | Gate | +|---|---|---| +| **W1** | GATE #1 constraint amendment (28-01) — bless the opt-in hosted departure from local-first. | **Operator gate #1** (below) | +| **W2** | Terraform root + 3 projects (reference existing WIF/state, 28-00); earnings build-gate (28-04). | | +| **W3** | Secret bindings + per-project budgets + Pub/Sub transport SAs (28-02). | **Budget-alert test notification** (below) | +| **W4** | Earnings capture + audio handoff (28-10); weather backfill CLI extension (28-20). | | +| **W5** | STT GPU **europe-west1** bounded (28-11); backfill fleet run (28-21); incremental + monitoring (28-22). | **GPU quota** + **backfill pilot sign-off** (below) | +| **W6** | Role/fact + Pub/Sub publisher (28-13); weather serving (28-30). | | +| **W7** | Earnings serving + SSE (28-12); fill `delivery="hosted"` seam (28-31). | **Operator gate #2** (below) | +| **W8** | TS hosted-fetch shim + MV3 extension wiring (28-40). | | +| **W9** | Docs + staleness fixes (28-41). | | + +## Operator gates + +These are the human sign-offs the plan marks as **blocking**. Nothing downstream +runs until each is cleared. + +### Gate #1 — Local-first departure sign-off (blocks W1, so blocks everything) + +The phase intentionally departs from the "no hosted backend" stance. The operator +must sign off that: + +- weather gets the same served-`hosted` tier earnings already blessed (D-27.6), and +- the amended grep-gate keeps the **default path hosted-call-free** (hosted is + opt-in only). + +Without this, nothing ships. + +### Budget-alert test notification (blocks first spend, W3) + +Per-project `google_billing_budget` alerts (50/90/100% USD → email +`vu@mostlyright.md` + Pub/Sub) must fire a **verified test notification** before +any project incurs its first spend: + +| Project | Budget | +|---|---| +| `mr-earnings-ingest` | $40 | +| `mr-serving` | $25 | +| `mostlyright-satellite` | $150 | + +### GPU quota (blocks STT, W5) + +STT runs on **Cloud Run GPU (NVIDIA L4) in `europe-west1`** — Cloud Run L4 GPU is +**not offered in `europe-west3`**, so the serving region and the GPU region +differ. The new-project L4 default quota in `europe-west1` is ~3. Before STT +deploys, either: + +- confirm the L4 quota and **bound STT concurrency ≤ the confirmed quota** (H8), or +- file a quota bump, or +- accept the **GCE L4 MIG min=0** fallback (Pub/Sub-depth autoscaled) + its + cold-start. + +### Backfill pilot cost sign-off (blocks the full fleet run, W5) + +The one-time 28-TB reduction is scoped to the **market-driven roster** (the +Kalshi ∪ Polymarket `StationCatalog` stations, ~66 US+intl — D-28.8), sharded +across array tasks with **durable GCS progress markers**. A slice is marked +complete **only after its derived parquet is uploaded to R2** (crash-safe Spot +resume, C4). Run a **pilot slice** and get a **cost sign-off** before submitting +the full backfill (H5). + +### Gate #2 — Public hosted exposure (W7) + +Deploying earnings serving + SSE puts a public, internet-facing endpoint live. +The operator signs off on the public exposure + legal posture before 28-12 ships, +and verifies `EARNINGS_HOSTED_URL` returns byte-identical rows. + +## Post-deploy verification + +- **Byte-identical contract:** hosted `/satellite` and `/transcripts` rows must + reconcile with the local `live` path (same schema, `delivery`/`source` carry + the channel) — never error on a channel mix. +- **SSE 3600s canary:** confirm `/stream` survives past the Cloud Run 60-min + timeout via `Last-Event-ID` reconnect + ring-buffer replay (no events lost). +- **Firewalls hold:** audio never reaches R2 or serving; raw 28 TB never leaves + the US. See [`hosted-api.md`](hosted-api.md#the-two-firewalls). +- **Monitoring:** failed-execution + data-freshness + `/capabilities` uptime + alerts route to the budget notification channel. + +## See also + +- [`../infra/README.md`](../infra/README.md) — Terraform/OpenTofu bootstrap + apply. +- [`hosted-api.md`](hosted-api.md) — the user-facing hosted surface + opt-in seams. diff --git a/docs/earnings-synced-deeplink-design.md b/docs/earnings-synced-deeplink-design.md new file mode 100644 index 00000000..63a6f74a --- /dev/null +++ b/docs/earnings-synced-deeplink-design.md @@ -0,0 +1,266 @@ +# Synced Deep-Link Earnings Experience — Design Doc + +**Vertical:** mostlyright earnings (v1.11.0 shipped) +**Author:** Lead product+eng designer (synthesis) +**Legal posture:** D-27.9 — text/derived-facts only; NEVER touch/store/serve/proxy audio-video; DEEP-LINK to issuer's own player. +**Date:** 2026-07-02 (rev. 2 — feasibility-corrected) + +--- + +## 0. What is and isn't proven (read this first) + +**Honest one-liner:** *Two APIs return the types we need. Whether we can reach them on a real issuer page, cross-origin, with the data actually populated, is entirely unproven — and that is the point of the spike.* + +What we have actually verified is narrow: that the IVS Player SDK **exposes** `getSyncTime()` → UTC ms @1s granularity, and that hls.js **exposes** `playingDate` → `Date` from `EXT-X-PROGRAM-DATE-TIME`. That is verification that *two APIs exist and return the documented types*. It is **not** verification that the system works. Every hard part of this design lives in the gap between "the API exists" and "our content script can reach it, on an actual Q4 issuer page, cross-origin, with PDT actually present and wallclock-accurate." + +Three of the four things an earlier draft called "verified" are unproven or structurally inapplicable to the **client**: + +| Claim | Real status | +|---|---| +| IVS `getSyncTime()` returns UTC ms | ✅ API exists. ❌ **Reachability on a real Q4 page is UNPROVEN.** The player instance is almost certainly closure-scoped inside Q4's minified bundle, not a reachable global. If unreachable, we fall back to `