From d2dd7c3edec26e93a61b23fb86dccf122ac33440 Mon Sep 17 00:00:00 2001 From: helloiamvu Date: Sat, 4 Jul 2026 01:38:55 +0200 Subject: [PATCH] =?UTF-8?q?fix(28):=20unbreak=20fast-suite=20=E2=80=94=20e?= =?UTF-8?q?arnings=20tests=20need=20google-cloud=20dev=20deps;=20guard=20R?= =?UTF-8?q?2/satellite=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #96 added services/earnings/jobs tests that need the Cloud Run deploy deps, but the CI fast-suite runs `uv sync --all-packages` (no optional extras), so they errored (ModuleNotFoundError: google / boto3) and reddened main. Root cause: those deploy deps aren't installed in the base workspace. Fixes (verified against the exact CI env — boto3 pruned, google.cloud present: 3925 passed, 466 skipped, 0 failed): - Add google-cloud-pubsub + google-cloud-storage to the workspace dev group (same pattern as fastapi in Phase 27) so the capture/STT/handoff job tests import the `google.cloud` namespace (clients are still faked). - Do NOT add boto3: it is the [satellite]-extra marker several satellite tests key their skip-guard on (test_satellite_leakage), so adding it would make those guards falsely fire and crash on the still-absent s3fs/xarray. - Guard the serving R2-read tests (test_serving_r2_read) on boto3 — they exercise the S3-compat botocore error paths, so they run in the satellite/deploy lanes. - Guard test_roster's station-resolution test (imports satellite/_backfill → _goes_s3 → boto3/s3fs/xarray) on the [satellite] extra, like test_cli_roster. - Fold in the batch.tf GOES-footprint note that missed the merge commit (staging race). Recovery from an admin-merge of #96 whose CI hadn't finished (my error) — this time verified green in a CI-mimicking venv before pushing. --- infra/batch.tf | 7 +++ .../weather/tests/satellite/test_roster.py | 19 ++++++ pyproject.toml | 14 +++++ .../earnings/tests/test_serving_r2_read.py | 16 +++++ uv.lock | 63 +++++++++++++++++++ 5 files changed, 119 insertions(+) diff --git a/infra/batch.tf b/infra/batch.tf index e2681b4..0c67647 100644 --- a/infra/batch.tf +++ b/infra/batch.tf @@ -89,6 +89,13 @@ resource "google_storage_bucket_iam_member" "backfill_progress_rw" { # # resolvable station (D-28.8). run-weather-backfill.yml # # sets the SAME TASK_COUNT=65 — keep them in LOCKSTEP # # with the roster. +# # NOTE: the container passes no --satellites, so each shard +# # runs under the DEFAULT GOES-East/West satellites (Americas +# # / E-Pacific footprint). Shards for stations outside that +# # footprint (EU/Asia/Africa) cleanly NO-OP with a logged +# # exclusion (the CLI filters them, not fetch-empty-then-mark- +# # complete). Global native-ring coverage (Himawari/Meteosat/ +# # VIIRS) is the 28-26 follow-up — add --satellites to cover it. # parallelism = 16 # bounded concurrent Spot slices # task_spec: # max_run_duration = 21600s # 6h per-shard ceiling, caps a runaway (T-28.21-02) diff --git a/packages/weather/tests/satellite/test_roster.py b/packages/weather/tests/satellite/test_roster.py index cca45c8..0891060 100644 --- a/packages/weather/tests/satellite/test_roster.py +++ b/packages/weather/tests/satellite/test_roster.py @@ -18,6 +18,20 @@ shard_roster, ) +# The station-resolution test imports satellite/_backfill, whose transport +# (_fetchers/_goes_s3) imports boto3 + s3fs + xarray at module scope — i.e. it needs +# the [satellite] optional extra. The base fast-suite (`uv sync --all-packages`, +# no extras) lacks them, so skip that one test there; the dedicated +# satellite-coverage job installs the extra and runs it. Mirrors test_cli_roster. +try: + import boto3 # noqa: F401 + import s3fs # noqa: F401 + import xarray # noqa: F401 + + _HAVE_SATELLITE_DEPS = True +except ImportError: # pragma: no cover - exercised only without the extra + _HAVE_SATELLITE_DEPS = False + def _live_kalshi_stations() -> set[str]: from mostlyright.markets.catalog.kalshi_stations import KALSHI_SETTLEMENT_STATIONS @@ -57,6 +71,11 @@ def test_non_satellite_stations_excluded_but_in_live_catalog() -> None: assert station not in SETTLEMENT_STATION_ROSTER +@pytest.mark.skipif( + not _HAVE_SATELLITE_DEPS, + reason="station resolution imports satellite/_backfill → needs the [satellite] extra " + "(boto3/s3fs/xarray); runs in the satellite-coverage CI job", +) def test_every_roster_station_resolves_to_a_satellite_station() -> None: """No silent empty shards: every roster station resolves to a StationInfo.""" from mostlyright.weather.satellite._backfill import _resolve_station_infos diff --git a/pyproject.toml b/pyproject.toml index b1412c6..6c6d97d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,20 @@ dev = [ # dependency of any of the three published distributions. `httpx` (already a # workspace dep) backs Starlette's TestClient. "fastapi>=0.115,<1", + # Phase 28 (28-13): the `services/earnings/jobs/` Cloud Run entrypoints are + # NON-published monorepo services, but their tests run in the fast-suite + # (`uv sync --all-packages` installs no optional extras), so their deploy deps + # must be dev/test deps of the WORKSPACE (same pattern as fastapi above): + # google-cloud-pubsub / google-cloud-storage back the capture + STT job tests + # (the `google.cloud` namespace must import even though the clients are faked). + # NOTE: boto3 is deliberately NOT added here — it is the [satellite]-extra + # MARKER several satellite tests key their skip-guard on (e.g. + # test_satellite_leakage), so adding it would make those guards falsely fire and + # then crash on the still-absent s3fs/xarray. Tests that genuinely need boto3 + # (the R2 read/sink path) guard on it themselves and run in the satellite/deploy + # lanes. NONE enter a PyPI dist (dev group only). + "google-cloud-pubsub>=2.18,<3", + "google-cloud-storage>=2.10,<4", ] # Phase 15 W1: documentation toolchain (Sphinx + autodoc + Markdown builder). diff --git a/services/earnings/tests/test_serving_r2_read.py b/services/earnings/tests/test_serving_r2_read.py index c66faa1..10ff444 100644 --- a/services/earnings/tests/test_serving_r2_read.py +++ b/services/earnings/tests/test_serving_r2_read.py @@ -21,6 +21,22 @@ from services.earnings.deps import ServingState from services.earnings.r2_read import EarningsR2Reader, R2LedgerSource +# The R2 read path is an S3-compat (boto3/botocore) client, and these tests raise +# botocore ClientErrors to exercise the NoSuchKey-vs-real-error split. boto3 is the +# [satellite]-extra marker (deliberately NOT a base fast-suite dep — see the root +# pyproject dev group), so skip this module when it is absent; it runs in the +# satellite/deploy lanes where the extra is installed. Mirrors test_cli_roster. +try: + import boto3 # noqa: F401 + + _HAVE_BOTO3 = True +except ImportError: # pragma: no cover - exercised only without the extra + _HAVE_BOTO3 = False + +pytestmark = pytest.mark.skipif( + not _HAVE_BOTO3, reason="serving R2 read tests need boto3/botocore ([satellite]-extra marker)" +) + def _parquet_bytes(rows: list[dict]) -> bytes: buf = io.BytesIO() diff --git a/uv.lock b/uv.lock index a68fa01..10768b2 100644 --- a/uv.lock +++ b/uv.lock @@ -1068,6 +1068,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/4a/98da8930ab109c73d9a5d13782a9ebb81ea8c111f6d534a567b71d23e52b/google_cloud_core-2.6.0-py3-none-any.whl", hash = "sha256:6d63ac8e5eca6d9e4319d0a1e2265fadcd7f1049904378caecfa01cf52dd869e", size = 29390, upload-time = "2026-05-07T08:02:34.672Z" }, ] +[[package]] +name = "google-cloud-pubsub" +version = "2.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "grpc-google-iam-v1" }, + { name = "grpcio" }, + { name = "grpcio-status" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/2b/4bf2c17e319ff65340389565b0e1b4d72696d87802b2f5f94390fbefa73c/google_cloud_pubsub-2.39.0.tar.gz", hash = "sha256:eed65e25f57f95bf3e02d96d7ee171688b23922471f9f21b5a91ed90e1282c0f", size = 402096, upload-time = "2026-06-03T15:28:26.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/20/dd0b27d4ad4577c062e77ff968ca3e2d404186cd78c8a2a53a0ef5fe5389/google_cloud_pubsub-2.39.0-py3-none-any.whl", hash = "sha256:7210d691a46d7a66559696899ebe6eb731e63de29b624964b3be4dd2d12d3e19", size = 324665, upload-time = "2026-06-03T15:27:41.119Z" }, +] + [[package]] name = "google-cloud-storage" version = "3.12.0" @@ -1799,6 +1819,8 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "fastapi" }, + { name = "google-cloud-pubsub" }, + { name = "google-cloud-storage" }, { name = "hypothesis" }, { name = "narwhals" }, { name = "packaging" }, @@ -1829,6 +1851,8 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "fastapi", specifier = ">=0.115,<1" }, + { name = "google-cloud-pubsub", specifier = ">=2.18,<3" }, + { name = "google-cloud-storage", specifier = ">=2.10,<4" }, { name = "hypothesis", specifier = ">=6.100" }, { name = "narwhals", specifier = ">=1.20,<2.0" }, { name = "packaging", specifier = ">=23" }, @@ -2126,6 +2150,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/f6/2bac21f722aa45d876d4a51f26bd0ef30e704068a3cd5021a5a7cd784271/onnxruntime-1.27.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:370d211e1ceeac4cd5f45301655463ac59e27cdc74d9f7aeb2d19ff4b7a76715", size = 18670781, upload-time = "2026-06-15T22:43:17.151Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/cc/e4c9584181f86494df0f6bdec1a4f3280c50db44704dc2a407e994fc87bb/opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1", size = 73476, upload-time = "2026-06-24T15:19:55.323Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/eb/5041074274ac0956b03637cc039d434569112468e875eddfcc9a0674ce06/opentelemetry_sdk-1.43.0.tar.gz", hash = "sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9", size = 254744, upload-time = "2026-06-24T15:20:08.467Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/e3/b17be23af124201c9f52eececd4cc8ddfed1597d37b4ee771895d325805c/opentelemetry_sdk-1.43.0-py3-none-any.whl", hash = "sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823", size = 178852, upload-time = "2026-06-24T15:19:52.169Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.64b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/30/5f26df29509eccd86b99b481ac9ffa39da49ba9577cc69071c552ae30447/opentelemetry_semantic_conventions-0.64b0.tar.gz", hash = "sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c", size = 148340, upload-time = "2026-06-24T15:20:09.267Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" }, +] + [[package]] name = "packaging" version = "26.2"