feat: add debugging to feed download process

api/src/feeds/impl/feeds_api_impl.py

@@ -37,6 +37,7 @@
     add_official_filter,
     get_gbfs_feeds_query,
 )
+from shared.common.availability_check_source import feed_availability_checks_filter
 from shared.common.seal_criteria import SealCriterionName
 from shared.common.error_handling import (
     availability_from_after_to,
@@ -367,7 +368,11 @@ def get_gtfs_feed_availability(
         if not feed:
             raise_http_error(404, gtfs_feed_not_found.format(id))
 
-        query = db_session.query(GtfsFeedAvailabilityCheck).filter(GtfsFeedAvailabilityCheck.feed_id == feed.id)
+        # Only the daily check answers "was the feed available"; download diagnostics recorded in
+        # the same table are for the operations API.
+        query = db_session.query(GtfsFeedAvailabilityCheck).filter(
+            GtfsFeedAvailabilityCheck.feed_id == feed.id, feed_availability_checks_filter()
+        )
         if from_dt:
             query = query.filter(GtfsFeedAvailabilityCheck.checked_at >= from_dt)
         if to_dt:

api/src/scripts/populate_db_test_data.py

@@ -332,6 +332,9 @@ def populate_test_datasets(self, filepath, db_session: "Session"):
                     latency_ms=check.get("latency_ms"),
                     error_type=check.get("error_type"),
                 )
+                # Left unset when absent so the column default applies; source is NOT NULL.
+                if "source" in check:
+                    availability_check.source = check["source"]
                 db_session.add(availability_check)
 
         db_session.commit()

api/src/shared/common/availability_check_source.py

@@ -0,0 +1,41 @@
+"""Who wrote a `gtfs_feed_availability_check` row, and which rows count as feed availability.
+
+The table holds more than the daily availability check: failed nightly dataset downloads and
+the download runs asked for by hand through the operations API are recorded there too, with
+their request and response detail, so they can be debugged side by side.
+
+Only the daily check answers "was the feed available". Every public reader of the table -
+the `/v1/gtfs_feeds/{id}/availability` endpoint and the seal's `available` criterion, both in
+the nightly run and in the backfill - filters through `feed_availability_checks_filter()`, so
+changing which sources count is a single edit to `FEED_AVAILABILITY_SOURCES`. The operations
+API is the one reader that sees every source.
+
+This module is under `shared/common` so the API and every function that writes or reads the
+table (`tasks_executor`, `batch_process_dataset`, `operations_api`) share one definition.
+"""
+
+from enum import Enum
+from typing import Final, Tuple
+
+from shared.database_gen.sqlacodegen_models import GtfsFeedAvailabilityCheck
+
+
+class AvailabilityCheckSource(str, Enum):
+    """Values stored in `gtfs_feed_availability_check.source`."""
+
+    # The daily lightweight check run by tasks_executor.
+    AVAILABILITY_CHECK = "availability_check"
+    # A nightly dataset download that failed, recorded by batch_process_dataset.
+    DATASET_DOWNLOAD = "dataset_download"
+    # A run on the nightly job asked for through the operations API, download-only or full,
+    # success included.
+    DATASET_DOWNLOAD_MANUAL = "dataset_download_manual"
+
+
+# The sources whose rows answer "was the feed available".
+FEED_AVAILABILITY_SOURCES: Final[Tuple[AvailabilityCheckSource, ...]] = (AvailabilityCheckSource.AVAILABILITY_CHECK,)
+
+
+def feed_availability_checks_filter():
+    """SQLAlchemy clause keeping only the checks that count as feed availability."""
+    return GtfsFeedAvailabilityCheck.source.in_([source.value for source in FEED_AVAILABILITY_SOURCES])

api/src/shared/db_models/gtfs_feed_availability_check_impl.py

@@ -27,4 +27,13 @@ def from_orm(cls, check: GtfsFeedAvailabilityCheckOrm | None) -> GtfsFeedAvailab
             status_code=check.status_code,
             latency_ms=float(check.latency_ms) if check.latency_ms is not None else None,
             error_type=check.error_type,
+            source=check.source,
+            error_message=check.error_message,
+            content_type=check.content_type,
+            is_zip=check.is_zip,
+            resolved_url=check.resolved_url,
+            request_headers=check.request_headers,
+            response_headers=check.response_headers,
+            redirect_urls=check.redirect_urls,
+            external_ip=check.external_ip,
         )

api/tests/integration/test_data/extra_test_data.json

@@ -280,6 +280,17 @@
       "status_code": null,
       "latency_ms": null,
       "error_type": "connection_timeout"
+    },
+    {
+      "feed_stable_id": "mdb-1",
+      "checked_at": "2025-05-20T10:00:00+00:00",
+      "request_url": "https://example.com/gtfs.zip",
+      "request_type": "http_get",
+      "success": false,
+      "status_code": 403,
+      "latency_ms": 640,
+      "error_type": "http_error",
+      "source": "dataset_download"
     }
   ]
 }

api/tests/integration/test_feeds_api.py

@@ -989,7 +989,10 @@ def test_feeds_have_expected_license_info(
 # ---- GTFS Feed Availability endpoint tests ----
 
 AVAILABILITY_FEED_ID = TEST_GTFS_FEED_STABLE_IDS[0]  # mdb-1 — has availability_checks fixture data
-AVAILABILITY_CHECKS_COUNT = 5  # number of checks added in extra_test_data.json for mdb-1
+# Daily availability checks added in extra_test_data.json for mdb-1. The fixture also has a
+# failed dataset download for mdb-1, which the public endpoint must not return.
+AVAILABILITY_CHECKS_COUNT = 5
+DATASET_DOWNLOAD_CHECKED_AT = "2025-05-20T10:00:00"
 
 
 def test_gtfs_feed_availability_basic(client: TestClient):
@@ -1142,6 +1145,20 @@ def test_gtfs_feed_availability_pagination_offset(client: TestClient):
     assert first_timestamps.isdisjoint(second_timestamps)
 
 
+def test_gtfs_feed_availability_excludes_download_diagnostics(client: TestClient):
+    """A failed dataset download recorded in the same table is not an availability check."""
+    response = client.request(
+        "GET",
+        f"/v1/gtfs_feeds/{AVAILABILITY_FEED_ID}/availability",
+        headers=authHeaders,
+        params={"from": "2025-05-01T00:00:00Z"},
+    )
+    assert response.status_code == 200
+    body = response.json()
+    assert body["total"] == 1
+    assert all(not c["checked_at"].startswith(DATASET_DOWNLOAD_CHECKED_AT) for c in body["checks"])
+
+
 def test_gtfs_feed_availability_not_found(client: TestClient):
     """Unknown feed ID returns 404."""
     response = client.request(

api/tests/unittest/models/test_availability_check_source_contract.py

@@ -0,0 +1,53 @@
+"""Contract tests pinning `gtfs_feed_availability_check.source` values. No database connection.
+
+`shared.common.availability_check_source` is the single definition of the sources the
+functions write and of the ones that count as feed availability. The public availability
+endpoint and the Seal of Reliability both filter through it; the operations API exposes every
+source as an enum on its `source` filter.
+
+Adding a source means updating, in lockstep: `AvailabilityCheckSource` and the
+`availability_source` parameter in `docs/OperationsAPI.yaml` plus a stub regen.
+"""
+
+import unittest
+from pathlib import Path
+
+import yaml
+
+from shared.common.availability_check_source import (
+    FEED_AVAILABILITY_SOURCES,
+    AvailabilityCheckSource,
+    feed_availability_checks_filter,
+)
+from shared.database_gen.sqlacodegen_models import GtfsFeedAvailabilityCheck
+
+OPERATIONS_SPEC = Path(__file__).resolve().parents[4] / "docs" / "OperationsAPI.yaml"
+
+
+class TestAvailabilityCheckSourceContract(unittest.TestCase):
+    def test_only_the_daily_check_counts_as_feed_availability(self):
+        """Changing this changes the public endpoint and the seal; it should be deliberate."""
+        self.assertEqual(FEED_AVAILABILITY_SOURCES, (AvailabilityCheckSource.AVAILABILITY_CHECK,))
+
+    def test_filter_selects_the_feed_availability_sources(self):
+        clause = feed_availability_checks_filter()
+        compiled = clause.compile(compile_kwargs={"literal_binds": True})
+        self.assertEqual(clause.left.table.name, GtfsFeedAvailabilityCheck.__tablename__)
+        self.assertEqual(clause.left.name, "source")
+        self.assertIn("'availability_check'", str(compiled))
+        for source in set(AvailabilityCheckSource) - set(FEED_AVAILABILITY_SOURCES):
+            self.assertNotIn(f"'{source.value}'", str(compiled))
+
+    def test_operations_spec_lists_every_source(self):
+        spec = yaml.safe_load(OPERATIONS_SPEC.read_text())
+        parameter = spec["components"]["parameters"]["availability_source"]
+        self.assertEqual(
+            set(parameter["schema"]["items"]["enum"]),
+            {source.value for source in AvailabilityCheckSource},
+        )
+
+    def test_db_default_is_the_daily_check(self):
+        """A row written without a source must still count as an availability check."""
+        column = GtfsFeedAvailabilityCheck.__table__.c.source
+        self.assertFalse(column.nullable)
+        self.assertIn(AvailabilityCheckSource.AVAILABILITY_CHECK.value, str(column.server_default.arg))