From f6cfe887d1a066c3f5648d75d2cc883777da896f Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 17 Aug 2026 15:22:39 -0500 Subject: [PATCH 1/5] feat(waterdata): expose public STAC API --- NEWS.md | 2 + README.md | 14 ++ dataretrieval/transport/http.py | 12 +- dataretrieval/waterdata/__init__.py | 18 ++ dataretrieval/waterdata/stac.py | 316 ++++++++++++++++++++++++++++ docs/source/reference/waterdata.rst | 6 + tests/contracts/public_api_test.py | 8 + tests/waterdata_stac_test.py | 268 +++++++++++++++++++++++ 8 files changed, 641 insertions(+), 3 deletions(-) create mode 100644 dataretrieval/waterdata/stac.py create mode 100644 tests/waterdata_stac_test.py diff --git a/NEWS.md b/NEWS.md index 137ba5bf2..261ca1db9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**08/17/2026:** Added raw access to the complete public USGS Water Data STAC surface through `waterdata.get_stac_catalog`, `get_stac_conformance`, `get_stac_collections`, `get_stac_collection`, `get_stac_items`, `get_stac_item`, `get_stac_queryables`, and `search_stac`. Search supports both advertised GET and POST representations; collection and item listing helpers expose every advertised filter and pagination parameter. Each helper returns the unchanged STAC/GeoJSON/JSON Schema document plus `BaseMetadata`, preserving standard links instead of flattening heterogeneous documents into a DataFrame. Calls share Water Data API-key host scoping, configured base-URL redirection, typed HTTP errors, and bounded retries. Internal `/_mgmt` health routes are intentionally not exposed as STAC capabilities. + **08/13/2026:** Warning categories now say what they mean. Two advisories about *upstream data* were emitted as `DeprecationWarning` and as an uncategorized `warnings.warn` respectively; both are now `DataCurrencyWarning` (a `UserWarning` subclass, exported as `dataretrieval.DataCurrencyWarning`). **Behavior change:** WQP's legacy-WQX notice moves from `DeprecationWarning` to `DataCurrencyWarning`. Because `legacy=True` is the default on every WQP getter and the notice is unconditional, a downstream project running `-W error::DeprecationWarning` previously could not call any WQP getter with default arguments; it now can. The flip side is visibility — `DeprecationWarning` is silent by default outside `__main__`, so this notice will now print to stderr for library and notebook callers who never saw it. Silence it with `warnings.filterwarnings("ignore", category=dataretrieval.DataCurrencyWarning)`, or set `legacy=False` where a WQX3.0 profile exists. The NWIS qw-endpoint retirement notice gains the same category (it previously had none, arriving as a bare `UserWarning` that deprecation filters ignored). `DeprecationWarning` now means only that a name in this package is going away, always with a replacement and, where published, a removal date; those horizons are declared once in `dataretrieval._deprecation.REMOVALS` rather than spelled at each call site. **08/11/2026:** Settings resolve through a layered chain instead of the environment alone, and a *configuration profile* is a named set of settings for **one adapter**. The new `dataretrieval.configuration` module resolves every setting in one order, highest first: a configuration passed to an active `dataretrieval.configure(...)` block, a profile that block selected, the setting's `API_USGS_*` environment variable, the adapter's `[]` table in `~/.dataretrieval/config.toml` (or `DATARETRIEVAL_CONFIG`), the file's top-level keys, the adapter's own built-in preference, then the package default. Precedence applies **per setting**, so a file that sets only `concurrency` leaves an environment `API_USGS_PAT` in effect, and a `[ngwmn]` table still inherits every top-level key it does not name. `configure()` takes configuration objects positionally, at most one per adapter and nothing else: `configure(Configuration(api_key=vault.read("usgs/pat")), WaterdataConfiguration.load("bulk"), NgwmnConfiguration(concurrency=4))`. The adapter a configuration targets is a property of its class, so a caller never restates it, and each adapter owns its class in the module that *reads* those settings (`waterdata.WaterdataConfiguration`, `ngwmn.NgwmnConfiguration`, `nwdc.NwdcConfiguration`, `wqp.WqpConfiguration`, `nldi.NldiConfiguration`, `streamstats.StreamstatsConfiguration`) — an adapter accepts only the settings it reads, so `[streamstats] parallel_chunks = 8` is an error rather than a line that quietly does nothing. The block is delivered through a `ContextVar`, so a credential set inside it cannot leak across threads or asyncio tasks, which is what makes it safe for a server or notebook handling several users' keys and is the thing assigning to `os.environ` could never do (issue #352). The file gains named profiles beside each adapter's default profile: `[waterdata]` is always in effect, `[waterdata.bulk]` only when a caller selects it with `WaterdataConfiguration.load("bulk")`, and a selected profile still inherits the default profile and the package-wide keys per setting. A profile named in code outranks the setting's environment variable — the one place the ladder inverts the environment-above-file rule, because losing a deliberate selection to a stale shell export is what a caller would file a bug about. An adapter's configuration may also carry a `base_url`, which redirects that adapter's requests for the duration of the block — a staging instance, a mirror, a recording proxy — and no other adapter's; for Water Data one value moves the OGC collections, the Samples database, the statistics service and the STAC catalog together. It is settable in a `configure()` block only: a `base_url` key in the file and an exported `API_USGS_BASE_URL` each raise rather than being read, since a redirect a config file or a shell profile can set is one no reader of the script can see. The API key does not follow a redirect — it is scoped to the single host that honors it — and is deliberately not per-adapter: it authenticates to the gateway fronting a host, and Water Data and NGWMN share that host, one key, and one hourly quota. `dataretrieval.show_configuration()` reports each setting's effective value and where it came from, naming the profile behind each value (`configure() block [waterdata.bulk]` rather than a bare block), listing the profiles a file defines whether or not this run selected any, and naming any adapter this process has not imported rather than omitting it — without ever printing the key. One parser per setting owns its grammar, so a value means the same thing whichever source wrote it. **Breaking change:** `RetryPolicy.from_env()` is now `RetryPolicy.from_configuration()` and resolves through the whole chain rather than the environment alone. **Behavior change:** a credential-shaped keyword passed to a getter's `**kwargs` query passthrough — Water Data's `**queryables` and every WQP getter's search filters — now raises `TypeError` naming `configure(Configuration(api_key=...))` instead of putting a secret in a URL that clients, proxies and logs retain. The names refused are `api_key=`, `token=`, `x_api_key=`, `password=`, `auth=`, `pat=` and similar spellings; a filter the server actually defines is unaffected. **Bug fix:** `API_USGS_STALL_TIMEOUT` was read straight from `os.environ`, so it could not be set by a `configure()` block or the config file and never appeared in `show_configuration()`; it now resolves through the chain like every other setting. Rationale in ADRs 0009, 0010 and 0011; terms in `CONTEXT.md`. diff --git a/README.md b/README.md index b45eb5e10..97135a462 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,20 @@ df, metadata = waterdata.get_continuous( print(f"Retrieved {len(df)} continuous gage height measurements") ``` +#### Search the Water Data STAC catalog + +Use the raw STAC helpers when you need catalog metadata or item documents rather +than parsed rating tables. Responses preserve the standard STAC links and are +returned with request metadata: + +```python +collections, metadata = waterdata.get_stac_collections() +items, metadata = waterdata.search_stac(collections=["ratings"], limit=5) + +print([collection["id"] for collection in collections["collections"]]) +print([item["id"] for item in items["features"]]) +``` + #### Speeding up large downloads with `parallel_chunks` By default the getters split a multi-value request only as far as the server's diff --git a/dataretrieval/transport/http.py b/dataretrieval/transport/http.py index b14d5bb61..ba267d7df 100644 --- a/dataretrieval/transport/http.py +++ b/dataretrieval/transport/http.py @@ -30,6 +30,7 @@ "get", "network_error", "open_async_client", + "request", "strip_api_key_from_untrusted_host", "strip_api_key_from_untrusted_host_async", ] @@ -83,8 +84,8 @@ def network_error(url: str | httpx.URL, exc: httpx.TransportError) -> NetworkErr return NetworkError(f"Could not reach the service at {url}: {detail}") -def get(url: str | httpx.URL, **kwargs: Any) -> httpx.Response: - """Issue one guarded synchronous GET and map transport failures.""" +def request(method: str, url: str | httpx.URL, **kwargs: Any) -> httpx.Response: + """Issue one guarded synchronous request and map transport failures.""" client_options: dict[str, Any] = { key: kwargs.pop(key) for key in ("follow_redirects", "timeout", "transport", "verify") @@ -93,11 +94,16 @@ def get(url: str | httpx.URL, **kwargs: Any) -> httpx.Response: client_options["event_hooks"] = {"request": [strip_api_key_from_untrusted_host]} try: with httpx.Client(**client_options) as client: - return client.get(url, **kwargs) + return client.request(method, url, **kwargs) except httpx.TransportError as exc: raise network_error(url, exc) from exc +def get(url: str | httpx.URL, **kwargs: Any) -> httpx.Response: + """Issue one guarded synchronous GET and map transport failures.""" + return request("GET", url, **kwargs) + + @asynccontextmanager async def open_async_client(**overrides: Any) -> AsyncIterator[httpx.AsyncClient]: """Open a short-lived async client with redirect-safe shared defaults.""" diff --git a/dataretrieval/waterdata/__init__.py b/dataretrieval/waterdata/__init__.py index 988d3e620..df962ea5d 100644 --- a/dataretrieval/waterdata/__init__.py +++ b/dataretrieval/waterdata/__init__.py @@ -36,6 +36,16 @@ from .configuration import WaterdataConfiguration from .nearest import get_nearest_continuous from .ratings import get_ratings +from .stac import ( + get_stac_catalog, + get_stac_collection, + get_stac_collections, + get_stac_conformance, + get_stac_item, + get_stac_items, + get_stac_queryables, + search_stac, +) from .types import ( CODE_SERVICES, PROFILE_LOOKUP, @@ -69,9 +79,17 @@ "get_queryables", "get_ratings", "get_reference_table", + "get_stac_catalog", + "get_stac_collection", + "get_stac_collections", + "get_stac_conformance", + "get_stac_item", + "get_stac_items", + "get_stac_queryables", "get_samples", "get_samples_summary", "get_stats_date_range", "get_stats_por", "get_time_series_metadata", + "search_stac", ] diff --git a/dataretrieval/waterdata/stac.py b/dataretrieval/waterdata/stac.py new file mode 100644 index 000000000..af6d2d8f5 --- /dev/null +++ b/dataretrieval/waterdata/stac.py @@ -0,0 +1,316 @@ +"""Raw access to the public USGS Water Data STAC API. + +STAC catalogs, JSON Schemas, Collections, and GeoJSON ItemCollections have +meaningful document-level fields and links that do not share one tabular shape. +These helpers therefore return each response document unchanged, alongside the +same response metadata object used by the package's DataFrame getters. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Sequence +from typing import Any, Literal +from urllib.parse import quote + +import httpx + +from dataretrieval._response_metadata import BaseMetadata +from dataretrieval.exceptions import DataRetrievalError +from dataretrieval.ogc.errors import _raise_for_non_200 +from dataretrieval.transport.http import ( + HTTPX_DEFAULTS, + default_headers, + request, +) +from dataretrieval.transport.retry import RetryPolicy, retry_sync +from dataretrieval.waterdata.endpoints import ratings_catalog_url + +STAC_SEARCH_METHOD = Literal["GET", "POST"] +STAC_FILTER_LANG = Literal["cql2-json", "cql2-text"] +STACDocument = dict[str, Any] + +__all__ = [ + "get_stac_catalog", + "get_stac_collection", + "get_stac_collections", + "get_stac_conformance", + "get_stac_item", + "get_stac_items", + "get_stac_queryables", + "search_stac", +] + + +def get_stac_catalog(*, ssl_check: bool = True) -> tuple[STACDocument, BaseMetadata]: + """Return the STAC landing-page Catalog document.""" + return _get("/", ssl_check=ssl_check) + + +def get_stac_conformance( + *, ssl_check: bool = True +) -> tuple[STACDocument, BaseMetadata]: + """Return the STAC and OGC conformance declarations.""" + return _get("/conformance", ssl_check=ssl_check) + + +def get_stac_collections( + *, + bbox: Sequence[int | float] | None = None, + datetime: str | None = None, + limit: int | None = None, + query: Any | None = None, + sortby: Any | None = None, + fields: Any | None = None, + filter: Any | None = None, + filter_crs: str | None = None, + filter_lang: STAC_FILTER_LANG | None = None, + q: str | None = None, + offset: int | None = None, + ssl_check: bool = True, +) -> tuple[STACDocument, BaseMetadata]: + """Return one page of STAC Collections matching collection-search filters. + + The returned document retains its standard ``links`` and pagination counts. + Pass a subsequent page's ``offset`` from those links to continue the walk. + Structured ``query``, ``sortby``, ``fields``, and ``filter`` values are JSON + encoded for the GET endpoint; already encoded strings pass through unchanged. + """ + params = _get_params( + bbox=_csv(bbox), + datetime=datetime, + limit=limit, + query=_get_value(query), + sortby=_get_value(sortby), + fields=_get_value(fields), + filter=_get_value(filter), + filter_crs=filter_crs, + filter_lang=filter_lang, + q=q, + offset=offset, + ) + return _get("/collections", params=params, ssl_check=ssl_check) + + +def get_stac_collection( + collection_id: str, *, ssl_check: bool = True +) -> tuple[STACDocument, BaseMetadata]: + """Return one STAC Collection by identifier.""" + return _get(f"/collections/{_path_part(collection_id)}", ssl_check=ssl_check) + + +def get_stac_items( + collection_id: str, + *, + limit: int | None = None, + bbox: Sequence[int | float] | None = None, + datetime: str | None = None, + query: Any | None = None, + sortby: Any | None = None, + fields: Any | None = None, + filter: Any | None = None, + filter_crs: str | None = None, + filter_lang: STAC_FILTER_LANG | None = None, + page_token: str | None = None, + ssl_check: bool = True, +) -> tuple[STACDocument, BaseMetadata]: + """Return one GeoJSON ItemCollection page from a STAC Collection. + + The response's standard ``next`` link is preserved. Its continuation value + can be supplied as ``page_token`` to retrieve the next page. + """ + params = _get_params( + limit=limit, + bbox=_csv(bbox), + datetime=datetime, + query=_get_value(query), + sortby=_get_value(sortby), + fields=_get_value(fields), + filter=_get_value(filter), + filter_crs=filter_crs, + filter_lang=filter_lang, + token=page_token, + ) + path = f"/collections/{_path_part(collection_id)}/items" + return _get(path, params=params, ssl_check=ssl_check) + + +def get_stac_item( + collection_id: str, + item_id: str, + *, + ssl_check: bool = True, +) -> tuple[STACDocument, BaseMetadata]: + """Return one GeoJSON STAC Item by collection and item identifier.""" + path = f"/collections/{_path_part(collection_id)}/items/{_path_part(item_id)}" + return _get(path, ssl_check=ssl_check) + + +def get_stac_queryables( + collection_id: str | None = None, + *, + ssl_check: bool = True, +) -> tuple[STACDocument, BaseMetadata]: + """Return catalog-wide or collection-specific STAC queryables. + + This is distinct from :func:`get_queryables`, which describes a Water Data + OGC API collection under ``/ogcapi/v0`` rather than this STAC catalog. + """ + path = ( + "/queryables" + if collection_id is None + else f"/collections/{_path_part(collection_id)}/queryables" + ) + return _get(path, ssl_check=ssl_check) + + +def search_stac( + *, + method: STAC_SEARCH_METHOD = "GET", + collections: str | Iterable[str] | None = None, + ids: str | Iterable[str] | None = None, + bbox: Sequence[int | float] | None = None, + intersects: STACDocument | str | None = None, + datetime: str | None = None, + limit: int | None = None, + conf: STACDocument | None = None, + query: Any | None = None, + sortby: Any | None = None, + fields: Any | None = None, + filter: Any | None = None, + filter_crs: str | None = None, + filter_lang: STAC_FILTER_LANG | None = None, + page_token: str | None = None, + ssl_check: bool = True, +) -> tuple[STACDocument, BaseMetadata]: + """Search STAC Items using the advertised GET or POST representation. + + GET accepts either wire-ready strings or Python structures, which are + comma-joined or JSON encoded as required. POST sends native arrays and + objects. ``conf`` is part of the POST schema only and is rejected for GET. + The returned GeoJSON ItemCollection is unchanged, including pagination + links and context fields. + """ + normalized_method = method.upper() + if normalized_method not in ("GET", "POST"): + raise ValueError(f"method must be GET or POST (got {method!r}).") + if normalized_method == "GET": + if conf is not None: + raise ValueError("conf is supported only by the POST STAC search.") + params = _get_params( + collections=_csv(_strings(collections)), + ids=_csv(_strings(ids)), + bbox=_csv(bbox), + intersects=_get_value(intersects), + datetime=datetime, + limit=limit, + query=_get_value(query), + sortby=_get_value(sortby), + fields=_get_value(fields), + filter=_get_value(filter), + filter_crs=filter_crs, + filter_lang=filter_lang, + token=page_token, + ) + return _get("/search", params=params, ssl_check=ssl_check) + + body = _wire_names( + _without_none( + collections=_strings(collections), + ids=_strings(ids), + bbox=None if bbox is None else list(bbox), + intersects=intersects, + datetime=datetime, + limit=limit, + conf=conf, + query=query, + sortby=sortby, + fields=fields, + filter=filter, + filter_crs=filter_crs, + filter_lang=filter_lang, + token=page_token, + ) + ) + return _request_document("POST", "/search", json_body=body, ssl_check=ssl_check) + + +def _get( + path: str, + *, + params: dict[str, Any] | None = None, + ssl_check: bool, +) -> tuple[STACDocument, BaseMetadata]: + return _request_document("GET", path, params=params, ssl_check=ssl_check) + + +def _request_document( + method: str, + path: str, + *, + params: dict[str, Any] | None = None, + json_body: STACDocument | None = None, + ssl_check: bool, +) -> tuple[STACDocument, BaseMetadata]: + url = f"{ratings_catalog_url()}{path}" + + def attempt() -> httpx.Response: + response = request( + method, + url, + params=params, + json=json_body, + headers=default_headers(url), + verify=ssl_check, + **HTTPX_DEFAULTS, + ) + _raise_for_non_200(response) + return response + + response = retry_sync(attempt, RetryPolicy.from_configuration(adapter="waterdata")) + try: + document = response.json() + except ValueError as exc: + raise DataRetrievalError( + f"The STAC service returned invalid JSON (URL: {response.url})." + ) from exc + if not isinstance(document, dict): + raise DataRetrievalError( + "The STAC service returned a JSON value instead of a document " + f"(URL: {response.url})." + ) + return document, BaseMetadata(response) + + +def _strings(value: str | Iterable[str] | None) -> list[str] | None: + if value is None: + return None + return [value] if isinstance(value, str) else list(value) + + +def _csv(value: Sequence[Any] | None) -> str | None: + return None if value is None else ",".join(map(str, value)) + + +def _get_value(value: Any | None) -> Any | None: + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Sequence) and all(isinstance(part, str) for part in value): + return ",".join(value) + return json.dumps(value, separators=(",", ":")) + + +def _without_none(**values: Any) -> dict[str, Any]: + return {name: value for name, value in values.items() if value is not None} + + +def _wire_names(values: dict[str, Any]) -> dict[str, Any]: + return {name.replace("_", "-"): value for name, value in values.items()} + + +def _get_params(**values: Any) -> dict[str, Any]: + return _wire_names(_without_none(**values)) + + +def _path_part(value: str) -> str: + return quote(value, safe="") diff --git a/docs/source/reference/waterdata.rst b/docs/source/reference/waterdata.rst index e6cbc5a12..580624ae9 100644 --- a/docs/source/reference/waterdata.rst +++ b/docs/source/reference/waterdata.rst @@ -6,3 +6,9 @@ dataretrieval.waterdata .. automodule:: dataretrieval.waterdata :members: :special-members: + +STAC documents +~~~~~~~~~~~~~~ + +.. automodule:: dataretrieval.waterdata.stac + :members: diff --git a/tests/contracts/public_api_test.py b/tests/contracts/public_api_test.py index b0ed4be4a..6d2a398f2 100644 --- a/tests/contracts/public_api_test.py +++ b/tests/contracts/public_api_test.py @@ -42,11 +42,19 @@ "get_queryables", "get_ratings", "get_reference_table", + "get_stac_catalog", + "get_stac_collection", + "get_stac_collections", + "get_stac_conformance", + "get_stac_item", + "get_stac_items", + "get_stac_queryables", "get_samples", "get_samples_summary", "get_stats_date_range", "get_stats_por", "get_time_series_metadata", + "search_stac", ] #: Arguments a caller must supply positionally or by keyword. Adding an entry diff --git a/tests/waterdata_stac_test.py b/tests/waterdata_stac_test.py new file mode 100644 index 000000000..d87e26828 --- /dev/null +++ b/tests/waterdata_stac_test.py @@ -0,0 +1,268 @@ +"""Offline coverage for the public USGS Water Data STAC API client.""" + +from __future__ import annotations + +import json +import re + +import pytest + +import dataretrieval +from dataretrieval import Configuration, HTTPError, ServiceUnavailable, waterdata +from dataretrieval.waterdata import WaterdataConfiguration + +_STAC = "https://api.waterdata.usgs.gov/stac/v0" + + +def test_catalog_conformance_and_queryables_resources(httpx_mock): + documents = [ + {"type": "Catalog", "id": "usgs-water-data-stac"}, + {"conformsTo": ["https://api.stacspec.org/v1.0.0/core"]}, + {"$schema": "https://json-schema.org/draft/2019-09/schema"}, + {"properties": {"file_type": {"type": "string"}}}, + ] + urls = [ + f"{_STAC}/", + f"{_STAC}/conformance", + f"{_STAC}/queryables", + f"{_STAC}/collections/ratings/queryables", + ] + for url, document in zip(urls, documents, strict=True): + httpx_mock.add_response(method="GET", url=url, json=document) + + calls = [ + waterdata.get_stac_catalog(), + waterdata.get_stac_conformance(), + waterdata.get_stac_queryables(), + waterdata.get_stac_queryables("ratings"), + ] + + for (document, metadata), expected, url in zip(calls, documents, urls, strict=True): + assert document == expected + assert metadata.url == url + + +def test_collection_and_item_resource_paths_escape_identifiers(httpx_mock): + collection = {"type": "Collection", "id": "rating curves"} + item = {"type": "Feature", "id": "USGS/01234567.exsa.rdb"} + httpx_mock.add_response( + method="GET", + url=f"{_STAC}/collections/rating%20curves", + json=collection, + ) + httpx_mock.add_response( + method="GET", + url=(f"{_STAC}/collections/rating%20curves/items/USGS%2F01234567.exsa.rdb"), + json=item, + ) + + assert waterdata.get_stac_collection("rating curves")[0] == collection + assert waterdata.get_stac_item("rating curves", "USGS/01234567.exsa.rdb")[0] == item + + +def test_get_collections_forwards_every_advertised_parameter(httpx_mock): + httpx_mock.add_response( + method="GET", + url=re.compile(rf"^{re.escape(_STAC)}/collections(?:\?.*)?$"), + json={}, + ) + + waterdata.get_stac_collections( + bbox=[-95, 40, -92, 42], + datetime="2026-01-01/..", + limit=25, + query={"title": {"eq": "ratings"}}, + sortby=[{"field": "id", "direction": "asc"}], + fields={"include": ["id"], "exclude": ["links"]}, + filter={"op": "=", "args": [{"property": "id"}, "ratings"]}, + filter_crs="http://www.opengis.net/def/crs/OGC/1.3/CRS84", + filter_lang="cql2-json", + q="rating curves", + offset=10, + ) + + params = httpx_mock.get_requests()[0].url.params + assert params["bbox"] == "-95,40,-92,42" + assert params["datetime"] == "2026-01-01/.." + assert params["limit"] == "25" + assert json.loads(params["query"]) == {"title": {"eq": "ratings"}} + assert json.loads(params["sortby"]) == [{"field": "id", "direction": "asc"}] + assert json.loads(params["fields"]) == { + "include": ["id"], + "exclude": ["links"], + } + assert json.loads(params["filter"]) == { + "op": "=", + "args": [{"property": "id"}, "ratings"], + } + assert params["filter-crs"].endswith("CRS84") + assert params["filter-lang"] == "cql2-json" + assert params["q"] == "rating curves" + assert params["offset"] == "10" + + +def test_get_collection_items_forwards_every_advertised_parameter(httpx_mock): + httpx_mock.add_response( + method="GET", + url=re.compile(rf"^{re.escape(_STAC)}/collections/ratings/items(?:\?.*)?$"), + json={"features": []}, + ) + + waterdata.get_stac_items( + "ratings", + limit=50, + bbox=[-95, 40, -92, 42], + datetime="2026-01-01/..", + query='{"file_type":{"eq":"exsa"}}', + sortby="-properties.updated", + fields="+id,-geometry", + filter="file_type = 'exsa'", + filter_crs="http://www.opengis.net/def/crs/OGC/1.3/CRS84", + filter_lang="cql2-text", + page_token="next-page", + ) + + params = httpx_mock.get_requests()[0].url.params + assert dict(params) == { + "limit": "50", + "bbox": "-95,40,-92,42", + "datetime": "2026-01-01/..", + "query": '{"file_type":{"eq":"exsa"}}', + "sortby": "-properties.updated", + "fields": "+id,-geometry", + "filter": "file_type = 'exsa'", + "filter-crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "filter-lang": "cql2-text", + "token": "next-page", + } + + +def test_get_search_serializes_get_parameters(httpx_mock): + httpx_mock.add_response( + method="GET", + url=re.compile(rf"^{re.escape(_STAC)}/search(?:\?.*)?$"), + json={"features": []}, + ) + + document, _ = waterdata.search_stac( + method="GET", + collections=["ratings", "other"], + ids=["one", "two"], + bbox=[-95, 40, -92, 42], + intersects={"type": "Point", "coordinates": [-93, 41]}, + datetime="2026-01-01/..", + limit=100, + query={"file_type": {"eq": "exsa"}}, + sortby=["-properties.updated"], + fields=["+id", "-geometry"], + filter="file_type = 'exsa'", + filter_crs="http://www.opengis.net/def/crs/OGC/1.3/CRS84", + filter_lang="cql2-text", + page_token="next-page", + ) + + assert document == {"features": []} + params = httpx_mock.get_requests()[0].url.params + assert params["collections"] == "ratings,other" + assert params["ids"] == "one,two" + assert params["bbox"] == "-95,40,-92,42" + assert json.loads(params["intersects"]) == { + "type": "Point", + "coordinates": [-93, 41], + } + assert params["datetime"] == "2026-01-01/.." + assert params["limit"] == "100" + assert json.loads(params["query"]) == {"file_type": {"eq": "exsa"}} + assert params["sortby"] == "-properties.updated" + assert params["fields"] == "+id,-geometry" + assert params["filter"] == "file_type = 'exsa'" + assert params["filter-crs"].endswith("CRS84") + assert params["filter-lang"] == "cql2-text" + assert params["token"] == "next-page" + + +def test_post_search_sends_native_json_parameters(httpx_mock): + httpx_mock.add_response(method="POST", url=f"{_STAC}/search", json={"features": []}) + cql = {"op": "=", "args": [{"property": "file_type"}, "exsa"]} + + waterdata.search_stac( + method="POST", + collections="ratings", + ids="USGS-01104475.exsa.rdb", + bbox=[-95, 40, -92, 42], + intersects={"type": "Point", "coordinates": [-93, 41]}, + datetime="2026-01-01/..", + limit=100, + conf={"invalid": "match"}, + query={"file_type": {"eq": "exsa"}}, + sortby=[{"field": "properties.updated", "direction": "desc"}], + fields={"include": ["id"], "exclude": ["geometry"]}, + filter=cql, + filter_crs="http://www.opengis.net/def/crs/OGC/1.3/CRS84", + filter_lang="cql2-json", + page_token="next-page", + ) + + request = httpx_mock.get_requests()[0] + assert request.headers["content-type"].startswith("application/json") + assert json.loads(request.content) == { + "collections": ["ratings"], + "ids": ["USGS-01104475.exsa.rdb"], + "bbox": [-95, 40, -92, 42], + "intersects": {"type": "Point", "coordinates": [-93, 41]}, + "datetime": "2026-01-01/..", + "limit": 100, + "conf": {"invalid": "match"}, + "query": {"file_type": {"eq": "exsa"}}, + "sortby": [{"field": "properties.updated", "direction": "desc"}], + "fields": {"include": ["id"], "exclude": ["geometry"]}, + "filter": cql, + "filter-crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + "filter-lang": "cql2-json", + "token": "next-page", + } + + +def test_search_rejects_invalid_method_and_post_only_conf(): + with pytest.raises(ValueError, match="GET or POST"): + waterdata.search_stac(method="PUT") # type: ignore[arg-type] + with pytest.raises(ValueError, match="conf.*POST"): + waterdata.search_stac(method="GET", conf={}) + + +def test_stac_uses_typed_errors_and_waterdata_retries(httpx_mock, monkeypatch): + monkeypatch.setenv("API_USGS_RETRIES", "1") + monkeypatch.setattr("dataretrieval.transport.retry._RETRY_BASE_BACKOFF", 0) + httpx_mock.add_response( + method="GET", url=f"{_STAC}/collections/missing", status_code=503 + ) + httpx_mock.add_response( + method="GET", + url=f"{_STAC}/collections/missing", + json={"type": "Collection", "id": "recovered"}, + ) + + assert waterdata.get_stac_collection("missing")[0]["id"] == "recovered" + + httpx_mock.add_response( + method="GET", url=f"{_STAC}/collections/absent", status_code=404 + ) + with pytest.raises(HTTPError) as excinfo: + waterdata.get_stac_collection("absent") + assert not isinstance(excinfo.value, ServiceUnavailable) + + +def test_stac_honors_redirect_and_scopes_api_key(httpx_mock, monkeypatch): + mirror = "https://mirror.example/waterdata" + monkeypatch.setenv("API_USGS_PAT", "not-a-secret") + httpx_mock.add_response(method="GET", url=f"{mirror}/stac/v0/", json={}) + httpx_mock.add_response(method="GET", url=f"{_STAC}/", json={}) + + with dataretrieval.configure(WaterdataConfiguration(base_url=mirror)): + waterdata.get_stac_catalog() + with dataretrieval.configure(Configuration(api_key="not-a-secret")): + waterdata.get_stac_catalog() + + redirected, direct = httpx_mock.get_requests() + assert "X-Api-Key" not in redirected.headers + assert direct.headers["X-Api-Key"] == "not-a-secret" From 1c6c1e2d0f6dda6a9db6c649325393732c8c426d Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 17 Aug 2026 16:11:04 -0500 Subject: [PATCH 2/5] docs(waterdata): harden STAC API contracts --- dataretrieval/transport/http.py | 24 +++- dataretrieval/waterdata/stac.py | 225 ++++++++++++++++++++++++++++---- tests/architecture_test.py | 1 + 3 files changed, 222 insertions(+), 28 deletions(-) diff --git a/dataretrieval/transport/http.py b/dataretrieval/transport/http.py index ba267d7df..c1678d9c2 100644 --- a/dataretrieval/transport/http.py +++ b/dataretrieval/transport/http.py @@ -85,7 +85,29 @@ def network_error(url: str | httpx.URL, exc: httpx.TransportError) -> NetworkErr def request(method: str, url: str | httpx.URL, **kwargs: Any) -> httpx.Response: - """Issue one guarded synchronous request and map transport failures.""" + """Issue one guarded synchronous HTTP request. + + Parameters + ---------- + method : str + HTTP method, such as ``"GET"`` or ``"POST"``. + url : str or httpx.URL + Request destination. + **kwargs : Any + Request arguments accepted by :meth:`httpx.Client.request`. Client + options such as ``verify`` and ``timeout`` are applied to the guarded + client instead. + + Returns + ------- + httpx.Response + The completed response. + + Raises + ------ + NetworkError + If no HTTP response is received. + """ client_options: dict[str, Any] = { key: kwargs.pop(key) for key in ("follow_redirects", "timeout", "transport", "verify") diff --git a/dataretrieval/waterdata/stac.py b/dataretrieval/waterdata/stac.py index af6d2d8f5..6a68942b8 100644 --- a/dataretrieval/waterdata/stac.py +++ b/dataretrieval/waterdata/stac.py @@ -43,14 +43,36 @@ def get_stac_catalog(*, ssl_check: bool = True) -> tuple[STACDocument, BaseMetadata]: - """Return the STAC landing-page Catalog document.""" + """Return the STAC landing-page Catalog document. + + Parameters + ---------- + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged STAC Catalog and response metadata. + """ return _get("/", ssl_check=ssl_check) def get_stac_conformance( *, ssl_check: bool = True ) -> tuple[STACDocument, BaseMetadata]: - """Return the STAC and OGC conformance declarations.""" + """Return the STAC and OGC conformance declarations. + + Parameters + ---------- + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged conformance document and response metadata. + """ return _get("/conformance", ssl_check=ssl_check) @@ -72,18 +94,49 @@ def get_stac_collections( """Return one page of STAC Collections matching collection-search filters. The returned document retains its standard ``links`` and pagination counts. - Pass a subsequent page's ``offset`` from those links to continue the walk. Structured ``query``, ``sortby``, ``fields``, and ``filter`` values are JSON - encoded for the GET endpoint; already encoded strings pass through unchanged. + encoded for the GET endpoint; encoded strings pass through unchanged. + + Parameters + ---------- + bbox : sequence of numbers, optional + Four- or six-coordinate bounding box intersecting each collection. + datetime : str, optional + RFC 3339 instant or interval intersecting each collection's extent. + limit : int, optional + Maximum collections returned in this page. + query : object or str, optional + STAC Query extension expression or its encoded JSON form. + sortby : object or str, optional + STAC sort expression or its encoded wire form. + fields : object or str, optional + STAC Fields extension include/exclude expression. + filter : object or str, optional + CQL2 JSON object or CQL2 text expression. + filter_crs : str, optional + CRS URI used by spatial literals in ``filter``. + filter_lang : {"cql2-json", "cql2-text"}, optional + Encoding used by ``filter``. + q : str, optional + Free-text collection search. + offset : int, optional + Collection offset for the requested page. + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged STAC Collections document and response metadata. """ params = _get_params( bbox=_csv(bbox), datetime=datetime, limit=limit, - query=_get_value(query), - sortby=_get_value(sortby), - fields=_get_value(fields), - filter=_get_value(filter), + query=_encode_get_value(query), + sortby=_encode_get_value(sortby), + fields=_encode_get_value(fields), + filter=_encode_get_value(filter), filter_crs=filter_crs, filter_lang=filter_lang, q=q, @@ -95,7 +148,20 @@ def get_stac_collections( def get_stac_collection( collection_id: str, *, ssl_check: bool = True ) -> tuple[STACDocument, BaseMetadata]: - """Return one STAC Collection by identifier.""" + """Return one STAC Collection by identifier. + + Parameters + ---------- + collection_id : str + Collection identifier from :func:`get_stac_collections`. + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged STAC Collection and response metadata. + """ return _get(f"/collections/{_path_part(collection_id)}", ssl_check=ssl_check) @@ -116,17 +182,49 @@ def get_stac_items( ) -> tuple[STACDocument, BaseMetadata]: """Return one GeoJSON ItemCollection page from a STAC Collection. - The response's standard ``next`` link is preserved. Its continuation value - can be supplied as ``page_token`` to retrieve the next page. + The response's standard ``next`` link is preserved. Supply its continuation + value as ``page_token`` to retrieve the next page. + + Parameters + ---------- + collection_id : str + Collection whose Items should be listed. + limit : int, optional + Maximum Items returned in this page. + bbox : sequence of numbers, optional + Four- or six-coordinate bounding box intersecting returned Items. + datetime : str, optional + RFC 3339 instant or interval intersecting returned Items. + query : object or str, optional + STAC Query extension expression or its encoded JSON form. + sortby : object or str, optional + STAC sort expression or its encoded wire form. + fields : object or str, optional + STAC Fields extension include/exclude expression. + filter : object or str, optional + CQL2 JSON object or CQL2 text expression. + filter_crs : str, optional + CRS URI used by spatial literals in ``filter``. + filter_lang : {"cql2-json", "cql2-text"}, optional + Encoding used by ``filter``. + page_token : str, optional + Opaque continuation value sent as the STAC ``token`` query parameter. + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged GeoJSON ItemCollection and response metadata. """ params = _get_params( limit=limit, bbox=_csv(bbox), datetime=datetime, - query=_get_value(query), - sortby=_get_value(sortby), - fields=_get_value(fields), - filter=_get_value(filter), + query=_encode_get_value(query), + sortby=_encode_get_value(sortby), + fields=_encode_get_value(fields), + filter=_encode_get_value(filter), filter_crs=filter_crs, filter_lang=filter_lang, token=page_token, @@ -141,7 +239,22 @@ def get_stac_item( *, ssl_check: bool = True, ) -> tuple[STACDocument, BaseMetadata]: - """Return one GeoJSON STAC Item by collection and item identifier.""" + """Return one GeoJSON STAC Item by collection and item identifier. + + Parameters + ---------- + collection_id : str + Collection containing the Item. + item_id : str + Item identifier. + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged GeoJSON STAC Item and response metadata. + """ path = f"/collections/{_path_part(collection_id)}/items/{_path_part(item_id)}" return _get(path, ssl_check=ssl_check) @@ -155,6 +268,19 @@ def get_stac_queryables( This is distinct from :func:`get_queryables`, which describes a Water Data OGC API collection under ``/ogcapi/v0`` rather than this STAC catalog. + + Parameters + ---------- + collection_id : str, optional + Collection whose queryables should be returned. Omit it for the + catalog-wide queryables document. + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged JSON Schema queryables document and response metadata. """ path = ( "/queryables" @@ -185,11 +311,56 @@ def search_stac( ) -> tuple[STACDocument, BaseMetadata]: """Search STAC Items using the advertised GET or POST representation. - GET accepts either wire-ready strings or Python structures, which are - comma-joined or JSON encoded as required. POST sends native arrays and - objects. ``conf`` is part of the POST schema only and is rejected for GET. - The returned GeoJSON ItemCollection is unchanged, including pagination - links and context fields. + GET accepts wire-ready strings or Python structures, which are comma-joined + or JSON encoded as required. POST sends native arrays and objects. The + returned GeoJSON ItemCollection remains unchanged, including its links. + + Parameters + ---------- + method : {"GET", "POST"}, default "GET" + STAC search representation to use. + collections : str or iterable of str, optional + Collection identifiers to search. + ids : str or iterable of str, optional + Item identifiers to return. + bbox : sequence of numbers, optional + Four- or six-coordinate bounding box intersecting returned Items. + intersects : dict or str, optional + GeoJSON geometry intersecting returned Items. Mutually exclusive with + ``bbox`` according to the service contract. + datetime : str, optional + RFC 3339 instant or interval intersecting returned Items. + limit : int, optional + Maximum Items returned in this page, capped upstream at 10,000. + conf : dict, optional + POST-only server configuration object. + query : object or str, optional + STAC Query extension expression. Structured values are native JSON for + POST and encoded for GET. + sortby : object or str, optional + STAC sort expression. + fields : object or str, optional + STAC Fields extension include/exclude expression. + filter : object or str, optional + CQL2 JSON object or CQL2 text expression. + filter_crs : str, optional + CRS URI used by spatial literals in ``filter``. + filter_lang : {"cql2-json", "cql2-text"}, optional + Encoding used by ``filter``. + page_token : str, optional + Opaque continuation value sent as the STAC ``token`` field. + ssl_check : bool, default True + Verify the server's SSL certificate. + + Returns + ------- + dict, BaseMetadata + The unchanged GeoJSON ItemCollection and response metadata. + + Raises + ------ + ValueError + If ``method`` is not GET or POST, or ``conf`` is supplied for GET. """ normalized_method = method.upper() if normalized_method not in ("GET", "POST"): @@ -201,13 +372,13 @@ def search_stac( collections=_csv(_strings(collections)), ids=_csv(_strings(ids)), bbox=_csv(bbox), - intersects=_get_value(intersects), + intersects=_encode_get_value(intersects), datetime=datetime, limit=limit, - query=_get_value(query), - sortby=_get_value(sortby), - fields=_get_value(fields), - filter=_get_value(filter), + query=_encode_get_value(query), + sortby=_encode_get_value(sortby), + fields=_encode_get_value(fields), + filter=_encode_get_value(filter), filter_crs=filter_crs, filter_lang=filter_lang, token=page_token, @@ -292,7 +463,7 @@ def _csv(value: Sequence[Any] | None) -> str | None: return None if value is None else ",".join(map(str, value)) -def _get_value(value: Any | None) -> Any | None: +def _encode_get_value(value: Any | None) -> Any | None: if value is None or isinstance(value, (str, int, float, bool)): return value if isinstance(value, Sequence) and all(isinstance(part, str) for part in value): diff --git a/tests/architecture_test.py b/tests/architecture_test.py index 4af377b77..f3eef2d80 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -431,6 +431,7 @@ def _waterdata_family_paths() -> tuple[str, ...]: "wqp.py", "waterdata/nearest.py", "waterdata/ratings.py", + "waterdata/stac.py", "waterdata/stats.py", "waterdata/types.py", *_WATERDATA_FAMILIES, From 53425b063a956cfd98bc496d2d037d28c835edb9 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 17 Aug 2026 16:14:56 -0500 Subject: [PATCH 3/5] docs: record raw STAC return contract --- docs/source/architecture/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index c292b63a7..a6d9e4568 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -235,6 +235,10 @@ service into one return shape: - ``waterdata.get_ratings`` returns a mapping of feature IDs to parsed rating ``DataFrame`` objects by default, or the raw STAC feature list when downloads are disabled. +- Water Data STAC catalog helpers return ``(dict, BaseMetadata)``. The mapping + preserves each upstream Catalog, Collection, GeoJSON ItemCollection, Item, or + JSON Schema document and its standard links without flattening unlike shapes + into one table. - NLDI navigation functions return ``GeoDataFrame`` objects directly, or raw GeoJSON-like dictionaries when ``as_json=True``; they do not add a metadata tuple. From 11a7d8f21277d215ee01b194b56b6273405ce618 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Tue, 18 Aug 2026 08:43:45 -0500 Subject: [PATCH 4/5] docs(waterdata): add STAC example notebook --- demos/USGS_WaterData_STAC_Examples.ipynb | 437 ++++++++++++++++++ .../USGS_WaterData_STAC_Examples.nblink | 3 + docs/source/examples/index.rst | 8 +- 3 files changed, 445 insertions(+), 3 deletions(-) create mode 100644 demos/USGS_WaterData_STAC_Examples.ipynb create mode 100644 docs/source/examples/USGS_WaterData_STAC_Examples.nblink diff --git a/demos/USGS_WaterData_STAC_Examples.ipynb b/demos/USGS_WaterData_STAC_Examples.ipynb new file mode 100644 index 000000000..2b25b8566 --- /dev/null +++ b/demos/USGS_WaterData_STAC_Examples.ipynb @@ -0,0 +1,437 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Discovering USGS Water Data with STAC\n", + "\n", + "The [USGS Water Data STAC API](https://api.waterdata.usgs.gov/stac/v0/) is a\n", + "catalog for discovering file-based water-data resources. STAC (the SpatioTemporal\n", + "Asset Catalog specification) organizes those resources as **Collections**,\n", + "**Items**, and downloadable **assets**. At the time of writing, the catalog\n", + "contains USGS stage-discharge rating files, but the discovery workflow shown here\n", + "also applies as collections are added.\n", + "\n", + "This notebook starts at the catalog landing page, discovers its collections,\n", + "searches for a known monitoring location, inspects an Item and its asset, and\n", + "walks one page of results at a time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from urllib.parse import parse_qs, urlparse\n", + "\n", + "import pandas as pd\n", + "\n", + "from dataretrieval import waterdata" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "> **Return values.** The generic STAC helpers return a `(document, metadata)`\n", + "> tuple. `document` is the unchanged JSON response as a Python dictionary, and\n", + "> `metadata` describes the HTTP response. Unlike observation-oriented\n", + "> `waterdata` functions, these helpers do not coerce Catalogs, JSON Schemas,\n", + "> GeoJSON Items, and their links into one lossy table shape. We create small\n", + "> pandas previews below only when a table helps us read part of a document." + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Start at the catalog\n", + "\n", + "A STAC landing page identifies the catalog and advertises the operations it\n", + "supports. `get_stac_catalog` retrieves that starting document:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "catalog, catalog_metadata = waterdata.get_stac_catalog()\n", + "\n", + "{\n", + " \"id\": catalog[\"id\"],\n", + " \"type\": catalog[\"type\"],\n", + " \"title\": catalog[\"title\"],\n", + " \"description\": catalog[\"description\"],\n", + " \"request_url\": catalog_metadata.url,\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "STAC documents are hypermedia documents: their `links` tell a client where to\n", + "find collections, conformance declarations, searches, queryables, and API\n", + "documentation. Preserving these links is one reason the helpers return raw\n", + "documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame(catalog[\"links\"]).reindex(columns=[\"rel\", \"type\", \"method\", \"href\"])" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "## Discover collections\n", + "\n", + "A Collection describes a related group of Items. List the available collections\n", + "instead of assuming which datasets the catalog currently contains:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "collections_document, _ = waterdata.get_stac_collections()\n", + "\n", + "pd.DataFrame(\n", + " [\n", + " {\n", + " \"id\": collection[\"id\"],\n", + " \"title\": collection[\"title\"],\n", + " \"description\": collection[\"description\"],\n", + " }\n", + " for collection in collections_document[\"collections\"]\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "Retrieve one Collection to inspect its spatial and temporal extent, license,\n", + "providers, and the kinds of assets attached to its Items. The current `ratings`\n", + "Collection contains base ratings, corrections, and expanded stage-discharge\n", + "tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "collection_id = \"ratings\"\n", + "collection, _ = waterdata.get_stac_collection(collection_id)\n", + "\n", + "{\n", + " \"id\": collection[\"id\"],\n", + " \"title\": collection[\"title\"],\n", + " \"license\": collection[\"license\"],\n", + " \"extent\": collection[\"extent\"],\n", + " \"item_assets\": collection[\"item_assets\"],\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### Discover queryable fields\n", + "\n", + "`get_stac_queryables` returns a JSON Schema describing fields a server exposes\n", + "for filtering. Passing a collection id requests that collection's schema;\n", + "omitting it requests the catalog-wide schema. This schema allows additional\n", + "properties, so Items may also carry collection-specific fields such as\n", + "`monitoring_location_id` and `file_type`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "queryables, _ = waterdata.get_stac_queryables(collection_id)\n", + "\n", + "pd.DataFrame.from_dict(queryables[\"properties\"], orient=\"index\").reindex(\n", + " columns=[\"title\", \"description\", \"type\", \"format\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "## Search for Items\n", + "\n", + "Suppose we need rating files for monitoring location `USGS-10109000`. A CQL2\n", + "text filter selects that location while `collections` limits the search to\n", + "ratings. `limit` is the maximum number of Items in this response page, not a\n", + "request to flatten every page into one result." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "site = \"USGS-10109000\"\n", + "items_document, search_metadata = waterdata.search_stac(\n", + " collections=[collection_id],\n", + " filter=f\"monitoring_location_id = '{site}'\",\n", + " filter_lang=\"cql2-text\",\n", + " limit=10,\n", + ")\n", + "\n", + "item_preview = []\n", + "for feature in items_document[\"features\"]:\n", + " properties = feature[\"properties\"]\n", + " longitude, latitude = feature[\"geometry\"][\"coordinates\"]\n", + " item_preview.append(\n", + " {\n", + " \"id\": feature[\"id\"],\n", + " \"file_type\": properties[\"file_type\"],\n", + " \"updated\": properties[\"datetime\"],\n", + " \"longitude\": longitude,\n", + " \"latitude\": latitude,\n", + " }\n", + " )\n", + "\n", + "print(f\"{len(item_preview)} Items from {search_metadata.url}\")\n", + "pd.DataFrame(item_preview)" + ] + }, + { + "cell_type": "markdown", + "id": "15", + "metadata": {}, + "source": [ + "The response is a GeoJSON `FeatureCollection`. Each feature is a STAC Item with\n", + "an id, geometry, descriptive properties, links, and one or more assets. Use\n", + "`get_stac_item` when you know the collection and Item ids and want that Item\n", + "directly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "first_feature = items_document[\"features\"][0]\n", + "item, item_metadata = waterdata.get_stac_item(collection_id, first_feature[\"id\"])\n", + "\n", + "{\n", + " \"id\": item[\"id\"],\n", + " \"collection\": item[\"collection\"],\n", + " \"properties\": item[\"properties\"],\n", + " \"request_url\": item_metadata.url,\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "### Follow an Item to its asset\n", + "\n", + "An asset entry describes the file associated with an Item. Its `href` is the\n", + "download URL, while `type`, `roles`, size, and description explain what the\n", + "file contains. Generic STAC helpers preserve this information but do not decide\n", + "how to parse an arbitrary asset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame.from_dict(item[\"assets\"], orient=\"index\").reindex(\n", + " columns=[\"title\", \"description\", \"type\", \"file:size\", \"roles\", \"href\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "For rating assets, `get_ratings` is the higher-level convenience function: it\n", + "uses this catalog, downloads the selected RDB asset, and parses it into a pandas\n", + "DataFrame. Use the generic STAC helpers for discovery and raw Item metadata; use\n", + "the specialized helper when you want analysis-ready rating values." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "ratings = waterdata.get_ratings(monitoring_location_id=site, file_type=\"exsa\")\n", + "rating = ratings[f\"{site}.exsa.rdb\"]\n", + "rating.head()" + ] + }, + { + "cell_type": "markdown", + "id": "21", + "metadata": {}, + "source": [ + "## GET and POST searches\n", + "\n", + "`search_stac` defaults to GET, which is convenient for simple, shareable query\n", + "URLs. Choose POST for structured JSON expressions or geometries that would be\n", + "awkward or too long in a URL. Here is the same location filter represented as\n", + "CQL2 JSON; POST sends the dictionary as native JSON rather than encoding it into\n", + "a query string." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "json_filter = {\n", + " \"op\": \"=\",\n", + " \"args\": [{\"property\": \"monitoring_location_id\"}, site],\n", + "}\n", + "post_document, _ = waterdata.search_stac(\n", + " method=\"POST\",\n", + " collections=[collection_id],\n", + " filter=json_filter,\n", + " filter_lang=\"cql2-json\",\n", + " limit=10,\n", + ")\n", + "\n", + "[feature[\"id\"] for feature in post_document[\"features\"]]" + ] + }, + { + "cell_type": "markdown", + "id": "23", + "metadata": {}, + "source": [ + "## Walk result pages deliberately\n", + "\n", + "Generic STAC calls return one standard response document at a time. If more\n", + "Items are available, the response includes a link whose relation is `next`.\n", + "Extract its opaque `token` and pass it back as `page_token`; do not construct or\n", + "modify the token yourself. This keeps page boundaries and STAC links visible to\n", + "the caller." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "first_page, _ = waterdata.search_stac(collections=[collection_id], limit=2)\n", + "next_link = next(link for link in first_page[\"links\"] if link[\"rel\"] == \"next\")\n", + "next_token = parse_qs(urlparse(next_link[\"href\"]).query)[\"token\"][0]\n", + "\n", + "second_page, _ = waterdata.search_stac(\n", + " collections=[collection_id],\n", + " limit=2,\n", + " page_token=next_token,\n", + ")\n", + "\n", + "pd.DataFrame(\n", + " {\n", + " \"first page\": [feature[\"id\"] for feature in first_page[\"features\"]],\n", + " \"second page\": [feature[\"id\"] for feature in second_page[\"features\"]],\n", + " }\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "25", + "metadata": {}, + "source": [ + "## Choosing a STAC helper\n", + "\n", + "| Goal | Helper | Document returned |\n", + "| --- | --- | --- |\n", + "| Discover API operations | `get_stac_catalog`, `get_stac_conformance` | Catalog or conformance document |\n", + "| Discover datasets | `get_stac_collections`, `get_stac_collection` | Collection list or Collection |\n", + "| Browse one Collection | `get_stac_items` | GeoJSON ItemCollection page |\n", + "| Retrieve a known Item | `get_stac_item` | GeoJSON STAC Item |\n", + "| Discover filter fields | `get_stac_queryables` | JSON Schema |\n", + "| Search across Collections | `search_stac` | GeoJSON ItemCollection page |\n", + "| Download parsed rating tables | `get_ratings` | Dictionary of pandas DataFrames |\n", + "\n", + "All generic helpers preserve the server's standard links. They also use the same\n", + "Water Data configuration, API-key handling, retries, and configured base URL as\n", + "the rest of `dataretrieval.waterdata`." + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "## More help\n", + "\n", + "- STAC API: \n", + "- STAC specification: \n", + "- `dataretrieval` API reference: \n", + "- See the *USGS Water Data Rating Curve Examples* notebook for a deeper\n", + " `get_ratings` walkthrough.\n", + "- Issues / questions: " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/source/examples/USGS_WaterData_STAC_Examples.nblink b/docs/source/examples/USGS_WaterData_STAC_Examples.nblink new file mode 100644 index 000000000..83ec2e1c8 --- /dev/null +++ b/docs/source/examples/USGS_WaterData_STAC_Examples.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../demos/USGS_WaterData_STAC_Examples.ipynb" +} diff --git a/docs/source/examples/index.rst b/docs/source/examples/index.rst index e7c2deb8a..ad519d9f5 100644 --- a/docs/source/examples/index.rst +++ b/docs/source/examples/index.rst @@ -17,9 +17,10 @@ covers a basic introduction to module functions and usage. USGS Water Data API vignettes ----------------------------- -These notebooks are Python ports of the new USGS Water Data API vignettes from -the R `dataRetrieval`_ package. Each introduces a family of Water Data API -functions and is executed against the live USGS Water Data API. +These notebooks include Python ports of the new USGS Water Data API vignettes +from the R `dataRetrieval`_ package and focused walkthroughs of other Water Data +services. Each introduces a family of functions and is executed against the live +USGS Water Data APIs. .. _dataRetrieval: https://doi-usgs.github.io/dataRetrieval/ @@ -31,6 +32,7 @@ functions and is executed against the live USGS Water Data API. USGS_WaterData_DailyStatistics_Examples USGS_WaterData_ContinuousData_Examples USGS_WaterData_ReferenceLists_Examples + USGS_WaterData_STAC_Examples USGS_NGWMN_Examples Simple uses of the ``dataretrieval`` package From f1d289bbe349f99095315e215e499096517dc337 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Tue, 18 Aug 2026 09:30:48 -0500 Subject: [PATCH 5/5] refactor(waterdata): namespace STAC API --- NEWS.md | 2 +- README.md | 4 +- dataretrieval/waterdata/__init__.py | 20 +------ dataretrieval/waterdata/stac.py | 41 +++++++------ demos/USGS_WaterData_STAC_Examples.ipynb | 38 ++++++------ .../decisions/0012-stac-namespace.rst | 55 ++++++++++++++++++ docs/source/architecture/decisions/index.rst | 1 + docs/source/architecture/index.rst | 19 +++--- tests/contracts/public_api_test.py | 9 +-- tests/waterdata_stac_test.py | 58 ++++++++++++++----- 10 files changed, 155 insertions(+), 92 deletions(-) create mode 100644 docs/source/architecture/decisions/0012-stac-namespace.rst diff --git a/NEWS.md b/NEWS.md index 261ca1db9..4b67e94bd 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -**08/17/2026:** Added raw access to the complete public USGS Water Data STAC surface through `waterdata.get_stac_catalog`, `get_stac_conformance`, `get_stac_collections`, `get_stac_collection`, `get_stac_items`, `get_stac_item`, `get_stac_queryables`, and `search_stac`. Search supports both advertised GET and POST representations; collection and item listing helpers expose every advertised filter and pagination parameter. Each helper returns the unchanged STAC/GeoJSON/JSON Schema document plus `BaseMetadata`, preserving standard links instead of flattening heterogeneous documents into a DataFrame. Calls share Water Data API-key host scoping, configured base-URL redirection, typed HTTP errors, and bounded retries. Internal `/_mgmt` health routes are intentionally not exposed as STAC capabilities. +**08/17/2026:** Added raw access to the complete public USGS Water Data STAC surface through the `waterdata.stac` namespace: `get_catalog`, `get_conformance`, `get_collections`, `get_collection`, `get_items`, `get_item`, `get_queryables`, and `search`. Search supports both advertised GET and POST representations; collection and item listing helpers expose every advertised filter and pagination parameter. Each helper returns the unchanged STAC/GeoJSON/JSON Schema document plus `BaseMetadata`, preserving standard links instead of flattening heterogeneous documents into a DataFrame. Calls share Water Data API-key host scoping, configured base-URL redirection, typed HTTP errors, and bounded retries. Internal `/_mgmt` health routes are intentionally not exposed as STAC capabilities. **08/13/2026:** Warning categories now say what they mean. Two advisories about *upstream data* were emitted as `DeprecationWarning` and as an uncategorized `warnings.warn` respectively; both are now `DataCurrencyWarning` (a `UserWarning` subclass, exported as `dataretrieval.DataCurrencyWarning`). **Behavior change:** WQP's legacy-WQX notice moves from `DeprecationWarning` to `DataCurrencyWarning`. Because `legacy=True` is the default on every WQP getter and the notice is unconditional, a downstream project running `-W error::DeprecationWarning` previously could not call any WQP getter with default arguments; it now can. The flip side is visibility — `DeprecationWarning` is silent by default outside `__main__`, so this notice will now print to stderr for library and notebook callers who never saw it. Silence it with `warnings.filterwarnings("ignore", category=dataretrieval.DataCurrencyWarning)`, or set `legacy=False` where a WQX3.0 profile exists. The NWIS qw-endpoint retirement notice gains the same category (it previously had none, arriving as a bare `UserWarning` that deprecation filters ignored). `DeprecationWarning` now means only that a name in this package is going away, always with a replacement and, where published, a removal date; those horizons are declared once in `dataretrieval._deprecation.REMOVALS` rather than spelled at each call site. diff --git a/README.md b/README.md index 97135a462..1f79b07ce 100644 --- a/README.md +++ b/README.md @@ -155,8 +155,8 @@ than parsed rating tables. Responses preserve the standard STAC links and are returned with request metadata: ```python -collections, metadata = waterdata.get_stac_collections() -items, metadata = waterdata.search_stac(collections=["ratings"], limit=5) +collections, metadata = waterdata.stac.get_collections() +items, metadata = waterdata.stac.search(collections=["ratings"], limit=5) print([collection["id"] for collection in collections["collections"]]) print([item["id"] for item in items["features"]]) diff --git a/dataretrieval/waterdata/__init__.py b/dataretrieval/waterdata/__init__.py index df962ea5d..abe1a09ab 100644 --- a/dataretrieval/waterdata/__init__.py +++ b/dataretrieval/waterdata/__init__.py @@ -12,6 +12,7 @@ from dataretrieval.ogc.filters import FILTER_LANG # Public API exports +from . import stac from .api import ( get_channel, get_codes, @@ -36,16 +37,6 @@ from .configuration import WaterdataConfiguration from .nearest import get_nearest_continuous from .ratings import get_ratings -from .stac import ( - get_stac_catalog, - get_stac_collection, - get_stac_collections, - get_stac_conformance, - get_stac_item, - get_stac_items, - get_stac_queryables, - search_stac, -) from .types import ( CODE_SERVICES, PROFILE_LOOKUP, @@ -63,6 +54,7 @@ "SERVICES", "WATERDATA_SERVICES", "parallel_chunks", + "stac", "get_channel", "get_codes", "get_combined_metadata", @@ -79,17 +71,9 @@ "get_queryables", "get_ratings", "get_reference_table", - "get_stac_catalog", - "get_stac_collection", - "get_stac_collections", - "get_stac_conformance", - "get_stac_item", - "get_stac_items", - "get_stac_queryables", "get_samples", "get_samples_summary", "get_stats_date_range", "get_stats_por", "get_time_series_metadata", - "search_stac", ] diff --git a/dataretrieval/waterdata/stac.py b/dataretrieval/waterdata/stac.py index 6a68942b8..c9d8831e2 100644 --- a/dataretrieval/waterdata/stac.py +++ b/dataretrieval/waterdata/stac.py @@ -31,18 +31,18 @@ STACDocument = dict[str, Any] __all__ = [ - "get_stac_catalog", - "get_stac_collection", - "get_stac_collections", - "get_stac_conformance", - "get_stac_item", - "get_stac_items", - "get_stac_queryables", - "search_stac", + "get_catalog", + "get_collection", + "get_collections", + "get_conformance", + "get_item", + "get_items", + "get_queryables", + "search", ] -def get_stac_catalog(*, ssl_check: bool = True) -> tuple[STACDocument, BaseMetadata]: +def get_catalog(*, ssl_check: bool = True) -> tuple[STACDocument, BaseMetadata]: """Return the STAC landing-page Catalog document. Parameters @@ -58,9 +58,7 @@ def get_stac_catalog(*, ssl_check: bool = True) -> tuple[STACDocument, BaseMetad return _get("/", ssl_check=ssl_check) -def get_stac_conformance( - *, ssl_check: bool = True -) -> tuple[STACDocument, BaseMetadata]: +def get_conformance(*, ssl_check: bool = True) -> tuple[STACDocument, BaseMetadata]: """Return the STAC and OGC conformance declarations. Parameters @@ -76,7 +74,7 @@ def get_stac_conformance( return _get("/conformance", ssl_check=ssl_check) -def get_stac_collections( +def get_collections( *, bbox: Sequence[int | float] | None = None, datetime: str | None = None, @@ -145,7 +143,7 @@ def get_stac_collections( return _get("/collections", params=params, ssl_check=ssl_check) -def get_stac_collection( +def get_collection( collection_id: str, *, ssl_check: bool = True ) -> tuple[STACDocument, BaseMetadata]: """Return one STAC Collection by identifier. @@ -153,7 +151,7 @@ def get_stac_collection( Parameters ---------- collection_id : str - Collection identifier from :func:`get_stac_collections`. + Collection identifier from :func:`get_collections`. ssl_check : bool, default True Verify the server's SSL certificate. @@ -165,7 +163,7 @@ def get_stac_collection( return _get(f"/collections/{_path_part(collection_id)}", ssl_check=ssl_check) -def get_stac_items( +def get_items( collection_id: str, *, limit: int | None = None, @@ -233,7 +231,7 @@ def get_stac_items( return _get(path, params=params, ssl_check=ssl_check) -def get_stac_item( +def get_item( collection_id: str, item_id: str, *, @@ -259,15 +257,16 @@ def get_stac_item( return _get(path, ssl_check=ssl_check) -def get_stac_queryables( +def get_queryables( collection_id: str | None = None, *, ssl_check: bool = True, ) -> tuple[STACDocument, BaseMetadata]: """Return catalog-wide or collection-specific STAC queryables. - This is distinct from :func:`get_queryables`, which describes a Water Data - OGC API collection under ``/ogcapi/v0`` rather than this STAC catalog. + This is distinct from the tabular + :func:`dataretrieval.waterdata.get_queryables` helper, which describes a Water + Data OGC API collection under ``/ogcapi/v0`` rather than this STAC catalog. Parameters ---------- @@ -290,7 +289,7 @@ def get_stac_queryables( return _get(path, ssl_check=ssl_check) -def search_stac( +def search( *, method: STAC_SEARCH_METHOD = "GET", collections: str | Iterable[str] | None = None, diff --git a/demos/USGS_WaterData_STAC_Examples.ipynb b/demos/USGS_WaterData_STAC_Examples.ipynb index 2b25b8566..663dd11ae 100644 --- a/demos/USGS_WaterData_STAC_Examples.ipynb +++ b/demos/USGS_WaterData_STAC_Examples.ipynb @@ -54,7 +54,7 @@ "## Start at the catalog\n", "\n", "A STAC landing page identifies the catalog and advertises the operations it\n", - "supports. `get_stac_catalog` retrieves that starting document:" + "supports. `waterdata.stac.get_catalog` retrieves that starting document:" ] }, { @@ -64,7 +64,7 @@ "metadata": {}, "outputs": [], "source": [ - "catalog, catalog_metadata = waterdata.get_stac_catalog()\n", + "catalog, catalog_metadata = waterdata.stac.get_catalog()\n", "\n", "{\n", " \"id\": catalog[\"id\"],\n", @@ -114,7 +114,7 @@ "metadata": {}, "outputs": [], "source": [ - "collections_document, _ = waterdata.get_stac_collections()\n", + "collections_document, _ = waterdata.stac.get_collections()\n", "\n", "pd.DataFrame(\n", " [\n", @@ -147,7 +147,7 @@ "outputs": [], "source": [ "collection_id = \"ratings\"\n", - "collection, _ = waterdata.get_stac_collection(collection_id)\n", + "collection, _ = waterdata.stac.get_collection(collection_id)\n", "\n", "{\n", " \"id\": collection[\"id\"],\n", @@ -165,7 +165,7 @@ "source": [ "### Discover queryable fields\n", "\n", - "`get_stac_queryables` returns a JSON Schema describing fields a server exposes\n", + "`waterdata.stac.get_queryables` returns a JSON Schema describing fields a server exposes\n", "for filtering. Passing a collection id requests that collection's schema;\n", "omitting it requests the catalog-wide schema. This schema allows additional\n", "properties, so Items may also carry collection-specific fields such as\n", @@ -179,7 +179,7 @@ "metadata": {}, "outputs": [], "source": [ - "queryables, _ = waterdata.get_stac_queryables(collection_id)\n", + "queryables, _ = waterdata.stac.get_queryables(collection_id)\n", "\n", "pd.DataFrame.from_dict(queryables[\"properties\"], orient=\"index\").reindex(\n", " columns=[\"title\", \"description\", \"type\", \"format\"]\n", @@ -207,7 +207,7 @@ "outputs": [], "source": [ "site = \"USGS-10109000\"\n", - "items_document, search_metadata = waterdata.search_stac(\n", + "items_document, search_metadata = waterdata.stac.search(\n", " collections=[collection_id],\n", " filter=f\"monitoring_location_id = '{site}'\",\n", " filter_lang=\"cql2-text\",\n", @@ -239,7 +239,7 @@ "source": [ "The response is a GeoJSON `FeatureCollection`. Each feature is a STAC Item with\n", "an id, geometry, descriptive properties, links, and one or more assets. Use\n", - "`get_stac_item` when you know the collection and Item ids and want that Item\n", + "`waterdata.stac.get_item` when you know the collection and Item ids and want that Item\n", "directly." ] }, @@ -251,7 +251,7 @@ "outputs": [], "source": [ "first_feature = items_document[\"features\"][0]\n", - "item, item_metadata = waterdata.get_stac_item(collection_id, first_feature[\"id\"])\n", + "item, item_metadata = waterdata.stac.get_item(collection_id, first_feature[\"id\"])\n", "\n", "{\n", " \"id\": item[\"id\"],\n", @@ -316,7 +316,7 @@ "source": [ "## GET and POST searches\n", "\n", - "`search_stac` defaults to GET, which is convenient for simple, shareable query\n", + "`waterdata.stac.search` defaults to GET, which is convenient for simple, shareable query\n", "URLs. Choose POST for structured JSON expressions or geometries that would be\n", "awkward or too long in a URL. Here is the same location filter represented as\n", "CQL2 JSON; POST sends the dictionary as native JSON rather than encoding it into\n", @@ -334,7 +334,7 @@ " \"op\": \"=\",\n", " \"args\": [{\"property\": \"monitoring_location_id\"}, site],\n", "}\n", - "post_document, _ = waterdata.search_stac(\n", + "post_document, _ = waterdata.stac.search(\n", " method=\"POST\",\n", " collections=[collection_id],\n", " filter=json_filter,\n", @@ -366,11 +366,11 @@ "metadata": {}, "outputs": [], "source": [ - "first_page, _ = waterdata.search_stac(collections=[collection_id], limit=2)\n", + "first_page, _ = waterdata.stac.search(collections=[collection_id], limit=2)\n", "next_link = next(link for link in first_page[\"links\"] if link[\"rel\"] == \"next\")\n", "next_token = parse_qs(urlparse(next_link[\"href\"]).query)[\"token\"][0]\n", "\n", - "second_page, _ = waterdata.search_stac(\n", + "second_page, _ = waterdata.stac.search(\n", " collections=[collection_id],\n", " limit=2,\n", " page_token=next_token,\n", @@ -393,12 +393,12 @@ "\n", "| Goal | Helper | Document returned |\n", "| --- | --- | --- |\n", - "| Discover API operations | `get_stac_catalog`, `get_stac_conformance` | Catalog or conformance document |\n", - "| Discover datasets | `get_stac_collections`, `get_stac_collection` | Collection list or Collection |\n", - "| Browse one Collection | `get_stac_items` | GeoJSON ItemCollection page |\n", - "| Retrieve a known Item | `get_stac_item` | GeoJSON STAC Item |\n", - "| Discover filter fields | `get_stac_queryables` | JSON Schema |\n", - "| Search across Collections | `search_stac` | GeoJSON ItemCollection page |\n", + "| Discover API operations | `waterdata.stac.get_catalog`, `waterdata.stac.get_conformance` | Catalog or conformance document |\n", + "| Discover datasets | `waterdata.stac.get_collections`, `waterdata.stac.get_collection` | Collection list or Collection |\n", + "| Browse one Collection | `waterdata.stac.get_items` | GeoJSON ItemCollection page |\n", + "| Retrieve a known Item | `waterdata.stac.get_item` | GeoJSON STAC Item |\n", + "| Discover filter fields | `waterdata.stac.get_queryables` | JSON Schema |\n", + "| Search across Collections | `waterdata.stac.search` | GeoJSON ItemCollection page |\n", "| Download parsed rating tables | `get_ratings` | Dictionary of pandas DataFrames |\n", "\n", "All generic helpers preserve the server's standard links. They also use the same\n", diff --git a/docs/source/architecture/decisions/0012-stac-namespace.rst b/docs/source/architecture/decisions/0012-stac-namespace.rst new file mode 100644 index 000000000..c7013ba17 --- /dev/null +++ b/docs/source/architecture/decisions/0012-stac-namespace.rst @@ -0,0 +1,55 @@ +ADR 0012: Namespace raw STAC catalog operations +================================================ + +Status +------ + +Accepted + +Context +------- + +The Water Data STAC API exposes protocol-shaped catalog discovery, item lookup, +queryables, and search operations. Prefixing every operation with the protocol +name would make the main Water Data facade wider, repeat that qualifier on every +operation, and still leave generic concepts such as queryables competing with +the facade's existing tabular OGC helpers. These STAC names were developed on an +unreleased feature branch, so they have no published compatibility contract. + +The analysis-ready ratings workflow has a different role: callers request a +monitoring location and receive parsed rating tables rather than navigating raw +STAC documents. + +Decision +-------- + +Expose raw catalog operations only through the public ``waterdata.stac`` +namespace: ``get_catalog``, ``get_conformance``, ``get_collections``, +``get_collection``, ``get_items``, ``get_item``, ``get_queryables``, and +``search``. Do not export flat prefixed aliases from ``waterdata`` because those +draft names were never released. + +Keep ``waterdata.get_ratings`` on the main facade as the analysis-ready ratings +API. Keep the existing tabular ``waterdata.get_queryables`` distinct from raw +``waterdata.stac.get_queryables``. + +Consequences +------------ + +- Related raw STAC operations are discoverable together without widening the + main facade by eight prefixed names. +- Call sites make the raw STAC boundary explicit. +- ``waterdata`` publicly exports one namespace object in place of eight + functions. +- Code written against the unreleased flat draft names must adopt the nested + namespace; no deprecation period applies. +- Documentation and executable public-surface snapshots must be updated when + the nested STAC surface changes. + +Compliance +---------- + +``tests/contracts/public_api_test.py`` freezes the ``waterdata`` export surface. +``tests/waterdata_stac_test.py`` freezes ``waterdata.stac.__all__``, verifies all +eight nested operations, and rejects every former flat draft name. The full +Sphinx build exercises the documented namespace and example notebook. diff --git a/docs/source/architecture/decisions/index.rst b/docs/source/architecture/decisions/index.rst index ad468728a..6f6e0198e 100644 --- a/docs/source/architecture/decisions/index.rst +++ b/docs/source/architecture/decisions/index.rst @@ -28,4 +28,5 @@ records sequentially. 0009-layered-configuration 0010-adapter-scoped-settings 0011-configuration-profiles + 0012-stac-namespace template diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index a6d9e4568..ef738e2f4 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -71,9 +71,14 @@ Public service facades compatibility facade over collection-family modules: ``time_series``, ``metadata``, ``measurements``, ``reference``, ``samples``, and ``cql``. Focused modules own ratings, nearest-value selection, statistics execution, - shared service policy, and type vocabularies. Internal modules import - protocol helpers from their canonical OGC modules rather than re-exporting - them through Water Data utilities. + shared service policy, and type vocabularies. Raw STAC catalog operations + are grouped under the public ``waterdata.stac`` namespace rather than + exported as flat prefixed helpers; analysis-ready ``waterdata.get_ratings`` + remains on the main facade. Internal modules import protocol helpers from + their canonical OGC modules rather than re-exporting them through Water Data + utilities. + + This public boundary is recorded in :doc:`decisions/0012-stac-namespace`. ``dataretrieval.ngwmn`` NGWMN facade. Its only OGC dependency is the public OGC facade, which it @@ -235,10 +240,10 @@ service into one return shape: - ``waterdata.get_ratings`` returns a mapping of feature IDs to parsed rating ``DataFrame`` objects by default, or the raw STAC feature list when downloads are disabled. -- Water Data STAC catalog helpers return ``(dict, BaseMetadata)``. The mapping - preserves each upstream Catalog, Collection, GeoJSON ItemCollection, Item, or - JSON Schema document and its standard links without flattening unlike shapes - into one table. +- ``waterdata.stac`` catalog helpers return ``(dict, BaseMetadata)``. The + mapping preserves each upstream Catalog, Collection, GeoJSON ItemCollection, + Item, or JSON Schema document and its standard links without flattening unlike + shapes into one table. - NLDI navigation functions return ``GeoDataFrame`` objects directly, or raw GeoJSON-like dictionaries when ``as_json=True``; they do not add a metadata tuple. diff --git a/tests/contracts/public_api_test.py b/tests/contracts/public_api_test.py index 6d2a398f2..8b5f5808f 100644 --- a/tests/contracts/public_api_test.py +++ b/tests/contracts/public_api_test.py @@ -26,6 +26,7 @@ "SERVICES", "WATERDATA_SERVICES", "parallel_chunks", + "stac", "get_channel", "get_codes", "get_combined_metadata", @@ -42,19 +43,11 @@ "get_queryables", "get_ratings", "get_reference_table", - "get_stac_catalog", - "get_stac_collection", - "get_stac_collections", - "get_stac_conformance", - "get_stac_item", - "get_stac_items", - "get_stac_queryables", "get_samples", "get_samples_summary", "get_stats_date_range", "get_stats_por", "get_time_series_metadata", - "search_stac", ] #: Arguments a caller must supply positionally or by keyword. Adding an entry diff --git a/tests/waterdata_stac_test.py b/tests/waterdata_stac_test.py index d87e26828..a212fdef4 100644 --- a/tests/waterdata_stac_test.py +++ b/tests/waterdata_stac_test.py @@ -12,6 +12,32 @@ from dataretrieval.waterdata import WaterdataConfiguration _STAC = "https://api.waterdata.usgs.gov/stac/v0" +_STAC_EXPORTS = [ + "get_catalog", + "get_collection", + "get_collections", + "get_conformance", + "get_item", + "get_items", + "get_queryables", + "search", +] +_OLD_FLAT_EXPORTS = [ + "get_stac_catalog", + "get_stac_collection", + "get_stac_collections", + "get_stac_conformance", + "get_stac_item", + "get_stac_items", + "get_stac_queryables", + "search_stac", +] + + +def test_stac_api_is_nested_without_flat_aliases(): + assert waterdata.stac.__all__ == _STAC_EXPORTS + assert all(hasattr(waterdata.stac, name) for name in _STAC_EXPORTS) + assert all(not hasattr(waterdata, name) for name in _OLD_FLAT_EXPORTS) def test_catalog_conformance_and_queryables_resources(httpx_mock): @@ -31,10 +57,10 @@ def test_catalog_conformance_and_queryables_resources(httpx_mock): httpx_mock.add_response(method="GET", url=url, json=document) calls = [ - waterdata.get_stac_catalog(), - waterdata.get_stac_conformance(), - waterdata.get_stac_queryables(), - waterdata.get_stac_queryables("ratings"), + waterdata.stac.get_catalog(), + waterdata.stac.get_conformance(), + waterdata.stac.get_queryables(), + waterdata.stac.get_queryables("ratings"), ] for (document, metadata), expected, url in zip(calls, documents, urls, strict=True): @@ -56,8 +82,8 @@ def test_collection_and_item_resource_paths_escape_identifiers(httpx_mock): json=item, ) - assert waterdata.get_stac_collection("rating curves")[0] == collection - assert waterdata.get_stac_item("rating curves", "USGS/01234567.exsa.rdb")[0] == item + assert waterdata.stac.get_collection("rating curves")[0] == collection + assert waterdata.stac.get_item("rating curves", "USGS/01234567.exsa.rdb")[0] == item def test_get_collections_forwards_every_advertised_parameter(httpx_mock): @@ -67,7 +93,7 @@ def test_get_collections_forwards_every_advertised_parameter(httpx_mock): json={}, ) - waterdata.get_stac_collections( + waterdata.stac.get_collections( bbox=[-95, 40, -92, 42], datetime="2026-01-01/..", limit=25, @@ -108,7 +134,7 @@ def test_get_collection_items_forwards_every_advertised_parameter(httpx_mock): json={"features": []}, ) - waterdata.get_stac_items( + waterdata.stac.get_items( "ratings", limit=50, bbox=[-95, 40, -92, 42], @@ -144,7 +170,7 @@ def test_get_search_serializes_get_parameters(httpx_mock): json={"features": []}, ) - document, _ = waterdata.search_stac( + document, _ = waterdata.stac.search( method="GET", collections=["ratings", "other"], ids=["one", "two"], @@ -185,7 +211,7 @@ def test_post_search_sends_native_json_parameters(httpx_mock): httpx_mock.add_response(method="POST", url=f"{_STAC}/search", json={"features": []}) cql = {"op": "=", "args": [{"property": "file_type"}, "exsa"]} - waterdata.search_stac( + waterdata.stac.search( method="POST", collections="ratings", ids="USGS-01104475.exsa.rdb", @@ -225,9 +251,9 @@ def test_post_search_sends_native_json_parameters(httpx_mock): def test_search_rejects_invalid_method_and_post_only_conf(): with pytest.raises(ValueError, match="GET or POST"): - waterdata.search_stac(method="PUT") # type: ignore[arg-type] + waterdata.stac.search(method="PUT") # type: ignore[arg-type] with pytest.raises(ValueError, match="conf.*POST"): - waterdata.search_stac(method="GET", conf={}) + waterdata.stac.search(method="GET", conf={}) def test_stac_uses_typed_errors_and_waterdata_retries(httpx_mock, monkeypatch): @@ -242,13 +268,13 @@ def test_stac_uses_typed_errors_and_waterdata_retries(httpx_mock, monkeypatch): json={"type": "Collection", "id": "recovered"}, ) - assert waterdata.get_stac_collection("missing")[0]["id"] == "recovered" + assert waterdata.stac.get_collection("missing")[0]["id"] == "recovered" httpx_mock.add_response( method="GET", url=f"{_STAC}/collections/absent", status_code=404 ) with pytest.raises(HTTPError) as excinfo: - waterdata.get_stac_collection("absent") + waterdata.stac.get_collection("absent") assert not isinstance(excinfo.value, ServiceUnavailable) @@ -259,9 +285,9 @@ def test_stac_honors_redirect_and_scopes_api_key(httpx_mock, monkeypatch): httpx_mock.add_response(method="GET", url=f"{_STAC}/", json={}) with dataretrieval.configure(WaterdataConfiguration(base_url=mirror)): - waterdata.get_stac_catalog() + waterdata.stac.get_catalog() with dataretrieval.configure(Configuration(api_key="not-a-secret")): - waterdata.get_stac_catalog() + waterdata.stac.get_catalog() redirected, direct = httpx_mock.get_requests() assert "X-Api-Key" not in redirected.headers