From 4a01ef39e166be2a2706e81289a2da5a125ee540 Mon Sep 17 00:00:00 2001 From: "MD. Mohiuddin Ahmed" Date: Mon, 5 Oct 2026 01:31:15 +0600 Subject: [PATCH 1/3] Add Pangeo CMIP6 ArchiveIndex accessor Adds packages/third_party/pangeo_site_archive with PangeoCMIP6, an ArchiveIndex that reads CMIP6 variables from the Pangeo ARCO cloud archive, plus offline unit tests, CI install/coverage entries and a docs section. Closes #284. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Unqqtp91DWVC8ro9Y724iN --- .github/workflows/python-app.yml | 2 +- docs/data.md | 14 ++ .../third_party/pangeo_site_archive/README.md | 3 + .../pangeo_site_archive/pyproject.toml | 43 +++++ .../src/site_archive_pangeo/__init__.py | 21 +++ .../src/site_archive_pangeo/pangeo_cmip6.py | 175 ++++++++++++++++++ .../tests/data/mini_catalog.csv | 8 + .../tests/test_pangeo_cmip6.py | 149 +++++++++++++++ requirements_cicd.txt | 1 + 9 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 packages/third_party/pangeo_site_archive/README.md create mode 100644 packages/third_party/pangeo_site_archive/pyproject.toml create mode 100644 packages/third_party/pangeo_site_archive/src/site_archive_pangeo/__init__.py create mode 100644 packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py create mode 100644 packages/third_party/pangeo_site_archive/tests/data/mini_catalog.csv create mode 100644 packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 3342d752..29e48cb9 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -40,7 +40,7 @@ jobs: pip install -e packages/bundled_models/fourcastnext pip install -e .[test] - pytest -m="not noci" --cov=packages/data --cov=packages/utils --cov=packages/pipeline --cov=packages/training --cov=packages/zoo --cov=packages/bundled_models/fourcastnext --ignore=packages/nci_site_archive + pytest -m="not noci" --cov=packages/data --cov=packages/utils --cov=packages/pipeline --cov=packages/training --cov=packages/zoo --cov=packages/bundled_models/fourcastnext --cov=packages/third_party/pangeo_site_archive --ignore=packages/nci_site_archive - name: Coveralls GitHub Action uses: coverallsapp/github-action@v2.3.0 with: diff --git a/docs/data.md b/docs/data.md index f5a3cbb8..8e15c3d8 100644 --- a/docs/data.md +++ b/docs/data.md @@ -40,6 +40,20 @@ The package `site_archive_nci` (which is present in some of the tutorials) is th Additionally, you can explore the [geonetwork](https://geonetwork.nci.org.au/geonetwork/srv/eng/catalog.search#/home) to find additional data sets of interest which you might like to utilise by writing a custom PyEarthTools data accessor. +### Accessing CMIP6 climate model data from the cloud + +The package `site_archive_pangeo` (in `packages/third_party/pangeo_site_archive`) reads CMIP6 model output from the [Pangeo CMIP6 cloud archive](https://pangeo-data.github.io/pangeo-cmip6-cloud/accessing_data.html) on Google Cloud. No account, download or HPC access is needed; the data is opened lazily, so only the times you request are transferred. + +```python +import pyearthtools.data +from site_archive_pangeo import PangeoCMIP6 + +index = PangeoCMIP6(["tas", "pr"], source_id="CanESM5", experiment_id="historical") +ds = index.retrieve(pyearthtools.data.Petdt("2000-01")) +``` + +Each variable is looked up in the Pangeo catalogue and the newest version is used. If a variable exists on several grids, pass `grid_label` to choose one. Scalar coordinates that differ between variables (for example `height`, which is 2 m for `tas` but 10 m for `uas`) are dropped when the variables are combined. + ### Connecting PyEarthTools to a new dataset in any HPC facility by coding a new data accessor For on-disk data access, you will need to create a new accessor based on the [`pyearthtools.data.ArchiveIndex`](project:./api/data/data_api.md#pyearthtools.data.indexes.ArchiveIndex) class (or [`pyearthtools.data.Index`](project:./api/data/data_api.md#pyearthtools.data.indexes.Index) for some use cases). Additional instructions for this still need to be written. In the meantime, refer to the [NCI site archive source code](https://github.com/ACCESS-Community-Hub/PyEarthTools/tree/develop/packages/nci_site_archive) for examples. diff --git a/packages/third_party/pangeo_site_archive/README.md b/packages/third_party/pangeo_site_archive/README.md new file mode 100644 index 00000000..4a6dd9bf --- /dev/null +++ b/packages/third_party/pangeo_site_archive/README.md @@ -0,0 +1,3 @@ +# PyEarthTools archive: Pangeo CMIP6 + +Accessor for the Pangeo analysis-ready, cloud-optimised (ARCO) CMIP6 data, hosted as Zarr stores on Google Cloud. diff --git a/packages/third_party/pangeo_site_archive/pyproject.toml b/packages/third_party/pangeo_site_archive/pyproject.toml new file mode 100644 index 00000000..86db42c9 --- /dev/null +++ b/packages/third_party/pangeo_site_archive/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools>=61", "setuptools-scm>=8.0"] + +[project] +name = "PyEarthTools-archive-Pangeo" +description = "PyEarthTools archive for the Pangeo CMIP6 analysis-ready, cloud-optimised (ARCO) data" +requires-python = ">=3.11, <3.14" +keywords = ["pyearthtools", "cmip6", "pangeo", "zarr", "climate"] +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "pyearthtools-data>=0.6.3", + "pandas", + "xarray", + "zarr", + "gcsfs", +] +dynamic = ["version", "readme"] + +[project.optional-dependencies] +dev = ["pytest>=7.0", "pytest-cov", "black==26.3.1", "ruff"] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.dynamic] +readme = {file = ["README.md"], content-type = "text/markdown"} + +[project.urls] +homepage = "https://pyearthtools.readthedocs.io/en/latest/" +documentation = "https://pyearthtools.readthedocs.io/en/latest/" +repository = "https://github.com/ACCESS-Community-Hub/PyEarthTools" + +[tool.isort] +profile = "black" + +[tool.black] +line-length = 120 diff --git a/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/__init__.py b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/__init__.py new file mode 100644 index 00000000..247cec44 --- /dev/null +++ b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/__init__.py @@ -0,0 +1,21 @@ +""" +Pangeo CMIP6 archive for PyEarthTools. + +| Name | Description | +| :--- | ----: | +| [PangeoCMIP6][site_archive_pangeo.PangeoCMIP6] | CMIP6 analysis-ready, cloud-optimised (ARCO) data | +""" + +from pyearthtools.data.archive import register_archive + +import site_archive_pangeo +from site_archive_pangeo.pangeo_cmip6 import PangeoCMIP6 + +register_archive("pangeo")(site_archive_pangeo) + +try: + from ._version import __version__ # type: ignore +except ImportError: + __version__ = "999" + +__all__ = ["PangeoCMIP6"] diff --git a/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py new file mode 100644 index 00000000..f8a32efd --- /dev/null +++ b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py @@ -0,0 +1,175 @@ +""" +Pangeo CMIP6 accessor. + +Reads the Pangeo analysis-ready, cloud-optimised (ARCO) CMIP6 data, which is published as one Zarr store +per variable and described by a single catalog CSV. +""" + +import os +from typing import Any, ClassVar + +import pandas +import pyearthtools.data +import xarray +from pyearthtools.data import Petdt +from pyearthtools.data.archive import register_archive +from pyearthtools.data.indexes import ArchiveIndex +from pyearthtools.data.transforms import Transform, TransformCollection + +DEFAULT_CATALOG = "https://storage.googleapis.com/cmip6/pangeo-cmip6.csv" + +_CATALOG_COLUMNS = [ + "source_id", + "experiment_id", + "member_id", + "table_id", + "variable_id", + "grid_label", + "zstore", + "version", +] + + +@register_archive( + "PangeoCMIP6", + sample_kwargs={"variables": ["tas"], "source_id": "CanESM5", "experiment_id": "historical"}, +) +class PangeoCMIP6(ArchiveIndex): + """ + Data accessor for the Pangeo CMIP6 analysis-ready, cloud-optimised (ARCO) data. + + A catalog CSV lists the Zarr store of every variable of every model run. This accessor selects one + model run (``source_id``, ``experiment_id``, ``member_id``, ``table_id``) and opens the store of each + requested variable lazily, so only the parts of the data that are used are downloaded. + + A store holds all times, so ``filesystem`` ignores the requested time and the time is selected after + loading. + + Examples: + >>> index = PangeoCMIP6(["tas"], source_id="CanESM5", experiment_id="historical") + >>> index.retrieve("2000-01") + """ + + _OPEN_ARGS: ClassVar[dict[str, Any]] = {"consolidated": True, "storage_options": {"token": "anon"}} + + def __init__( + self, + variables: list[str] | str, + *, + source_id: str, + experiment_id: str, + member_id: str = "r1i1p1f1", + table_id: str = "Amon", + grid_label: str | None = None, + catalog: str | os.PathLike = DEFAULT_CATALOG, + transforms: Transform | TransformCollection | None = None, + ): + """ + Set up the Pangeo CMIP6 accessor. + + The catalog is not read here, only when data is first requested. + + Args: + variables: Variables to retrieve, e.g. ``["tas", "pr"]``. + source_id: Model name, e.g. ``"CanESM5"``. + experiment_id: Experiment, e.g. ``"historical"`` or ``"ssp585"``. + member_id: Ensemble member. Defaults to ``"r1i1p1f1"``. + table_id: Frequency and realm table, e.g. ``"Amon"`` for monthly atmosphere. Defaults to ``"Amon"``. + grid_label: Grid, e.g. ``"gn"`` (native) or ``"gr"`` (regridded). Needed when a variable exists + on more than one grid. Defaults to None. + catalog: Location of the Pangeo catalog CSV, a URL or a local path. + transforms: Extra transforms to apply after loading. + + Raises: + ValueError: If no variables are given. + """ + self._variables = [variables] if isinstance(variables, str) else list(variables) + if not self._variables: + raise ValueError("At least one variable is required.") + + self._query = { + "source_id": source_id, + "experiment_id": experiment_id, + "member_id": member_id, + "table_id": table_id, + } + self._grid_label = grid_label + self._catalog_src = catalog + self._catalog = None # read lazily, see _load_catalog + + extra_transforms = TransformCollection() if transforms is None else TransformCollection(transforms) + super().__init__( + transforms=TransformCollection([pyearthtools.data.transforms.variables.Trim(self._variables)]) + + extra_transforms, + ) + self.record_initialisation() + + def _load_catalog(self) -> pandas.DataFrame: + """Read the catalog once, keeping only the rows of the requested variables and model run.""" + if self._catalog is None: + catalog = pandas.read_csv(self._catalog_src, usecols=_CATALOG_COLUMNS) + catalog = catalog[catalog["variable_id"].isin(self._variables)] + for column, value in self._query.items(): + catalog = catalog[catalog[column] == value] + self._catalog = catalog.reset_index(drop=True) + return self._catalog + + def filesystem(self, querytime: str | Petdt) -> dict[str, str]: + """ + Find the Zarr store of each variable. + + Args: + querytime: Ignored, as a store holds all times. + + Raises: + ValueError: If a variable has no store, or exists on several grids and no ``grid_label`` was given. + + Returns: + Store URL of each variable, using the latest version in the catalog. + """ + catalog = self._load_catalog() + stores = {} + for variable in self._variables: + rows = catalog[catalog["variable_id"] == variable] + if self._grid_label is not None: + rows = rows[rows["grid_label"] == self._grid_label] + if rows.empty: + raise ValueError(f"No Pangeo CMIP6 store for variable {variable!r} matching {self._query}") + grids = sorted(rows["grid_label"].unique()) + if len(grids) > 1: + raise ValueError( + f"Variable {variable!r} exists on several grids {grids}; pass grid_label to choose one." + ) + stores[variable] = str(rows.sort_values("version").iloc[-1]["zstore"]) + return stores + + def load(self, files: dict[str, str], **kwargs) -> xarray.Dataset: + """ + Open the store of each variable lazily, keep only that variable, and merge them. + + Args: + files: Store URL of each variable, as found by ``filesystem``. + **kwargs: Passed to ``xarray.open_zarr``. + + Returns: + Dataset holding all times of all the variables. Scalar coordinates that differ between the variables + are dropped, e.g. ``height``, which is 2 m for ``tas`` but 10 m for ``uas``. + """ + open_args = {**self._OPEN_ARGS, **kwargs} + datasets = [xarray.open_zarr(url, **open_args)[[variable]] for variable, url in files.items()] + return xarray.merge(datasets, join="exact", compat="minimal") + + def exists(self, *args, **kwargs) -> bool: + """Whether a store is found for every variable. Stores are URLs, so they are not checked on disk.""" + try: + self.search(*args) + except ValueError: + return False + return True + + @property + def _desc_(self): + return { + "singleline": "Pangeo CMIP6 analysis-ready, cloud-optimised data accessor", + "Documentation": "https://pangeo-data.github.io/pangeo-cmip6-cloud/accessing_data.html", + } diff --git a/packages/third_party/pangeo_site_archive/tests/data/mini_catalog.csv b/packages/third_party/pangeo_site_archive/tests/data/mini_catalog.csv new file mode 100644 index 00000000..5a3fa5c2 --- /dev/null +++ b/packages/third_party/pangeo_site_archive/tests/data/mini_catalog.csv @@ -0,0 +1,8 @@ +activity_id,institution_id,source_id,experiment_id,member_id,table_id,variable_id,grid_label,zstore,dcpp_init_year,version +CMIP,TEST-INST,TEST-MODEL,historical,r1i1p1f1,Amon,tas,gn,gs://test-bucket/tas/gn/v20190101/,,20190101 +CMIP,TEST-INST,TEST-MODEL,historical,r1i1p1f1,Amon,tas,gn,gs://test-bucket/tas/gn/v20200101/,,20200101 +CMIP,TEST-INST,TEST-MODEL,historical,r1i1p1f1,Amon,pr,gn,gs://test-bucket/pr/gn/v20190101/,,20190101 +CMIP,TEST-INST,TEST-MODEL,historical,r1i1p1f1,Amon,psl,gn,gs://test-bucket/psl/gn/v20190101/,,20190101 +CMIP,TEST-INST,TEST-MODEL,historical,r1i1p1f1,Amon,psl,gr,gs://test-bucket/psl/gr/v20190101/,,20190101 +ScenarioMIP,TEST-INST,TEST-MODEL,ssp585,r1i1p1f1,Amon,tas,gn,gs://test-bucket/ssp/tas/gn/v20190101/,,20190101 +CMIP,TEST-INST,TEST-MODEL,historical,r1i1p1f1,Amon,uas,gn,gs://test-bucket/uas/gn/v20190101/,,20190101 diff --git a/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py b/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py new file mode 100644 index 00000000..1adcbab4 --- /dev/null +++ b/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py @@ -0,0 +1,149 @@ +import pathlib + +import pandas +import pyearthtools.data +import pytest +import site_archive_pangeo +import xarray +from pyearthtools.data import Petdt +from site_archive_pangeo import PangeoCMIP6 + +CATALOG = str(pathlib.Path(__file__).parent / "data" / "mini_catalog.csv") +TIME = Petdt("2000-02") +TAS_URL = "gs://test-bucket/tas/gn/v20200101/" +PR_URL = "gs://test-bucket/pr/gn/v20190101/" + + +def _index(variables, **kwargs): + kwargs = {"source_id": "TEST-MODEL", "experiment_id": "historical", **kwargs} + return PangeoCMIP6(variables, catalog=CATALOG, **kwargs) + + +def _fake_open_zarr(seen): + """Stand-in for xarray.open_zarr, returning a tiny dataset for each store URL.""" + times = pandas.date_range("2000-01-01", periods=3, freq="MS") + + def fake(url, **kwargs): + seen[url] = kwargs + name = url.rstrip("/").split("/")[-3] + return xarray.Dataset( + {name: ("time", [1.0, 2.0, 3.0]), "time_bnds": ("time", [0.0, 0.0, 0.0])}, + coords={"time": times}, + ) + + return fake + + +def test_registered_in_archive(): + assert pyearthtools.data.archive.PangeoCMIP6 is PangeoCMIP6 + assert pyearthtools.data.archive.pangeo is site_archive_pangeo + + +def test_sample_construction_needs_no_network(monkeypatch): + def no_network(*args, **kwargs): + raise AssertionError("the catalog must not be read at construction") + + monkeypatch.setattr(pandas, "read_csv", no_network) + assert isinstance(PangeoCMIP6.sample(), PangeoCMIP6) + + +def test_description(): + assert "Documentation" in _index(["tas"])._desc_ + + +def test_filesystem_picks_latest_version(): + assert _index(["tas", "pr"]).filesystem(TIME) == {"tas": TAS_URL, "pr": PR_URL} + + +def test_single_variable_as_string(): + assert _index("pr").filesystem(TIME) == {"pr": PR_URL} + + +def test_experiment_is_filtered(): + stores = _index(["tas"], experiment_id="ssp585").filesystem(TIME) + assert stores == {"tas": "gs://test-bucket/ssp/tas/gn/v20190101/"} + + +def test_catalog_is_read_once(monkeypatch): + calls = [] + real_read_csv = pandas.read_csv + + def counting_read_csv(*args, **kwargs): + calls.append(args) + return real_read_csv(*args, **kwargs) + + monkeypatch.setattr(pandas, "read_csv", counting_read_csv) + index = _index(["tas"]) + index.filesystem(TIME) + index.filesystem(TIME) + assert len(calls) == 1 + + +def test_no_variables_rejected(): + with pytest.raises(ValueError, match="variable"): + _index([]) + + +def test_unknown_variable_raises(): + with pytest.raises(ValueError, match="nonsense"): + _index(["nonsense"]).filesystem(TIME) + + +def test_several_grids_need_a_grid_label(): + with pytest.raises(ValueError, match="grid_label"): + _index(["psl"]).filesystem(TIME) + assert _index(["psl"], grid_label="gr").filesystem(TIME) == {"psl": "gs://test-bucket/psl/gr/v20190101/"} + with pytest.raises(ValueError, match="psl"): + _index(["psl"], grid_label="xx").filesystem(TIME) + + +def test_exists_does_not_check_the_disk(): + assert _index(["tas"]).exists(TIME) is True + assert _index(["nonsense"]).exists(TIME) is False + + +def test_get_wraps_lookup_errors(): + with pytest.raises(pyearthtools.data.DataNotFoundError): + _index(["nonsense"]).get(TIME) + + +def test_load_merges_one_store_per_variable(monkeypatch): + seen = {} + monkeypatch.setattr(xarray, "open_zarr", _fake_open_zarr(seen)) + index = _index(["tas", "pr"]) + + dataset = index.load(index.filesystem(TIME), chunks={}) + + assert set(dataset.data_vars) == {"tas", "pr"} + assert set(seen) == {TAS_URL, PR_URL} + for kwargs in seen.values(): + assert kwargs["consolidated"] is True + assert kwargs["storage_options"] == {"token": "anon"} + assert kwargs["chunks"] == {} + + +def test_retrieve_selects_time_and_keeps_requested_variables(monkeypatch): + monkeypatch.setattr(xarray, "open_zarr", _fake_open_zarr({})) + + dataset = _index(["tas"]).retrieve("2000-02") + + assert list(dataset.data_vars) == ["tas"] + assert dataset["tas"].values.tolist() == [2.0] + + +def test_load_drops_conflicting_scalar_coordinates(monkeypatch): + """tas (2 m) and uas (10 m) carry different scalar height coordinates, which must not stop the merge.""" + heights = {"tas": 2.0, "uas": 10.0} + times = pandas.date_range("2000-01-01", periods=3, freq="MS") + + def fake(url, **kwargs): + name = url.rstrip("/").split("/")[-3] + return xarray.Dataset({name: ("time", [1.0, 2.0, 3.0])}, coords={"time": times, "height": heights[name]}) + + monkeypatch.setattr(xarray, "open_zarr", fake) + index = _index(["tas", "uas"]) + + dataset = index.load(index.filesystem(TIME)) + + assert set(dataset.data_vars) == {"tas", "uas"} + assert "height" not in dataset.coords diff --git a/requirements_cicd.txt b/requirements_cicd.txt index 1d7fc87e..256afc17 100644 --- a/requirements_cicd.txt +++ b/requirements_cicd.txt @@ -5,4 +5,5 @@ -e packages/tutorial -e packages/zoo -e packages/bundled_models/fourcastnext +-e packages/third_party/pangeo_site_archive -e .[test] From 5ae0b9651a59cec4f9e90a05e8d2292ba10a5a36 Mon Sep 17 00:00:00 2001 From: "MD. Mohiuddin Ahmed" Date: Sat, 10 Oct 2026 03:02:21 +0600 Subject: [PATCH 2/3] Add PangeoCMIP6.catalog() static method Returns the Pangeo CMIP6 catalogue as a pandas.DataFrame so users can discover available data. Includes a unit test. --- .../src/site_archive_pangeo/pangeo_cmip6.py | 15 ++++++++++++++- .../tests/test_pangeo_cmip6.py | 9 +++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py index f8a32efd..94719e62 100644 --- a/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py +++ b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py @@ -107,7 +107,7 @@ def __init__( def _load_catalog(self) -> pandas.DataFrame: """Read the catalog once, keeping only the rows of the requested variables and model run.""" if self._catalog is None: - catalog = pandas.read_csv(self._catalog_src, usecols=_CATALOG_COLUMNS) + catalog = self.catalog(self._catalog_src) catalog = catalog[catalog["variable_id"].isin(self._variables)] for column, value in self._query.items(): catalog = catalog[catalog[column] == value] @@ -159,6 +159,19 @@ def load(self, files: dict[str, str], **kwargs) -> xarray.Dataset: datasets = [xarray.open_zarr(url, **open_args)[[variable]] for variable, url in files.items()] return xarray.merge(datasets, join="exact", compat="minimal") + @staticmethod + def catalog(catalog: str | os.PathLike = DEFAULT_CATALOG) -> pandas.DataFrame: + """Return the Pangeo CMIP6 catalogue, so the available data can be discovered. + + Args: + catalog: CSV catalogue to read. Defaults to the public Pangeo CMIP6 catalogue. + + Returns: + DataFrame with one row per variable of one model run, with the columns `source_id`, + `experiment_id`, `member_id`, `table_id`, `variable_id`, `grid_label`, `zstore` and `version`. + """ + return pandas.read_csv(catalog, usecols=_CATALOG_COLUMNS) + def exists(self, *args, **kwargs) -> bool: """Whether a store is found for every variable. Stores are URLs, so they are not checked on disk.""" try: diff --git a/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py b/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py index 1adcbab4..e53dac58 100644 --- a/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py +++ b/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py @@ -147,3 +147,12 @@ def fake(url, **kwargs): assert set(dataset.data_vars) == {"tas", "uas"} assert "height" not in dataset.coords + + +def test_catalog_lists_available_data(): + catalog = PangeoCMIP6.catalog(CATALOG) + + assert isinstance(catalog, pandas.DataFrame) + assert {"source_id", "experiment_id", "variable_id", "zstore"} <= set(catalog.columns) + assert set(catalog["variable_id"]) == {"tas", "pr", "psl", "uas"} + assert len(catalog) == 7 From 795f14b768dab033c1820097b1de0dfdf9533fd5 Mon Sep 17 00:00:00 2001 From: "MD. Mohiuddin Ahmed" Date: Thu, 15 Oct 2026 00:59:36 +0600 Subject: [PATCH 3/3] Make querytime explicit in PangeoCMIP6.exists --- .../src/site_archive_pangeo/pangeo_cmip6.py | 4 ++-- .../pangeo_site_archive/tests/test_pangeo_cmip6.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py index 94719e62..cabdff2a 100644 --- a/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py +++ b/packages/third_party/pangeo_site_archive/src/site_archive_pangeo/pangeo_cmip6.py @@ -172,10 +172,10 @@ def catalog(catalog: str | os.PathLike = DEFAULT_CATALOG) -> pandas.DataFrame: """ return pandas.read_csv(catalog, usecols=_CATALOG_COLUMNS) - def exists(self, *args, **kwargs) -> bool: + def exists(self, querytime: str | Petdt) -> bool: """Whether a store is found for every variable. Stores are URLs, so they are not checked on disk.""" try: - self.search(*args) + self.search(querytime) except ValueError: return False return True diff --git a/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py b/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py index e53dac58..25290800 100644 --- a/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py +++ b/packages/third_party/pangeo_site_archive/tests/test_pangeo_cmip6.py @@ -102,6 +102,11 @@ def test_exists_does_not_check_the_disk(): assert _index(["nonsense"]).exists(TIME) is False +def test_exists_accepts_querytime_keyword(): + assert _index(["tas"]).exists(querytime=TIME) is True + assert _index(["nonsense"]).exists(querytime=TIME) is False + + def test_get_wraps_lookup_errors(): with pytest.raises(pyearthtools.data.DataNotFoundError): _index(["nonsense"]).get(TIME)