diff --git a/Changelog b/Changelog index fb7631133..e00fdd361 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,8 @@ + version 1.7.6 (not yet released) + ================================ + * Allow mode flag (`extra_mode_flag` Dataset kwarg) to be passed to nc_open() to support + additional dispatch layers available via the NetCDF Expansion Pack (NEP) (issue #1490). + version 1.7.5.1.post1 (tag v1.7.5.1.post1rel) ============================================= * fix installation of plugins into linux wheels. diff --git a/MANIFEST.in b/MANIFEST.in index 38a3c86a5..f8188bf58 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,6 +10,7 @@ include examples/README.md exclude examples/data include test/*py include test/*nc +include test/*grb2 include src/netCDF4/__init__.py include src/netCDF4/_netCDF4.pyx exclude src/netCDF4/_netCDF4.c diff --git a/src/netCDF4/__init__.pyi b/src/netCDF4/__init__.pyi index bb0c30831..9fd8b3eff 100644 --- a/src/netCDF4/__init__.pyi +++ b/src/netCDF4/__init__.pyi @@ -187,6 +187,16 @@ is_native_big: bool default_encoding: Final = "utf-8" unicode_error: Final = "replace" default_fillvals: dict[str, int | float | str] +NEP_UDF_GEOTIFF_BIGTIFF: Final = 0x0040 +NEP_UDF_GEOTIFF_STANDARD: Final = 0x0080 +NEP_UDF_GRIB2: Final = 0x10000 +NEP_UDF_FITS: Final = 0x80000 +NEP_UDF_CDF: Final = 0x100000 +NEP_UDF_PDS4: Final = 0x200000 +NEP_UDF_DICOM: Final = 0x400000 +NEP_UDF_PDB: Final = 0x800000 +NEP_UDF_MMCIF: Final = 0x1000000 +NEP_UDF_NEXTCDF4: Final = 0x2000000 # date2index, date2num, and num2date are actually provided by cftime and if stubs for # cftime are completed these should be removed. @@ -235,6 +245,7 @@ class Dataset: comm: Any = None, info: Any = None, auto_complex: bool = False, + extra_mode_flag: int | None = None, **kwargs: Any, ): ... @property diff --git a/src/netCDF4/_netCDF4.pyi b/src/netCDF4/_netCDF4.pyi index d0bdb389f..7b865b052 100644 --- a/src/netCDF4/_netCDF4.pyi +++ b/src/netCDF4/_netCDF4.pyi @@ -8,6 +8,16 @@ from . import ( Group, MFDataset, MFTime, + NEP_UDF_CDF, + NEP_UDF_DICOM, + NEP_UDF_FITS, + NEP_UDF_GEOTIFF_BIGTIFF, + NEP_UDF_GEOTIFF_STANDARD, + NEP_UDF_GRIB2, + NEP_UDF_MMCIF, + NEP_UDF_NEXTCDF4, + NEP_UDF_PDB, + NEP_UDF_PDS4, NetCDF4MissingFeatureException, Variable, VLType, diff --git a/src/netCDF4/_netCDF4.pyx b/src/netCDF4/_netCDF4.pyx index c9e46e4ca..60f7a755f 100644 --- a/src/netCDF4/_netCDF4.pyx +++ b/src/netCDF4/_netCDF4.pyx @@ -1,4 +1,4 @@ -"""Version 1.7.5.1.post1 +"""Version 1.7.6 ------------- # Introduction @@ -1283,7 +1283,7 @@ import sys import functools from typing import Union -__version__ = "1.7.5.1.post1" +__version__ = "1.7.6" # Initialize numpy import posixpath @@ -1612,6 +1612,23 @@ if _nclibversion >= Version("4.6.2"): else: # prior to 4.6.2 this flag doesn't work, so make the same as NC_DISKLESS NC_PERSIST = NC_DISKLESS +# mode flags for the user-defined formats (UDFs) provided by the +# NetCDF Expansion Pack (NEP), names match those in nep.h. +# Used with the extra_mode_flag Dataset kwarg (requires netcdf-c >= 4.10.0). +NEP_UDF_GEOTIFF_BIGTIFF = 0x0040 # NC_UDF0 +NEP_UDF_GEOTIFF_STANDARD = 0x0080 # NC_UDF1 +NEP_UDF_GRIB2 = 0x10000 # NC_UDF2 +NEP_UDF_FITS = 0x80000 # NC_UDF3 +NEP_UDF_CDF = 0x100000 # NC_UDF4 +NEP_UDF_PDS4 = 0x200000 # NC_UDF5 +NEP_UDF_DICOM = 0x400000 # NC_UDF6 +NEP_UDF_PDB = 0x800000 # NC_UDF7 +NEP_UDF_MMCIF = 0x1000000 # NC_UDF8 +NEP_UDF_NEXTCDF4 = 0x2000000 # NC_UDF9 +_nep_udf_flags = (NEP_UDF_GEOTIFF_BIGTIFF, NEP_UDF_GEOTIFF_STANDARD, + NEP_UDF_GRIB2, NEP_UDF_FITS, NEP_UDF_CDF, NEP_UDF_PDS4, + NEP_UDF_DICOM, NEP_UDF_PDB, NEP_UDF_MMCIF, NEP_UDF_NEXTCDF4) + # next two lines do nothing, preserved for backwards compatibility. default_encoding = 'utf-8' unicode_error = 'replace' @@ -2306,11 +2323,13 @@ strings. def __init__(self, filename, mode='r', clobber=True, format='NETCDF4', diskless=False, persist=False, keepweakref=False, memory=None, encoding=None, parallel=False, - comm=None, info=None, auto_complex=False, **kwargs): + comm=None, info=None, auto_complex=False, + extra_mode_flag=None, **kwargs): """ **`__init__(self, filename, mode="r", clobber=True, diskless=False, persist=False, keepweakref=False, memory=None, encoding=None, - parallel=False, comm=None, info=None, format='NETCDF4')`** + parallel=False, comm=None, info=None, format='NETCDF4', + auto_complex=False, extra_mode_flag=None)`** `Dataset` constructor. @@ -2403,6 +2422,15 @@ strings. means MPI_INFO_NULL will be used. Ignored if `parallel=False`. **`auto_complex`**: if `True`, then automatically convert complex number types + + **`extra_mode_flag`**: integer mode flag selecting a user-defined format + (UDF) dispatch layer, such as those provided by the NetCDF Expansion Pack + (NEP). Only needed for files that cannot be identified by their magic + number. Must be one of the `NEP_UDF_*` module constants (for example + `NEP_UDF_MMCIF`), and the UDF must be registered with netcdf-c (for NEP, + via the .ncrc file). Only allowed with `mode='r'`, and not with + `diskless`, `memory` or `parallel`. Requires netcdf-c >= 4.10.0. + Default `None`. """ cdef int grpid, ierr, numgrps, numdims, numvars, cdef size_t initialsize @@ -2478,6 +2506,19 @@ strings. if write_mode and persist: cmode |= NC_WRITE | NC_PERSIST + # extra mode flag for user-defined formats, only used for plain + # read-only nc_open (cmode == NC_NOWRITE). Need to check read_mode + # too, since NC_CLOBBER (mode='w') is also zero. + if extra_mode_flag is not None: + if _nclibversion < Version("4.10.0"): + raise NetCDF4MissingFeatureException("extra_mode_flag", "4.10.0") + if not read_mode or cmode != NC_NOWRITE or memory is not None or parallel: + raise ValueError("extra_mode_flag only allowed with mode='r', " + "and not with diskless, memory or parallel") + if extra_mode_flag not in _nep_udf_flags: + raise ValueError(f"extra_mode_flag must be one of the NEP_UDF_* " + f"constants, got {extra_mode_flag!r}") + if write_mode: _set_default_format(format=format) if memory is not None: @@ -2521,6 +2562,8 @@ strings. with nogil: ierr = nc_open_par(path, cmode | NC_MPIIO, mpicomm, mpiinfo, &grpid) else: + if extra_mode_flag is not None: + cmode |= extra_mode_flag with nogil: ierr = nc_open(path, cmode, &grpid) diff --git a/test/gfs_sample.grb2 b/test/gfs_sample.grb2 new file mode 100644 index 000000000..16a09083c Binary files /dev/null and b/test/gfs_sample.grb2 differ diff --git a/test/test_extra_mode_flag.py b/test/test_extra_mode_flag.py new file mode 100644 index 000000000..a2be4cbca --- /dev/null +++ b/test/test_extra_mode_flag.py @@ -0,0 +1,114 @@ +import unittest, os, tempfile +from packaging.version import Version +from numpy.testing import assert_array_almost_equal +import netCDF4 + +# test extra_mode_flag Dataset kwarg (issue #1490), used to select a +# user-defined format (UDF) dispatch layer for files with no magic number. +# The read tests only run if the NetCDF Expansion Pack (NEP) mmCIF reader +# is registered with netcdf-c (e.g. via NCRCENV_RC pointing to NEP's .ncrc). + +MMCIF = """\ +data_TINY +# +_entry.id TINY +# +loop_ +_atom_site.group_PDB +_atom_site.id +_atom_site.type_symbol +_atom_site.label_atom_id +_atom_site.label_alt_id +_atom_site.label_comp_id +_atom_site.label_asym_id +_atom_site.label_entity_id +_atom_site.label_seq_id +_atom_site.pdbx_PDB_ins_code +_atom_site.Cartn_x +_atom_site.Cartn_y +_atom_site.Cartn_z +_atom_site.occupancy +_atom_site.B_iso_or_equiv +_atom_site.auth_seq_id +_atom_site.auth_asym_id +_atom_site.pdbx_PDB_model_num +ATOM 1 N N . VAL A 1 1 ? 6.204 16.869 4.854 1.00 49.05 1 A 1 +ATOM 2 C CA . VAL A 1 1 ? 6.913 17.759 4.607 1.00 43.14 1 A 1 +ATOM 3 C C . VAL A 1 1 ? 8.504 17.378 4.797 1.00 24.80 1 A 1 +# +""" +CARTN_X = [6.204, 6.913, 8.504] + +has_udf = Version(netCDF4.__netcdf4libversion__.split('-')[0]) >= Version('4.10.0') + +def _write(text): + f = tempfile.NamedTemporaryFile(suffix='.cif', delete=False, mode='w') + f.write(text) + f.close() + return f.name + +def _has_nep_mmcif(): + if not has_udf: + return False + filename = _write(MMCIF) + try: + netCDF4.Dataset(filename).close() + return True + except OSError: + return False + finally: + os.remove(filename) + +has_nep_mmcif = _has_nep_mmcif() + + +@unittest.skipUnless(has_udf, "extra_mode_flag requires netcdf-c >= 4.10.0") +class ExtraModeFlagArgsTestCase(unittest.TestCase): + + def setUp(self): + self.file = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name + netCDF4.Dataset(self.file, 'w').close() + + def tearDown(self): + os.remove(self.file) + + def runTest(self): + """only allowed for plain read-only opens with a NEP_UDF_* flag""" + flag = netCDF4.NEP_UDF_MMCIF + for kwargs in (dict(mode='w'), dict(mode='a'), dict(mode='r+'), + dict(mode='rs'), dict(mode='r', diskless=True)): + with self.assertRaises(ValueError, msg=str(kwargs)): + netCDF4.Dataset(self.file, extra_mode_flag=flag, **kwargs) + with open(self.file, 'rb') as f: + buf = f.read() + with self.assertRaises(ValueError): + netCDF4.Dataset('inmemory.nc', memory=buf, extra_mode_flag=flag) + # not a NEP_UDF_* flag (NC_WRITE) + with self.assertRaises(ValueError): + netCDF4.Dataset(self.file, extra_mode_flag=0x0001) + # extra_mode_flag=None is the default + netCDF4.Dataset(self.file, extra_mode_flag=None).close() + + +@unittest.skipUnless(has_nep_mmcif, "NEP mmCIF reader not available") +class ExtraModeFlagNEPTestCase(unittest.TestCase): + + def setUp(self): + # without the leading data_ magic number netcdf-c can't detect the format + self.file = _write("# no magic number\n" + MMCIF) + + def tearDown(self): + os.remove(self.file) + + def runTest(self): + """open mmCIF file with no magic number using NEP_UDF_MMCIF""" + with self.assertRaises(OSError): + netCDF4.Dataset(self.file) + with netCDF4.Dataset(self.file, extra_mode_flag=netCDF4.NEP_UDF_MMCIF) as ds: + assert ds.entry_id == 'TINY' + assert len(ds.dimensions['atom']) == 3 + assert_array_almost_equal(ds['atom_site_Cartn_x'][0], CARTN_X) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_nep_grib2.py b/test/test_nep_grib2.py new file mode 100644 index 000000000..3b88cdf87 --- /dev/null +++ b/test/test_nep_grib2.py @@ -0,0 +1,62 @@ +import unittest, os +import numpy as np +from numpy.testing import assert_allclose +import netCDF4 + +# test reading a GRIB2 file with the NetCDF Expansion Pack (NEP) GRIB2 +# reader (issue #1490). Only runs if the NEP GRIB2 reader is registered +# with netcdf-c (e.g. via NCRCENV_RC pointing to NEP's .ncrc). +# gfs_sample.grb2 is the first 3 messages of pygrib's sampledata/gfs.grb +# (10 hPa HGT, TMP and RH on a 2.5 degree global grid). + +FILE_NAME = os.path.join(os.path.dirname(__file__), 'gfs_sample.grb2') +NLATS, NLONS = 73, 144 + +# reference values from pygrib: (value at [0,0], [36,72], [-1,-1], min, max) +REFERENCE = { + 'HGT': (29989.89, 31001.2, 27951.64, 27900.99, 31664.09), + 'TMP': (213.7, 233.1, 221.0, 209.3, 257.1), + 'RH': (0.0, 0.03, 0.06, 0.0, 0.21), +} + +def _has_nep_grib2(): + try: + netCDF4.Dataset(FILE_NAME).close() + return True + except OSError: + return False + +has_nep_grib2 = _has_nep_grib2() + + +@unittest.skipUnless(has_nep_grib2, "NEP GRIB2 reader not available") +class NEPGrib2TestCase(unittest.TestCase): + + def check(self, ds): + assert ds.Conventions == 'GRIB2' + assert ds.GRIB2_edition == 2 + assert list(ds.variables) == list(REFERENCE) + for name, (v00, vmid, vlast, vmin, vmax) in REFERENCE.items(): + var = ds[name] + assert var.long_name == name + data = var[:] + assert data.size == NLATS * NLONS + # NEP currently defines the dimensions in the wrong order + # (y=144, x=73), but the data are in the usual (lat, lon) + # order, so reshape before comparing. + data = data.reshape(NLATS, NLONS) + assert_allclose([data[0, 0], data[NLATS//2, NLONS//2], data[-1, -1], + data.min(), data.max()], + [v00, vmid, vlast, vmin, vmax], rtol=1.e-5, atol=1.e-5) + + def runTest(self): + """read GRIB2 file, with format detected by magic number""" + with netCDF4.Dataset(FILE_NAME) as ds: + self.check(ds) + # selecting the GRIB2 reader explicitly gives the same result + with netCDF4.Dataset(FILE_NAME, extra_mode_flag=netCDF4.NEP_UDF_GRIB2) as ds: + self.check(ds) + + +if __name__ == '__main__': + unittest.main()