add extra_mode_flag Dataset kwarg for user-defined formats (issue #1490)

Changelog

@@ -1,3 +1,8 @@
+ version 1.7.6 (not yet released)
+ ================================
+ * Allow mode flag (`extra_mode_flag` Dataset kwarg) to be passed to nc_open() to support 
+ additional dispatch layers available via the NetCDF Expansion Pack (NEP) (issue #1490).
+
  version 1.7.5.1.post1 (tag v1.7.5.1.post1rel)
  =============================================
  * fix installation of plugins into linux wheels.

MANIFEST.in

@@ -10,6 +10,7 @@ include examples/README.md
 exclude examples/data
 include test/*py
 include test/*nc
+include test/*grb2
 include src/netCDF4/__init__.py
 include src/netCDF4/_netCDF4.pyx
 exclude src/netCDF4/_netCDF4.c

src/netCDF4/__init__.pyi

@@ -187,6 +187,16 @@ is_native_big: bool
 default_encoding: Final = "utf-8"
 unicode_error: Final = "replace"
 default_fillvals: dict[str, int | float | str]
+NEP_UDF_GEOTIFF_BIGTIFF: Final = 0x0040
+NEP_UDF_GEOTIFF_STANDARD: Final = 0x0080
+NEP_UDF_GRIB2: Final = 0x10000
+NEP_UDF_FITS: Final = 0x80000
+NEP_UDF_CDF: Final = 0x100000
+NEP_UDF_PDS4: Final = 0x200000
+NEP_UDF_DICOM: Final = 0x400000
+NEP_UDF_PDB: Final = 0x800000
+NEP_UDF_MMCIF: Final = 0x1000000
+NEP_UDF_NEXTCDF4: Final = 0x2000000
 
 # date2index, date2num, and num2date are actually provided by cftime and if stubs for
 # cftime are completed these should be removed.
@@ -235,6 +245,7 @@ class Dataset:
         comm: Any = None,
         info: Any = None,
         auto_complex: bool = False,
+        extra_mode_flag: int | None = None,
         **kwargs: Any,
     ): ...
     @property

src/netCDF4/_netCDF4.pyi

@@ -8,6 +8,16 @@ from . import (
     Group,
     MFDataset,
     MFTime,
+    NEP_UDF_CDF,
+    NEP_UDF_DICOM,
+    NEP_UDF_FITS,
+    NEP_UDF_GEOTIFF_BIGTIFF,
+    NEP_UDF_GEOTIFF_STANDARD,
+    NEP_UDF_GRIB2,
+    NEP_UDF_MMCIF,
+    NEP_UDF_NEXTCDF4,
+    NEP_UDF_PDB,
+    NEP_UDF_PDS4,
     NetCDF4MissingFeatureException,
     Variable,
     VLType,

src/netCDF4/_netCDF4.pyx

@@ -1,4 +1,4 @@
-"""Version 1.7.5.1.post1
+"""Version 1.7.6
 -------------
 
 # Introduction
@@ -1283,7 +1283,7 @@ import sys
 import functools
 from typing import Union
 
-__version__ = "1.7.5.1.post1"
+__version__ = "1.7.6"
 
 # Initialize numpy
 import posixpath
@@ -1612,6 +1612,23 @@ if _nclibversion >= Version("4.6.2"):
 else:  # prior to 4.6.2 this flag doesn't work, so make the same as NC_DISKLESS
     NC_PERSIST = NC_DISKLESS
 
+# mode flags for the user-defined formats (UDFs) provided by the
+# NetCDF Expansion Pack (NEP), names match those in nep.h.
+# Used with the extra_mode_flag Dataset kwarg (requires netcdf-c >= 4.10.0).
+NEP_UDF_GEOTIFF_BIGTIFF = 0x0040     # NC_UDF0
+NEP_UDF_GEOTIFF_STANDARD = 0x0080    # NC_UDF1
+NEP_UDF_GRIB2 = 0x10000              # NC_UDF2
+NEP_UDF_FITS = 0x80000               # NC_UDF3
+NEP_UDF_CDF = 0x100000               # NC_UDF4
+NEP_UDF_PDS4 = 0x200000              # NC_UDF5
+NEP_UDF_DICOM = 0x400000             # NC_UDF6
+NEP_UDF_PDB = 0x800000               # NC_UDF7
+NEP_UDF_MMCIF = 0x1000000            # NC_UDF8
+NEP_UDF_NEXTCDF4 = 0x2000000         # NC_UDF9
+_nep_udf_flags = (NEP_UDF_GEOTIFF_BIGTIFF, NEP_UDF_GEOTIFF_STANDARD,
+                  NEP_UDF_GRIB2, NEP_UDF_FITS, NEP_UDF_CDF, NEP_UDF_PDS4,
+                  NEP_UDF_DICOM, NEP_UDF_PDB, NEP_UDF_MMCIF, NEP_UDF_NEXTCDF4)
+
 # next two lines do nothing, preserved for backwards compatibility.
 default_encoding = 'utf-8'
 unicode_error = 'replace'
@@ -2306,11 +2323,13 @@ strings.
     def __init__(self, filename, mode='r', clobber=True, format='NETCDF4',
                      diskless=False, persist=False, keepweakref=False,
                      memory=None, encoding=None, parallel=False,
-                     comm=None, info=None, auto_complex=False, **kwargs):
+                     comm=None, info=None, auto_complex=False,
+                     extra_mode_flag=None, **kwargs):
         """
         **`__init__(self, filename, mode="r", clobber=True, diskless=False,
         persist=False, keepweakref=False, memory=None, encoding=None,
-        parallel=False, comm=None, info=None, format='NETCDF4')`**
+        parallel=False, comm=None, info=None, format='NETCDF4',
+        auto_complex=False, extra_mode_flag=None)`**
 
         `Dataset` constructor.
 
@@ -2403,6 +2422,15 @@ strings.
         means MPI_INFO_NULL will be used.  Ignored if `parallel=False`.
 
         **`auto_complex`**: if `True`, then automatically convert complex number types
+
+        **`extra_mode_flag`**: integer mode flag selecting a user-defined format
+        (UDF) dispatch layer, such as those provided by the NetCDF Expansion Pack
+        (NEP). Only needed for files that cannot be identified by their magic
+        number. Must be one of the `NEP_UDF_*` module constants (for example
+        `NEP_UDF_MMCIF`), and the UDF must be registered with netcdf-c (for NEP,
+        via the .ncrc file). Only allowed with `mode='r'`, and not with
+        `diskless`, `memory` or `parallel`. Requires netcdf-c >= 4.10.0.
+        Default `None`.
         """
         cdef int grpid, ierr, numgrps, numdims, numvars,
         cdef size_t initialsize
@@ -2478,6 +2506,19 @@ strings.
             if write_mode and persist:
                 cmode |= NC_WRITE | NC_PERSIST
 
+        # extra mode flag for user-defined formats, only used for plain
+        # read-only nc_open (cmode == NC_NOWRITE). Need to check read_mode
+        # too, since NC_CLOBBER (mode='w') is also zero.
+        if extra_mode_flag is not None:
+            if _nclibversion < Version("4.10.0"):
+                raise NetCDF4MissingFeatureException("extra_mode_flag", "4.10.0")
+            if not read_mode or cmode != NC_NOWRITE or memory is not None or parallel:
+                raise ValueError("extra_mode_flag only allowed with mode='r', "
+                                 "and not with diskless, memory or parallel")
+            if extra_mode_flag not in _nep_udf_flags:
+                raise ValueError(f"extra_mode_flag must be one of the NEP_UDF_* "
+                                 f"constants, got {extra_mode_flag!r}")
+
         if write_mode:
             _set_default_format(format=format)
             if memory is not None:
@@ -2521,6 +2562,8 @@ strings.
                 with nogil:
                     ierr = nc_open_par(path, cmode | NC_MPIIO, mpicomm, mpiinfo, &grpid)
             else:
+                if extra_mode_flag is not None:
+                    cmode |= extra_mode_flag
                 with nogil:
                     ierr = nc_open(path, cmode, &grpid)
 

test/test_extra_mode_flag.py

@@ -0,0 +1,114 @@
+import unittest, os, tempfile
+from packaging.version import Version
+from numpy.testing import assert_array_almost_equal
+import netCDF4
+
+# test extra_mode_flag Dataset kwarg (issue #1490), used to select a
+# user-defined format (UDF) dispatch layer for files with no magic number.
+# The read tests only run if the NetCDF Expansion Pack (NEP) mmCIF reader
+# is registered with netcdf-c (e.g. via NCRCENV_RC pointing to NEP's .ncrc).
+
+MMCIF = """\
+data_TINY
+#
+_entry.id TINY
+#
+loop_
+_atom_site.group_PDB
+_atom_site.id
+_atom_site.type_symbol
+_atom_site.label_atom_id
+_atom_site.label_alt_id
+_atom_site.label_comp_id
+_atom_site.label_asym_id
+_atom_site.label_entity_id
+_atom_site.label_seq_id
+_atom_site.pdbx_PDB_ins_code
+_atom_site.Cartn_x
+_atom_site.Cartn_y
+_atom_site.Cartn_z
+_atom_site.occupancy
+_atom_site.B_iso_or_equiv
+_atom_site.auth_seq_id
+_atom_site.auth_asym_id
+_atom_site.pdbx_PDB_model_num
+ATOM 1 N N . VAL A 1 1 ? 6.204 16.869 4.854 1.00 49.05 1 A 1
+ATOM 2 C CA . VAL A 1 1 ? 6.913 17.759 4.607 1.00 43.14 1 A 1
+ATOM 3 C C . VAL A 1 1 ? 8.504 17.378 4.797 1.00 24.80 1 A 1
+#
+"""
+CARTN_X = [6.204, 6.913, 8.504]
+
+has_udf = Version(netCDF4.__netcdf4libversion__.split('-')[0]) >= Version('4.10.0')
+
+def _write(text):
+    f = tempfile.NamedTemporaryFile(suffix='.cif', delete=False, mode='w')
+    f.write(text)
+    f.close()
+    return f.name
+
+def _has_nep_mmcif():
+    if not has_udf:
+        return False
+    filename = _write(MMCIF)
+    try:
+        netCDF4.Dataset(filename).close()
+        return True
+    except OSError:
+        return False
+    finally:
+        os.remove(filename)
+
+has_nep_mmcif = _has_nep_mmcif()
+
+
+@unittest.skipUnless(has_udf, "extra_mode_flag requires netcdf-c >= 4.10.0")
+class ExtraModeFlagArgsTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.file = tempfile.NamedTemporaryFile(suffix='.nc', delete=False).name
+        netCDF4.Dataset(self.file, 'w').close()
+
+    def tearDown(self):
+        os.remove(self.file)
+
+    def runTest(self):
+        """only allowed for plain read-only opens with a NEP_UDF_* flag"""
+        flag = netCDF4.NEP_UDF_MMCIF
+        for kwargs in (dict(mode='w'), dict(mode='a'), dict(mode='r+'),
+                       dict(mode='rs'), dict(mode='r', diskless=True)):
+            with self.assertRaises(ValueError, msg=str(kwargs)):
+                netCDF4.Dataset(self.file, extra_mode_flag=flag, **kwargs)
+        with open(self.file, 'rb') as f:
+            buf = f.read()
+        with self.assertRaises(ValueError):
+            netCDF4.Dataset('inmemory.nc', memory=buf, extra_mode_flag=flag)
+        # not a NEP_UDF_* flag (NC_WRITE)
+        with self.assertRaises(ValueError):
+            netCDF4.Dataset(self.file, extra_mode_flag=0x0001)
+        # extra_mode_flag=None is the default
+        netCDF4.Dataset(self.file, extra_mode_flag=None).close()
+
+
+@unittest.skipUnless(has_nep_mmcif, "NEP mmCIF reader not available")
+class ExtraModeFlagNEPTestCase(unittest.TestCase):
+
+    def setUp(self):
+        # without the leading data_ magic number netcdf-c can't detect the format
+        self.file = _write("# no magic number\n" + MMCIF)
+
+    def tearDown(self):
+        os.remove(self.file)
+
+    def runTest(self):
+        """open mmCIF file with no magic number using NEP_UDF_MMCIF"""
+        with self.assertRaises(OSError):
+            netCDF4.Dataset(self.file)
+        with netCDF4.Dataset(self.file, extra_mode_flag=netCDF4.NEP_UDF_MMCIF) as ds:
+            assert ds.entry_id == 'TINY'
+            assert len(ds.dimensions['atom']) == 3
+            assert_array_almost_equal(ds['atom_site_Cartn_x'][0], CARTN_X)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_nep_grib2.py

@@ -0,0 +1,62 @@
+import unittest, os
+import numpy as np
+from numpy.testing import assert_allclose
+import netCDF4
+
+# test reading a GRIB2 file with the NetCDF Expansion Pack (NEP) GRIB2
+# reader (issue #1490). Only runs if the NEP GRIB2 reader is registered
+# with netcdf-c (e.g. via NCRCENV_RC pointing to NEP's .ncrc).
+# gfs_sample.grb2 is the first 3 messages of pygrib's sampledata/gfs.grb
+# (10 hPa HGT, TMP and RH on a 2.5 degree global grid).
+
+FILE_NAME = os.path.join(os.path.dirname(__file__), 'gfs_sample.grb2')
+NLATS, NLONS = 73, 144
+
+# reference values from pygrib: (value at [0,0], [36,72], [-1,-1], min, max)
+REFERENCE = {
+    'HGT': (29989.89, 31001.2, 27951.64, 27900.99, 31664.09),
+    'TMP': (213.7, 233.1, 221.0, 209.3, 257.1),
+    'RH': (0.0, 0.03, 0.06, 0.0, 0.21),
+}
+
+def _has_nep_grib2():
+    try:
+        netCDF4.Dataset(FILE_NAME).close()
+        return True
+    except OSError:
+        return False
+
+has_nep_grib2 = _has_nep_grib2()
+
+
+@unittest.skipUnless(has_nep_grib2, "NEP GRIB2 reader not available")
+class NEPGrib2TestCase(unittest.TestCase):
+
+    def check(self, ds):
+        assert ds.Conventions == 'GRIB2'
+        assert ds.GRIB2_edition == 2
+        assert list(ds.variables) == list(REFERENCE)
+        for name, (v00, vmid, vlast, vmin, vmax) in REFERENCE.items():
+            var = ds[name]
+            assert var.long_name == name
+            data = var[:]
+            assert data.size == NLATS * NLONS
+            # NEP currently defines the dimensions in the wrong order
+            # (y=144, x=73), but the data are in the usual (lat, lon)
+            # order, so reshape before comparing.
+            data = data.reshape(NLATS, NLONS)
+            assert_allclose([data[0, 0], data[NLATS//2, NLONS//2], data[-1, -1],
+                             data.min(), data.max()],
+                            [v00, vmid, vlast, vmin, vmax], rtol=1.e-5, atol=1.e-5)
+
+    def runTest(self):
+        """read GRIB2 file, with format detected by magic number"""
+        with netCDF4.Dataset(FILE_NAME) as ds:
+            self.check(ds)
+        # selecting the GRIB2 reader explicitly gives the same result
+        with netCDF4.Dataset(FILE_NAME, extra_mode_flag=netCDF4.NEP_UDF_GRIB2) as ds:
+            self.check(ds)
+
+
+if __name__ == '__main__':
+    unittest.main()