diff --git a/doc/whats-new.rst b/doc/whats-new.rst index d1505bfa081..cab451196ed 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -53,6 +53,10 @@ Deprecations Bug Fixes ~~~~~~~~~ +- Fix the ``netcdf4`` engine silently writing byte-swapped values for + non-native-endian numeric attribute arrays, such as attributes of netCDF-3 + files read with the ``scipy`` engine (:pull:`11543`). + By `glaziermag `_. - Fix async zarr tests using ``wraps`` with ``autospec=True`` on async methods, which caused ``AsyncMock`` objects to leak through instead of real array data (:pull:`11232`). diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index af9804669c9..ff9a29a0707 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -252,6 +252,15 @@ def _force_native_endianness(var): return var +def _force_native_endianness_attr(value): + # netCDF4-python writes non-native-endian attribute arrays without + # byte-swapping, silently corrupting the stored values (variable data + # is handled by _force_native_endianness above). + if isinstance(value, np.ndarray) and value.dtype.byteorder not in ("=", "|"): + value = value.astype(value.dtype.newbyteorder("=")) + return value + + def _extract_nc4_variable_encoding( variable: Variable, raise_on_invalid=False, @@ -632,6 +641,7 @@ def set_dimension(self, name, length, is_unlimited=False): self.ds.createDimension(name, size=dim_length) def set_attribute(self, key, value): + value = _force_native_endianness_attr(value) if self.format != "NETCDF4": value = encode_nc3_attr_value(value) if _is_list_of_strings(value): @@ -652,7 +662,7 @@ def prepare_variable( self, name, variable: Variable, check_encoding=False, unlimited_dims=None ): _ensure_no_forward_slash_in_name(name) - attrs = variable.attrs.copy() + attrs = {k: _force_native_endianness_attr(v) for k, v in variable.attrs.items()} fill_value = attrs.pop("_FillValue", None) datatype: np.dtype | ncEnumType | h5EnumType datatype = _get_datatype( diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 6773c79b003..51a6154b606 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -7252,6 +7252,31 @@ def test_load_single_value_h5netcdf(tmp_path: Path) -> None: ds2["test"][0].load() +@requires_netCDF4 +def test_roundtrip_non_native_endian_attrs(tmp_path: Path) -> None: + """Test that non-native-endian numeric attribute values round-trip. + + The scipy backend returns attribute values of netCDF-3 files as big-endian + arrays; netCDF4-python does not byte-swap attribute arrays on write, so + writing them back with the netCDF4 backend silently stored byte-swapped + values (e.g. valid_range [0.0, 1.0] became [0.0, 3.03865e-319]). + """ + ds = xr.Dataset( + { + "x": xr.DataArray( + [1.0], + dims=("d",), + attrs={"valid_range": np.array([0.0, 1.0], dtype=">f8")}, + ) + }, + attrs={"levels": np.array([1, 2], dtype=">i4")}, + ) + ds.to_netcdf(tmp_path / "test.nc", engine="netcdf4") + with xr.open_dataset(tmp_path / "test.nc", engine="netcdf4") as ds2: + assert_array_equal(ds2["x"].attrs["valid_range"], [0.0, 1.0]) + assert_array_equal(ds2.attrs["levels"], [1, 2]) + + @requires_zarr @requires_dask @pytest.mark.parametrize(