diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py index 7d86a03ba8..7f21872034 100644 --- a/src/zarr/codecs/gzip.py +++ b/src/zarr/codecs/gzip.py @@ -28,6 +28,13 @@ def parse_gzip_level(data: JSON) -> int: return parsed +def _gzip_streams_equal_except_mtime(a: bytes, b: bytes) -> bool: + if len(a) != len(b): + return False + + return a[:4] == b[:4] and a[8:] == b[8:] + + @dataclass(frozen=True) class GzipCodec(BytesBytesCodec): """gzip codec""" diff --git a/tests/test_codecs/test_gzip.py b/tests/test_codecs/test_gzip.py index 8932ba5e59..52c4a835d2 100644 --- a/tests/test_codecs/test_gzip.py +++ b/tests/test_codecs/test_gzip.py @@ -5,6 +5,7 @@ from zarr.abc.codec import SupportsSyncCodec from zarr.abc.store import Store from zarr.codecs import GzipCodec +from zarr.codecs.gzip import _gzip_streams_equal_except_mtime from zarr.core.array_spec import ArrayConfig, ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.dtype import get_data_type_from_native_dtype @@ -50,3 +51,39 @@ def test_gzip_codec_sync_roundtrip() -> None: decoded = codec._decode_sync(encoded, spec) result = np.frombuffer(decoded.as_numpy_array(), dtype="float64") np.testing.assert_array_equal(arr, result) + + +def test_gzip_streams_equal_except_mtime() -> None: + prefix = b"\x1f\x8b\x08\x00" + mtime = b"\x01\x02\x03\x04" + suffix = b"\x00\xff\x10\x20" + + # Identical streams are equal. + assert _gzip_streams_equal_except_mtime( + prefix + mtime + suffix, + prefix + mtime + suffix, + ) + + # Streams with different MTIME values are still equal. + assert _gzip_streams_equal_except_mtime( + prefix + b"\x01\x02\x03\x04" + suffix, + prefix + b"\x05\x06\x07\x08" + suffix, + ) + + # Differences after MTIME are detected. + assert not _gzip_streams_equal_except_mtime( + prefix + mtime + suffix, + prefix + mtime + b"\x00\xff\x10\x21", + ) + + # Differences before MTIME are detected. + assert not _gzip_streams_equal_except_mtime( + prefix + mtime + suffix, + b"\x1f\x8b\x09\x00" + mtime + suffix, + ) + + # Different lengths are detected. + assert not _gzip_streams_equal_except_mtime( + prefix + mtime + suffix, + prefix + mtime + suffix + b"\x00", + ) diff --git a/tests/test_fused_pipeline.py b/tests/test_fused_pipeline.py index 5c712fa97a..87bfaf71b4 100644 --- a/tests/test_fused_pipeline.py +++ b/tests/test_fused_pipeline.py @@ -18,7 +18,7 @@ ) from zarr.abc.store import Store, _store_supports_sync_io from zarr.codecs.bytes import BytesCodec -from zarr.codecs.gzip import GzipCodec +from zarr.codecs.gzip import GzipCodec, _gzip_streams_equal_except_mtime from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec from zarr.core.codec_pipeline import FusedCodecPipeline @@ -904,7 +904,19 @@ def test_async_chunk_transform_matches_sync(codecs: tuple[Any, ...]) -> None: async_bytes = asyncio.run(async_t.encode_chunk(value, spec)) assert sync_bytes is not None assert async_bytes is not None - np.testing.assert_array_equal(async_bytes.to_bytes(), sync_bytes.to_bytes()) + + has_timestamp_codec = any(isinstance(c, GzipCodec) for c in evolved) + + if has_timestamp_codec: + assert _gzip_streams_equal_except_mtime( + async_bytes.to_bytes(), + sync_bytes.to_bytes(), + ) + else: + np.testing.assert_array_equal( + async_bytes.to_bytes(), + sync_bytes.to_bytes(), + ) sync_arr = sync_t.decode_chunk(async_bytes, spec) async_arr = asyncio.run(async_t.decode_chunk(async_bytes, spec))