Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/zarr/codecs/gzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ def parse_gzip_level(data: JSON) -> int:
return parsed


def _gzip_streams_equal_except_mtime(a: bytes, b: bytes) -> bool:
if len(a) != len(b):
return False

return a[:4] == b[:4] and a[8:] == b[8:]


@dataclass(frozen=True)
class GzipCodec(BytesBytesCodec):
"""gzip codec"""
Expand Down
37 changes: 37 additions & 0 deletions tests/test_codecs/test_gzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from zarr.abc.codec import SupportsSyncCodec
from zarr.abc.store import Store
from zarr.codecs import GzipCodec
from zarr.codecs.gzip import _gzip_streams_equal_except_mtime
from zarr.core.array_spec import ArrayConfig, ArraySpec
from zarr.core.buffer import default_buffer_prototype
from zarr.core.dtype import get_data_type_from_native_dtype
Expand Down Expand Up @@ -50,3 +51,39 @@ def test_gzip_codec_sync_roundtrip() -> None:
decoded = codec._decode_sync(encoded, spec)
result = np.frombuffer(decoded.as_numpy_array(), dtype="float64")
np.testing.assert_array_equal(arr, result)


def test_gzip_streams_equal_except_mtime() -> None:
prefix = b"\x1f\x8b\x08\x00"
mtime = b"\x01\x02\x03\x04"
suffix = b"\x00\xff\x10\x20"

# Identical streams are equal.
assert _gzip_streams_equal_except_mtime(
prefix + mtime + suffix,
prefix + mtime + suffix,
)

# Streams with different MTIME values are still equal.
assert _gzip_streams_equal_except_mtime(
prefix + b"\x01\x02\x03\x04" + suffix,
prefix + b"\x05\x06\x07\x08" + suffix,
)

# Differences after MTIME are detected.
assert not _gzip_streams_equal_except_mtime(
prefix + mtime + suffix,
prefix + mtime + b"\x00\xff\x10\x21",
)

# Differences before MTIME are detected.
assert not _gzip_streams_equal_except_mtime(
prefix + mtime + suffix,
b"\x1f\x8b\x09\x00" + mtime + suffix,
)

# Different lengths are detected.
assert not _gzip_streams_equal_except_mtime(
prefix + mtime + suffix,
prefix + mtime + suffix + b"\x00",
)
16 changes: 14 additions & 2 deletions tests/test_fused_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
)
from zarr.abc.store import Store, _store_supports_sync_io
from zarr.codecs.bytes import BytesCodec
from zarr.codecs.gzip import GzipCodec
from zarr.codecs.gzip import GzipCodec, _gzip_streams_equal_except_mtime
from zarr.codecs.transpose import TransposeCodec
from zarr.codecs.zstd import ZstdCodec
from zarr.core.codec_pipeline import FusedCodecPipeline
Expand Down Expand Up @@ -904,7 +904,19 @@ def test_async_chunk_transform_matches_sync(codecs: tuple[Any, ...]) -> None:
async_bytes = asyncio.run(async_t.encode_chunk(value, spec))
assert sync_bytes is not None
assert async_bytes is not None
np.testing.assert_array_equal(async_bytes.to_bytes(), sync_bytes.to_bytes())

has_timestamp_codec = any(isinstance(c, GzipCodec) for c in evolved)

if has_timestamp_codec:
assert _gzip_streams_equal_except_mtime(
async_bytes.to_bytes(),
sync_bytes.to_bytes(),
)
else:
np.testing.assert_array_equal(
async_bytes.to_bytes(),
sync_bytes.to_bytes(),
)

sync_arr = sync_t.decode_chunk(async_bytes, spec)
async_arr = asyncio.run(async_t.decode_chunk(async_bytes, spec))
Expand Down
Loading