feat: name packages that provide a codec zarr cannot find

changes/4277.feature.md

@@ -0,0 +1,28 @@
+`zarr.registry.get_codec_class` now raises `zarr.errors.UnknownCodecError` instead of `KeyError`
+when no implementation is registered for a codec, and `zarr.core.config.BadConfigError` instead of
+`KeyError` when the implementation named in `config["codecs"][name]` is not registered.
+`zarr.registry.get_numcodec` raises `UnknownCodecError` instead of the `ValueError` numcodecs
+raises for an unregistered Zarr format 2 codec id (`numcodecs.errors.UnknownCodecError` on
+numcodecs 0.15.1 and later). All of these are subclasses of `ValueError`, so `except ValueError`
+is unaffected, but `except KeyError` and `except numcodecs.errors.UnknownCodecError` are.
+
+These errors now name Python packages known to provide the codec, so that a user who cannot read
+an array learns what to install:
+
+```
+An implementation for codec 'wavpack' is not available. Register one explicitly using the codec
+registry (see ...), or install a Python package that registers a codec implementation with
+numcodecs. Known packages supporting this codec: wavpack-numcodecs.
+```
+
+The tables covering this live in `src/zarr/registry.py`, one per Zarr format, and include the
+codecs `numcodecs` gates behind its own optional dependencies (`zfpy`, `pcodec`, `crc32c`,
+`msgpack2`). Codec authors can add their published package to them.
+
+A codec whose `from_dict` raises `KeyError` on a malformed configuration now surfaces as
+`zarr.errors.MetadataValidationError` naming the codec and the missing key. Previously it was
+reported as `UnknownCodecError: Unknown codec: '<configuration key>'`, presenting a configuration
+key as though it were a codec name, and on the `zarr.open` path a bare `KeyError` could be
+swallowed by the array-then-group fallback and reported as an unrelated group error.
+
+`zarr.errors.UnknownCodecError` is now exported from `zarr.errors`.

docs/user-guide/extending.md

@@ -63,6 +63,13 @@ New codecs need to have their own unique identifier. To avoid naming collisions,
 strongly recommended to prefix the codec identifier with a unique name. For example,
 the codecs from `numcodecs` are prefixed with `numcodecs.`, e.g. `numcodecs.delta`.
 
+If someone opens an array that uses your codec without your package installed, Zarr raises
+[`zarr.errors.UnknownCodecError`][] explaining how to register an implementation. Zarr also
+keeps a small table of codec names and the published packages that provide them, and names
+those packages in that error. Once your package is on PyPI, please open a pull request adding
+it to the codec-package tables in `src/zarr/registry.py`, so that users get a message telling them
+exactly what to install.
+
 !!! note
     Note that the extension mechanism for the Zarr format 3 is still under development.
     Requirements for custom codecs including the choice of codec identifiers might

src/zarr/core/metadata/v3.py

@@ -36,7 +36,7 @@
 from zarr.core.dtype.common import check_dtype_spec_v3
 from zarr.core.json_parse import parse_field, validate_json_value
 from zarr.core.metadata.common import parse_attributes
-from zarr.errors import MetadataValidationError, NodeTypeValidationError, UnknownCodecError
+from zarr.errors import MetadataValidationError, NodeTypeValidationError
 from zarr.registry import get_codec_class
 
 if TYPE_CHECKING:
@@ -74,10 +74,17 @@ def parse_codecs(data: object) -> tuple[Codec, ...]:
         else:
             name_parsed, _ = parse_named_configuration(c, require_configuration=False)
 
+            codec_cls = get_codec_class(name_parsed)
             try:
-                out += (get_codec_class(name_parsed).from_dict(c),)
+                out += (codec_cls.from_dict(c),)
             except KeyError as e:
-                raise UnknownCodecError(f"Unknown codec: {e.args[0]!r}") from e
+                # A codec's `from_dict` may index its configuration directly, so a malformed
+                # configuration surfaces as a KeyError. Convert it: a bare KeyError escaping
+                # metadata parsing is swallowed by the array-then-group fallback in
+                # `zarr.api.asynchronous.open`, which then reports an unrelated group error.
+                raise MetadataValidationError(
+                    f"Invalid configuration for codec {name_parsed!r}: missing key {e.args[0]!r}."
+                ) from e
 
     return out
 

src/zarr/errors.py

@@ -12,6 +12,7 @@
     "MetadataValidationError",
     "NegativeStepError",
     "NodeTypeValidationError",
+    "UnknownCodecError",
     "UnstableSpecificationWarning",
     "VindexInvalidSelectionError",
     "ZarrDeprecationWarning",

src/zarr/metadata/migrate_v3.py

@@ -29,6 +29,7 @@
 from zarr.core.metadata.v2 import ArrayV2Metadata
 from zarr.core.metadata.v3 import ArrayV3Metadata, RegularChunkGridMetadata
 from zarr.core.sync import sync
+from zarr.errors import UnknownCodecError
 from zarr.registry import get_codec_class
 from zarr.storage import StorePath
 from zarr.types import AnyArray
@@ -273,7 +274,7 @@ def _find_numcodecs_zarr3(numcodecs_codec: numcodecs.abc.Codec) -> Codec:
 
     try:
         codec_v3 = get_codec_class(numcodec_name)
-    except KeyError as exc:
+    except UnknownCodecError as exc:
         raise ValueError(
             f"Couldn't find corresponding zarr.codecs.numcodecs codec for {numcodecs_codec.codec_id}"
         ) from exc

src/zarr/registry.py

@@ -2,12 +2,13 @@
 
 import warnings
 from collections import defaultdict
+from collections.abc import Mapping
 from importlib.metadata import entry_points as get_entry_points
 from typing import TYPE_CHECKING, Any
 
 from zarr.core.config import BadConfigError, config
 from zarr.core.dtype import data_type_registry
-from zarr.errors import ZarrUserWarning
+from zarr.errors import UnknownCodecError, ZarrUserWarning
 
 if TYPE_CHECKING:
     from importlib.metadata import EntryPoint
@@ -23,7 +24,7 @@
     from zarr.abc.numcodec import Numcodec
     from zarr.core.buffer import Buffer, NDBuffer
     from zarr.core.chunk_key_encodings import ChunkKeyEncoding
-    from zarr.core.common import JSON
+    from zarr.core.common import JSON, ZarrFormat
 
 __all__ = [
     "Registry",
@@ -39,6 +40,131 @@
     "register_pipeline",
 ]
 
+_ZARR_CODEC_DOCS_URL = "https://zarr.readthedocs.io/en/stable/user-guide/extending/#custom-codecs"
+_NUMCODECS_CODEC_DOCS_URL = (
+    "https://numcodecs.readthedocs.io/en/stable/registry.html#numcodecs.registry.register_codec"
+)
+
+# Codecs zarr-python does not implement, mapped to the names of Python packages that do.
+# These tables exist purely to make the "no implementation for this codec" error actionable;
+# nothing here affects which codecs zarr can actually read or write. Values are what you would
+# pass to `pip install`. Only add an entry you have verified against the package's declared
+# entry points, and only for a package that is actually published.
+#
+# The two Zarr formats resolve codecs through different registries, so they get different
+# tables: a name can mean one thing as a Zarr format 3 codec name and another as a Zarr
+# format 2 codec id. `imagecodecs_*` is exactly that -- `virtual-tiff` declares 15 of those
+# names under `zarr.codecs`, while `imagecodecs-numcodecs` declares all 81 under
+# `numcodecs.codecs`, so the format 2 side can use a prefix and the format 3 side cannot.
+
+# Zarr format 3 codec names (entry point group "zarr.codecs").
+_CODEC_PACKAGES: dict[str, tuple[str, ...]] = {
+    "gribberish": ("gribberish",),
+    # `virtual-tiff` declares these 15 `imagecodecs_*` names, out of the 81 that exist as
+    # numcodecs ids. They are listed exactly rather than by prefix so that the other 66 get no
+    # hint instead of a hint pointing at a package that does not provide them.
+    "imagecodecs_deflate": ("virtual-tiff",),
+    "imagecodecs_delta": ("virtual-tiff",),
+    "imagecodecs_floatpred": ("virtual-tiff",),
+    "imagecodecs_jetraw": ("virtual-tiff",),
+    "imagecodecs_jpeg": ("virtual-tiff",),
+    "imagecodecs_jpeg2k": ("virtual-tiff",),
+    "imagecodecs_jpeg8": ("virtual-tiff",),
+    "imagecodecs_jpegxl": ("virtual-tiff",),
+    "imagecodecs_jpegxr": ("virtual-tiff",),
+    "imagecodecs_lerc": ("virtual-tiff",),
+    "imagecodecs_lzw": ("virtual-tiff",),
+    "imagecodecs_packbits": ("virtual-tiff",),
+    "imagecodecs_png": ("virtual-tiff",),
+    "imagecodecs_webp": ("virtual-tiff",),
+    "imagecodecs_zstd": ("virtual-tiff",),
+    "n5_default": ("zarr-n5",),
+}
+
+# As `_CODEC_PACKAGES`, but each key is matched against the start of the codec name. Packages
+# that provide many codecs namespace them behind a shared prefix, so one entry covers them all.
+_CODEC_PACKAGE_PREFIXES: dict[str, tuple[str, ...]] = {
+    "any-numcodecs.": ("zarr-any-numcodecs",),
+    "omfiles.": ("omfiles",),
+    "virtual_tiff.": ("virtual-tiff",),
+}
+
+# Zarr format 2 codec ids (entry point group "numcodecs.codecs"). `numcodecs` itself gates
+# several of its own codecs behind optional dependencies, so the package to install for those
+# is an extra of numcodecs rather than a third-party distribution.
+_NUMCODEC_PACKAGES: dict[str, tuple[str, ...]] = {
+    "FITSAscii": ("kerchunk",),
+    "FITSVarBintable": ("kerchunk",),
+    "crc32c": ("numcodecs[crc32c]",),
+    "fill_hdf_strings": ("kerchunk",),
+    "grib": ("kerchunk",),
+    "msgpack2": ("numcodecs[msgpack]",),
+    "pcodec": ("numcodecs[pcodec]",),
+    "rawgrib": ("gribscan",),
+    "record_member": ("kerchunk",),
+    "vc-delta3d": ("vc-delta3d",),
+    "wavpack": ("wavpack-numcodecs",),
+    "zfpy": ("numcodecs[zfpy]",),
+}
+
+# As `_NUMCODEC_PACKAGES`, but matched against the start of the codec id.
+_NUMCODEC_PACKAGE_PREFIXES: dict[str, tuple[str, ...]] = {
+    "gribscan.": ("gribscan",),
+    "imagecodecs_": ("imagecodecs-numcodecs",),
+}
+
+
+def _packages_for_codec(name: str, *, zarr_format: ZarrFormat) -> tuple[str, ...]:
+    """
+    Names of Python packages known to provide an implementation of the codec ``name``.
+
+    Returns an empty tuple if we don't know of any.
+
+    Parameters
+    ----------
+    name : str
+        The codec name (Zarr format 3) or codec id (Zarr format 2) we failed to resolve.
+    zarr_format : ZarrFormat
+        Which registry the codec was looked up in.
+    """
+    if zarr_format == 2:
+        exact, prefixes = _NUMCODEC_PACKAGES, _NUMCODEC_PACKAGE_PREFIXES
+    else:
+        exact, prefixes = _CODEC_PACKAGES, _CODEC_PACKAGE_PREFIXES
+    if name in exact:
+        return exact[name]
+    for prefix, packages in prefixes.items():
+        if name.startswith(prefix):
+            return packages
+    return ()
+
+
+def _missing_codec_message(name: str, *, zarr_format: ZarrFormat) -> str:
+    """
+    Build the error message raised when no implementation of the codec ``name`` is available.
+
+    Parameters
+    ----------
+    name : str
+        The codec name (Zarr format 3) or codec id (Zarr format 2) we failed to resolve.
+    zarr_format : ZarrFormat
+        Which registry the codec was looked up in. Zarr format 2 codecs are resolved through
+        numcodecs, so that case points at the numcodecs registry rather than at zarr's.
+    """
+    if zarr_format == 2:
+        docs_url, registry = _NUMCODECS_CODEC_DOCS_URL, "numcodecs"
+    else:
+        docs_url, registry = _ZARR_CODEC_DOCS_URL, "zarr"
+    msg = (
+        f"An implementation for codec {name!r} is not available. Register one explicitly "
+        f"using the codec registry (see {docs_url}), or install a Python package that "
+        f"registers a codec implementation with {registry}."
+    )
+    packages = _packages_for_codec(name, zarr_format=zarr_format)
+    if packages:
+        msg += f" Known packages supporting this codec: {', '.join(packages)}."
+    return msg
+
 
 class Registry[T](dict[str, type[T]]):
     def __init__(self) -> None:
@@ -168,7 +294,7 @@ def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]:
 
     codec_classes = _codec_registries[key]
     if not codec_classes:
-        raise KeyError(key)
+        raise UnknownCodecError(_missing_codec_message(key, zarr_format=3))
     config_entry = config.get("codecs", {}).get(key)
     if config_entry is None:
         if len(codec_classes) == 1:
@@ -179,11 +305,17 @@ def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]:
             category=ZarrUserWarning,
         )
         return list(codec_classes.values())[-1]
-    selected_codec_cls = codec_classes[config_entry]
-
-    if selected_codec_cls:
-        return selected_codec_cls
-    raise KeyError(key)
+    selected_codec_cls = codec_classes.get(config_entry)
+    if selected_codec_cls is None:
+        # Not UnknownCodecError: the codec is known, the implementation named in the config is
+        # not registered. That is a configuration problem, which is what the sibling getters in
+        # this module raise BadConfigError for.
+        raise BadConfigError(
+            f"Codec {key!r} is configured to use the implementation {config_entry!r}, which is "
+            f"not registered. Registered implementations of this codec: "
+            f"{sorted(codec_classes)}."
+        )
+    return selected_codec_cls
 
 
 def _resolve_codec(data: dict[str, JSON]) -> Codec:
@@ -321,6 +453,13 @@ def get_numcodec(data: CodecJSON_V2[str]) -> Numcodec:
     -------
     codec : Numcodec
 
+    Raises
+    ------
+    UnknownCodecError
+        If ``data`` carries a string ``"id"`` that is not registered with numcodecs. Any other
+        failure, including a registered codec rejecting its configuration and a ``data`` that is
+        not a mapping, propagates from numcodecs unchanged.
+
     Examples
     --------
     ```python
@@ -331,6 +470,19 @@ def get_numcodec(data: CodecJSON_V2[str]) -> Numcodec:
     ```
     """
 
-    from numcodecs.registry import get_codec
-
+    from numcodecs.registry import codec_registry, entries, get_codec
+
+    # Check whether numcodecs can resolve the id *before* handing off, rather than catching what
+    # `get_codec` raises. Catching cannot tell "this id is unregistered" from "a registered codec
+    # rejected its configuration" or from "a wrapper codec failed to resolve an inner codec", and
+    # relabelling either of those with this id would attach a package hint that is simply wrong.
+    # This mirrors the two lookups `get_codec` performs (it then tests the result for
+    # truthiness rather than membership, which only differs for a falsy registry value).
+    # Widened to `object` deliberately: `data` is annotated as a TypedDict, but this is a public
+    # function and callers pass whatever they like. numcodecs coerces with `dict(config)` and
+    # raises for anything that is not a mapping, which is the behaviour to preserve.
+    raw: object = data
+    codec_id = raw.get("id") if isinstance(raw, Mapping) else None
+    if isinstance(codec_id, str) and codec_id not in codec_registry and codec_id not in entries:
+        raise UnknownCodecError(_missing_codec_message(codec_id, zarr_format=2))
     return get_codec(data)  # type: ignore[no-any-return]

tests/test_codecs/test_numcodecs.py

@@ -8,15 +8,10 @@
 import pytest
 from numcodecs import GZip
 
-try:
-    from numcodecs.errors import UnknownCodecError
-except ImportError:
-    # Older versions of numcodecs don't have a separate errors module
-    UnknownCodecError = ValueError
-
 from zarr import config, create_array, open_array
 from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls
 from zarr.codecs import numcodecs as _numcodecs
+from zarr.errors import UnknownCodecError
 from zarr.registry import get_codec_class, get_numcodec
 
 if TYPE_CHECKING:
@@ -297,12 +292,7 @@ def test_generic_checksum(codec_class: type[_numcodecs._NumcodecsBytesBytesCodec
 def test_generic_bytes_codec(codec_class: type[_numcodecs._NumcodecsArrayBytesCodec]) -> None:
     try:
         codec_class()._codec  # noqa: B018
-    except ValueError as e:  # pragma: no cover
-        if "codec not available" in str(e):
-            pytest.xfail(f"{codec_class.codec_name} is not available: {e}")  # type: ignore[misc]
-        else:
-            raise
-    except ImportError as e:  # pragma: no cover
+    except (UnknownCodecError, ImportError) as e:  # pragma: no cover
         pytest.xfail(f"{codec_class.codec_name} is not available: {e}")  # type: ignore[misc]
 
     data = np.arange(0, 256, dtype="float32").reshape((16, 16))

tests/test_config.py

@@ -24,7 +24,7 @@
 from zarr.core.codec_pipeline import BatchedCodecPipeline
 from zarr.core.config import BadConfigError, config
 from zarr.core.indexing import SelectorTuple
-from zarr.errors import ChunkNotFoundError, ZarrUserWarning
+from zarr.errors import ChunkNotFoundError, UnknownCodecError, ZarrUserWarning
 from zarr.registry import (
     fully_qualified_name,
     get_buffer_class,
@@ -334,7 +334,7 @@ class NewCodec2(BytesCodec):
         pass
 
     # error if codec is not registered
-    with pytest.raises(KeyError):
+    with pytest.raises(UnknownCodecError):
         get_codec_class("missing_codec")
 
     # no warning if only one implementation is available