diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index 45aa5268..554a9d0d 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -22,19 +22,27 @@ def double(arr: NDArray) -> NDArray: from __future__ import annotations -from collections.abc import Mapping, Sequence -from typing import Annotated, Any, List, Union +import sys +from collections.abc import Callable, Mapping, Sequence +from typing import Annotated, Any, List, Literal, Self, Union, cast import numpy as np from pydantic import ( + Base64Bytes, + BaseModel, ConfigDict, + GetCoreSchemaHandler, + GetJsonSchemaHandler, PlainSerializer, PlainValidator, RootModel, SerializerFunctionWrapHandler, WithJsonSchema, WrapSerializer, + model_validator, ) +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import PydanticCustomError, core_schema from typing_extensions import TypeAlias # Define a nested list of floats with 0-6 dimensions @@ -148,3 +156,358 @@ class DenumpifyingDict(RootModel): root: Annotated[Mapping, WrapSerializer(denumpify_serialiser)] model_config = ConfigDict(arbitrary_types_allowed=True) + + +ElementType: TypeAlias = Literal["bool", "uint", "int", "float", "complex"] + + +ELEMENT_TYPES_BY_KIND: dict[str, ElementType] = { + "b": "bool", + "u": "uint", + "i": "int", + "f": "float", + "c": "complex", +} +"""A mapping from numpy ``dtype.kind`` to element type names. + +The single-letter codes are documented at `numpy.dtype.kind` but are +specific to `numpy`. LabThings uses slightly more verbose notation +that ought to be clear to someone working in any language. +""" + + +KINDS_BY_ELEMENT_TYPE = {etype: kind for kind, etype in ELEMENT_TYPES_BY_KIND.items()} +"""A mapping from element type names to `numpy.dtype.kind` values.""" + + +INT_LENGTHS = (1, 2, 4, 8) +"""Allowed lengths (in bytes) for integer types.""" + + +FLOAT_LENGTHS = (2, 4, 8, np.dtype(np.longdouble).itemsize) +"""Allowed lengths (in bytes) for floating point types. + +Note that the highest-precision type is 12 or 16 bytes depending on +the platform. +""" + + +COMPLEX_LENGTHS = (8, 16, np.dtype(np.clongdouble).itemsize) +"""Allowed lengths (in bytes) for complex types. + +Note that the highest-precision type is 12 or 16 bytes depending on +the platform. +""" + + +def _check_length(length: int, allowed: tuple[int, ...], etype: ElementType) -> bool: + """Check a datatype length is in the list of allowed lengths. + + :param length: the specified element length. + :param allowed: the allowed lengths. + :param etype: the element type. + :return: `True` if the length is allowed. + :raises PydanticCustomError: if the length is not allowed. + """ + if length in allowed: + return True + raise PydanticCustomError( + "unsupported_length", + "'element_length' must be in {allowed_lengths} for {element_type} " + "but '{element_length}' was specified.", + { + "element_length": length, + "allowed_lengths": allowed, + "element_type": etype, + }, + ) + + +def byte_order_name(native: bool) -> Literal["little", "big"]: + """Return the name for a byte order. + + :param native: whether to return the machine's byte order. + :return: the machine's byte order, or the other order, as a string. + """ + if native: + return sys.byteorder + else: + return "little" if sys.byteorder == "big" else "big" + + +class Base64NDArrayModel(BaseModel): + """Numeric arrays that can be serialised to JSON.""" + + model_config = {"use_attribute_docstrings": True} + + element_type: ElementType + """The datatype of each array element.""" + + element_length: int + """The length of each array element in bytes. + + Valid lengths are platform-dependent, but usually include + 1, 2, 4, 8 for integers and 2, 4, 8 and either 12 or 16 for + floating point numbers. Boolean arrays always use one byte + per element. + """ + + byte_order: Literal["little", "big", None] = None + """Whether the data is in little-endian or big-endian format. + + Note that order is irrelevant for data types that are only + one byte long, i.e. ``element_length==1``. In that case, `None` + is the only allowed value, though implementations may decide + to ignore this field rather than raise an error. + + For data types longer than one byte, ``byte_order`` must always + be specified and must be either `"little"` or `"big"`. + """ + + shape: tuple[int, ...] + """The shape of the array, i.e. the length in each dimension.""" + + array_order: Literal["c-contiguous"] + """Whether the array elements are in C or fortran order. + + Currently only c order is supported. + """ + + data: Base64Bytes + """The data for the array, Base64 encoded. + + When accessing a valid `Base64NDArrayModel` in Python, or when creating + one using `model_construct`, this field holds the array's data as a `bytes` + object. When constructing the model with `model_validate` or the constructor, + the value of ``data`` should be a Base64 encoded `bytes` object. + + The value in a JSON string will always be Base64 encoded. + """ + + @property + def _kind(self) -> str: + """A one-character string matching `numpy.dtype.kind`.""" + return KINDS_BY_ELEMENT_TYPE[self.element_type] + + @property + def _endian_char(self) -> str: + """A one-character string describing endianness.""" + if self.byte_order is None: + return "|" # one-byte values have no endianness + return "<" if self.byte_order == "little" else ">" + + @property + def _descr(self) -> list[tuple[str, str]]: + """The expected value of `numpy.dtype.descr` for this array. + + This property may be compared with `array.dtype.descr` and ought + to match exactly. If it doesn't, we have found an edge case and + it's likely validation/serialisation will fail. + """ + return [("", f"{self._endian_char}{self._kind}{self.element_length}")] + + @model_validator(mode="after") + def check_dtype(self) -> Self: + """Check that the data type is supported. + + This validator ensures that a byte order is specified for element lengths + of more than one byte, and that no byte order is specified for one-byte + element types. + + We also check that the element length is valid for the type specified. + + Finally, we construct a `numpy.dtype` object to check that the + datatype is supported. + + :return: the array model (with `byte_order` coerced if needed). + :raises PydanticCustomError: if the model is not valid. + """ + if self.element_length > 1: + if self.byte_order is None: + raise PydanticCustomError( + "missing_byte_order", + "'byte_order' must be 'little' or 'big' when element_length > 1.", + ) + else: + # For one-byte values, ignore the specified `byte_order` + # We could raise a validation error if it's not None, but + # coercing feels reasonable. + self.byte_order = None + if self.element_type == "bool": + _check_length(self.element_length, (1,), "bool") + elif self.element_type in ("int", "uint"): + _check_length(self.element_length, INT_LENGTHS, self.element_type) + elif self.element_type == "float": + _check_length(self.element_length, FLOAT_LENGTHS, self.element_type) + elif self.element_type == "complex": + _check_length(self.element_length, COMPLEX_LENGTHS, self.element_type) + + # Finally, check we can construct a `dtype` object. If this fails, + # something has gone wrong with the checks above. + np.dtype(self._descr) + + return self + + @model_validator(mode="after") + def check_size(self) -> Self: + """Check that the specified size matches the data. + + :return: the array model, unchanged. + :raises PydanticCustomError: if the length of the data is incorrect. + """ + size = self.element_length + for dimension in self.shape: + size *= dimension + if len(self.data) != size: + raise PydanticCustomError( + "length_mismatch", + "The data supplied was {length} bytes but we expected {expected}.", + {"length": len(self.data), "expected": size}, + ) + return self + + @classmethod + def from_array(cls, array: np.ndarray) -> Self: + """Construct a Base64NDArrayModel from a numpy array. + + Note that this will raise an exception if the array is not in a supported + format. This class only deals with boolean or numeric arrays. + + :param array: a `numpy.ndarray` object. + :return: a validated model holding the array's data. + :raises ValueError: if the array has an unsupported type. + """ + dtype = array.dtype + if dtype.kind not in ELEMENT_TYPES_BY_KIND: + raise ValueError(f"The array has dtype '{dtype}' which is not supported.") + _model = cls.model_construct( + element_type=ELEMENT_TYPES_BY_KIND[dtype.kind], + element_length=dtype.itemsize, + byte_order=None if dtype.itemsize == 1 else byte_order_name(dtype.isnative), + shape=array.shape, + array_order="c-contiguous", + data=array.tobytes(order="C"), + ) + # This comment block explains the cast statement below. + # _model will have type Self, because that's the type of `cls.model_construct`. + # MyPy infers the type as Base64ArrayModel rather than Self, which is correct + # in the base class but is incorrect for a subclass. The cast below types it + # correctly. For `Base64ArrayModel` this is a no-op, as `Self` refers to the + # same type. In a subclass, this could potentially be helpful. + # FWIW `BaseModel.model_construct` is correctly typed as returning + # Self so this feels like a MyPy/Pydantic interaction problem. + model = cast(Self, _model) + if model._descr != dtype.descr: + raise ValueError( + "The array dtype couldn't be matched. " + f"Input dtype.descr='{dtype.descr}', Expected '{model._descr}'." + ) + return model + + def to_array(self) -> np.ndarray: + """Construct a numpy array from this model. + + :return: the data, reinflated to a `numpy.ndarray` object. + """ + dtype = np.dtype(f"{self._endian_char}{self._kind}{self.element_length}") + return np.frombuffer(self.data, dtype=dtype).reshape(self.shape) + + +class NDArrayAnnotation: + """A class that enables serialisation and validation of `numpy` arrays. + + This class is intended to be used as part of an annotated type, i.e. + ``Annotated[np.ndarray, NDArrayAnnotation]``. + + For documentation on how this works, see the relevant section of the + `pydantic` docs: + + https://pydantic.dev/docs/validation/dev/concepts/types/#as-an-annotation + """ + + @classmethod + def __get_pydantic_core_schema__( + cls, source: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + """Get the pydantic core schema for this type. + + This magic method allows `pydantic` to serialise `.Blob` + instances, and generate a JSONSchema for them. + + We tell `pydantic` to base its handling of `Blob` on the + `.BlobModel` schema, with custom validation and serialisation. + Validation and serialisation behaviour is described in the docstrings + of `.Blob._validate` and `.Blob._serialise`. + + The JSONSchema is generated for `.BlobModel` but is then refined + in `__get_pydantic_json_schema__` to include the ``media_type`` + and ``description`` defaults. + + :param source: The source type being converted. + :param handler: The pydantic core schema handler. + :return: The pydantic core schema for the URLFor type. + """ + return core_schema.no_info_wrap_validator_function( + cls._validate, + Base64NDArrayModel.__pydantic_core_schema__, + serialization=core_schema.wrap_serializer_function_ser_schema( + cls._serialise, + is_field_serializer=False, + info_arg=False, + when_used="always", + ), + ) + + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + """Customise the JSON Schema to include the media_type. + + :param core_schema: The core schema. + :param handler: The pydantic JSON schema handler. + :return: The JSON schema for the type, with media_type included. + """ + json_schema = handler(core_schema) + json_schema = handler.resolve_ref_schema(json_schema) + # Set a nicer title + json_schema["title"] = "Numeric Array" + return json_schema + + @classmethod + def _validate( + cls, value: Any, handler: Callable[[Any], Base64NDArrayModel] + ) -> np.ndarray: + r"""Validate and convert a value to a `numpy.ndarray` instance. + + :param value: The input value, as passed in or loaded from JSON. + :param handler: A function that runs the validation logic of Base64NDArrayModel. + + If the value is already a `~numpy.ndarray`, it will be returned directly. + Otherwise, we first validate the input using the `Base64NDArrayModel` schema. + + :return: a `~numpy.ndarray` object. + """ + # If the value is already a Blob, return it directly + if isinstance(value, np.ndarray): + return value + # If not, validate it using the array model + model = handler(value) + return model.to_array() + + @classmethod + def _serialise( + cls, obj: np.ndarray, handler: Callable[[Base64NDArrayModel], Mapping[str, str]] + ) -> Mapping[str, str]: + """Serialise the Blob to a dictionary. + + See `.Blob.to_blobmodel` for a description of how we serialise. + + :param obj: the `.Blob` instance to serialise. + :param handler: the handler (provided by pydantic) takes a BlobModel + and converts it to a dictionary. The handler runs the serialiser of + the core schema we've wrapped, in this case the BlobModel serialiser. + :return: a JSON-serialisable dictionary with a URL that allows + the `.Blob` to be downloaded from the `.BlobManager`. + """ + return handler(Base64NDArrayModel.from_array(obj)) diff --git a/tests/test_types/__init__.py b/tests/test_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_types/test_numpy.py b/tests/test_types/test_numpy.py new file mode 100644 index 00000000..66197887 --- /dev/null +++ b/tests/test_types/test_numpy.py @@ -0,0 +1,247 @@ +"""Test the numpy types. + +This module tests the new model and annotated type for numpy arrays. +It's intended that the old tests (`../test_numpy_type.py`) will be +removed once the old types are retired. +""" + +import base64 +from datetime import datetime +from typing import Annotated + +import numpy as np +import pytest +from pydantic import RootModel, ValidationError + +from labthings_fastapi.types.numpy import Base64NDArrayModel, NDArrayAnnotation + +ARRAYS = ( + [ # Unsigned integer types + np.arange(0, 255, dtype=d) for d in [np.uint8, np.uint16, np.uint32, np.uint64] + ] + + [ # Signed integer types + np.arange(-100, 100, dtype=d) for d in [np.int8, np.int16, np.int32, np.int64] + ] + + [ # Floating point types + 3 ** np.arange(-5, 5, dtype=d) + for d in [np.float16, np.float32, np.float64, np.float128] + ] + + [ # Complex types + np.array([0.4 + 0.7j, 35 + 85j, 89, -49j], dtype=d) + for d in [np.complex64, np.complex128, np.complex256] + ] + + [ # Boolean arrays + np.array([False, True, False, True], dtype=np.bool) + ] + + [ # Scalars + np.array(1000), # scalar int + np.array(10.0), # scalar float + ] + + [ # Non-numeric float values + np.array([0.0, 1.0, np.inf, -np.inf, np.nan]) + ] + + [ # Both little- and big-endian arrays + np.array([0, -4, 16, 255], dtype=d) for d in ["i2", "i8"] + ] +) + + +UNSUPPORTED_ARRAYS = [ + # Record arrays aren't supported + np.array([(1.0, 2), (3.0, 4)], dtype=[("x", "1 byte must have byte_order + { + "element_type": "int", + "element_length": 2, + "array_order": "c-contiguous", + "shape": (1,), + "data": b"", + }, + "byte_order", + ), + ( # 3 byte ints are not a thing! + { + "element_type": "int", + "element_length": 3, + "byte_order": "little", + "array_order": "c-contiguous", + "shape": (1,), + "data": b"", + }, + "unsupported_length", + ), + ( # Mismatched length + { + "element_type": "uint", + "element_length": 1, + "byte_order": "little", + "array_order": "c-contiguous", + "shape": (3,), + "data": base64.b64encode(b"0000"), + }, + "length_mismatch", + ), + ], +) +def test_invalid_json(input, msg): + """Test invalid input fails at validation stage, not afterwards.""" + with pytest.raises(ValidationError, match=msg): + Base64NDArrayModel.model_validate(input) + + +def test_descr_mismatch(): + """Check the error that occurs when `dtype.descr` doesn't match. + + This is an error I don't ever expect in reality. We already check + the relevant properties of an array's datatype, and raise an + error if it's not supported (see previous test). + + As a belt-and-braces check, we generate the `numpy.dtype.descr` + value we'd expect given the datatype, element size, and endianness + that we have recorded, and check our array has the expected + value. + + This `numpy`-specific string is what gets recorded in a ``npz`` file + to allow the data to be reinflated, so it's a very good check that + we have correctly recorded the array's parameters. + + Any array that's not properly supported will fail before we check + ``descr`` so this test mocks the property to trigger the error. + """ + + class ArrayModelWithBrokenDescr(Base64NDArrayModel): + """This feels simpler and clearer than mocking the class.""" + + @property + def _descr(self) -> list[tuple[str, str]]: + return [("", "!!!")] + + with pytest.raises(ValueError, match="dtype.descr"): + ArrayModelWithBrokenDescr.from_array(np.array([1, 2, 3])) + + +@pytest.mark.parametrize("array", ARRAYS) +def test_rootmodel_annotation(array): + """Test that the annotated type validates and serialises correctly in Python.""" + model = NDArrayRootModel(array) + dumped = model.model_dump() + assert set(dumped.keys()) == FIELDS + reconstructed = NDArrayRootModel.model_validate(dumped) + reconstructed_array = reconstructed.root + assert array.dtype == reconstructed_array.dtype + assert array.dtype.descr == reconstructed_array.dtype.descr + assert np.array_equal(array, reconstructed_array, equal_nan=True) + + +@pytest.mark.parametrize("array", ARRAYS) +def test_rootmodel_annotation_json(array): + """Test that the annotated type validates and serialises correctly in JSON.""" + model = NDArrayRootModel(array) + dumped = model.model_dump_json() + for field in FIELDS: + assert field in dumped + reconstructed = NDArrayRootModel.model_validate_json(dumped) + reconstructed_array = reconstructed.root + assert array.dtype == reconstructed_array.dtype + assert array.dtype.descr == reconstructed_array.dtype.descr + assert np.array_equal(array, reconstructed_array, equal_nan=True) + + +def test_annotation_jsonschema(): + """Make sure the JSON Schema is as expected.""" + schema = NDArrayRootModel.model_json_schema() + assert schema["title"] == "Numeric Array" # customised by the annotation + assert schema["properties"]["element_type"]["description"] == ( + # Check that docstrings are correctly pulled through as descriptions + "The datatype of each array element." + ) + assert set(schema["properties"]["element_type"]["enum"]) == { + # Check the datatype options match what's expected + "bool", + "int", + "uint", + "float", + "complex", + } + assert schema["properties"]["element_length"]["type"] == "integer" + assert set(schema["properties"]["byte_order"]["enum"]) == {"little", "big", None} + # We only support c-contiguous arrays. Check that's clear. + assert schema["properties"]["array_order"]["const"] == "c-contiguous" + assert schema["properties"]["shape"]["type"] == "array" + assert schema["properties"]["shape"]["items"] == {"type": "integer"} + assert schema["properties"]["data"]["format"] == "base64" + for v in schema["properties"].values(): + assert len(v["description"]) > 0, "All properties should have a description."