From 1d829785eebd0275395ecadeb31b10edd26a53bf Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Tue, 10 Nov 2026 22:13:49 +0000 Subject: [PATCH 1/8] Pydantic serialisation of numpy arrays to base64 This commit introduces a Pydantic model for numpy arrays, and a corresponding class that may be used as an annotated type to serialise/validate arrays via that model, while keeping the type annotation as `np.ndarray`. There's a reasonable level of testing of both classes, and all of the supported datatypes are round-tripped via both simple Python objects and via JSON strings. --- src/labthings_fastapi/types/numpy.py | 232 ++++++++++++++++++++++++++- tests/test_types/__init__.py | 0 tests/test_types/test_numpy.py | 127 +++++++++++++++ 3 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 tests/test_types/__init__.py create mode 100644 tests/test_types/test_numpy.py diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index 45aa5268..e86217b2 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -22,12 +22,16 @@ def double(arr: NDArray) -> NDArray: from __future__ import annotations -from collections.abc import Mapping, Sequence -from typing import Annotated, Any, List, Union +import sys +from collections.abc import Callable, Mapping, Sequence +from typing import Annotated, Any, List, Literal, Self, Union, cast import numpy as np from pydantic import ( + Base64Bytes, + BaseModel, ConfigDict, + GetCoreSchemaHandler, PlainSerializer, PlainValidator, RootModel, @@ -35,6 +39,7 @@ def double(arr: NDArray) -> NDArray: WithJsonSchema, WrapSerializer, ) +from pydantic_core import core_schema from typing_extensions import TypeAlias # Define a nested list of floats with 0-6 dimensions @@ -148,3 +153,226 @@ class DenumpifyingDict(RootModel): root: Annotated[Mapping, WrapSerializer(denumpify_serialiser)] model_config = ConfigDict(arbitrary_types_allowed=True) + + +ElementType: TypeAlias = Literal["bool", "uint", "int", "float", "complex"] + + +ELEMENT_TYPES_BY_KIND: dict[str, ElementType] = { + "b": "bool", + "u": "uint", + "i": "int", + "f": "float", + "c": "complex", +} +"""A mapping from numpy ``dtype.kind`` to element type names. + +The single-letter codes are documented at `numpy.dtype.kind` but are +specific to `numpy`. LabThings uses slightly more verbose notation +that ought to be clear to someone working in any language. +""" + + +KINDS_BY_ELEMENT_TYPE = {etype: kind for kind, etype in ELEMENT_TYPES_BY_KIND.items()} +"""A mapping from element type names to `numpy.dtype.kind` values.""" + + +def byte_order_name(native: bool) -> Literal["little", "big"]: + """Return the name for a byte order. + + :param native: whether to return the machine's byte order. + :return: the machine's byte order, or the other order, as a string. + """ + if native: + return sys.byteorder + else: + return "little" if sys.byteorder == "big" else "big" + + +class Base64NDArrayModel(BaseModel): + """A Pydantic model for numpy arrays.""" + + element_type: ElementType + """The datatype of each array element.""" + + element_length: int + """The length of each array element in bytes.""" + + byte_order: Literal["little", "big"] + """Whether the data is in little-endian or big-endian format.""" + + shape: tuple[int, ...] + """The shape of the array, i.e. the length in each dimension.""" + + array_order: Literal["c-contiguous", "fortran-contiguous"] + """Whether the array elements are in C or fortran order.""" + + data: Base64Bytes + """The data for the array, Base64 encoded.""" + + @property + def _kind(self) -> str: + """A one-character string describing the element_type.""" + return KINDS_BY_ELEMENT_TYPE[self.element_type] + + @property + def _endian_char(self) -> str: + """A one-character string describing endianness.""" + if self.element_length == 1: + return "|" # one-byte values have no endianness + return "<" if self.byte_order == "little" else ">" + + @property + def _descr(self) -> list[tuple[str, str]]: + """The expected value of `numpy.dtype.descr` for this array. + + This property may be compared with `array.dtype.descr` and ought + to match exactly. If it doesn't, we have found an edge case. + """ + return [("", f"{self._endian_char}{self._kind}{self.element_length}")] + + @classmethod + def from_array(cls, array: np.ndarray) -> Self: + """Construct a Base64NDArrayModel from a numpy array. + + Note that this will raise an exception if the array is not in a supported + format. This class only deals with boolean or numeric arrays. + + :param array: a `numpy.ndarray` object. + :return: a validated model holding the array's data. + :raises ValueError: if the array has an unsupported type. + """ + dtype = array.dtype + if dtype.kind not in ELEMENT_TYPES_BY_KIND: + raise ValueError(f"The array has dtype '{dtype}' which is not supported.") + _model = cls.model_construct( + element_type=ELEMENT_TYPES_BY_KIND[dtype.kind], + element_length=dtype.itemsize, + byte_order=byte_order_name(dtype.isnative), + shape=array.shape, + array_order="c-contiguous", + data=array.tobytes(order="C"), + ) + # This comment block explains the cast statement below. + # _model will have type Self, because that's the type of `cls.model_construct`. + # MyPy infers the type as Base64ArrayModel rather than Self, which is correct + # in the base class but is incorrect for a subclass. The cast below types it + # correctly. For `Base64ArrayModel` this is a no-op, as `Self` refers to the + # same type. In a subclass, this could potentially be helpful. + # FWIW `BaseModel.model_construct` is correctly typed as returning + # Self so this feels like a MyPy/Pydantic interaction problem. + model = cast(Self, _model) + if model._descr != dtype.descr: + raise ValueError( + "The array dtype couldn't be matched. " + f"Input dtype.descr='{dtype.descr}', Expected '{model._descr}'." + ) + return model + + def to_array(self) -> np.ndarray: + """Construct a numpy array from this model. + + :return: the data, reinflated to a `numpy.ndarray` object. + """ + dtype = np.dtype(f"{self._endian_char}{self._kind}{self.element_length}") + return np.frombuffer(self.data, dtype=dtype).reshape(self.shape) + + +class NDArrayAnnotation: + """A class that enables serialisation and validation of `numpy` arrays. + + This class is intended to be used as part of an annotated type, i.e. + ``Annotated[np.ndarray, NDArrayAnnotation]``. + + For documentation on how this works, see the relevant section of the + `pydantic` docs: + + https://pydantic.dev/docs/validation/dev/concepts/types/#as-an-annotation + """ + + @classmethod + def __get_pydantic_core_schema__( + cls, source: type[Any], handler: GetCoreSchemaHandler + ) -> core_schema.CoreSchema: + """Get the pydantic core schema for this type. + + This magic method allows `pydantic` to serialise `.Blob` + instances, and generate a JSONSchema for them. + + We tell `pydantic` to base its handling of `Blob` on the + `.BlobModel` schema, with custom validation and serialisation. + Validation and serialisation behaviour is described in the docstrings + of `.Blob._validate` and `.Blob._serialise`. + + The JSONSchema is generated for `.BlobModel` but is then refined + in `__get_pydantic_json_schema__` to include the ``media_type`` + and ``description`` defaults. + + :param source: The source type being converted. + :param handler: The pydantic core schema handler. + :return: The pydantic core schema for the URLFor type. + """ + return core_schema.no_info_wrap_validator_function( + cls._validate, + Base64NDArrayModel.__pydantic_core_schema__, + serialization=core_schema.wrap_serializer_function_ser_schema( + cls._serialise, + is_field_serializer=False, + info_arg=False, + when_used="always", + ), + ) + + # @classmethod + # def __get_pydantic_json_schema__( + # cls, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler + # ) -> JsonSchemaValue: + # """Customise the JSON Schema to include the media_type. + + # :param core_schema: The core schema. + # :param handler: The pydantic JSON schema handler. + # :return: The JSON schema for the type, with media_type included. + # """ + # json_schema = handler(core_schema) + # json_schema = handler.resolve_ref_schema(json_schema) + # # Set the title to the class name, not BlobModel + # json_schema["title"] = "Numeric Array" + # return json_schema + + @classmethod + def _validate( + cls, value: Any, handler: Callable[[Any], Base64NDArrayModel] + ) -> np.ndarray: + r"""Validate and convert a value to a `numpy.ndarray` instance. + + :param value: The input value, as passed in or loaded from JSON. + :param handler: A function that runs the validation logic of Base64NDArrayModel. + + If the value is already a `~numpy.ndarray`, it will be returned directly. + Otherwise, we first validate the input using the `Base64NDArrayModel` schema. + + :return: a `~numpy.ndarray` object. + """ + # If the value is already a Blob, return it directly + if isinstance(value, np.ndarray): + return value + # If not, validate it using the array model + model = handler(value) + return model.to_array() + + @classmethod + def _serialise( + cls, obj: np.ndarray, handler: Callable[[Base64NDArrayModel], Mapping[str, str]] + ) -> Mapping[str, str]: + """Serialise the Blob to a dictionary. + + See `.Blob.to_blobmodel` for a description of how we serialise. + + :param obj: the `.Blob` instance to serialise. + :param handler: the handler (provided by pydantic) takes a BlobModel + and converts it to a dictionary. The handler runs the serialiser of + the core schema we've wrapped, in this case the BlobModel serialiser. + :return: a JSON-serialisable dictionary with a URL that allows + the `.Blob` to be downloaded from the `.BlobManager`. + """ + return handler(Base64NDArrayModel.from_array(obj)) diff --git a/tests/test_types/__init__.py b/tests/test_types/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_types/test_numpy.py b/tests/test_types/test_numpy.py new file mode 100644 index 00000000..b4c02c3b --- /dev/null +++ b/tests/test_types/test_numpy.py @@ -0,0 +1,127 @@ +"""Test the numpy types. + +This module tests the new model and annotated type for numpy arrays. +It's intended that the old tests (`../test_numpy_type.py`) will be +removed once the old types are retired. +""" + +from datetime import datetime +from typing import Annotated + +import numpy as np +import pytest +from pydantic import RootModel + +from labthings_fastapi.types.numpy import Base64NDArrayModel, NDArrayAnnotation + +ARRAYS = ( + [np.arange(0, 255, dtype=d) for d in [np.uint8, np.uint16, np.uint32, np.uint64]] + + [np.arange(-100, 100, dtype=d) for d in [np.int8, np.int16, np.int32, np.int64]] + + [ + 3 ** np.arange(-5, 5, dtype=d) + for d in [np.float16, np.float32, np.float64, np.float128] + ] + + [ + np.array([0.4 + 0.7j, 35 + 85j, 89, -49j], dtype=d) + for d in [np.complex64, np.complex128, np.complex256] + ] + + [np.array([False, True, False, True], dtype=np.bool)] + + [ + np.array(1000), # scalar int + np.array(10.0), # scalar float + ] + + [ + np.array([0.0, 1.0, np.inf, -np.inf, np.nan]) # special float values + ] +) + + +UNSUPPORTED_ARRAYS = [ + # Record arrays aren't supported + np.array([(1.0, 2), (3.0, 4)], dtype=[("x", " Date: Thu, 12 Nov 2026 12:20:20 +0000 Subject: [PATCH 2/8] Improve test coverage by adding endianness I now test against more arrays, some of which have foreign byte order. I've also removed the suggestion that we cope with fortran-ordered arrays, because the reconstruction currently doesn't. --- src/labthings_fastapi/types/numpy.py | 7 +++++-- tests/test_types/test_numpy.py | 29 +++++++++++++++++++--------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index e86217b2..cfa36107 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -204,8 +204,11 @@ class Base64NDArrayModel(BaseModel): shape: tuple[int, ...] """The shape of the array, i.e. the length in each dimension.""" - array_order: Literal["c-contiguous", "fortran-contiguous"] - """Whether the array elements are in C or fortran order.""" + array_order: Literal["c-contiguous"] + """Whether the array elements are in C or fortran order. + + Currently only c order is supported. + """ data: Base64Bytes """The data for the array, Base64 encoded.""" diff --git a/tests/test_types/test_numpy.py b/tests/test_types/test_numpy.py index b4c02c3b..a5effbc3 100644 --- a/tests/test_types/test_numpy.py +++ b/tests/test_types/test_numpy.py @@ -15,23 +15,32 @@ from labthings_fastapi.types.numpy import Base64NDArrayModel, NDArrayAnnotation ARRAYS = ( - [np.arange(0, 255, dtype=d) for d in [np.uint8, np.uint16, np.uint32, np.uint64]] - + [np.arange(-100, 100, dtype=d) for d in [np.int8, np.int16, np.int32, np.int64]] - + [ + [ # Unsigned integer types + np.arange(0, 255, dtype=d) for d in [np.uint8, np.uint16, np.uint32, np.uint64] + ] + + [ # Signed integer types + np.arange(-100, 100, dtype=d) for d in [np.int8, np.int16, np.int32, np.int64] + ] + + [ # Floating point types 3 ** np.arange(-5, 5, dtype=d) for d in [np.float16, np.float32, np.float64, np.float128] ] - + [ + + [ # Complex types np.array([0.4 + 0.7j, 35 + 85j, 89, -49j], dtype=d) for d in [np.complex64, np.complex128, np.complex256] ] - + [np.array([False, True, False, True], dtype=np.bool)] - + [ + + [ # Boolean arrays + np.array([False, True, False, True], dtype=np.bool) + ] + + [ # Scalars np.array(1000), # scalar int np.array(10.0), # scalar float ] - + [ - np.array([0.0, 1.0, np.inf, -np.inf, np.nan]) # special float values + + [ # Non-numeric float values + np.array([0.0, 1.0, np.inf, -np.inf, np.nan]) + ] + + [ # Both little- and big-endian arrays + np.array([0, -4, 16, 255], dtype=d) for d in ["i2", "i8"] ] ) @@ -48,6 +57,8 @@ np.array(["an", "array", "of", "strings"]), # Bytes aren't supported np.array(b"01234"), + # A record array with one record might confuse things + np.array([1.0, 2.0], dtype=[("name", " Date: Thu, 12 Nov 2026 17:07:52 +0000 Subject: [PATCH 3/8] Fully cover the diff This tests a sanity check that should never fail in practice. --- tests/test_types/test_numpy.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_types/test_numpy.py b/tests/test_types/test_numpy.py index a5effbc3..15551209 100644 --- a/tests/test_types/test_numpy.py +++ b/tests/test_types/test_numpy.py @@ -103,6 +103,37 @@ def test_array_unsupported(array): Base64NDArrayModel.from_array(array) +def test_descr_mismatch(): + """Check the error that occurs when `dtype.descr` doesn't match. + + This is an error I don't ever expect in reality. We already check + the relevant properties of an array's datatype, and raise an + error if it's not supported (see previous test). + + As a belt-and-braces check, we generate the `numpy.dtype.descr` + value we'd expect given the datatype, element size, and endianness + that we have recorded, and check our array has the expected + value. + + This `numpy`-specific string is what gets recorded in a ``npz`` file + to allow the data to be reinflated, so it's a very good check that + we have correctly recorded the array's parameters. + + Any array that's not properly supported will fail before we check + ``descr`` so this test mocks the property to trigger the error. + """ + + class ArrayModelWithBrokenDescr(Base64NDArrayModel): + """This feels simpler and clearer than mocking the class.""" + + @property + def _descr(self) -> list[tuple[str, str]]: + return [("", "!!!")] + + with pytest.raises(ValueError, match="dtype.descr"): + ArrayModelWithBrokenDescr.from_array(np.array([1, 2, 3])) + + @pytest.mark.parametrize("array", ARRAYS) def test_rootmodel_annotation(array): """Test that the annotated type validates and serialises correctly in Python.""" From 87ce9b748fd29c1d7b196527aea8516735cc211f Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 16 Nov 2026 23:23:10 +0000 Subject: [PATCH 4/8] Add validation of data type Previously, some errors would only be caught when we converted from a model to an array. These errors ought now to get caught earlier. I've added a few test cases for invalid values. I'm pretty confident this tests all the cases. We could come up with more, but I'm not sure how much it gains. --- src/labthings_fastapi/types/numpy.py | 155 +++++++++++++++++++++++---- tests/test_types/test_numpy.py | 57 +++++++++- 2 files changed, 191 insertions(+), 21 deletions(-) diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index cfa36107..59410a2e 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -32,14 +32,17 @@ def double(arr: NDArray) -> NDArray: BaseModel, ConfigDict, GetCoreSchemaHandler, + GetJsonSchemaHandler, PlainSerializer, PlainValidator, RootModel, SerializerFunctionWrapHandler, WithJsonSchema, WrapSerializer, + model_validator, ) -from pydantic_core import core_schema +from pydantic.json_schema import JsonSchemaValue +from pydantic_core import PydanticCustomError, core_schema from typing_extensions import TypeAlias # Define a nested list of floats with 0-6 dimensions @@ -177,6 +180,49 @@ class DenumpifyingDict(RootModel): """A mapping from element type names to `numpy.dtype.kind` values.""" +INT_LENGTHS = (1, 2, 4, 8) +"""Allowed lengths (in bytes) for integer types.""" + + +FLOAT_LENGTHS = (2, 4, 8, np.dtype(np.longdouble).itemsize) +"""Allowed lengths (in bytes) for floating point types. + +Note that the highest-precision type is 12 or 16 bytes depending on +the platform. +""" + + +COMPLEX_LENGTHS = (8, 16, np.dtype(np.clongdouble).itemsize) +"""Allowed lengths (in bytes) for complex types. + +Note that the highest-precision type is 12 or 16 bytes depending on +the platform. +""" + + +def _check_length(length: int, allowed: tuple[int, ...], etype: ElementType) -> bool: + """Check a datatype length is in the list of allowed lengths. + + :param length: the specified element length. + :param allowed: the allowed lengths. + :param etype: the element type. + :return: `True` if the length is allowed. + :raises PydanticCustomError: if the length is not allowed. + """ + if length in allowed: + return True + raise PydanticCustomError( + "unsupported_length", + "'element_length' must be in {allowed_lengths} for {element_type} " + "but '{element_length}' was specified.", + { + "element_length": length, + "allowed_lengths": allowed, + "element_type": etype, + }, + ) + + def byte_order_name(native: bool) -> Literal["little", "big"]: """Return the name for a byte order. @@ -198,8 +244,17 @@ class Base64NDArrayModel(BaseModel): element_length: int """The length of each array element in bytes.""" - byte_order: Literal["little", "big"] - """Whether the data is in little-endian or big-endian format.""" + byte_order: Literal["little", "big"] | None = None + """Whether the data is in little-endian or big-endian format. + + Note that order is irrelevant for data types that are only + one byte long, i.e. ``element_length==1``. In that case, `None` + is the only allowed value, though implementations may decide + to ignore this field. + + For data types longer than one byte, ``byte_order`` must always + be specified and must be either `"little"` or `"big"`. + """ shape: tuple[int, ...] """The shape of the array, i.e. the length in each dimension.""" @@ -221,7 +276,7 @@ def _kind(self) -> str: @property def _endian_char(self) -> str: """A one-character string describing endianness.""" - if self.element_length == 1: + if self.byte_order is None: return "|" # one-byte values have no endianness return "<" if self.byte_order == "little" else ">" @@ -234,6 +289,66 @@ def _descr(self) -> list[tuple[str, str]]: """ return [("", f"{self._endian_char}{self._kind}{self.element_length}")] + @model_validator(mode="after") + def check_dtype(self) -> Self: + """Check that the data type is supported. + + This validator ensures that a byte order is specified for element lengths + of more than one byte, and that no byte order is specified for one-byte + element types. + + We also check that the element length is valid for the type specified. + + Finally, we construct a `numpy.dtype` object to check that the + datatype is supported. + + :return: the array model (with `byte_order` coerced if needed). + :raises PydanticCustomError: if the model is not valid. + """ + if self.element_length > 1: + if self.byte_order is None: + raise PydanticCustomError( + "missing_byte_order", + "'byte_order' must be 'little' or 'big' when element_length > 1.", + ) + else: + # For one-byte values, ignore the specified `byte_order` + # We could raise a validation error if it's not None, but + # coercing feels reasonable. + self.byte_order = None + if self.element_type == "bool": + _check_length(self.element_length, (1,), "bool") + elif self.element_type in ("int", "uint"): + _check_length(self.element_length, INT_LENGTHS, self.element_type) + elif self.element_type == "float": + _check_length(self.element_length, FLOAT_LENGTHS, self.element_type) + elif self.element_type == "complex": + _check_length(self.element_length, COMPLEX_LENGTHS, self.element_type) + + # Finally, check we can construct a `dtype` object. If this fails, + # something has gone wrong with the checks above. + np.dtype(self._descr) + + return self + + @model_validator(mode="after") + def check_size(self) -> Self: + """Check that the specified size matches the data. + + :return: the array model, unchanged. + :raises PydanticCustomError: if the length of the data is incorrect. + """ + size = self.element_length + for dimension in self.shape: + size *= dimension + if len(self.data) != size: + raise PydanticCustomError( + "length_mismatch", + "The data supplied was {length} bytes but we expected {expected}.", + {"length": len(self.data), "expected": size}, + ) + return self + @classmethod def from_array(cls, array: np.ndarray) -> Self: """Construct a Base64NDArrayModel from a numpy array. @@ -251,7 +366,7 @@ def from_array(cls, array: np.ndarray) -> Self: _model = cls.model_construct( element_type=ELEMENT_TYPES_BY_KIND[dtype.kind], element_length=dtype.itemsize, - byte_order=byte_order_name(dtype.isnative), + byte_order=None if dtype.itemsize == 1 else byte_order_name(dtype.isnative), shape=array.shape, array_order="c-contiguous", data=array.tobytes(order="C"), @@ -326,21 +441,21 @@ def __get_pydantic_core_schema__( ), ) - # @classmethod - # def __get_pydantic_json_schema__( - # cls, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler - # ) -> JsonSchemaValue: - # """Customise the JSON Schema to include the media_type. - - # :param core_schema: The core schema. - # :param handler: The pydantic JSON schema handler. - # :return: The JSON schema for the type, with media_type included. - # """ - # json_schema = handler(core_schema) - # json_schema = handler.resolve_ref_schema(json_schema) - # # Set the title to the class name, not BlobModel - # json_schema["title"] = "Numeric Array" - # return json_schema + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + """Customise the JSON Schema to include the media_type. + + :param core_schema: The core schema. + :param handler: The pydantic JSON schema handler. + :return: The JSON schema for the type, with media_type included. + """ + json_schema = handler(core_schema) + json_schema = handler.resolve_ref_schema(json_schema) + # Set a nicer title + json_schema["title"] = "Numeric Array" + return json_schema @classmethod def _validate( diff --git a/tests/test_types/test_numpy.py b/tests/test_types/test_numpy.py index 15551209..1fbac0f4 100644 --- a/tests/test_types/test_numpy.py +++ b/tests/test_types/test_numpy.py @@ -5,12 +5,13 @@ removed once the old types are retired. """ +import base64 from datetime import datetime from typing import Annotated import numpy as np import pytest -from pydantic import RootModel +from pydantic import RootModel, ValidationError from labthings_fastapi.types.numpy import Base64NDArrayModel, NDArrayAnnotation @@ -103,6 +104,60 @@ def test_array_unsupported(array): Base64NDArrayModel.from_array(array) +@pytest.mark.parametrize( + ("input", "msg"), + [ + ( # Booleans must be 1 byte long + { + "element_type": "bool", + "element_length": 2, + "byte_order": "little", + "array_order": "c-contiguous", + "shape": (1,), + "data": b"", + }, + "unsupported_length", + ), + ( # Ints >1 byte must have byte_order + { + "element_type": "int", + "element_length": 2, + "array_order": "c-contiguous", + "shape": (1,), + "data": b"", + }, + "byte_order", + ), + ( # 3 byte ints are not a thing! + { + "element_type": "int", + "element_length": 3, + "byte_order": "little", + "array_order": "c-contiguous", + "shape": (1,), + "data": b"", + }, + "unsupported_length", + ), + ( # Mismatched length + { + "element_type": "uint", + "element_length": 1, + "byte_order": "little", + "array_order": "c-contiguous", + "shape": (3,), + "data": base64.b64encode(b"0000"), + }, + "length_mismatch", + ), + ], +) +def test_invalid_json(input, msg): + """Test invalid input fails at validation stage, not afterwards.""" + with pytest.raises(ValidationError, match=msg): + Base64NDArrayModel.model_validate(input) + + def test_descr_mismatch(): """Check the error that occurs when `dtype.descr` doesn't match. From a7020b3b4bd35071514e6b7a034f2871ca5d7dd1 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Mon, 16 Nov 2026 23:34:56 +0000 Subject: [PATCH 5/8] Make field docstrings appear in the schema. --- src/labthings_fastapi/types/numpy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index 59410a2e..64a3bc04 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -236,7 +236,9 @@ def byte_order_name(native: bool) -> Literal["little", "big"]: class Base64NDArrayModel(BaseModel): - """A Pydantic model for numpy arrays.""" + """Numeric arrays that can be serialised to JSON.""" + + model_config = {"use_attribute_docstrings": True} element_type: ElementType """The datatype of each array element.""" From 739e46fcbb3afda0910a6ef88e5bd1861dc3a24f Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Tue, 17 Nov 2026 09:48:01 +0000 Subject: [PATCH 6/8] Add detail to field docstrings --- src/labthings_fastapi/types/numpy.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index 64a3bc04..b036a148 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -244,7 +244,13 @@ class Base64NDArrayModel(BaseModel): """The datatype of each array element.""" element_length: int - """The length of each array element in bytes.""" + """The length of each array element in bytes. + + Valid lengths are platform-dependent, but usually include + 1, 2, 4, 8 for integers and 2, 4, 8 and either 12 or 16 for + floating point numbers. Boolean arrays always use one byte + per element. + """ byte_order: Literal["little", "big"] | None = None """Whether the data is in little-endian or big-endian format. @@ -252,7 +258,7 @@ class Base64NDArrayModel(BaseModel): Note that order is irrelevant for data types that are only one byte long, i.e. ``element_length==1``. In that case, `None` is the only allowed value, though implementations may decide - to ignore this field. + to ignore this field rather than raise an error. For data types longer than one byte, ``byte_order`` must always be specified and must be either `"little"` or `"big"`. @@ -268,11 +274,19 @@ class Base64NDArrayModel(BaseModel): """ data: Base64Bytes - """The data for the array, Base64 encoded.""" + """The data for the array, Base64 encoded. + + When accessing a valid `Base64NDArrayModel` in Python, or when creating + one using `model_construct`, this field holds the array's data as a `bytes` + object. When constructing the model with `model_validate` or the constructor, + the value of ``data`` should be a Base64 encoded `bytes` object. + + The value in a JSON string will always be Base64 encoded. + """ @property def _kind(self) -> str: - """A one-character string describing the element_type.""" + """A one-character string matching `numpy.dtype.kind`.""" return KINDS_BY_ELEMENT_TYPE[self.element_type] @property @@ -287,7 +301,8 @@ def _descr(self) -> list[tuple[str, str]]: """The expected value of `numpy.dtype.descr` for this array. This property may be compared with `array.dtype.descr` and ought - to match exactly. If it doesn't, we have found an edge case. + to match exactly. If it doesn't, we have found an edge case and + it's likely validation/serialisation will fail. """ return [("", f"{self._endian_char}{self._kind}{self.element_length}")] From 342109fc37ec30529935992e1b533ba376a1e46d Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Tue, 17 Nov 2026 09:48:11 +0000 Subject: [PATCH 7/8] Test generated JSON Schema --- tests/test_types/test_numpy.py | 42 +++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/test_types/test_numpy.py b/tests/test_types/test_numpy.py index 1fbac0f4..8e86b39e 100644 --- a/tests/test_types/test_numpy.py +++ b/tests/test_types/test_numpy.py @@ -73,6 +73,10 @@ } +class NDArrayRootModel(RootModel): + root: Annotated[np.ndarray, NDArrayAnnotation] + + @pytest.mark.parametrize("array", ARRAYS) def test_array_roundtrip_json(array): """Check an array can be serialised and reconstructed from JSON.""" @@ -192,10 +196,6 @@ def _descr(self) -> list[tuple[str, str]]: @pytest.mark.parametrize("array", ARRAYS) def test_rootmodel_annotation(array): """Test that the annotated type validates and serialises correctly in Python.""" - - class NDArrayRootModel(RootModel): - root: Annotated[np.ndarray, NDArrayAnnotation] - model = NDArrayRootModel(array) dumped = model.model_dump() assert set(dumped.keys()) == FIELDS @@ -209,10 +209,6 @@ class NDArrayRootModel(RootModel): @pytest.mark.parametrize("array", ARRAYS) def test_rootmodel_annotation_json(array): """Test that the annotated type validates and serialises correctly in JSON.""" - - class NDArrayRootModel(RootModel): - root: Annotated[np.ndarray, NDArrayAnnotation] - model = NDArrayRootModel(array) dumped = model.model_dump_json() for field in FIELDS: @@ -222,3 +218,33 @@ class NDArrayRootModel(RootModel): assert array.dtype == reconstructed_array.dtype assert array.dtype.descr == reconstructed_array.dtype.descr assert np.array_equal(array, reconstructed_array, equal_nan=True) + + +def test_annotation_jsonschema(): + """Make sure the JSON Schema is as expected.""" + schema = NDArrayRootModel.model_json_schema() + assert schema["title"] == "Numeric Array" # customised by the annotation + assert schema["properties"]["element_type"]["description"] == ( + # Check that docstrings are correctly pulled through as descriptions + "The datatype of each array element." + ) + assert set(schema["properties"]["element_type"]["enum"]) == { + # Check the datatype options match what's expected + "bool", + "int", + "uint", + "float", + "complex", + } + assert schema["properties"]["element_length"]["type"] == "integer" + assert schema["properties"]["byte_order"]["anyOf"] == [ + {"enum": ["little", "big"], "type": "string"}, + {"type": "null"}, + ] + # We only support c-contiguous arrays. Check that's clear. + assert schema["properties"]["array_order"]["const"] == "c-contiguous" + assert schema["properties"]["shape"]["type"] == "array" + assert schema["properties"]["shape"]["items"] == {"type": "integer"} + assert schema["properties"]["data"]["format"] == "base64" + for v in schema["properties"].values(): + assert len(v["description"]) > 0, "All properties should have a description." From e2fa623d0404208cf54ef902d0e0d1d174f4d266 Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Tue, 17 Nov 2026 20:35:22 +0000 Subject: [PATCH 8/8] Put `None` in an enum rather than union This results in a simpler JSONSchema definition. The allowed values are identical. --- src/labthings_fastapi/types/numpy.py | 2 +- tests/test_types/test_numpy.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/labthings_fastapi/types/numpy.py b/src/labthings_fastapi/types/numpy.py index b036a148..554a9d0d 100644 --- a/src/labthings_fastapi/types/numpy.py +++ b/src/labthings_fastapi/types/numpy.py @@ -252,7 +252,7 @@ class Base64NDArrayModel(BaseModel): per element. """ - byte_order: Literal["little", "big"] | None = None + byte_order: Literal["little", "big", None] = None """Whether the data is in little-endian or big-endian format. Note that order is irrelevant for data types that are only diff --git a/tests/test_types/test_numpy.py b/tests/test_types/test_numpy.py index 8e86b39e..66197887 100644 --- a/tests/test_types/test_numpy.py +++ b/tests/test_types/test_numpy.py @@ -237,10 +237,7 @@ def test_annotation_jsonschema(): "complex", } assert schema["properties"]["element_length"]["type"] == "integer" - assert schema["properties"]["byte_order"]["anyOf"] == [ - {"enum": ["little", "big"], "type": "string"}, - {"type": "null"}, - ] + assert set(schema["properties"]["byte_order"]["enum"]) == {"little", "big", None} # We only support c-contiguous arrays. Check that's clear. assert schema["properties"]["array_order"]["const"] == "c-contiguous" assert schema["properties"]["shape"]["type"] == "array"