diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b2784511f..ccf17239b 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -19,6 +19,7 @@ from datetime import datetime from enum import Enum from fnmatch import fnmatchcase +from functools import cached_property import json import os.path from pathlib import Path, PurePosixPath @@ -26,11 +27,12 @@ import re from time import sleep, time from types import TracebackType -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast import click from dandischema import models import dandischema.consts +from dandischema.metadata import migrate from packaging.version import Version as PackagingVersion from pydantic import BaseModel, Field, PrivateAttr import requests @@ -709,13 +711,8 @@ def check_schema_version(self, schema_version: str | None = None) -> None: """ if schema_version is None: schema_version = models.get_schema_version() - server_info = self.get("/info/") - server_schema_version = server_info.get("schema_version") - if not server_schema_version: - raise RuntimeError( - "Server did not provide schema_version in /info/;" - f" returned {server_info!r}" - ) + server_info = self.server_info + server_schema_version = self.server_schema_version server_ver, our_ver = PackagingVersion(server_schema_version), PackagingVersion( schema_version ) @@ -737,28 +734,178 @@ def check_schema_version(self, schema_version: str | None = None) -> None: # TODO: check current server behavior which is likely to just not care! # So that is where server might need to provide support for upgrades upon # providing metadata. - elif ( - server_ver.major == 0 and server_ver.release[:2] != our_ver.release[:2] - ) or ( - server_ver.major != our_ver.major - ): # MAJOR, MINOR within 0.x.y and MAJOR within 1.x.y - raise SchemaVersionError( - f"Server uses older incompatible schema version {server_schema_version};" - f" client supports {schema_version}." - ) elif server_ver < our_ver: - # Compatible older server version -- all good, but inform the user - # TODO: potentially downgrade the record to match the schema, - # see https://github.com/dandi/dandi-schema/issues/343 + # Server is older. If dandischema knows a downgrade path + # (`_maybe_downgrade_metadata` will use it on upload) OR if the + # versions are within the same MAJOR.MINOR (0.x.y) / MAJOR (1.x.y) + # compatibility band, just warn; otherwise raise as incompatible. + can_downgrade = ( + server_schema_version in dandischema.consts.ALLOWED_TARGET_SCHEMAS + ) + if not can_downgrade and ( + ( + server_ver.major == 0 + and server_ver.release[:2] != our_ver.release[:2] + ) + or (server_ver.major != our_ver.major) + ): + raise SchemaVersionError( + f"Server uses older incompatible schema version {server_schema_version};" + f" client supports {schema_version}." + ) + if can_downgrade: + msg_downgrade = ( + "Library will attempt (but might fail) to downgrade outgoing " + "metadata to this schema version on upload. " + ) + else: + msg_downgrade = ( + "Library DOES NOT support downgrade to this schema version. " + ) lgr.warning( - "Server uses schema version %s older than client's %s (dandischema library %s). " - "Server might fail to validate such assets and you might not be able to " - "publish this dandiset until server is upgraded. " - "Alternatively, you may downgrade dandischema and reupload.", + "Server uses schema version %s older than client's %s " + "(dandischema library %s). " + "%s" + "Server might fail to validate " + "such assets and you might not be able to publish this dandiset " + "until server is upgraded. Alternatively, you may downgrade " + "dandischema and reupload.", server_ver, our_ver, dandischema.__version__, + msg_downgrade, + ) + + @cached_property + def server_info(self) -> dict[str, Any]: + """ + Cached response from the server's ``/info/`` endpoint. + """ + info = self.get("/info/") + assert isinstance(info, dict) + return info + + @cached_property + def server_schema_version(self) -> str: + """ + The DANDI schema version reported by the server's ``/info/`` endpoint. + """ + schema_version = self.server_info.get("schema_version") + if not schema_version: + raise RuntimeError( + f"Server did not provide schema_version in /info/; " + f"returned {self.server_info!r}" + ) + assert isinstance(schema_version, str) + return schema_version + + def _maybe_downgrade_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: + """ + If the server reports an older ``schema_version`` than the one embedded + in ``metadata``, and dandischema knows how to migrate down to that + version (i.e. it is in + ``dandischema.consts.ALLOWED_TARGET_SCHEMAS``), return a downgraded + copy of the metadata. Otherwise return ``metadata`` unchanged. + + If the downgrade path exists but ``dandischema.metadata.migrate`` + refuses (e.g. a field added post-server-version is populated and + cannot be simply stripped), log a warning and return the original + metadata — the server will then decide. + + Ref: https://github.com/dandi/dandi-schema/issues/343 + """ + obj_ver = metadata.get("schemaVersion") + if not obj_ver: + return metadata + server_ver_str = self.server_schema_version + if ( + PackagingVersion(server_ver_str) >= PackagingVersion(obj_ver) + or server_ver_str not in dandischema.consts.ALLOWED_TARGET_SCHEMAS + ): + return metadata + try: + downgraded = migrate( + metadata, to_version=server_ver_str, skip_validation=True + ) + except ValueError as exc: + lgr.warning( + "Could not downgrade metadata from schema version %s to server's " + "%s: %s. Sending original metadata; server may reject it.", + obj_ver, + server_ver_str, + exc, ) + return metadata + return cast(Dict[str, Any], downgraded) + + def _maybe_downgrade_request_metadata(self, body: Any) -> Any: + """ + Intercept an outgoing request body: if it wraps a DANDI metadata + dict under the ``"metadata"`` key, downgrade that dict to the + server's schema version via ``_maybe_downgrade_metadata``. + + A body is treated as carrying DANDI metadata iff it is a + ``dict`` whose ``"metadata"`` value is itself a ``dict`` with + both ``schemaKey`` (a str, e.g. ``"Dandiset"``, ``"BareAsset"``) + and ``schemaVersion`` (a str) — this is the invariant shape of + every DANDI-metadata payload the archive accepts. Bodies that + don't match are passed through unchanged. + + Called from `request()`, so every metadata-sending endpoint + (``create_dandiset``, ``*.set_raw_metadata``, ``iter_upload`` + asset-create/replace, plus any future ones) gets the downgrade + automatically without spot-patching. + """ + if not isinstance(body, dict): + return body + md = body.get("metadata") + if not ( + isinstance(md, dict) + and isinstance(md.get("schemaKey"), str) + and isinstance(md.get("schemaVersion"), str) + ): + return body + downgraded = self._maybe_downgrade_metadata(md) + if downgraded is md: + return body + return {**body, "metadata": downgraded} + + def request( + self, + method: str, + path: str, + params: dict | None = None, + data: Any = None, + files: dict | None = None, + json: Any = None, + headers: dict | None = None, + json_resp: bool = True, + retry_statuses: Sequence[int] = (), + retry_if: Callable[[requests.Response], Any] | None = None, + **kwargs: Any, + ) -> Any: + """ + Override of `RESTFullAPIClient.request` that runs + `_maybe_downgrade_request_metadata` over the outgoing ``json`` + body so DANDI metadata gets downgraded to the server's schema + version on the way out. See that method's docstring for the + shape check used and the ref to dandi-schema#343. + """ + if json is not None: + json = self._maybe_downgrade_request_metadata(json) + return super().request( + method, + path, + params=params, + data=data, + files=files, + json=json, + headers=headers, + json_resp=json_resp, + retry_statuses=retry_statuses, + retry_if=retry_if, + **kwargs, + ) def get_asset(self, asset_id: str) -> BaseRemoteAsset: """ diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index b50640b12..8da28852e 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -2,6 +2,7 @@ import builtins from datetime import datetime, timezone +import json import logging from pathlib import Path import random @@ -11,7 +12,9 @@ import anys import click +import dandischema from dandischema.models import UUID_PATTERN, DigestType, get_schema_version +from packaging.version import Version as _PackagingVersion import pytest from pytest_mock import MockerFixture import requests @@ -31,14 +34,25 @@ DandiAPIClient, RemoteAsset, RemoteBlobAsset, + RemoteDandiset, RemoteZarrAsset, Version, + VersionStatus, ) from ..download import download from ..exceptions import NotFoundError, SchemaVersionError from ..files import GenericAsset, dandi_file from ..utils import list_paths +# These tests exercise the 0.8.0 -> 0.7.0 downgrade wiring, which requires +# both the 0.8.0-era metadata schema and the migration paths for `sameAs` / +# `releaseNotes` -- all of which landed together in dandischema 0.14.0. +# `py3-lowest` resolves to `dandischema==0.12.0` and simply skips them. +_needs_downgrade = pytest.mark.skipif( + _PackagingVersion(dandischema.__version__) < _PackagingVersion("0.14.0"), + reason="Downgrade tests require dandischema >= 0.14.0", +) + def test_upload( new_dandiset: SampleDandiset, simple1_nwb: Path, tmp_path: Path @@ -308,6 +322,11 @@ def test_remote_asset_json_dict(text_dandiset: SampleDandiset) -> None: True, "Server uses older incompatible schema version 0.6.7; client supports 0.7.0", ), + # ...unless the older server version is in + # `ALLOWED_TARGET_SCHEMAS` (i.e. `_maybe_downgrade_metadata` can + # downgrade to it on upload) -- then a warning, not an error. + pytest.param("0.7.0", "0.8.0", False, None, marks=_needs_downgrade), + pytest.param("0.6.10", "0.8.0", False, None, marks=_needs_downgrade), # After 1.x -- rely on MAJOR. ("1.0.0", "1.2.3", False, None), ("1.6.7", "1.7.0", False, None), @@ -356,6 +375,231 @@ def test_check_schema_version( client.check_schema_version(local_schema_version) +def _server_info(schema_version: str) -> dict: + return { + "schema_version": schema_version, + "version": "0.0.0", + "services": {"api": {"url": "https://test.nil/api"}}, + "cli-minimal-version": "0.0.0", + "cli-bad-versions": [], + } + + +def _mock_server_info(schema_version: str) -> None: + info = _server_info(schema_version) + responses.add(responses.GET, "https://test.nil/server-info", json=info) + responses.add(responses.GET, "https://test.nil/api/info/", json=info) + + +@pytest.mark.ai_generated +@pytest.mark.parametrize( + "server_schema,obj_schema,expected_schema,dropped,kept", + [ + # server on same version -- no touch, sameAs preserved + ("0.8.0", "0.8.0", "0.8.0", [], ["sameAs", "releaseNotes"]), + # server one minor behind -- downgrade to 0.7.0, drop sameAs (added 0.7.1), + # keep releaseNotes (added 0.7.0) + pytest.param( + "0.7.0", + "0.8.0", + "0.7.0", + ["sameAs"], + ["releaseNotes"], + marks=_needs_downgrade, + ), + # server on 0.6.10 -- both sameAs and releaseNotes go + pytest.param( + "0.6.10", + "0.8.0", + "0.6.10", + ["sameAs", "releaseNotes"], + [], + marks=_needs_downgrade, + ), + # server on an unsupported (too-old) target -- leave metadata alone + ("0.6.5", "0.8.0", "0.8.0", [], ["sameAs", "releaseNotes"]), + ], +) +@responses.activate +def test__maybe_downgrade_metadata( + server_schema: str, + obj_schema: str, + expected_schema: str, + dropped: list[str], + kept: list[str], +) -> None: + _mock_server_info(server_schema) + client = DandiAPIClient("https://test.nil/api") + metadata = { + "schemaKey": "Dandiset", + "schemaVersion": obj_schema, + "name": "n", + "description": "d", + "identifier": "DANDI:000001", + "sameAs": [], + "releaseNotes": "", + } + out = client._maybe_downgrade_metadata(metadata) + assert out["schemaVersion"] == expected_schema + for f in dropped: + assert f not in out, f"expected {f!r} to be stripped" + for f in kept: + assert f in out, f"expected {f!r} to be kept" + + +@pytest.mark.ai_generated +@_needs_downgrade +@pytest.mark.parametrize( + "server_schema,obj_schema,populated_field,populated_value", + [ + # sameAs was added in 0.8.0 -- server on 0.7.0 cannot accept it, + # and a non-empty value cannot be simply stripped. + ("0.7.0", "0.8.0", "sameAs", ["DANDI:000002"]), + # releaseNotes was added in 0.7.0 -- server on 0.6.10 cannot accept + # it, and a non-empty value cannot be simply stripped. + ("0.6.10", "0.7.0", "releaseNotes", "Some release notes."), + ], +) +@responses.activate +def test__maybe_downgrade_metadata_falls_through_on_populated_field( + server_schema: str, + obj_schema: str, + populated_field: str, + populated_value: Any, + caplog: pytest.LogCaptureFixture, +) -> None: + """ + When ``dandischema.metadata.migrate`` refuses because a post-server-version + field carries a non-empty value that cannot be simply stripped, the client + must log a warning and return the original metadata unchanged so the + server can decide. + """ + _mock_server_info(server_schema) + client = DandiAPIClient("https://test.nil/api") + metadata = { + "schemaKey": "Dandiset", + "schemaVersion": obj_schema, + "name": "n", + "description": "d", + "identifier": "DANDI:000001", + populated_field: populated_value, + } + with caplog.at_level(logging.WARNING, logger="dandi"): + out = client._maybe_downgrade_metadata(metadata) + assert out is metadata + assert out["schemaVersion"] == obj_schema + assert out[populated_field] == populated_value + assert any( + "Could not downgrade metadata" in rec.message and populated_field in rec.message + for rec in caplog.records + ) + + +@pytest.mark.ai_generated +@responses.activate +def test__maybe_downgrade_request_metadata_shape_check() -> None: + """ + The request-body interceptor only touches bodies whose `metadata` + sub-dict has *both* `schemaKey` and `schemaVersion` as strings. + Anything else -- non-dict body, missing `metadata` key, `metadata` + that isn't a dict, `metadata` missing either shape marker -- must + pass through unchanged (identity). + """ + _mock_server_info("0.7.0") + client = DandiAPIClient("https://test.nil/api") + non_matches = [ + None, + "a string body", + 42, + [{"metadata": {"schemaKey": "Dandiset", "schemaVersion": "0.8.0"}}], + {}, + {"metadata": "not a dict"}, + {"metadata": {"schemaVersion": "0.8.0"}}, # missing schemaKey + {"metadata": {"schemaKey": "Dandiset"}}, # missing schemaVersion + {"metadata": {"schemaKey": "Dandiset", "schemaVersion": 800}}, # non-str + {"other_key": {"schemaKey": "Dandiset", "schemaVersion": "0.8.0"}}, + ] + for body in non_matches: + assert client._maybe_downgrade_request_metadata(body) is body + + +@pytest.mark.ai_generated +@_needs_downgrade +@responses.activate +def test__maybe_downgrade_request_metadata_downgrades() -> None: + """ + When the body shape matches, the interceptor swaps in a downgraded + metadata dict, returning a *new* body (does not mutate input). + """ + _mock_server_info("0.7.0") + client = DandiAPIClient("https://test.nil/api") + md = { + "schemaKey": "Dandiset", + "schemaVersion": "0.8.0", + "name": "n", + "sameAs": [], + } + body = {"metadata": md, "name": "n"} + out = client._maybe_downgrade_request_metadata(body) + assert out is not body # new body + assert out["metadata"]["schemaVersion"] == "0.7.0" + assert "sameAs" not in out["metadata"] + assert out["name"] == "n" + # original body untouched + assert body["metadata"] is md + assert md["schemaVersion"] == "0.8.0" + + +@pytest.mark.ai_generated +@_needs_downgrade +@responses.activate +def test_set_raw_metadata_downgrades_on_older_server() -> None: + """ + When the server reports an older schema version than the metadata carries, + `RemoteVersion.set_raw_metadata` must downgrade the metadata before the + outgoing PUT. Reproduces the CI failure seen with dandi-schema #342. + """ + _mock_server_info("0.7.0") + + captured: dict = {} + + def _put_callback(request: Any) -> tuple[int, dict, str]: + captured["body"] = json.loads(request.body) + return (200, {}, json.dumps({})) + + responses.add_callback( + responses.PUT, + re.compile(r"^https://test\.nil/api/dandisets/000001/versions/draft/$"), + callback=_put_callback, + content_type="application/json", + ) + + client = DandiAPIClient("https://test.nil/api") + ver = Version( + identifier="draft", + name="draft", + asset_count=0, + size=0, + created=datetime.now(timezone.utc), + modified=datetime.now(timezone.utc), + status=VersionStatus.PENDING, + ) + d = RemoteDandiset(client=client, identifier="000001", version=ver) + d.set_raw_metadata( + { + "schemaKey": "Dandiset", + "schemaVersion": "0.8.0", + "name": "n", + "description": "d", + "identifier": "DANDI:000001", + "sameAs": [], + } + ) + sent = captured["body"]["metadata"] + assert sent["schemaVersion"] == "0.7.0" + assert "sameAs" not in sent + + def test_get_dandisets(text_dandiset: SampleDandiset) -> None: dandisets = list(text_dandiset.client.get_dandisets()) assert text_dandiset.dandiset_id in [d.identifier for d in dandisets] diff --git a/pyproject.toml b/pyproject.toml index bc00d6ac6..b98d4f7a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,10 @@ dependencies = [ "bids-validator-deno >= 2.0.5", "click >= 8.2", "click-didyoumean >= 0.3.1", - "dandischema ~= 0.12.0", + # Exclude 0.13.0: it lacks the downgrade migrations (SIMPLE_DOWNGRADES + # entries for releaseNotes / sameAs) that the client relies on to talk + # to older Archive servers. 0.14.0 restored them. + "dandischema >= 0.12.0, != 0.13.0, < 0.15.0", "etelemetry >= 0.2.2", "fasteners >= 0.19", "fscacher >= 0.3.0", diff --git a/tox.ini b/tox.ini index 97cf30772..f40c8230a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] requires = tox-uv >= 1.11 -envlist = lint,typing,py3 +envlist = lint,typing,py3,py3-lowest [testenv] setenv =