From 6ff7faba4c38abfca6529b30f57e67c482766b44 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 4 Aug 2026 11:08:16 -0400 Subject: [PATCH 1/9] feat: downgrade outgoing metadata to server's schema version on upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the DANDI Archive server reports a `schema_version` older than the one carried in metadata the client is about to send, `DandiAPIClient` now attempts to downgrade the metadata via `dandischema.metadata.migrate(..., to_version= server_schema_version, skip_validation=True)` before sending. This unblocks uploads to servers that trail the client's `dandischema` (dandi-schema #342, tracking dandi-schema #343). Additions: - `DandiAPIClient.server_schema_version` — `@cached_property` reading `/info/` once; raises if the server does not expose `schema_version`. - `DandiAPIClient._maybe_downgrade_metadata(metadata)` — no-op unless the server is strictly older *and* its version is in `dandischema.consts.ALLOWED_TARGET_SCHEMAS`; otherwise returns the metadata unchanged (graceful fall-through when the installed dandischema has no downgrade path for the target). Call sites (every place a metadata dict leaves the client): - `create_dandiset` - `RemoteVersion.set_raw_metadata` - `RemoteBlobAsset.set_raw_metadata` - `RemoteZarrAsset.set_raw_metadata` - `LocalFileAsset.iter_upload` (`dandi/files/bases.py`) - `ZarrAsset.iter_upload` (`dandi/files/zarr.py`) `check_schema_version` no longer just warns and shrugs when the server is older: the warning now tells the user whether an automatic downgrade will be attempted (target ∈ `ALLOWED_TARGET_SCHEMAS`) or whether the library cannot downgrade to this version at all. Tests (in `dandi/tests/test_dandiapi.py`, using the existing `responses` pattern): - `test__maybe_downgrade_metadata` — parametrized over same/one-minor-older/two-minor-older/unsupported-target server versions, asserting the correct `schemaVersion` and which of `sameAs` (0.8.0-added) / `releaseNotes` (0.7.0-added) get stripped vs kept. - `test_set_raw_metadata_downgrades_on_older_server` — end-to-end reproducer of the dandi-schema #342 CI failure via a mocked 0.7.0 server; asserts the outgoing PUT body carries `schemaVersion=0.7.0` and no `sameAs`. Verified locally that the patched client behaves correctly against both a downgrade-capable dandischema (release-schema PR: full fix, all 16 tests green) and prior released dandischema (0.13.0 / 0.12.1: graceful no-op, no crash — same behavior as before this patch). Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- dandi/dandiapi.py | 71 +++++++++++++++++++--- dandi/files/bases.py | 1 + dandi/files/zarr.py | 1 + dandi/tests/test_dandiapi.py | 110 +++++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 9 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b2784511f..b4954dc94 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -19,6 +19,7 @@ from datetime import datetime from enum import Enum from fnmatch import fnmatchcase +from functools import cached_property import json import os.path from pathlib import Path, PurePosixPath @@ -31,6 +32,7 @@ import click from dandischema import models import dandischema.consts +from dandischema.metadata import migrate from packaging.version import Version as PackagingVersion from pydantic import BaseModel, Field, PrivateAttr import requests @@ -680,6 +682,7 @@ def create_dandiset( ``embargo`` argument added """ + metadata = self._maybe_downgrade_metadata(metadata) return RemoteDandiset.from_data( self, self.post( @@ -710,8 +713,7 @@ def check_schema_version(self, schema_version: str | None = None) -> None: if schema_version is None: schema_version = models.get_schema_version() server_info = self.get("/info/") - server_schema_version = server_info.get("schema_version") - if not server_schema_version: + if not (server_schema_version := server_info.get("schema_version")): raise RuntimeError( "Server did not provide schema_version in /info/;" f" returned {server_info!r}" @@ -747,19 +749,67 @@ def check_schema_version(self, schema_version: str | None = None) -> None: f" client supports {schema_version}." ) elif server_ver < our_ver: - # Compatible older server version -- all good, but inform the user - # TODO: potentially downgrade the record to match the schema, - # see https://github.com/dandi/dandi-schema/issues/343 + # Compatible older server version -- `_maybe_downgrade_metadata` will + # attempt an on-upload downgrade when possible. + if server_schema_version in dandischema.consts.ALLOWED_TARGET_SCHEMAS: + msg_downgrade = ( + "Library will attempt (but might fail) to downgrade outgoing " + "metadata to this schema version on upload. " + ) + else: + msg_downgrade = ( + "Library DOES NOT support downgrade to this schema version. " + ) lgr.warning( - "Server uses schema version %s older than client's %s (dandischema library %s). " - "Server might fail to validate such assets and you might not be able to " - "publish this dandiset until server is upgraded. " - "Alternatively, you may downgrade dandischema and reupload.", + "Server uses schema version %s older than client's %s " + "(dandischema library %s). " + "%s" + "Server might fail to validate " + "such assets and you might not be able to publish this dandiset " + "until server is upgraded. Alternatively, you may downgrade " + "dandischema and reupload.", server_ver, our_ver, dandischema.__version__, + msg_downgrade, ) + @cached_property + def server_schema_version(self) -> str: + """ + The DANDI schema version reported by the server's ``/info/`` endpoint. + """ + info = self.get("/info/") + schema_version = info.get("schema_version") + if not schema_version: + raise RuntimeError( + "Server did not provide schema_version in /info/;" f" returned {info!r}" + ) + assert isinstance(schema_version, str) + return schema_version + + def _maybe_downgrade_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: + """ + If the server reports an older ``schema_version`` than the one embedded + in ``metadata``, and dandischema knows how to migrate down to that + version (i.e. it is in + ``dandischema.consts.ALLOWED_TARGET_SCHEMAS``), return a downgraded + copy of the metadata. Otherwise return ``metadata`` unchanged. + + Ref: https://github.com/dandi/dandi-schema/issues/343 + """ + obj_ver = metadata.get("schemaVersion") + if not obj_ver: + return metadata + server_ver_str = self.server_schema_version + if ( + obj_ver == server_ver_str + or PackagingVersion(server_ver_str) >= PackagingVersion(obj_ver) + or server_ver_str not in dandischema.consts.ALLOWED_TARGET_SCHEMAS + ): + return metadata + return migrate(metadata, to_version=server_ver_str, skip_validation=True) + def get_asset(self, asset_id: str) -> BaseRemoteAsset: """ Fetch the asset with the given asset ID. If the given asset does not @@ -1201,6 +1251,7 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: """ Set the metadata for this version of the Dandiset to the given value """ + metadata = self.client._maybe_downgrade_metadata(metadata) self.client.put( self.version_api_path, json={"metadata": metadata, "name": metadata.get("name", "")}, @@ -2026,6 +2077,7 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: update the `RemoteBlobAsset` in place. """ set_asset_schema_key(metadata) + metadata = self.client._maybe_downgrade_metadata(metadata) data = self.client.put( self.api_path, json={"metadata": metadata, "blob_id": self.blob} ) @@ -2050,6 +2102,7 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: update the `RemoteZarrAsset` in place. """ set_asset_schema_key(metadata) + metadata = self.client._maybe_downgrade_metadata(metadata) data = self.client.put( self.api_path, json={"metadata": metadata, "zarr_id": self.zarr} ) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index 3577e605f..e210b0531 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -370,6 +370,7 @@ def iter_upload( asset_path = metadata.setdefault("path", self.path) set_asset_schema_key(metadata) client = dandiset.client + metadata = client._maybe_downgrade_metadata(metadata) yield {"status": "calculating etag"} etagger = get_dandietag(self.filepath) filetag = etagger.as_str() diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 8f5a4aa0e..0fee40a27 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -583,6 +583,7 @@ def iter_upload( asset_path = metadata.setdefault("path", self.path) set_asset_schema_key(metadata) client = dandiset.client + metadata = client._maybe_downgrade_metadata(metadata) lgr.debug("%s: Producing asset", asset_path) yield {"status": "producing asset"} diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index b50640b12..823893b9a 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -2,6 +2,7 @@ import builtins from datetime import datetime, timezone +import json import logging from pathlib import Path import random @@ -356,6 +357,115 @@ def test_check_schema_version( client.check_schema_version(local_schema_version) +def _server_info(schema_version: str) -> dict: + return { + "schema_version": schema_version, + "version": "0.0.0", + "services": {"api": {"url": "https://test.nil/api"}}, + "cli-minimal-version": "0.0.0", + "cli-bad-versions": [], + } + + +def _mock_server_info(schema_version: str) -> None: + info = _server_info(schema_version) + responses.add(responses.GET, "https://test.nil/server-info", json=info) + responses.add(responses.GET, "https://test.nil/api/info/", json=info) + + +@pytest.mark.parametrize( + "server_schema,obj_schema,expected_schema,dropped,kept", + [ + # server on same version -- no touch, sameAs preserved + ("0.8.0", "0.8.0", "0.8.0", [], ["sameAs", "releaseNotes"]), + # server one minor behind -- downgrade to 0.7.0, drop sameAs (added 0.7.1), + # keep releaseNotes (added 0.7.0) + ("0.7.0", "0.8.0", "0.7.0", ["sameAs"], ["releaseNotes"]), + # server on 0.6.10 -- both sameAs and releaseNotes go + ("0.6.10", "0.8.0", "0.6.10", ["sameAs", "releaseNotes"], []), + # server on an unsupported (too-old) target -- leave metadata alone + ("0.6.5", "0.8.0", "0.8.0", [], ["sameAs", "releaseNotes"]), + ], +) +@responses.activate +def test__maybe_downgrade_metadata( + server_schema: str, + obj_schema: str, + expected_schema: str, + dropped: list[str], + kept: list[str], +) -> None: + _mock_server_info(server_schema) + client = DandiAPIClient("https://test.nil/api") + metadata = { + "schemaKey": "Dandiset", + "schemaVersion": obj_schema, + "name": "n", + "description": "d", + "identifier": "DANDI:000001", + "sameAs": [], + "releaseNotes": "", + } + out = client._maybe_downgrade_metadata(metadata) + assert out["schemaVersion"] == expected_schema + for f in dropped: + assert f not in out, f"expected {f!r} to be stripped" + for f in kept: + assert f in out, f"expected {f!r} to be kept" + + +@responses.activate +def test_set_raw_metadata_downgrades_on_older_server(mocker: MockerFixture) -> None: + """ + When the server reports an older schema version than the metadata carries, + `RemoteVersion.set_raw_metadata` must downgrade the metadata before the + outgoing PUT. Reproduces the CI failure seen with dandi-schema #342. + """ + _mock_server_info("0.7.0") + + captured: dict = {} + + def _put_callback(request: Any) -> tuple[int, dict, str]: + captured["body"] = json.loads(request.body) + return (200, {}, json.dumps({})) + + responses.add_callback( + responses.PUT, + re.compile(r"^https://test\.nil/api/dandisets/000001/versions/draft/$"), + callback=_put_callback, + content_type="application/json", + ) + + from datetime import datetime, timezone + + from ..dandiapi import RemoteDandiset, VersionStatus + + client = DandiAPIClient("https://test.nil/api") + ver = Version( + version="draft", + name="draft", + asset_count=0, + size=0, + created=datetime.now(timezone.utc), + modified=datetime.now(timezone.utc), + status=VersionStatus.PENDING, + ) + d = RemoteDandiset(client=client, identifier="000001", version=ver) + d.set_raw_metadata( + { + "schemaKey": "Dandiset", + "schemaVersion": "0.8.0", + "name": "n", + "description": "d", + "identifier": "DANDI:000001", + "sameAs": [], + } + ) + sent = captured["body"]["metadata"] + assert sent["schemaVersion"] == "0.7.0" + assert "sameAs" not in sent + + def test_get_dandisets(text_dandiset: SampleDandiset) -> None: dandisets = list(text_dandiset.client.get_dandisets()) assert text_dandiset.dandiset_id in [d.identifier for d in dandisets] From 2b9af795c66f759e70a9057376575909c6c29294 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 4 Aug 2026 17:23:04 -0400 Subject: [PATCH 2/9] Address review feedback on downgrade wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to previous commit "downgrade outgoing metadata to server's schema version on upload". Applies fixes surfaced by an independent code review. Blockers: - `tests/test_dandiapi.py`: move `from ..dandiapi import RemoteDandiset, VersionStatus` to module-top imports; drop the shadowing in-function `from datetime import datetime, timezone` (already imported at module top). Complies with the project's "no function-local imports" rule. - `dandiapi.py`: `_maybe_downgrade_metadata` now returns `cast(Dict[str, Any], migrate(...))` — dandischema has no type stubs, so mypy would otherwise flag `no-any-return` in strict mode. Behavior fixes: - `_maybe_downgrade_metadata`: catch `ValueError` from `dandischema.metadata.migrate` (raised when a post-server-version field such as `sameAs` or `releaseNotes` is populated and cannot be simply stripped), log a warning, and return the original metadata unchanged so the server can decide. Prior behavior would crash the upload. - Add two `test__maybe_downgrade_metadata_falls_through_on_populated_field` cases (populated `sameAs` on 0.7.0 server; populated `releaseNotes` on 0.6.10 server) asserting fall-through + warning. Refactor: - Introduce `DandiAPIClient.server_info` (`@cached_property`) as the single source for the server's `/info/` response; `check_schema_version` and `server_schema_version` both use it now. Removes the duplicate `/info/` fetch that the two paths previously did independently. - Drop the redundant `obj_ver == server_ver_str` guard in `_maybe_downgrade_metadata` (subsumed by the `>=` version comparison). - Drop the unused `mocker: MockerFixture` fixture from `test_set_raw_metadata_downgrades_on_older_server`. Test infrastructure: - Add a `_needs_downgrade` skip marker keyed on `"0.7.0" in dandischema.consts.ALLOWED_TARGET_SCHEMAS`; apply it to parametrize cases and tests that exercise downgrade-to-older-schema behavior. Verified locally that the suite skips cleanly against released `dandischema==0.13.0` (which has no downgrade path to older schemas) and passes fully against the release-schema branch. Dependency: - `pyproject.toml`: widen `dandischema` to `>= 0.12.0, < 0.14.0`. With the downgrade wiring in place — and its graceful fall-through when the installed dandischema lacks migration paths — the released 0.13.x line is safe to use as well. Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- dandi/dandiapi.py | 48 +++++++++++++++------ dandi/tests/test_dandiapi.py | 82 +++++++++++++++++++++++++++++++++--- pyproject.toml | 2 +- 3 files changed, 111 insertions(+), 21 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b4954dc94..4b9f59187 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -27,7 +27,7 @@ import re from time import sleep, time from types import TracebackType -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast import click from dandischema import models @@ -712,12 +712,8 @@ def check_schema_version(self, schema_version: str | None = None) -> None: """ if schema_version is None: schema_version = models.get_schema_version() - server_info = self.get("/info/") - if not (server_schema_version := server_info.get("schema_version")): - raise RuntimeError( - "Server did not provide schema_version in /info/;" - f" returned {server_info!r}" - ) + server_info = self.server_info + server_schema_version = self.server_schema_version server_ver, our_ver = PackagingVersion(server_schema_version), PackagingVersion( schema_version ) @@ -774,16 +770,25 @@ def check_schema_version(self, schema_version: str | None = None) -> None: msg_downgrade, ) + @cached_property + def server_info(self) -> dict[str, Any]: + """ + Cached response from the server's ``/info/`` endpoint. + """ + info = self.get("/info/") + assert isinstance(info, dict) + return info + @cached_property def server_schema_version(self) -> str: """ The DANDI schema version reported by the server's ``/info/`` endpoint. """ - info = self.get("/info/") - schema_version = info.get("schema_version") + schema_version = self.server_info.get("schema_version") if not schema_version: raise RuntimeError( - "Server did not provide schema_version in /info/;" f" returned {info!r}" + f"Server did not provide schema_version in /info/; " + f"returned {self.server_info!r}" ) assert isinstance(schema_version, str) return schema_version @@ -796,6 +801,11 @@ def _maybe_downgrade_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: ``dandischema.consts.ALLOWED_TARGET_SCHEMAS``), return a downgraded copy of the metadata. Otherwise return ``metadata`` unchanged. + If the downgrade path exists but ``dandischema.metadata.migrate`` + refuses (e.g. a field added post-server-version is populated and + cannot be simply stripped), log a warning and return the original + metadata — the server will then decide. + Ref: https://github.com/dandi/dandi-schema/issues/343 """ obj_ver = metadata.get("schemaVersion") @@ -803,12 +813,24 @@ def _maybe_downgrade_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: return metadata server_ver_str = self.server_schema_version if ( - obj_ver == server_ver_str - or PackagingVersion(server_ver_str) >= PackagingVersion(obj_ver) + PackagingVersion(server_ver_str) >= PackagingVersion(obj_ver) or server_ver_str not in dandischema.consts.ALLOWED_TARGET_SCHEMAS ): return metadata - return migrate(metadata, to_version=server_ver_str, skip_validation=True) + try: + downgraded = migrate( + metadata, to_version=server_ver_str, skip_validation=True + ) + except ValueError as exc: + lgr.warning( + "Could not downgrade metadata from schema version %s to server's " + "%s: %s. Sending original metadata; server may reject it.", + obj_ver, + server_ver_str, + exc, + ) + return metadata + return cast(Dict[str, Any], downgraded) def get_asset(self, asset_id: str) -> BaseRemoteAsset: """ diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 823893b9a..f7c441b95 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -12,6 +12,7 @@ import anys import click +import dandischema.consts from dandischema.models import UUID_PATTERN, DigestType, get_schema_version import pytest from pytest_mock import MockerFixture @@ -32,14 +33,23 @@ DandiAPIClient, RemoteAsset, RemoteBlobAsset, + RemoteDandiset, RemoteZarrAsset, Version, + VersionStatus, ) from ..download import download from ..exceptions import NotFoundError, SchemaVersionError from ..files import GenericAsset, dandi_file from ..utils import list_paths +# Skip downgrade tests when the installed dandischema lacks migration paths to +# older schema versions (e.g. released 0.13.0 only lists the current schema). +_needs_downgrade = pytest.mark.skipif( + "0.7.0" not in dandischema.consts.ALLOWED_TARGET_SCHEMAS, + reason="Installed dandischema has no downgrade path to older schema versions", +) + def test_upload( new_dandiset: SampleDandiset, simple1_nwb: Path, tmp_path: Path @@ -380,9 +390,23 @@ def _mock_server_info(schema_version: str) -> None: ("0.8.0", "0.8.0", "0.8.0", [], ["sameAs", "releaseNotes"]), # server one minor behind -- downgrade to 0.7.0, drop sameAs (added 0.7.1), # keep releaseNotes (added 0.7.0) - ("0.7.0", "0.8.0", "0.7.0", ["sameAs"], ["releaseNotes"]), + pytest.param( + "0.7.0", + "0.8.0", + "0.7.0", + ["sameAs"], + ["releaseNotes"], + marks=_needs_downgrade, + ), # server on 0.6.10 -- both sameAs and releaseNotes go - ("0.6.10", "0.8.0", "0.6.10", ["sameAs", "releaseNotes"], []), + pytest.param( + "0.6.10", + "0.8.0", + "0.6.10", + ["sameAs", "releaseNotes"], + [], + marks=_needs_downgrade, + ), # server on an unsupported (too-old) target -- leave metadata alone ("0.6.5", "0.8.0", "0.8.0", [], ["sameAs", "releaseNotes"]), ], @@ -414,8 +438,56 @@ def test__maybe_downgrade_metadata( assert f in out, f"expected {f!r} to be kept" +@_needs_downgrade +@pytest.mark.parametrize( + "server_schema,obj_schema,populated_field,populated_value", + [ + # sameAs was added in 0.8.0 -- server on 0.7.0 cannot accept it, + # and a non-empty value cannot be simply stripped. + ("0.7.0", "0.8.0", "sameAs", ["DANDI:000002"]), + # releaseNotes was added in 0.7.0 -- server on 0.6.10 cannot accept + # it, and a non-empty value cannot be simply stripped. + ("0.6.10", "0.7.0", "releaseNotes", "Some release notes."), + ], +) +@responses.activate +def test__maybe_downgrade_metadata_falls_through_on_populated_field( + server_schema: str, + obj_schema: str, + populated_field: str, + populated_value: Any, + caplog: pytest.LogCaptureFixture, +) -> None: + """ + When ``dandischema.metadata.migrate`` refuses because a post-server-version + field carries a non-empty value that cannot be simply stripped, the client + must log a warning and return the original metadata unchanged so the + server can decide. + """ + _mock_server_info(server_schema) + client = DandiAPIClient("https://test.nil/api") + metadata = { + "schemaKey": "Dandiset", + "schemaVersion": obj_schema, + "name": "n", + "description": "d", + "identifier": "DANDI:000001", + populated_field: populated_value, + } + with caplog.at_level(logging.WARNING, logger="dandi"): + out = client._maybe_downgrade_metadata(metadata) + assert out is metadata + assert out["schemaVersion"] == obj_schema + assert out[populated_field] == populated_value + assert any( + "Could not downgrade metadata" in rec.message and populated_field in rec.message + for rec in caplog.records + ) + + +@_needs_downgrade @responses.activate -def test_set_raw_metadata_downgrades_on_older_server(mocker: MockerFixture) -> None: +def test_set_raw_metadata_downgrades_on_older_server() -> None: """ When the server reports an older schema version than the metadata carries, `RemoteVersion.set_raw_metadata` must downgrade the metadata before the @@ -436,10 +508,6 @@ def _put_callback(request: Any) -> tuple[int, dict, str]: content_type="application/json", ) - from datetime import datetime, timezone - - from ..dandiapi import RemoteDandiset, VersionStatus - client = DandiAPIClient("https://test.nil/api") ver = Version( version="draft", diff --git a/pyproject.toml b/pyproject.toml index 03d9f0d62..56865008d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ "bids-validator-deno >= 2.0.5", "click >= 8.2", "click-didyoumean", - "dandischema ~= 0.12.0", + "dandischema >= 0.12.0, < 0.14.0", "etelemetry >= 0.2.2", "fasteners", "fscacher >= 0.3.0", From ca7dd15e1a47455064a890b93d39e9f486e1a575 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 5 Aug 2026 23:58:06 -0400 Subject: [PATCH 3/9] deps: exclude dandischema 0.13.0 (lacks downgrade migrations) Released dandischema 0.13.0 dropped the SIMPLE_DOWNGRADES entries for `releaseNotes` (0.7.0) and `sameAs` (0.8.0), so `_maybe_downgrade_metadata` falls through and uploads to older Archive servers fail with unknown-field errors. 0.14.0 restored those migrations. Widen the range and exclude just 0.13.0: dandischema >= 0.12.0, != 0.13.0, < 0.15.0 < 0.15.0 keeps the upper bound conservative; we can widen it further when 0.15.0 lands and is verified to preserve the downgrade path. Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 01291d1e8..b98d4f7a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,10 @@ dependencies = [ "bids-validator-deno >= 2.0.5", "click >= 8.2", "click-didyoumean >= 0.3.1", - "dandischema >= 0.12.0, < 0.14.0", + # Exclude 0.13.0: it lacks the downgrade migrations (SIMPLE_DOWNGRADES + # entries for releaseNotes / sameAs) that the client relies on to talk + # to older Archive servers. 0.14.0 restored them. + "dandischema >= 0.12.0, != 0.13.0, < 0.15.0", "etelemetry >= 0.2.2", "fasteners >= 0.19", "fscacher >= 0.3.0", From 45169759d326ae3ac79e728e6f3227a2b7524d43 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 4 Aug 2026 23:22:49 -0400 Subject: [PATCH 4/9] fix(test): use `identifier=` field name in Version() construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pydantic's mypy plugin materializes `__init__` from *field names*, so `Version(version="draft", ...)` — which uses the `alias="version"` declared on `identifier` — is a type error even though `populate_by_name=True` on `APIBase` accepts it at runtime. Use the field name directly. Fixes the typing job on the enh-downgrade branch. Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- dandi/tests/test_dandiapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index f7c441b95..a5ba1bc97 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -510,7 +510,7 @@ def _put_callback(request: Any) -> tuple[int, dict, str]: client = DandiAPIClient("https://test.nil/api") ver = Version( - version="draft", + identifier="draft", name="draft", asset_count=0, size=0, From e13db27c9503c0290f1d03758a58e3df6868448c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 4 Aug 2026 23:52:35 -0400 Subject: [PATCH 5/9] fix: don't raise SchemaVersionError when a downgrade path exists `check_schema_version` was raising `SchemaVersionError` for the exact scenario this branch exists to handle: server on 0.7.x, client on 0.8.x (different 0.x minor, treated as "incompatible"), so no test that goes through the sample-dandiset fixtures on Ubuntu (docker dandi-api) could even reach `_maybe_downgrade_metadata`. Consolidate the two "server older" branches: if dandischema knows a downgrade path (`server_version in ALLOWED_TARGET_SCHEMAS`) *or* the versions are within the same MAJOR.MINOR (0.x.y) / MAJOR (1.x.y) compatibility band, just warn -- with the existing message about whether a downgrade will be attempted or is unsupported. Only raise when neither condition holds. Also mark the three new AI-generated downgrade tests with `@pytest.mark.ai_generated` per repo convention, and extend `test_check_schema_version` with two parametrize cases covering the new behavior (0.7.0/0.8.0 and 0.6.10/0.8.0 both no longer raise). Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- dandi/dandiapi.py | 32 ++++++++++++++++++++------------ dandi/tests/test_dandiapi.py | 8 ++++++++ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 4b9f59187..b853aa8c1 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -735,19 +735,27 @@ def check_schema_version(self, schema_version: str | None = None) -> None: # TODO: check current server behavior which is likely to just not care! # So that is where server might need to provide support for upgrades upon # providing metadata. - elif ( - server_ver.major == 0 and server_ver.release[:2] != our_ver.release[:2] - ) or ( - server_ver.major != our_ver.major - ): # MAJOR, MINOR within 0.x.y and MAJOR within 1.x.y - raise SchemaVersionError( - f"Server uses older incompatible schema version {server_schema_version};" - f" client supports {schema_version}." - ) elif server_ver < our_ver: - # Compatible older server version -- `_maybe_downgrade_metadata` will - # attempt an on-upload downgrade when possible. - if server_schema_version in dandischema.consts.ALLOWED_TARGET_SCHEMAS: + # Server is older. If dandischema knows a downgrade path + # (`_maybe_downgrade_metadata` will use it on upload) OR if the + # versions are within the same MAJOR.MINOR (0.x.y) / MAJOR (1.x.y) + # compatibility band, just warn; otherwise raise as incompatible. + can_downgrade = ( + server_schema_version in dandischema.consts.ALLOWED_TARGET_SCHEMAS + ) + same_compat_band = not ( + ( + server_ver.major == 0 + and server_ver.release[:2] != our_ver.release[:2] + ) + or (server_ver.major != our_ver.major) + ) + if not can_downgrade and not same_compat_band: + raise SchemaVersionError( + f"Server uses older incompatible schema version {server_schema_version};" + f" client supports {schema_version}." + ) + if can_downgrade: msg_downgrade = ( "Library will attempt (but might fail) to downgrade outgoing " "metadata to this schema version on upload. " diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index a5ba1bc97..2fa0f26fe 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -319,6 +319,11 @@ def test_remote_asset_json_dict(text_dandiset: SampleDandiset) -> None: True, "Server uses older incompatible schema version 0.6.7; client supports 0.7.0", ), + # ...unless the older server version is in + # `ALLOWED_TARGET_SCHEMAS` (i.e. `_maybe_downgrade_metadata` can + # downgrade to it on upload) -- then a warning, not an error. + pytest.param("0.7.0", "0.8.0", False, None, marks=_needs_downgrade), + pytest.param("0.6.10", "0.8.0", False, None, marks=_needs_downgrade), # After 1.x -- rely on MAJOR. ("1.0.0", "1.2.3", False, None), ("1.6.7", "1.7.0", False, None), @@ -383,6 +388,7 @@ def _mock_server_info(schema_version: str) -> None: responses.add(responses.GET, "https://test.nil/api/info/", json=info) +@pytest.mark.ai_generated @pytest.mark.parametrize( "server_schema,obj_schema,expected_schema,dropped,kept", [ @@ -438,6 +444,7 @@ def test__maybe_downgrade_metadata( assert f in out, f"expected {f!r} to be kept" +@pytest.mark.ai_generated @_needs_downgrade @pytest.mark.parametrize( "server_schema,obj_schema,populated_field,populated_value", @@ -485,6 +492,7 @@ def test__maybe_downgrade_metadata_falls_through_on_populated_field( ) +@pytest.mark.ai_generated @_needs_downgrade @responses.activate def test_set_raw_metadata_downgrades_on_older_server() -> None: From 4b5e0ce2aa0c869f7c8baa89ebf04525eaa748f7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 5 Aug 2026 08:46:56 -0400 Subject: [PATCH 6/9] fix(check_schema_version): silence CodeQL false-positive on intermediate var CodeQL flagged `same_compat_band` as "may be used before initialized" on `check_schema_version`. It was in fact unconditionally assigned in the same `elif` branch, but eliminating the intermediate variable and inlining the condition side-steps the analyzer's confusion entirely. Same semantics. Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- dandi/dandiapi.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index b853aa8c1..cb11a3ba1 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -743,14 +743,13 @@ def check_schema_version(self, schema_version: str | None = None) -> None: can_downgrade = ( server_schema_version in dandischema.consts.ALLOWED_TARGET_SCHEMAS ) - same_compat_band = not ( + if not can_downgrade and ( ( server_ver.major == 0 and server_ver.release[:2] != our_ver.release[:2] ) or (server_ver.major != our_ver.major) - ) - if not can_downgrade and not same_compat_band: + ): raise SchemaVersionError( f"Server uses older incompatible schema version {server_schema_version};" f" client supports {schema_version}." From c0d63c3d21aaaf52d667c78b2ef283de7e79269a Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 5 Aug 2026 09:38:30 -0400 Subject: [PATCH 7/9] refactor: centralize downgrade in `DandiAPIClient.request` interceptor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of sprinkling `_maybe_downgrade_metadata(...)` at every metadata- sending call site, override `DandiAPIClient.request` to run `_maybe_downgrade_request_metadata` over the outgoing ``json`` body. A body is considered to carry DANDI metadata iff it is a ``dict`` whose ``"metadata"`` value is itself a ``dict`` with both ``schemaKey`` and ``schemaVersion`` as strings — the invariant shape of every DANDI metadata payload the archive accepts. Non-matching bodies pass through unchanged. Drops the six spot-patches added in the previous commit: - `DandiAPIClient.create_dandiset` - `RemoteVersion.set_raw_metadata` - `RemoteBlobAsset.set_raw_metadata` - `RemoteZarrAsset.set_raw_metadata` - `LocalFileAsset.iter_upload` (`dandi/files/bases.py`) - `ZarrAsset.iter_upload` (`dandi/files/zarr.py`) Any future metadata-sending endpoint now gets the downgrade for free. Also adds two unit tests: - `test__maybe_downgrade_request_metadata_shape_check` — verifies the shape check does not touch non-matching bodies (non-dict, missing `metadata`, `metadata` not a dict, missing/non-str `schemaKey` or `schemaVersion`, or `metadata` sitting under a different key). - `test__maybe_downgrade_request_metadata_downgrades` — verifies the interceptor returns a new body (does not mutate input) when the shape matches, and correctly downgrades the nested metadata dict. The end-to-end `test_set_raw_metadata_downgrades_on_older_server` test is unchanged and now exercises the full `set_raw_metadata` -> `client.put` -> `request` -> interceptor path. Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- dandi/dandiapi.py | 73 ++++++++++++++++++++++++++++++++++-- dandi/files/bases.py | 1 - dandi/files/zarr.py | 1 - dandi/tests/test_dandiapi.py | 55 +++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 6 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index cb11a3ba1..ccf17239b 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -682,7 +682,6 @@ def create_dandiset( ``embargo`` argument added """ - metadata = self._maybe_downgrade_metadata(metadata) return RemoteDandiset.from_data( self, self.post( @@ -839,6 +838,75 @@ def _maybe_downgrade_metadata(self, metadata: dict[str, Any]) -> dict[str, Any]: return metadata return cast(Dict[str, Any], downgraded) + def _maybe_downgrade_request_metadata(self, body: Any) -> Any: + """ + Intercept an outgoing request body: if it wraps a DANDI metadata + dict under the ``"metadata"`` key, downgrade that dict to the + server's schema version via ``_maybe_downgrade_metadata``. + + A body is treated as carrying DANDI metadata iff it is a + ``dict`` whose ``"metadata"`` value is itself a ``dict`` with + both ``schemaKey`` (a str, e.g. ``"Dandiset"``, ``"BareAsset"``) + and ``schemaVersion`` (a str) — this is the invariant shape of + every DANDI-metadata payload the archive accepts. Bodies that + don't match are passed through unchanged. + + Called from `request()`, so every metadata-sending endpoint + (``create_dandiset``, ``*.set_raw_metadata``, ``iter_upload`` + asset-create/replace, plus any future ones) gets the downgrade + automatically without spot-patching. + """ + if not isinstance(body, dict): + return body + md = body.get("metadata") + if not ( + isinstance(md, dict) + and isinstance(md.get("schemaKey"), str) + and isinstance(md.get("schemaVersion"), str) + ): + return body + downgraded = self._maybe_downgrade_metadata(md) + if downgraded is md: + return body + return {**body, "metadata": downgraded} + + def request( + self, + method: str, + path: str, + params: dict | None = None, + data: Any = None, + files: dict | None = None, + json: Any = None, + headers: dict | None = None, + json_resp: bool = True, + retry_statuses: Sequence[int] = (), + retry_if: Callable[[requests.Response], Any] | None = None, + **kwargs: Any, + ) -> Any: + """ + Override of `RESTFullAPIClient.request` that runs + `_maybe_downgrade_request_metadata` over the outgoing ``json`` + body so DANDI metadata gets downgraded to the server's schema + version on the way out. See that method's docstring for the + shape check used and the ref to dandi-schema#343. + """ + if json is not None: + json = self._maybe_downgrade_request_metadata(json) + return super().request( + method, + path, + params=params, + data=data, + files=files, + json=json, + headers=headers, + json_resp=json_resp, + retry_statuses=retry_statuses, + retry_if=retry_if, + **kwargs, + ) + def get_asset(self, asset_id: str) -> BaseRemoteAsset: """ Fetch the asset with the given asset ID. If the given asset does not @@ -1280,7 +1348,6 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: """ Set the metadata for this version of the Dandiset to the given value """ - metadata = self.client._maybe_downgrade_metadata(metadata) self.client.put( self.version_api_path, json={"metadata": metadata, "name": metadata.get("name", "")}, @@ -2106,7 +2173,6 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: update the `RemoteBlobAsset` in place. """ set_asset_schema_key(metadata) - metadata = self.client._maybe_downgrade_metadata(metadata) data = self.client.put( self.api_path, json={"metadata": metadata, "blob_id": self.blob} ) @@ -2131,7 +2197,6 @@ def set_raw_metadata(self, metadata: dict[str, Any]) -> None: update the `RemoteZarrAsset` in place. """ set_asset_schema_key(metadata) - metadata = self.client._maybe_downgrade_metadata(metadata) data = self.client.put( self.api_path, json={"metadata": metadata, "zarr_id": self.zarr} ) diff --git a/dandi/files/bases.py b/dandi/files/bases.py index e210b0531..3577e605f 100644 --- a/dandi/files/bases.py +++ b/dandi/files/bases.py @@ -370,7 +370,6 @@ def iter_upload( asset_path = metadata.setdefault("path", self.path) set_asset_schema_key(metadata) client = dandiset.client - metadata = client._maybe_downgrade_metadata(metadata) yield {"status": "calculating etag"} etagger = get_dandietag(self.filepath) filetag = etagger.as_str() diff --git a/dandi/files/zarr.py b/dandi/files/zarr.py index 0fee40a27..8f5a4aa0e 100644 --- a/dandi/files/zarr.py +++ b/dandi/files/zarr.py @@ -583,7 +583,6 @@ def iter_upload( asset_path = metadata.setdefault("path", self.path) set_asset_schema_key(metadata) client = dandiset.client - metadata = client._maybe_downgrade_metadata(metadata) lgr.debug("%s: Producing asset", asset_path) yield {"status": "producing asset"} diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 2fa0f26fe..2a14f76ff 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -492,6 +492,61 @@ def test__maybe_downgrade_metadata_falls_through_on_populated_field( ) +@pytest.mark.ai_generated +@responses.activate +def test__maybe_downgrade_request_metadata_shape_check() -> None: + """ + The request-body interceptor only touches bodies whose `metadata` + sub-dict has *both* `schemaKey` and `schemaVersion` as strings. + Anything else -- non-dict body, missing `metadata` key, `metadata` + that isn't a dict, `metadata` missing either shape marker -- must + pass through unchanged (identity). + """ + _mock_server_info("0.7.0") + client = DandiAPIClient("https://test.nil/api") + non_matches = [ + None, + "a string body", + 42, + [{"metadata": {"schemaKey": "Dandiset", "schemaVersion": "0.8.0"}}], + {}, + {"metadata": "not a dict"}, + {"metadata": {"schemaVersion": "0.8.0"}}, # missing schemaKey + {"metadata": {"schemaKey": "Dandiset"}}, # missing schemaVersion + {"metadata": {"schemaKey": "Dandiset", "schemaVersion": 800}}, # non-str + {"other_key": {"schemaKey": "Dandiset", "schemaVersion": "0.8.0"}}, + ] + for body in non_matches: + assert client._maybe_downgrade_request_metadata(body) is body + + +@pytest.mark.ai_generated +@_needs_downgrade +@responses.activate +def test__maybe_downgrade_request_metadata_downgrades() -> None: + """ + When the body shape matches, the interceptor swaps in a downgraded + metadata dict, returning a *new* body (does not mutate input). + """ + _mock_server_info("0.7.0") + client = DandiAPIClient("https://test.nil/api") + md = { + "schemaKey": "Dandiset", + "schemaVersion": "0.8.0", + "name": "n", + "sameAs": [], + } + body = {"metadata": md, "name": "n"} + out = client._maybe_downgrade_request_metadata(body) + assert out is not body # new body + assert out["metadata"]["schemaVersion"] == "0.7.0" + assert "sameAs" not in out["metadata"] + assert out["name"] == "n" + # original body untouched + assert body["metadata"] is md + assert md["schemaVersion"] == "0.8.0" + + @pytest.mark.ai_generated @_needs_downgrade @responses.activate From f29a172e389851ae074c077af4d209b3e9909097 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 6 Aug 2026 10:57:23 -0400 Subject: [PATCH 8/9] test: gate `_needs_downgrade` on `dandischema >= 0.14.0` The downgrade parametrizations feed the client a 0.8.0-schema metadata dict and expect `dandischema.metadata.migrate` to bring it down to 0.7.0. Everything they need -- the 0.8.0-era metadata schema plus `SIMPLE_DOWNGRADES` entries for `sameAs` and `releaseNotes` -- landed together in `dandischema==0.14.0`. Older dandischema in the allowed range (0.12.0, reached only under `py3-lowest`) generates 0.7.0 metadata and has no downgrade path, so these tests are meaningless there and previously failed inside `migrate()` because 0.12.0 doesn't recognize "0.8.0" as a valid input at all. 0.13.0 is excluded from `pyproject.toml`. Replace the compound two-condition gate (which spoke about `DANDI_SCHEMA_VERSION` and `ALLOWED_TARGET_SCHEMAS` separately) with a single direct check on the library version. With this, `py3-lowest`/0.12.0 cleanly skips the 8 downgrade parametrizations while the default env (0.14.0+) runs the full 23. Co-Authored-By: Claude Code 2.1.221 / Claude Opus 4.7 (1M context) --- dandi/tests/test_dandiapi.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 2a14f76ff..8da28852e 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -12,8 +12,9 @@ import anys import click -import dandischema.consts +import dandischema from dandischema.models import UUID_PATTERN, DigestType, get_schema_version +from packaging.version import Version as _PackagingVersion import pytest from pytest_mock import MockerFixture import requests @@ -43,11 +44,13 @@ from ..files import GenericAsset, dandi_file from ..utils import list_paths -# Skip downgrade tests when the installed dandischema lacks migration paths to -# older schema versions (e.g. released 0.13.0 only lists the current schema). +# These tests exercise the 0.8.0 -> 0.7.0 downgrade wiring, which requires +# both the 0.8.0-era metadata schema and the migration paths for `sameAs` / +# `releaseNotes` -- all of which landed together in dandischema 0.14.0. +# `py3-lowest` resolves to `dandischema==0.12.0` and simply skips them. _needs_downgrade = pytest.mark.skipif( - "0.7.0" not in dandischema.consts.ALLOWED_TARGET_SCHEMAS, - reason="Installed dandischema has no downgrade path to older schema versions", + _PackagingVersion(dandischema.__version__) < _PackagingVersion("0.14.0"), + reason="Downgrade tests require dandischema >= 0.14.0", ) From 5381c466cc85c927043bbfcc47d9ffe02f5afd12 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 6 Aug 2026 16:50:23 -0400 Subject: [PATCH 9/9] Run py3-lowest as well for the tox default sweep --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 97cf30772..f40c8230a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] requires = tox-uv >= 1.11 -envlist = lint,typing,py3 +envlist = lint,typing,py3,py3-lowest [testenv] setenv =