diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 3b4848d12..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[bumpversion] -current_version = 0.0.1 -tag = True -commit = False - -[bumpversion:file:./pyproject.toml] -search = version = "{current_version}" -replace = version = "{new_version}" diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 000000000..d0c0e2917 --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,17 @@ +# Based on pydata/xarray +codecov: + require_ci_to_pass: no + +coverage: + status: + project: + default: + # Require 1% coverage, i.e., always succeed + target: 1 + patch: false + changes: false + +comment: + layout: diff, flags, files + behavior: once + require_base: no diff --git a/.cruft.json b/.cruft.json new file mode 100644 index 000000000..49cb95efd --- /dev/null +++ b/.cruft.json @@ -0,0 +1,44 @@ +{ + "template": "https://github.com/scverse/cookiecutter-scverse", + "commit": "eb4523fde2e18bcfc121ff2cd722c9037ff8b910", + "checkout": null, + "context": { + "cookiecutter": { + "project_name": "spatialdata", + "package_name": "spatialdata", + "project_description": "Spatial data format.", + "author_full_name": "scverse", + "author_email": "giov.pll@gmail.com", + "github_user": "scverse", + "github_repo": "spatialdata", + "license": "BSD 3-Clause License", + "ide_integration": true, + "issue_categorization": "labels", + "_copy_without_render": [ + ".github/workflows/build.yaml", + ".github/workflows/test.yaml", + "docs/_templates/autosummary/**.rst" + ], + "_exclude_on_template_update": [ + "CHANGELOG.md", + "LICENSE", + "README.md", + "docs/api.md", + "docs/index.md", + "docs/notebooks/example.ipynb", + "docs/references.bib", + "docs/references.md", + "src/**", + "tests/**" + ], + "_render_devdocs": false, + "_jinja2_env_vars": { + "lstrip_blocks": true, + "trim_blocks": true + }, + "_template": "https://github.com/scverse/cookiecutter-scverse", + "_commit": "eb4523fde2e18bcfc121ff2cd722c9037ff8b910" + } + }, + "directory": null +} diff --git a/.editorconfig b/.editorconfig index 284cec449..33c3807c3 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,5 +9,8 @@ trim_trailing_whitespace = true insert_final_newline = true line_length = 120 +[{*.{yml,yaml,toml},.cruft.json}] +indent_size = 2 + [Makefile] indent_style = tab diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 392612654..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: "" -labels: "" -assignees: "" ---- - -**Recommendation: attach a minimal working example** -Generally, the easier it is for us to reproduce the issue, the faster we can work on it. It is not required, but if you can, please: - -1. Reproduce using the [`blobs` dataset](https://spatialdata.scverse.org/en/stable/api/datasets.html#spatialdata.datasets.blobs) - - ```python - from spatialdata.datasets import blobs - - sdata = blobs() - ``` - - You can also use [`blobs_annotating_element`](https://spatialdata.scverse.org/en/stable/api/datasets.html#spatialdata.datasets.blobs_annotating_element) for more - control: - - ``` - from spatialdata.datasets import blobs_annotating_element - sdata = blobs_annotating_element('blobs_labels') - ``` - -2. If the above is not possible, reproduce using a public dataset and explain how we can download the data. -3. If the data is private, consider sharing an anonymized version/subset via a [Zulip private message](https://scverse.zulipchat.com/#user/480560), or provide screenshots/GIFs showing the behavior. - -**Describe the bug** -A clear and concise description of what the bug is; please report only one bug per issue. - -**To Reproduce** -Steps to reproduce the behavior: - -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Desktop (optional):** - -- OS: [e.g. macOS, Windows, Linux] -- Version [e.g. 22] - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..59698ad32 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,116 @@ +name: Bug report +description: Report something that is broken or incorrect +labels: bug +body: + - type: markdown + attributes: + value: | + **Note**: Please read [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) + detailing how to provide the necessary information for us to reproduce your bug. In brief: + * Please provide exact steps how to reproduce the bug in a clean Python environment. + * In case it's not clear what's causing this bug, please provide the data or the data generation procedure. + * Replicate problems on public datasets or share data subsets when full sharing isn't possible. + + Generally, the easier it is for us to reproduce the issue, the faster we can work on it. If you can, please: + + 1. Reproduce using the [`blobs` dataset](https://spatialdata.scverse.org/en/stable/api/datasets.html#spatialdata.datasets.blobs): + + ```python + from spatialdata.datasets import blobs + + sdata = blobs() + ``` + + You can also use [`blobs_annotating_element`](https://spatialdata.scverse.org/en/stable/api/datasets.html#spatialdata.datasets.blobs_annotating_element) for more control: + + ```python + from spatialdata.datasets import blobs_annotating_element + + sdata = blobs_annotating_element("blobs_labels") + ``` + + 2. If the above is not possible, reproduce using a public dataset and explain how we can download the data. + 3. If the data is private, consider sharing an anonymized version/subset via a [Zulip private message](https://scverse.zulipchat.com/#user/480560), or provide screenshots/GIFs showing the behavior. + + Please report only one bug per issue. + + - type: textarea + id: report + attributes: + label: Report + description: A clear and concise description of what the bug is. + validations: + required: true + + - type: textarea + id: versions + attributes: + label: Versions + description: | + Which version of packages. + + Please install `session-info2`, run the following command in a notebook, + click the “Copy as Markdown” button, then paste the results into the text box below. + + ```python + In[1]: import session_info2; session_info2.session_info(dependencies=True) + ``` + + Alternatively, run this in a console: + + ```python + >>> import session_info2; print(session_info2.session_info(dependencies=True)._repr_mimebundle_()["text/markdown"]) + ``` + render: python + placeholder: | + anndata 0.11.3 + ---- ---- + charset-normalizer 3.4.1 + coverage 7.7.0 + psutil 7.0.0 + dask 2024.7.1 + jaraco.context 5.3.0 + numcodecs 0.15.1 + jaraco.functools 4.0.1 + Jinja2 3.1.6 + sphinxcontrib-jsmath 1.0.1 + sphinxcontrib-htmlhelp 2.1.0 + toolz 1.0.0 + session-info2 0.1.2 + PyYAML 6.0.2 + llvmlite 0.44.0 + scipy 1.15.2 + pandas 2.2.3 + sphinxcontrib-devhelp 2.0.0 + h5py 3.13.0 + tblib 3.0.0 + setuptools-scm 8.2.0 + more-itertools 10.3.0 + msgpack 1.1.0 + sparse 0.15.5 + wrapt 1.17.2 + jaraco.collections 5.1.0 + numba 0.61.0 + pyarrow 19.0.1 + pytz 2025.1 + MarkupSafe 3.0.2 + crc32c 2.7.1 + sphinxcontrib-qthelp 2.0.0 + sphinxcontrib-serializinghtml 2.0.0 + zarr 2.18.4 + asciitree 0.3.3 + six 1.17.0 + sphinxcontrib-applehelp 2.0.0 + numpy 2.1.3 + cloudpickle 3.1.1 + sphinxcontrib-bibtex 2.6.3 + natsort 8.4.0 + jaraco.text 3.12.1 + setuptools 76.1.0 + Deprecated 1.2.18 + packaging 24.2 + python-dateutil 2.9.0.post0 + ---- ---- + Python 3.13.2 | packaged by conda-forge | (main, Feb 17 2025, 14:10:22) [GCC 13.3.0] + OS Linux-6.11.0-109019-tuxedo-x86_64-with-glibc2.39 + Updated 2025-03-18 15:47 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..5b62547f9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Scverse Community Forum + url: https://discourse.scverse.org/ + about: If you have questions about “How to do X”, please ask them here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 2bc5d5f71..000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: "" -labels: "" -assignees: "" ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..aafdf2a29 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,11 @@ +name: Feature request +description: Propose a new feature for spatialdata +labels: enhancement +body: + - type: textarea + id: description + attributes: + label: Description of feature + description: Please describe your suggestion for a new feature. It might help to describe a problem or use case, plus any alternatives that you have considered. + validations: + required: true diff --git a/.github/codecov.yml b/.github/codecov.yml deleted file mode 100644 index 872442c76..000000000 --- a/.github/codecov.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Based on pydata/xarray -codecov: - require_ci_to_pass: false - -coverage: - status: - project: - default: - # Require 1% coverage, i.e., always succeed - target: 1 - patch: false - changes: false - -comment: - layout: "diff, flags, files" - behavior: once - require_base: false diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..322793040 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +version: 2 +updates: + - package-ecosystem: pre-commit + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + pre-commit: + patterns: ["*"] + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + actions-deps: + patterns: + - "*" diff --git a/.github/release.yml b/.github/release.yml index fe7b92b74..4070fe995 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -1,29 +1,29 @@ changelog: - exclude: - labels: - - release-ignore - authors: - - pre-commit-ci - - pre-commit-ci[bot] - categories: - - title: Added - labels: - - "release-added" - - title: Changed - labels: - - "release-changed" - - title: Deprecated - labels: - - "release-deprecated" - - title: Removed - labels: - - "release-removed" - - title: Fixed - labels: - - "release-fixed" - - title: Security - labels: - - "release-security" - - title: Other Changes - labels: - - "*" + exclude: + labels: + - release-ignore + authors: + - pre-commit-ci + - pre-commit-ci[bot] + categories: + - title: Added + labels: + - "release-added" + - title: Changed + labels: + - "release-changed" + - title: Deprecated + labels: + - "release-deprecated" + - title: Removed + labels: + - "release-removed" + - title: Fixed + labels: + - "release-fixed" + - title: Security + labels: + - "release-security" + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 000000000..6cca7d4d7 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,30 @@ +name: Check Build + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + package: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + filter: blob:none + fetch-depth: 0 + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - name: Build package + run: uv build + - name: Check package + run: uvx twine check --strict dist/*.whl diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index ca3ac2398..031f1e480 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -1,100 +1,106 @@ name: Build Docker image on: - workflow_dispatch: -# schedule: -# - cron: '0 0 * * *' # run daily at midnight UTC + workflow_dispatch: + # schedule: + # - cron: "0 0 * * *" # run daily at midnight UTC env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read jobs: - build: - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -e {0} # -e to fail on error - - permissions: - contents: read - packages: write - attestations: write - id-token: write - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - name: Upgrade pip - run: pip install pip - - - name: Get latest versions - id: get_versions - run: | - SPATIALDATA_VERSION=$(pip index versions spatialdata | grep "Available versions" | sed 's/Available versions: //' | awk -F', ' '{print $1}') - SPATIALDATA_IO_VERSION=$(pip index versions spatialdata-io | grep "Available versions" | sed 's/Available versions: //' | awk -F', ' '{print $1}') - SPATIALDATA_PLOT_VERSION=$(pip index versions spatialdata-plot | grep "Available versions" | sed 's/Available versions: //' | awk -F', ' '{print $1}') - echo "SPATIALDATA_VERSION=${SPATIALDATA_VERSION}" >> $GITHUB_ENV - echo "SPATIALDATA_IO_VERSION=${SPATIALDATA_IO_VERSION}" >> $GITHUB_ENV - echo "SPATIALDATA_PLOT_VERSION=${SPATIALDATA_PLOT_VERSION}" >> $GITHUB_ENV - - - name: Check if image tag exists - id: check_tag - env: - IMAGE_TAG_SUFFIX: spatialdata${{ env.SPATIALDATA_VERSION }}_spatialdata-io${{ env.SPATIALDATA_IO_VERSION }}_spatialdata-plot${{ env.SPATIALDATA_PLOT_VERSION }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Define the API URL - API_URL="https://api.github.com/orgs/scverse/packages/container/spatialdata/versions" - - # Fetch all existing versions - existing_tags=$(curl -s -H "Authorization: token $GITHUB_TOKEN" $API_URL | jq -r '.[].metadata.container.tags[]') - - # Debug: Output all existing tags - echo "Existing tags:" - echo "$existing_tags" - - # Check if the constructed tag exists - if echo "$existing_tags" | grep -q "$IMAGE_TAG_SUFFIX"; then - echo "Image tag $IMAGE_TAG_SUFFIX already exists. Skipping build." - echo "skip_build=true" >> $GITHUB_ENV - else - echo "Image tag $IMAGE_TAG_SUFFIX does not exist. Proceeding with build." - echo "skip_build=false" >> $GITHUB_ENV - echo "IMAGE_TAG_SUFFIX=${IMAGE_TAG_SUFFIX}" >> $GITHUB_ENV - fi - - - name: Login to GitHub Container Registry - if: ${{ env.skip_build == 'false' }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - uses: docker/build-push-action@v5 - if: ${{ env.skip_build == 'false' }} - env: - IMAGE_TAG: ${{ env.REGISTRY }}/scverse/spatialdata:${{ env.IMAGE_TAG_SUFFIX }} - with: - context: . - file: ./Dockerfile - push: true - cache-from: type=registry,ref=${{ env.REGISTRY }}/scverse/spatialdata:buildcache - cache-to: type=inline,ref=${{ env.REGISTRY }}/scverse/spatialdata:buildcache - build-args: | - SPATIALDATA_VERSION=${{ env.SPATIALDATA_VERSION }} - SPATIALDATA_IO_VERSION=${{ env.SPATIALDATA_IO_VERSION }} - SPATIALDATA_PLOT_VERSION=${{ env.SPATIALDATA_PLOT_VERSION }} - tags: ${{ env.IMAGE_TAG }} + build: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -e {0} # -e to fail on error + + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + filter: blob:none + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.x" + + - name: Upgrade pip + run: pip install pip + + - name: Get latest versions + id: get_versions + run: | + SPATIALDATA_VERSION=$(pip index versions spatialdata | grep "Available versions" | sed 's/Available versions: //' | awk -F', ' '{print $1}') + SPATIALDATA_IO_VERSION=$(pip index versions spatialdata-io | grep "Available versions" | sed 's/Available versions: //' | awk -F', ' '{print $1}') + SPATIALDATA_PLOT_VERSION=$(pip index versions spatialdata-plot | grep "Available versions" | sed 's/Available versions: //' | awk -F', ' '{print $1}') + echo "SPATIALDATA_VERSION=${SPATIALDATA_VERSION}" >> $GITHUB_ENV + echo "SPATIALDATA_IO_VERSION=${SPATIALDATA_IO_VERSION}" >> $GITHUB_ENV + echo "SPATIALDATA_PLOT_VERSION=${SPATIALDATA_PLOT_VERSION}" >> $GITHUB_ENV + + - name: Check if image tag exists + id: check_tag + env: + IMAGE_TAG_SUFFIX: spatialdata${{ env.SPATIALDATA_VERSION }}_spatialdata-io${{ env.SPATIALDATA_IO_VERSION }}_spatialdata-plot${{ env.SPATIALDATA_PLOT_VERSION }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Define the API URL + API_URL="https://api.github.com/orgs/scverse/packages/container/spatialdata/versions" + + # Fetch all existing versions + existing_tags=$(curl -s -H "Authorization: token $GITHUB_TOKEN" $API_URL | jq -r '.[].metadata.container.tags[]') + + # Debug: Output all existing tags + echo "Existing tags:" + echo "$existing_tags" + + # Check if the constructed tag exists + if echo "$existing_tags" | grep -q "$IMAGE_TAG_SUFFIX"; then + echo "Image tag $IMAGE_TAG_SUFFIX already exists. Skipping build." + echo "skip_build=true" >> $GITHUB_ENV + else + echo "Image tag $IMAGE_TAG_SUFFIX does not exist. Proceeding with build." + echo "skip_build=false" >> $GITHUB_ENV + echo "IMAGE_TAG_SUFFIX=${IMAGE_TAG_SUFFIX}" >> $GITHUB_ENV + fi + + - name: Login to GitHub Container Registry + if: ${{ env.skip_build == 'false' }} + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + if: ${{ env.skip_build == 'false' }} + env: + IMAGE_TAG: ${{ env.REGISTRY }}/scverse/spatialdata:${{ env.IMAGE_TAG_SUFFIX }} + with: + context: . + file: ./Dockerfile + push: true + cache-from: type=registry,ref=${{ env.REGISTRY }}/scverse/spatialdata:buildcache + cache-to: type=inline,ref=${{ env.REGISTRY }}/scverse/spatialdata:buildcache + build-args: | + SPATIALDATA_VERSION=${{ env.SPATIALDATA_VERSION }} + SPATIALDATA_IO_VERSION=${{ env.SPATIALDATA_IO_VERSION }} + SPATIALDATA_PLOT_VERSION=${{ env.SPATIALDATA_PLOT_VERSION }} + tags: ${{ env.IMAGE_TAG }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 18493bca3..bb5a0bbf7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,31 +1,33 @@ name: Release on: - release: - types: [published] + release: + types: [published] + +# Use "trusted publishing", see https://docs.pypi.org/trusted-publishers/ +permissions: {} jobs: - package_and_release: - runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/v') - steps: - - uses: actions/checkout@v6 - - name: Set up Python 3.12 - uses: actions/setup-python@v6 - with: - python-version: "3.12" - cache: pip - - name: Install build dependencies - run: python -m pip install --upgrade pip wheel twine build - - name: Build package - run: python -m build - - name: Check package - run: twine check --strict dist/*.whl - - name: Install hatch - run: pip install hatch - - name: Build project for distribution - run: hatch build - - name: Publish a Python distribution to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} + release: + name: Upload release to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/spatialdata + permissions: + contents: read + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + filter: blob:none + fetch-depth: 0 + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + enable-cache: false + - name: Build package + run: uv build + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bc65f074a..7f92f751b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,70 +1,111 @@ name: Test on: - push: - branches: [main] - tags: - - "v*" - pull_request: - branches: "*" + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "0 5 1,15 * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read jobs: - test: - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash # bash also on windows + # [tool.hatch.envs.hatch-test.matrix] in pyproject.toml is the single source of truth: the same environments run here and locally. + get-environments: + runs-on: ubuntu-slim + outputs: + envs: ${{ steps.get-envs.outputs.envs }} + steps: + - &clone + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + filter: blob:none + fetch-depth: 0 + persist-credentials: false + - &setup-uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - name: Get test environments + id: get-envs + run: | + ENVS_JSON=$(uvx hatch env show --json | jq -c 'to_entries + | map( + select(.key | startswith("hatch-test")) + | { + name: .key, + label: (if (.key | contains("pre")) then .key + " (PRE-RELEASE DEPENDENCIES)" else .key end), + python: .value.python + } + )') + echo "envs=${ENVS_JSON}" | tee $GITHUB_OUTPUT + + test: + needs: get-environments + permissions: + id-token: write # for codecov OIDC + contents: read - strategy: - fail-fast: false - matrix: - include: - - {os: windows-latest, python: "3.12", dask-version: "2026.3.0", name: "min dask"} - - {os: windows-latest, python: "3.14", dask-version: "latest"} - - {os: ubuntu-latest, python: "3.12", dask-version: "latest"} - - {os: ubuntu-latest, python: "3.13", dask-version: "latest"} - - {os: ubuntu-latest, python: "3.14", dask-version: "latest"} - - {os: macos-latest, python: "3.12", dask-version: "latest"} - - {os: macos-latest, python: "3.14", prerelease: "allow", name: "prerelease"} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + env: ${{ fromJSON(needs.get-environments.outputs.envs) }} + + name: ${{ matrix.env.label }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + continue-on-error: ${{ contains(matrix.env.name, 'pre') }} # make "all-green" pass even if pre-release job fails + env: + UV_PYTHON: ${{ matrix.env.python }} + + steps: + - *clone + - *setup-uv + - name: create hatch environment + run: uvx hatch env create ${{ matrix.env.name }} + - name: list all all installed package versions + run: uvx hatch run ${{ matrix.env.name }}:uv pip list + - name: run tests using hatch env: - OS: ${{ matrix.os }} - PYTHON: ${{ matrix.python }} - DASK_VERSION: ${{ matrix.dask-version }} - PRERELEASE: ${{ matrix.prerelease }} + MPLBACKEND: agg + PLATFORM: ${{ matrix.os }} + DISPLAY: :42 + run: uvx hatch run ${{ matrix.env.name }}:run-cov -v --color=yes -n auto --dist worksteal --run-network + - name: generate coverage report + run: | + # See https://coverage.readthedocs.io/page/config.html#run-patch + test -f .coverage || uvx hatch run ${{ matrix.env.name }}:cov-combine + uvx hatch run ${{ matrix.env.name }}:cov-report # report visibly + uvx hatch run ${{ matrix.env.name }}:coverage xml # create report for upload + - name: Upload coverage + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 + with: + fail_ci_if_error: true + use_oidc: true + + lint: + name: Pre-commit checks + runs-on: ubuntu-latest + steps: + - *clone + - *setup-uv + - run: uv tool install hatch + - run: uvx prek run --all-files --show-diff-on-failure --color=always - steps: - - uses: actions/checkout@v6 - - uses: astral-sh/setup-uv@v7 - id: setup-uv - with: - version: "latest" - python-version: ${{ matrix.python }} - - name: Install dependencies - run: | - if [[ "${PRERELEASE}" == "allow" ]]; then - uv add git+https://github.com/scverse/anndata.git - uv add pandas>=3.dev0 - fi - if [[ -n "${DASK_VERSION}" ]]; then - if [[ "${DASK_VERSION}" == "latest" ]]; then - uv add dask - else - uv add dask==${DASK_VERSION} - fi - fi - uv sync --group=test - - name: Test - env: - MPLBACKEND: agg - PLATFORM: ${{ matrix.os }} - DISPLAY: :42 - run: | - uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v6 - with: - name: coverage - verbose: true - fail_ci_if_error: true - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # One required check for branch protection, so it does not need updating whenever the matrix changes. + check: + name: Tests pass in all hatch environments + if: always() + needs: + - get-environments + - test + - lint + runs-on: ubuntu-latest + steps: + - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.gitignore b/.gitignore index 23fa4b3d2..d157ddaae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,16 @@ # Temp files .DS_Store *~ +buck-out/ .dmypy.json +temp/ # Compiled files +.venv/ +uv.lock +pixi.lock __pycache__/ +.*cache/ # Distribution / packaging /build/ @@ -12,27 +18,23 @@ __pycache__/ /*.egg-info/ # Tests and coverage -/.pytest_cache/ -/.cache/ /data/ +/node_modules/ +/.coverage* +/coverage.xml # docs -docs/_build -!docs/api/.md -docs/**/generated +/docs/generated/ +/docs/_build/ +docs/**/generated/ docs/_static/datasets_data.js # IDEs /.idea/ -.vscode # data *.zarr/ -# temp files -temp/ - - # symlinks (luca) for extending the refactoring to satellite projects napari-spatialdata spatialdata-io @@ -52,12 +54,3 @@ _version.py # benchmarking and profiling .asv/ profile.speedscope.json - -# other -node_modules/ - -.mypy_cache -.ruff_cache -uv.lock -pixi.lock - diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index 64f98e8ba..000000000 --- a/.mypy.ini +++ /dev/null @@ -1,27 +0,0 @@ -[mypy] -python_version = 3.12 - -ignore_errors = False -warn_redundant_casts = True -warn_unused_configs = True -warn_unused_ignores = False - -disallow_untyped_calls = False -disallow_untyped_defs = True -disallow_incomplete_defs = True -disallow_any_generics = True - -strict_optional = True -strict_equality = True -warn_return_any = True -warn_unreachable = False -check_untyped_defs = True -; because of docrep -allow_untyped_decorators = True -no_implicit_optional = True -no_implicit_reexport = True -no_warn_no_return = True - -show_error_codes = True -show_column_numbers = True -error_summary = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99e507546..5774a30cf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,27 +1,59 @@ fail_fast: false default_language_version: - python: python3 + python: python3 default_stages: - - pre-commit - - pre-push + - pre-commit + - pre-push minimum_pre_commit_version: 2.16.0 -ci: - skip: [] repos: - - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.6 - hooks: - - id: prettier - exclude: ^.github/workflows/test.yaml - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v2.3.1 - hooks: - - id: mypy - additional_dependencies: [numpy, types-requests] - exclude: tests/|docs/ - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.4 - hooks: - - id: ruff - args: [--fix, --exit-non-zero-on-fix] - - id: ruff-format + - repo: https://github.com/biomejs/pre-commit + rev: v2.5.10 + hooks: + - id: biome-format + exclude: ^\.cruft\.json$ # inconsistent indentation with cruft - file never to be modified manually. + groups: [format] + - repo: https://github.com/tox-dev/pyproject-fmt + rev: v2.28.1 + hooks: + - id: pyproject-fmt + groups: [format] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.4 + hooks: + - id: ruff-check + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + groups: [format] + - repo: local + hooks: + - id: mypy + name: mypy + entry: hatch check types + language: system + types: [python] + files: ^(src|tests)/ + require_serial: true + pass_filenames: false + groups: [typecheck] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: detect-private-key + - id: check-ast + - id: end-of-file-fixer + groups: [format] + - id: mixed-line-ending + args: [--fix=lf] + groups: [format] + - id: trailing-whitespace + groups: [format] + - id: check-case-conflict + # Check that there are no merge conflicts (could be generated by template sync) + - id: check-merge-conflict + args: [--assume-in-merge] + + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.29.0 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d49a2c156..2f14a7802 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,23 +1,23 @@ -# https://docs.readthedocs.io/en/stable/config-file/v2.html +# https://docs.readthedocs.io/page/config-file/v2.html version: 2 build: - os: ubuntu-24.04 - tools: - python: "3.13" - jobs: - post_checkout: - # unshallow so version can be derived from tag - - git fetch --unshallow || true - create_environment: - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - build: - html: - - uv sync --group=docs --extra=torch - - uv run make --directory=docs html - - mv docs/_build $READTHEDOCS_OUTPUT + os: ubuntu-24.04 + tools: + python: "3.14" + nodejs: latest + jobs: + post_checkout: + # unshallow so the version can be derived from the git tag + - git fetch --unshallow || true + create_environment: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - uvx hatch run docs:build + - mv docs/_build $READTHEDOCS_OUTPUT submodules: - include: - - "docs/tutorials/notebooks" - recursive: true + include: + - "docs/tutorials/notebooks" + recursive: true diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..caaeb4f73 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,18 @@ +{ + "recommendations": [ + // GitHub integration + "github.vscode-github-actions", + "github.vscode-pull-request-github", + // Language support + "ms-python.python", + "ms-python.vscode-pylance", + "ms-toolsai.jupyter", + "tamasfe.even-better-toml", + // Dependency management + "ninoseki.vscode-mogami", + // Linting and formatting + "editorconfig.editorconfig", + "charliermarsh.ruff", + "biomejs.biome", + ], +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..36d187461 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,33 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Build Documentation", + "type": "debugpy", + "request": "launch", + "module": "sphinx", + "args": ["-M", "html", ".", "_build"], + "cwd": "${workspaceFolder}/docs", + "console": "internalConsole", + "justMyCode": false, + }, + { + "name": "Python: Debug Test", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "purpose": ["debug-test"], + "console": "internalConsole", + "justMyCode": false, + "env": { + "PYTEST_ADDOPTS": "--color=yes", + }, + "presentation": { + "hidden": true, + }, + }, + ], +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..e034b91f7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,18 @@ +{ + "[python][json][jsonc]": { + "editor.formatOnSave": true, + }, + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.codeActionsOnSave": { + "source.fixAll": "always", + "source.organizeImports": "always", + }, + }, + "[json][jsonc]": { + "editor.defaultFormatter": "biomejs.biome", + }, + "python.analysis.typeCheckingMode": "basic", + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": ["-vv", "--color=yes"], +} diff --git a/README.md b/README.md index 50bd90069..616715a46 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ # SpatialData: an open and universal framework for processing spatial omics data. [![Tests][badge-tests]][link-tests] -[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/scverse/spatialdata/main.svg)](https://results.pre-commit.ci/latest/github/scverse/spatialdata/main) [![codecov](https://codecov.io/gh/scverse/spatialdata/branch/main/graph/badge.svg?token=X19DRSIMCU)](https://codecov.io/gh/scverse/spatialdata) [![documentation badge](https://readthedocs.org/projects/scverse-spatialdata/badge/?version=latest)](https://spatialdata.scverse.org/en/latest/) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.20056407.svg)](https://doi.org/10.5281/zenodo.20056407) @@ -71,10 +70,6 @@ Update Feb 2025: `spatialdata` cannot be currently be installed via `conda` beca mamba install -c conda-forge spatialdata napari-spatialdata spatialdata-io spatialdata-plot ``` -## Limitations - -- Code only manually tested for Windows machines. Currently the framework is being developed using Linux, macOS and Windows machines, but it is automatically tested only for Linux and macOS machines. - ## Contact To get involved in the discussion, or if you need help to get started, you are welcome to use the following options. diff --git a/benchmarks/README.md b/benchmarks/README.md index 22b98eec7..5af8369c5 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -9,7 +9,7 @@ Note that to run code, your current working directory should be the SpatialData The benchmarks use the [airspeed velocity](https://asv.readthedocs.io/en/stable/) (asv) framework. Install it with the `benchmark` option: ``` -pip install -e . --group dev --group test --group docs --group benchmark +pip install -e . --group dev --group test --group doc --group benchmark ``` ## Usage diff --git a/biome.jsonc b/biome.jsonc new file mode 100644 index 000000000..731044ba3 --- /dev/null +++ b/biome.jsonc @@ -0,0 +1,22 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.2.0/schema.json", + "vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true }, + "formatter": { "useEditorconfig": true }, + "overrides": [ + { + "includes": ["./.vscode/*.json", "**/*.jsonc"], + "json": { + "formatter": { "trailingCommas": "all" }, + "parser": { + "allowComments": true, + "allowTrailingCommas": true, + }, + }, + }, + { + // asv reads its config with a JSON parser that understands `//` comments but not trailing commas. + "includes": ["./asv.conf.json"], + "json": { "parser": { "allowComments": true } }, + }, + ], +} diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d03e94a01..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= python3 -msphinx -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -clean: - rm -r "$(BUILDDIR)" - rm -r "generated" diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 754b98ad7..ccb03c738 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -10,3 +10,8 @@ margin-bottom: 1.5rem; height: 280px; } + +/* Reduce the font size in data frames - See https://github.com/scverse/cookiecutter-scverse/issues/193 */ +div.cell_output table.dataframe { + font-size: 0.8em; +} diff --git a/CHANGELOG.md b/docs/_templates/.gitkeep similarity index 100% rename from CHANGELOG.md rename to docs/_templates/.gitkeep diff --git a/docs/conf.py b/docs/conf.py index 593dc88cd..ae6e531f6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,17 +1,11 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -from __future__ import annotations - -# -- Path setup -------------------------------------------------------------- +import shutil import sys from datetime import datetime from importlib.metadata import metadata from pathlib import Path, PurePosixPath +from sphinxcontrib import katex + HERE = Path(__file__).parent sys.path.insert(0, str(HERE / "extensions")) sys.path.insert(0, str(HERE / "tutorials" / "notebooks" / "extensions")) @@ -20,33 +14,30 @@ # -- Project information ----------------------------------------------------- info = metadata("spatialdata") -project_name = info["Name"] +project = info["Name"] author = info["Author"] copyright = f"{datetime.now():%Y}, {author}" version = info["Version"] -# repository_url = f"https://github.com/scverse/{project_name}" +urls = dict(pu.split(", ") for pu in info.get_all("Project-URL")) +repository_url = urls["Source"] -# The full version, including alpha/beta/rc tags release = info["Version"] bibtex_bibfiles = ["references.bib"] bibtex_reference_style = "author_year" templates_path = ["_templates"] -nitpicky = True # Warn about broken links needs_sphinx = "4.0" html_context = { "display_github": True, # Integrate GitHub - "github_user": "scverse", # Username - "github_repo": project_name, # Repo name - "github_version": "main", # Version - "conf_py_path": "/docs/", # Path in the checkout to the docs root + "github_user": "scverse", + "github_repo": project, + "github_version": "main", + "conf_py_path": "/docs/", } # -- General configuration --------------------------------------------------- -# Add any Sphinx extension module names here, as strings. -# They can be extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ "git_ref", # needs to be before scanpydoc.rtd_github_links "scanpydoc.rtd_github_links", # needs to be before sphinx.ext.linkcode @@ -55,13 +46,15 @@ "sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.autosummary", + "sphinx.ext.linkcode", "sphinx.ext.napoleon", "sphinxcontrib.bibtex", + "sphinxcontrib.katex", "sphinx_autodoc_typehints", - "sphinx.ext.mathjax", - "sphinx.ext.linkcode", - "IPython.sphinxext.ipython_console_highlighting", "sphinx_design", + "IPython.sphinxext.ipython_console_highlighting", + "sphinxext.opengraph", + "scverse_misc.sphinx_ext", *[p.stem for p in (HERE / "extensions").glob("*.py")], *[p.stem for p in (HERE / "tutorials" / "notebooks" / "extensions").glob("*.py")], ] @@ -83,7 +76,7 @@ napoleon_include_init_with_doc = False napoleon_use_rtype = True # having a separate entry generally helps readability napoleon_use_param = True -myst_heading_anchors = 3 # create anchors for h1-h3 +myst_heading_anchors = 6 # create anchors for h1-h6 myst_enable_extensions = [ "amsmath", "colon_fence", @@ -97,6 +90,7 @@ nb_execution_mode = "off" nb_merge_streams = True typehints_defaults = "braces" +always_use_bars_union = True # use `|` instead of `Union` in types even when building with Python ≤3.14 source_suffix = { ".rst": "restructuredtext", @@ -105,29 +99,27 @@ } intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), "anndata": ("https://anndata.readthedocs.io/en/stable/", None), - "numpy": ("https://numpy.org/doc/stable/", None), - "geopandas": ("https://geopandas.org/en/stable/", None), - "xarray": ("https://docs.xarray.dev/en/stable/", None), - "datatree": ("https://datatree.readthedocs.io/en/latest/", None), + "annsel": ("https://annsel.readthedocs.io/en/latest/", None), "dask": ("https://docs.dask.org/en/latest/", None), + "datatree": ("https://datatree.readthedocs.io/en/latest/", None), + "geopandas": ("https://geopandas.org/en/stable/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "scanpy": ("https://scanpy.readthedocs.io/en/stable/", None), "shapely": ("https://shapely.readthedocs.io/en/stable", None), - "annsel": ("https://annsel.readthedocs.io/en/latest/", None), + "xarray": ("https://docs.xarray.dev/en/stable/", None), } - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. exclude_patterns = [ "_build", "Thumbs.db", + ".DS_Store", "**.ipynb_checkpoints", "tutorials/notebooks/index.md", "tutorials/notebooks/README.md", "tutorials/notebooks/references.md", "tutorials/notebooks/notebooks/paper_reproducibility/*", - "tutorials/notebooks/notebooks/paper_reproducibility/*", "tutorials/notebooks/notebooks/developers_resources/storage_format/*.ipynb", "tutorials/notebooks/notebooks/developers_resources/storage_format/Readme.md", "tutorials/notebooks/notebooks/examples/technology_stereoseq.ipynb", @@ -135,12 +127,8 @@ "tutorials/notebooks/notebooks/examples/technology_cosmx.ipynb", "tutorials/notebooks/notebooks/examples/stereoseq_data/*", ] -# Ignore warnings. -nitpicky = False # TODO: solve upstream. -# nitpick_ignore = [ -# ("py:class", "spatial_image.SpatialImage"), -# ("py:class", "multiscale_spatial_image.multiscale_spatial_image.MultiscaleSpatialImage"), -# ] + +nitpicky = False # TODO: solve upstream, then set back to True to warn about broken links. # no solution yet (7.4.7); using the workaround shown here: https://github.com/sphinx-doc/sphinx/issues/12589 suppress_warnings = [ "autosummary.import_cycle", @@ -149,42 +137,25 @@ # -- Options for HTML output ------------------------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# html_theme = "sphinx_book_theme" -# html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] -html_title = project_name +html_css_files = ["css/custom.css"] + +html_title = project html_logo = "_static/img/spatialdata_horizontal.png" html_theme_options = { - "navigation_with_keys": True, + "repository_url": repository_url, + "use_repository_button": True, + "path_to_docs": "docs/", + "navigation_with_keys": False, "show_toc_level": 4, - # "repository_url": repository_url, - # "use_repository_button": True, } pygments_style = "default" +katex_prerender = shutil.which(katex.NODEJS_BINARY) is not None nitpick_ignore = [ - # If building the documentation fails because of a missing link that is outside your control, - # you can add an exception to this list. + # Add an entry here when a missing link is outside our control. ("py:class", "igraph.Graph"), ] - - -def setup(app): - """App setup hook.""" - app.add_config_value( - "recommonmark_config", - { - "auto_toc_tree_section": "Contents", - "enable_auto_toc_tree": True, - "enable_math": True, - "enable_inline_math": False, - "enable_eval_rst": True, - }, - True, - ) - app.add_css_file("css/custom.css") diff --git a/docs/contributing.md b/docs/contributing.md index 6026d4e1f..955333fed 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,266 +1,390 @@ # Contributing guide -Scanpy provides extensive [developer documentation][scanpy developer guide], most of which applies to this repo, too. -This document will not reproduce the entire content from there. Instead, it aims at summarizing the most important -information to get you started on contributing. +This document aims at summarizing the most important information for getting you started on contributing to this project. +We assume that you are already familiar with git and with making pull requests on GitHub. -We assume that you are already familiar with git and with making pull requests on GitHub. If not, please refer -to the [scanpy developer guide][]. +For more extensive tutorials, that also cover the absolute basics, please refer to other resources such as the [pyopensci tutorials][], the [scientific Python tutorials][], or the [scanpy developer guide][]. + +[pyopensci tutorials]: https://www.pyopensci.org/learn.html +[scientific Python tutorials]: https://learn.scientific-python.org/development/tutorials/ +[scanpy developer guide]: https://scanpy.scverse.org/page/dev/ + +:::{tip} The *hatch* project manager + +We highly recommend to familiarize yourself with [`hatch`][hatch]. +Hatch is a Python project manager that + +- manages virtual environments, separately for development, testing and building the documentation. + Separating the environments is useful to avoid dependency conflicts. +- allows to run tests locally in different environments (e.g. different python versions) +- allows to run tasks defined in `pyproject.toml`, e.g. to build documentation. + +While the project is setup with `hatch` in mind, it is still possible to use different tools to manage dependencies, such as `uv` or `pip`. + +::: + +[hatch]: https://hatch.pypa.io/latest/ ## Installing dev dependencies -In addition to the packages needed to _use_ this package, you need additional python packages to _run tests_ and _build -the documentation_. It's easy to install them using `pip`: +In addition to the packages needed to _use_ this package, you need additional python packages to [run tests](#writing-tests) and [build the documentation](#docs-building). + +:::::{tab-set} +::::{tab-item} Hatch +:sync: hatch + +On the command line, you typically interact with hatch through its command line interface (CLI). +Running one of the following commands will automatically resolve the environments for testing and building the documentation in the background: ```bash -pip install -e . --group dev --group test --group docs +hatch test # defined in the table [tool.hatch.envs.hatch-test] in pyproject.toml +hatch run docs:build # defined in the table [tool.hatch.envs.docs] ``` -## Code-style +### VS Code + +If you are using VS code, install the [hatch-code][] extension. +Additionally, make sure that the `vscode-python-environments` extension is installed (should be by default) and `"python.useEnvironmentsExtension": true` is activated in your `settings.json`. + +Next, open the "Python Environment Managers" sidebar. +You can do so by opening the command palette (Ctrl+Shift+P) and searching for `Python: Focus on Environment Managers View`. +It will show a collapsible list where you can expand "Hatch" and activate an environment by clicking on the checkmark next to it. +As the main development environment, we recommend to use `hatch-test` with the latest supported Python version. -This template uses [pre-commit][] to enforce consistent code-styles. On every commit, pre-commit checks will either -automatically fix issues with the code, or raise an error message. +### Other IDEs -To enable pre-commit locally, simply run +For other IDEs, you’ll have to point the editor at the paths to the virtual environments manually. +To get a list of all environments for your projects, run ```bash -pre-commit install +hatch env show -i ``` -in the root of the repository. Pre-commit will automatically download all dependencies when it is run for the first time. +This will list “Standalone” environments and a table of “Matrix” environments like the following: -Alternatively, you can rely on the [pre-commit.ci][] service enabled on GitHub. If you didn't run `pre-commit` before -pushing changes to GitHub it will automatically commit fixes to your pull request, or show an error message. +``` ++------------+---------+--------------------------+----------+---------------------------------+-------------+ +| Name | Type | Envs | Features | Dependencies | Scripts | ++------------+---------+--------------------------+----------+---------------------------------+-------------+ +| hatch-test | virtual | hatch-test.py3.12-stable | dev | coverage-enable-subprocess==1.0 | cov-combine | +| | | hatch-test.py3.14-stable | test | coverage[toml]~=7.4 | cov-report | +| | | hatch-test.py3.14-pre | | pytest-mock~=3.12 | run | +| | | | | pytest-randomly~=3.15 | run-cov | +| | | | | pytest-rerunfailures~=14.0 | | +| | | | | pytest-xdist[psutil]~=3.5 | | +| | | | | pytest~=8.1 | | ++------------+---------+--------------------------+----------+---------------------------------+-------------+ +``` + +From the `Envs` column, select the environment name you want to use for development. +As the main development environment, we recommend to use `hatch-test` with the latest supported Python version. +In this example, it would be `hatch-test.py3.14-stable`. -If pre-commit.ci added a commit on a branch you still have been working on locally, simply use +Next, create the environment with ```bash -git pull --rebase +hatch env create hatch-test.py3.14-stable ``` -to integrate the changes into yours. -While the [pre-commit.ci][] is useful, we strongly encourage installing and running pre-commit locally first to understand its usage. +Then, obtain the path to the environment using -Finally, most editors have an _autoformat on save_ feature. Consider enabling this option for [black][black-editors] -and [prettier][prettier-editors]. +```bash +hatch env find hatch-test.py3.14-stable +``` -[black-editors]: https://black.readthedocs.io/en/stable/integrations/editors.html -[prettier-editors]: https://prettier.io/docs/en/editors.html +and manually point it to the python binary. -## Writing tests -```{note} -Remember to first install the package with `pip install -e . --group dev --group test` +:::: + +::::{tab-item} uv +:sync: uv + +A popular choice for managing virtual environments is [uv][]. +The main disadvantage compared to hatch is that it supports only a single environment per project at a time, which requires you to mix the dependencies for running tests and building docs. +This can have undesired side-effects, such as requiring to install a lower version of a library your project depends on, only because an outdated sphinx plugin pins an older version. + +To initialize a virtual environment in the `.venv` directory of your project, simply run + +```bash +uv sync --group=test --group=doc --extra=torch ``` -This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package. +The `.venv` directory is typically automatically discovered by IDEs such as VS Code. + +:::: -Most IDEs integrate with pytest and provide a GUI to run tests. Alternatively, you can run all tests from the command line by executing +::::{tab-item} Pip +:sync: pip + +Pip is nowadays mostly superseded by environment manager such as [hatch][]. +However, for the sake of completeness, and since it’s ubiquitously available, we describe how you can manage environments manually using `pip`: ```bash -pytest +python3 -m venv .venv +source .venv/bin/activate +pip install -e . --group dev --group test --group doc ``` -in the root of the repository. Continuous integration will automatically run the tests on all pull requests. +The `.venv` directory is typically automatically discovered by IDEs such as VS Code. -### Continuous integration +:::: +::::: -Continuous integration will automatically run the tests on all pull requests and test against the minimum and maximum supported Python version. +[hatch environments]: https://hatch.pypa.io/latest/tutorials/environment/basic-usage/ +[hatch-code]: https://marketplace.visualstudio.com/items?itemName=PyPA.hatch +[uv]: https://docs.astral.sh/uv/ -Additionally, there's a CI job that tests against pre-releases of all dependencies (if there are any). The purpose of this check is to detect incompatibilities of new package versions early on and gives you time to fix the issue or reach out to the developers of the dependency before the package is released to a wider audience. +## Code-style -[scanpy-test-docs]: https://scanpy.readthedocs.io/en/latest/dev/testing.html#writing-tests +This package uses [pre-commit][]-style hooks to enforce consistent code-styles. +We recommend running them with [prek][], a fast, drop-in replacement for `pre-commit` that reads the same `.pre-commit-config.yaml`. +On every commit, the checks will either automatically fix issues with the code, or raise an error message. -By including this additional information, the document now provides a more comprehensive overview of the continuous integration process related to testing. +To enable the checks locally, run -### Integration testing +```bash +hatch run hatch-check-code:prek install +# or +uvx prek install +``` -Cross-repo integration testing is available in the [spatialdata-integration-testing](https://github.com/scverse/spatialdata-integration-testing/) repo. Please follow the instructions in the Readme (which also includes a video overview). +in the root of the repository. +prek will automatically download all dependencies when it is run for the first time. -## Publishing a release +If you didn’t run the checks locally, the `Pre-commit checks` job of the GitHub Actions CI runs them on your pull request and reports any failures. +We strongly encourage installing and running the checks locally first to understand their usage. -### Updating the version number +Finally, most editors have an _autoformat on save_ feature. +Consider enabling this option for [ruff][ruff-editors] and [biome][biome-editors]. -Before making a release, you need to update the version number. Please adhere to [Semantic Versioning][semver], in brief +[pre-commit]: https://pre-commit.com/ +[prek]: https://prek.j178.dev/ +[ruff-editors]: https://docs.astral.sh/ruff/integrations/ +[biome-editors]: https://biomejs.dev/guides/integrate-in-editor/ -> Given a version number MAJOR.MINOR.PATCH, increment the: -> -> 1. MAJOR version when you make incompatible API changes, -> 2. MINOR version when you add functionality in a backwards compatible manner, and -> 3. PATCH version when you make backwards compatible bug fixes. -> -> Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. -> For pre-release please use the aX suffix, such as v0.7.0a0, v0.7.0a1. Do not use the devX suffix since it doesn't support multiple incremental versions. +(writing-tests)= + +## Writing tests -You can find the [labels for pre-release in this page](https://packaging.python.org/en/latest/discussions/versioning/#valid-version-numbers). +This package uses [pytest][] for automated testing. +Please write {doc}`scanpy:dev/testing` for every function added to the package. -You can either use [bump2version][] to automatically create a git tag with the updated version number, or manually create the tag yourself (locally or from the GitHub interface when making a release). -If you use `bump2version`, you can run one of the following commands in the root of the repository +Most IDEs integrate with pytest and provide a GUI to run tests. +If you set up your virtual environments as described in [installing dev dependencies](#installing-dev-dependencies), test cases should be automatically discovered by your IDE. + +Alternatively, you can run all tests from the command line by executing + +:::::{tab-set} +::::{tab-item} Hatch +:sync: hatch ```bash -bump2version patch -bump2version minor -bump2version major +hatch test # test with the highest supported Python version +# or +hatch test --all # test with all supported Python versions ``` -Once you are done, run +:::: + +::::{tab-item} uv +:sync: uv +```bash +uv run pytest ``` -git push --tags + +:::: + +::::{tab-item} Pip +:sync: pip + +```bash +source .venv/bin/activate +pytest ``` -to publish the created tag on GitHub. +:::: +::::: -It's important that the tag for a pre-release follows this naming convention as it will determine if the package is displayed as [pre-release or release](https://pypi.org/project/spatialdata/#history) in PyPI. +in the root of the repository. -[bump2version]: https://github.com/c4urself/bump2version +[pytest]: https://docs.pytest.org/ + +### Continuous integration + +Continuous integration via GitHub actions will automatically run the tests on all pull requests and test against the minimum and maximum supported Python version. + +Additionally, there’s a CI job that tests against pre-releases of all dependencies (if there are any). +The purpose of this check is to detect incompatibilities of new package versions early on and gives you time to fix the issue or reach out to the developers of the dependency before the package is released to a wider audience. + +The CI job is defined in `.github/workflows/test.yaml`, however the single point of truth for CI jobs is the Hatch test matrix defined in `pyproject.toml`. +This means that local testing via hatch and remote testing on CI tests against the same python versions and uses the same environments. + +### Integration testing + +Cross-repo integration testing is available in the [spatialdata-integration-testing][] repo. +Please follow the instructions in its readme, which also includes a video overview. + +[spatialdata-integration-testing]: https://github.com/scverse/spatialdata-integration-testing/ + +## Publishing a release + +### Choosing the version number + +`spatialdata` derives its version from the git tag through [hatch-vcs][], so cutting a release amounts to creating a tag. +Please adhere to [Semantic Versioning][semver], in brief + +> Given a version number MAJOR.MINOR.PATCH, increment the: +> +> 1. MAJOR version when you make incompatible API changes, +> 2. MINOR version when you add functionality in a backwards compatible manner, and +> 3. PATCH version when you make backwards compatible bug fixes. +> +> Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. + +For pre-releases please use the `aX` suffix, such as `v0.7.0a0` or `v0.7.0a1`. +Do not use the `devX` suffix, since it does not support multiple incremental versions. +The [valid version numbers page][pypa-versioning] lists the labels you can choose from. +The naming of the tag matters: it determines whether the package is displayed as [pre-release or release](https://pypi.org/project/spatialdata/#history) on PyPI. + +[hatch-vcs]: https://github.com/ofek/hatch-vcs +[pypa-versioning]: https://packaging.python.org/en/latest/discussions/versioning/#valid-version-numbers ### Making a release on GitHub and publishing to PyPI -#### Recommended: Create the release via GitHub +#### Recommended: create the release via GitHub -- Go to the [Releases page on GitHub](https://github.com/scverse/spatialdata/releases) and press the “Draft a new release” button. +- Go to the [releases page on GitHub][releases] and press the “Draft a new release” button. - Press “Choose a tag” and create a new tag. - Please name the tag with the same string you intend for the release, including the `v` prefix. -- Alternatively, go to the [Tags page on GitHub](https://github.com/scverse/spatialdata/tags), select the latest tag, and press “Create release from tag”. +- Alternatively, go to the [tags page on GitHub][tags], select the latest tag, and press “Create release from tag”. - Please name the release with the same string used for the tag (including the `v` prefix). - Both approaches lead to the same page and view. From there: - Specify whether the release is a pre-release and whether it should be set as the latest release (use the checkboxes accordingly). - Fill in the release notes (explained in the next section). - Press “Publish release” to make the release available on GitHub. -- A [GitHub Action](https://github.com/scverse/spatialdata/blob/main/.github/workflows/release.yaml) will automatically build the package and [upload it to PyPI](https://pypi.org/project/spatialdata/#history). - - The action may fail; check the [workflow status badge in the README](https://github.com/scverse/spatialdata/actions/workflows/release.yaml). +- The [release workflow][] will then build the package and [upload it to PyPI](https://pypi.org/project/spatialdata/#history) using [trusted publishing][]. + - The workflow may fail; check the [release workflow status badge in the README](https://github.com/scverse/spatialdata/actions/workflows/release.yaml). -#### Not recommended: Manual tag-first workflow - -- If you already tagged and pushed a commit as explained above and want to create a release from that tag, you can go to the [Tags page on GitHub](https://github.com/scverse/spatialdata/tags), select the latest tag, and press “Create release from tag”. - - Please name the release with the same string used for the tag (including the `v` prefix). +[releases]: https://github.com/scverse/spatialdata/releases +[tags]: https://github.com/scverse/spatialdata/tags +[release workflow]: https://github.com/scverse/spatialdata/blob/main/.github/workflows/release.yaml +[trusted publishing]: https://docs.pypi.org/trusted-publishers/ #### Writing release notes -We recommend using the button "Generate release notes" to automatically collect all the information of the pull requests that are part of the release. -The release notes serve as a changelog for the user of the package so it's important to have them curated and well-organized. This is explained in depth below. +We recommend using the “Generate release notes” button to automatically collect the information of all pull requests that are part of the release. +The release notes serve as a changelog for the users of the package, so it is important to have them curated and well-organized. -Here is an example of automatically generated release notes for a previous release (v0.2.3): +The automatically generated notes are grouped by change type through our [release configuration file](https://github.com/scverse/spatialdata/blob/main/.github/release.yml), which infers the type from GitHub labels and ignores pull requests opened by bots. +We recommend opening the pull requests included in the release and adding the appropriate [release labels](https://github.com/scverse/spatialdata/labels?q=release-) before generating the notes. -``` -## What's Changed -* Add clip parameter to polygon_query; tests missing by @LucaMarconato in https://github.com/scverse/spatialdata/pull/670 -* Add sort parameter to points model by @LucaMarconato in https://github.com/scverse/spatialdata/pull/672 -* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/scverse/spatialdata/pull/673 -* Docs for datasets (blobs, raccoon) by @LucaMarconato in https://github.com/scverse/spatialdata/pull/674 -* Update issue templates by @LucaMarconato in https://github.com/scverse/spatialdata/pull/675 -* Minor fixes: `id()` -> `is`, inplace category subset `AnnData` relational query by @LucaMarconato in https://github.com/scverse/spatialdata/pull/681 -* Added ColorLike to _types.py by @timtreis in https://github.com/scverse/spatialdata/pull/689 -* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/scverse/spatialdata/pull/685 -* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/scverse/spatialdata/pull/690 -* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/scverse/spatialdata/pull/698 -* Fix labels multiscales method by @aeisenbarth in https://github.com/scverse/spatialdata/pull/697 - - -**Full Changelog**: https://github.com/scverse/spatialdata/compare/v0.2.2...v0.2.3 -``` - -The release notes above can be hard to read, but this is addressed by our [configuration file](https://github.com/scverse/spatialdata/blob/main/.github/release.yml). It organizes release notes by change type, inferred from GitHub labels, and ignores PRs from bots. We recommend opening the PRs included in the release and adding the appropriate labels. The automatic generation will then group PRs by [release labels](https://github.com/scverse/spatialdata/labels?q=release-) and list each PR on a separate line. Here is an example output: - -``` - - -## What's Changed -### Major -* Adding `attrs` at the `SpatialData` object level by @quentinblampey in https://github.com/scverse/spatialdata/pull/711 -### Minor -* Add asv benchmark code by @berombau in https://github.com/scverse/spatialdata/pull/784 -* relabel block by @ArneDefauw in https://github.com/scverse/spatialdata/pull/664 -* validate tables while parsing by @melonora in https://github.com/scverse/spatialdata/pull/808 -### Fixed -* relaxed fsspec version by @LucaMarconato in https://github.com/scverse/spatialdata/pull/798 -* fix for to_polygons when using processes instead of threads in dask by @ArneDefauw in https://github.com/scverse/spatialdata/pull/756 -* Fix `transform_to_data_extent` converting labels to images by @aeisenbarth in https://github.com/scverse/spatialdata/pull/791 -* fix join non matching table by @melonora in https://github.com/scverse/spatialdata/pull/813 - - -**Full Changelog**: https://github.com/scverse/spatialdata/compare/v0.2.6...v0.2.7 -``` - -Use informative titles for PRs, as these will serve as section titles in the release notes (rename the PRs if necessary). You can also manually edit the release notes before publishing them to improve readability. +Use informative titles for pull requests, as these serve as the entries in the release notes; rename them if necessary. +You can also manually edit the release notes before publishing them to improve readability. -Some additional considerations +Some additional considerations: -- **Important!** If a PR is large and its title isn't informative or requires multiple lines, **do not** add a release tag. Instead, at the end of the first message of the PR discussion, please include a markdown section with title `# Release notes` with a brief description of the intended release notes. This will allow the person making a release to manually add the PR content to the release notes during the release process. -- Please avoid redundancy and do not add the same release notes to consecutive pre-releases/releases/post-releases. -- When automatically generating the release notes, you can use the button "Previous tag: ..." to choose which PRs will be included in the release notes. -- Finally, you can see an example of a release in action in from Luca [this short video tutorial](https://www.loom.com/share/7097455bc0b9449fbe72d53fc778cbf9). +- **Important!** If a pull request is large and its title is not informative or requires multiple lines, **do not** add a release label. + Instead, at the end of the first message of the pull request discussion, include a markdown section titled `# Release notes` with a brief description of the intended release notes. + This allows the person making the release to manually add the content to the release notes. +- Please avoid redundancy and do not repeat the same release notes across consecutive pre-releases, releases and post-releases. +- When generating the release notes, you can use the “Previous tag: …” button to choose which pull requests are included. +- You can see an example of a release in action in [this short video tutorial](https://www.loom.com/share/7097455bc0b9449fbe72d53fc778cbf9). ### Publishing to conda-forge -Shortly after you make a release in PyPI, a new PR will be automatically made in the conda-forge "feedstock repository" for the package (this has been previously setup). The PR will contain a checklist of which tasks should be done to be able to merge the PR. Once the PR is merged, the package will be available in the conda-forge channel. +Shortly after the release lands on PyPI, a pull request is automatically opened in the conda-forge feedstock repository for the package. +The pull request contains a checklist of the tasks that need to be done before it can be merged. +Once it is merged, the new version becomes available in the conda-forge channel. -Practically, the changes that usually needs to be done are comparing the package requirements in `pyproject.toml` from your repository, with the packages and versions in the `meta.yaml` file in the conda-forge feedstock repository. If there are any differences, you should update the `meta.yaml` file accordingly. After that, the CI will run and if green the PR can be merged. +In practice, the change that is usually needed is reconciling the requirements in `pyproject.toml` with the packages and versions in the feedstock’s `meta.yaml`. +After updating `meta.yaml`, the CI runs, and if it is green the pull request can be merged. ## Writing documentation -Please write documentation for new or changed features and use-cases. This project uses [sphinx][] with the following features: +Please write documentation for new or changed features and use-cases. +This project uses [sphinx][] with the following features: -- the [myst][] extension allows to write documentation in markdown/Markedly Structured Text +- The [myst][] extension allows to write documentation in markdown/Markedly Structured Text - [Numpy-style docstrings][numpydoc] (through the [napoloen][numpydoc-napoleon] extension). - Jupyter notebooks as tutorials through [myst-nb][] (See [Tutorials with myst-nb](#tutorials-with-myst-nb-and-jupyter-notebooks)) -- [Sphinx autodoc typehints][], to automatically reference annotated input and output types +- [sphinx-autodoc-typehints][], to automatically reference annotated input and output types +- Citations (like {cite:p}`Virshup_2023`) can be included with [sphinxcontrib-bibtex](https://sphinxcontrib-bibtex.readthedocs.io/) -See the [scanpy developer docs](https://scanpy.readthedocs.io/en/latest/dev/documentation.html) for more information -on how to write documentation. +See scanpy’s {doc}`scanpy:dev/documentation` for more information on how to write your own. + +[sphinx]: https://www.sphinx-doc.org/ +[myst]: https://myst-parser.readthedocs.io/page/intro.html +[myst-nb]: https://myst-nb.readthedocs.io/ +[numpydoc-napoleon]: https://www.sphinx-doc.org/page/usage/extensions/napoleon.html +[numpydoc]: https://numpydoc.readthedocs.io/page/format.html +[sphinx-autodoc-typehints]: https://github.com/tox-dev/sphinx-autodoc-typehints ### Tutorials with myst-nb and jupyter notebooks The documentation is set-up to render jupyter notebooks stored in the `docs/notebooks` directory using [myst-nb][]. Currently, only notebooks in `.ipynb` format are supported that will be included with both their input and output cells. -It is your reponsibility to update and re-run the notebook whenever necessary. +It is your responsibility to update and re-run the notebook whenever necessary. + +If you are interested in automatically running notebooks as part of the continuous integration, please check out [this feature request][issue-render-notebooks] in the `cookiecutter-scverse` repository. -If you are interested in automatically running notebooks as part of the continuous integration, please check -out [this feature request](https://github.com/scverse/cookiecutter-scverse/issues/40) in the `cookiecutter-scverse` -repository. +[issue-render-notebooks]: https://github.com/scverse/cookiecutter-scverse/issues/40 #### Hints -- If you refer to objects from other packages, please add an entry to `intersphinx_mapping` in `docs/conf.py`. Only - if you do so can sphinx automatically create a link to the external documentation. -- If building the documentation fails because of a missing link that is outside your control, you can add an entry to - the `nitpick_ignore` list in `docs/conf.py` +- If you refer to objects from other packages, please add an entry to `intersphinx_mapping` in `docs/conf.py`. + Only if you do so can sphinx automatically create a link to the external documentation. +- If building the documentation fails because of a missing link that is outside your control, you can add an entry to the `nitpick_ignore` list in `docs/conf.py` + +(docs-building)= -#### Building the docs locally +### Building the docs locally + +:::::{tab-set} +::::{tab-item} Hatch +:sync: hatch + +```bash +hatch run docs:build +hatch run docs:open +``` + +:::: + +::::{tab-item} uv +:sync: uv ```bash cd docs -make html -open _build/html/index.html +uv run sphinx-build -M html . _build -W +(xdg-)open _build/html/index.html ``` -### Debugging and profiling +:::: -There are various tools available to help you understand the existing code base and your new code contributions. For debugging code there are multiple resources available: [Scientific Python](https://lectures.scientific-python.org/advanced/debugging/index.html), [VSCode](https://code.visualstudio.com/docs/python/debugging) and [PyCharm](https://www.jetbrains.com/help/pycharm/debugging-your-first-python-application.html). +::::{tab-item} Pip +:sync: pip -To find out the time or memory performance of your code, profilers can help. Again, various resources from [Scientific Python](https://lectures.scientific-python.org/advanced/optimizing/index.html), [napari](https://napari.org/stable/developers/contributing/performance/index.html), [PyCharm](https://www.jetbrains.com/help/pycharm/profiler.html) and [Dask](https://distributed.dask.org/en/latest/diagnosing-performance.html) can be helpful. +```bash +source .venv/bin/activate +cd docs +sphinx-build -M html . _build -W +(xdg-)open _build/html/index.html +``` - +:::: +::::: -[scanpy developer guide]: https://scanpy.readthedocs.io/en/latest/dev/index.html -[github quickstart guide]: https://docs.github.com/en/get-started/quickstart/create-a-repo?tool=webui -[codecov]: https://about.codecov.io/sign-up/ -[codecov docs]: https://docs.codecov.com/docs -[codecov bot]: https://docs.codecov.com/docs/team-bot -[codecov app]: https://github.com/apps/codecov -[pre-commit.ci]: https://pre-commit.ci/ -[readthedocs.org]: https://readthedocs.org/ -[myst-nb]: https://myst-nb.readthedocs.io/en/latest/ -[jupytext]: https://jupytext.readthedocs.io/en/latest/ -[pre-commit]: https://pre-commit.com/ -[anndata]: https://github.com/scverse/anndata -[mudata]: https://github.com/scverse/mudata -[pytest]: https://docs.pytest.org/ -[semver]: https://semver.org/ -[sphinx]: https://www.sphinx-doc.org/en/master/ -[myst]: https://myst-parser.readthedocs.io/en/latest/intro.html -[numpydoc-napoleon]: https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html -[numpydoc]: https://numpydoc.readthedocs.io/en/latest/format.html -[sphinx autodoc typehints]: https://github.com/tox-dev/sphinx-autodoc-typehints -[pypi]: https://pypi.org/ +## Debugging and profiling + +Various tools are available to help you understand the existing code base and your own contributions. +For debugging, see the resources from [Scientific Python](https://lectures.scientific-python.org/advanced/debugging/index.html), [VS Code](https://code.visualstudio.com/docs/python/debugging) and [PyCharm](https://www.jetbrains.com/help/pycharm/debugging-your-first-python-application.html). + +To find out the time or memory performance of your code, profilers can help. +Again, there are resources from [Scientific Python](https://lectures.scientific-python.org/advanced/optimizing/index.html), [napari](https://napari.org/stable/developers/contributing/performance/index.html), [PyCharm](https://www.jetbrains.com/help/pycharm/profiler.html) and [Dask](https://distributed.dask.org/en/latest/diagnosing-performance.html). +The `benchmark` dependency group and the `profiling` pixi environment declared in `pyproject.toml` provide [asv][], [memray][] and [py-spy][]; see `benchmarks/README.md` for how to run the benchmark suite. + +[asv]: https://asv.readthedocs.io/ +[memray]: https://bloomberg.github.io/memray/ +[py-spy]: https://github.com/benfred/py-spy diff --git a/docs/references.bib b/docs/references.bib index 20c3d84fe..69bd48cd9 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -38,3 +38,14 @@ @article{marconatoSpatialDataOpenUniversal2024 doi = {10.1038/s41592-024-02212-x}, abstract = {Spatially resolved omics technologies are transforming our understanding of biological tissues. However, the handling of uni- and multimodal spatial omics datasets remains a challenge owing to large data volumes, heterogeneity of data types and the lack of flexible, spatially aware data structures. Here we introduce SpatialData, a framework that establishes a unified and extensible multiplatform file-format, lazy representation of larger-than-memory data, transformations and alignment to common coordinate systems. SpatialData facilitates spatial annotations and cross-modal aggregation and analysis, the utility of which is illustrated in the context of multiple vignettes, including integrative analysis on a multimodal Xenium and Visium breast cancer study.} } + +@article{Virshup_2023, + doi = {10.1038/s41587-023-01733-8}, + url = {https://doi.org/10.1038%2Fs41587-023-01733-8}, + year = 2023, + month = {apr}, + publisher = {Springer Science and Business Media {LLC}}, + author = {Isaac Virshup and Danila Bredikhin and Lukas Heumos and Giovanni Palla and Gregor Sturm and Adam Gayoso and Ilia Kats and Mikaela Koutrouli and Philipp Angerer and Volker Bergen and Pierre Boyeau and Maren Büttner and Gokcen Eraslan and David Fischer and Max Frank and Justin Hong and Michal Klein and Marius Lange and Romain Lopez and Mohammad Lotfollahi and Malte D. Luecken and Fidel Ramirez and Jeffrey Regier and Sergei Rybakov and Anna C. Schaar and Valeh Valiollah Pour Amiri and Philipp Weiler and Galen Xing and Bonnie Berger and Dana Pe'er and Aviv Regev and Sarah A. Teichmann and Francesca Finotello and F. Alexander Wolf and Nir Yosef and Oliver Stegle and Fabian J. Theis and}, + title = {The scverse project provides a computational ecosystem for single-cell omics data analysis}, + journal = {Nature Biotechnology} +} diff --git a/docs/user_guide.md b/docs/user_guide.md index 3f0323513..28a19d063 100755 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -15,7 +15,7 @@ If you want to read more about the framework, please have a look at our publicat pip install spatialdata ``` -This command installs barebone SpatialData. +This command installs barebone SpatialData. For a more detailed description on the installation process including all bells and whistles, see [here](https://spatialdata.scverse.org/en/stable/installation.html). @@ -119,7 +119,7 @@ For more details, including information on how to add annotations for these regi

I have cells in my dataset, how do I annotate them? (usage of AnnData)

 -One of the most obvious things to do for spatial omics data is to annotate cells using the [AnnData](https://anndata.readthedocs.io/en/stable/) format (called tables in SpatialData). These tables can contain count/intensity data, all types of annotations, and make it possible to make use of [scanpy](https://scanpy.readthedocs.io/en/stable/) functionality (normalization/clustering/DE calculation). +One of the most obvious things to do for spatial omics data is to annotate cells using the [AnnData](https://anndata.readthedocs.io/en/stable/) format (called tables in SpatialData). These tables can contain count/intensity data, all types of annotations, and make it possible to make use of [scanpy](https://scanpy.readthedocs.io/en/stable/) functionality (normalization/clustering/DE calculation). If you want more technical details on how to create a table from scratch to annotate your shapes/labels/points, you can have a look [here](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/notebooks/examples/models2.html#tables). diff --git a/pyproject.toml b/pyproject.toml index 0a1471a15..c86545959 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,284 +1,386 @@ [build-system] build-backend = "hatchling.build" -requires = ["hatchling", "hatch-vcs"] - +requires = [ "hatch-vcs", "hatchling" ] [project] name = "spatialdata" description = "Spatial data format." -authors = [ - {name = "scverse"}, -] +readme = "README.md" +license = { file = "LICENSE" } maintainers = [ - {name = "scverse", email = "giov.pll@gmail.com"}, + { name = "scverse", email = "giov.pll@gmail.com" }, +] +authors = [ + { name = "scverse" }, ] -urls.Documentation = "https://spatialdata.scverse.org/en/latest" -urls.Source = "https://github.com/scverse/spatialdata.git" -urls.Home-page = "https://github.com/scverse/spatialdata.git" requires-python = ">=3.12" -dynamic= [ - "version" # allow version to be set by git tags +classifiers = [ + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Typing :: Typed", ] -license = {file = "LICENSE"} -readme = "README.md" +dynamic = [ "version" ] dependencies = [ - "anndata>=0.9.1", - "annsel>=0.1.2", - "click", - "dask-image", - "dask>=2026.3.0", - "distributed>=2026.3.0", - "datashader", - "fsspec[s3,http]", - "geopandas>=0.14", - "multiscale_spatial_image==2.0.3", - "networkx", - "numba>=0.55.0", - "numpy", - "ome_zarr>=0.16.0", - "pandas", - "pooch", - "pyarrow", - "rich", - "setuptools", - "shapely>=2.0.1", - "spatial_image>=1.2.3", - "scikit-image", - "scipy!=1.17.0", - "scverse-misc[datasets]>=0.1.0", - "typing_extensions>=4.8.0", - "universal_pathlib>=0.2.6", - "xarray>=2024.10.0", - "xarray-spatial>=0.3.5", - "zarr>=3.0.0", + "anndata>=0.9.1", + "annsel>=0.1.2", + "click", + "dask>=2026.3", + "dask-image", + "datashader", + "distributed>=2026.3", + "fsspec[http,s3]", + "geopandas>=0.14", + "multiscale-spatial-image==2.0.3", + "networkx", + "numba>=0.55", + "numpy", + "ome-zarr>=0.16", + "pandas", + "pooch", + "pyarrow", + "rich", + "scikit-image", + "scipy!=1.17", + "scverse-misc[datasets]>=0.1", + # for debug logging (referenced from the issue template) + "session-info2", + "setuptools", + "shapely>=2.0.1", + "spatial-image>=1.2.3", + "typing-extensions>=4.8", + "universal-pathlib>=0.2.6", + "xarray>=2024.10", + "xarray-spatial>=0.3.5", + "zarr>=3", ] -[project.optional-dependencies] -torch = [ - "torch" +optional-dependencies.extra = [ + "napari-spatialdata[all]", + "spatialdata-io", + "spatialdata-plot", ] -extra = [ - "napari-spatialdata[all]", - "spatialdata-plot", - "spatialdata-io", +optional-dependencies.torch = [ + "torch", ] +urls.Documentation = "https://spatialdata.scverse.org/en/latest" +urls.Homepage = "https://github.com/scverse/spatialdata" +urls.Source = "https://github.com/scverse/spatialdata" [dependency-groups] dev = [ - "bump2version", + "twine>=4.0.2", ] test = [ - "pytest", - "pytest-cov", - "pytest-mock", - "pytest-xdist", - "torch", -] -docs = [ - "sphinx>=4.5", - "sphinx-autobuild", - "sphinx-book-theme>=1.0.0", - "myst-nb", - "sphinxcontrib-bibtex>=1.0.0", - "sphinx-autodoc-typehints", - "sphinx-design", - "scanpydoc", - # For notebooks - "ipython>=8.6.0", - "sphinx-copybutton", - "sphinx-pytest", + "coverage>=7.10", + "pytest", + "pytest-cov", # For VS Code’s coverage functionality + "pytest-mock", + "pytest-xdist", + "torch", ] benchmark = [ - "asv", - "memray", - "profimp", + "asv", + "memray", + "profimp", ] - -[tool.coverage.run] -source = ["spatialdata"] -omit = [ - "**/test_*.py", +doc = [ + "ipykernel", + "ipython>=8.6", + "myst-nb>=1.1", + "pandas", + "scanpydoc", + "scverse-misc[sphinx]>=0.1.2", + "sphinx>=8.1", + "sphinx-autobuild", + "sphinx-autodoc-typehints", + "sphinx-book-theme>=1", + "sphinx-copybutton", + "sphinx-design", + "sphinx-pytest", + "sphinxcontrib-bibtex>=1", + "sphinxcontrib-katex", + "sphinxext-opengraph", +] +typecheck = [ + "mypy", + "pandas-stubs", + "scipy-stubs", + "types-geopandas", + "types-networkx", + "types-requests", + "types-shapely", ] -[tool.pytest] -testpaths = ["tests"] -strict = true -addopts = [ - "--import-mode=importlib", # allow using test files with same name - "-s", # print output from tests - "-p no:napari", # napari registers a pytest plugin via its entry point; disable it here since spatialdata tests don't need it +[tool.hatch] +version.source = "vcs" +metadata.allow-direct-references = true +build.hooks.vcs.version-file = "_version.py" +build.targets.wheel.packages = [ "src/spatialdata" ] +envs.default.installer = "uv" +envs.default.dependency-groups = [ "dev" ] +envs.docs.features = [ "torch" ] +envs.docs.scripts.build = "sphinx-build -M html docs docs/_build -W {args}" +envs.docs.scripts.clean = "git clean -fdX -- {args:docs}" +envs.docs.scripts.open = "python -m webbrowser -t docs/_build/html/index.html" +envs.docs.dependency-groups = [ "doc" ] +envs.hatch-test.matrix = [ + # Test the lowest and highest supported Python versions with normal deps + { deps = [ "stable" ], python = [ "3.12", "3.14" ] }, + # Test the lowest supported versions of the dependencies we pin a floor for + { deps = [ "min" ], python = [ "3.12" ] }, + # Test the newest supported Python version also with pre-release deps + { deps = [ "pre" ], python = [ "3.14" ] }, ] -# These are all markers coming from xarray, dask or anndata. Added here to silence warnings. -markers = [ - "slow: marks tests as slow (deselect with '-m \"not slow\"')", - "network: marks tests that require network access; skipped by default, run with '--run-network'", - "gpu: run test on GPU using CuPY.", - "array_api: used by anndata.tests.helpers, not us", - "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on", +# If the matrix variable `deps` is set to "pre", set the environment variable `UV_PRERELEASE` to "allow". +envs.hatch-test.overrides.matrix.deps.env-vars = [ + { value = "allow", key = "UV_PRERELEASE", if = [ "pre" ] }, ] -# info on how to use this https://stackoverflow.com/questions/57925071/how-do-i-avoid-getting-deprecationwarning-from-inside-dependencies-with-pytest -filterwarnings = [ - # "error", # if 3rd party libs raise DeprecationWarnings, TODO: filter them individually below - # "ignore:.*U.*mode is deprecated:DeprecationWarning", +# If the matrix variable `deps` is set to "min", pin the dependencies to their declared floor. +envs.hatch-test.overrides.matrix.deps.extra-dependencies = [ + { value = "dask==2026.3.0", if = [ "min" ] }, + { value = "distributed==2026.3.0", if = [ "min" ] }, ] - -[tool.jupytext] -formats = "ipynb,md" - -[tool.hatch.build.targets.wheel] -packages = ['src/spatialdata'] - -[tool.hatch.version] -source = "vcs" - -[tool.hatch.build.hooks.vcs] -version-file = "_version.py" - -[tool.hatch.metadata] -allow-direct-references = true - -[tool.ruff] -exclude = [ - ".git", - ".tox", - "__pycache__", - "build", - "docs/_build", - "dist", - "setup.py", - +envs.hatch-test.dependency-groups = [ "dev", "test" ] +envs.hatch-check-fmt.dependencies = [ "prek" ] +envs.hatch-check-fmt.scripts.format-check = "echo 'try `hatch check fmt --fix`'; false" +envs.hatch-check-fmt.scripts.format-fix = "prek run -a --group=format --no-group=typecheck" +envs.hatch-check-code.dependencies = [ "prek" ] +envs.hatch-check-code.scripts.lint-check = "echo 'try `hatch check code --fix`'; false" +envs.hatch-check-code.scripts.lint-fix = "prek run -a --no-group=format --no-group=typecheck" +envs.hatch-check-types.scripts.check = [ "mypy" ] +envs.hatch-check-types.dependency-groups = [ "typecheck", "test" ] +# Not part of the CI matrix; run locally with `hatch run test-anndata-pandas:test`. +envs.test-anndata-pandas.extra-dependencies = [ "zarr>=3" ] +envs.test-anndata-pandas.scripts.test = [ "pip list|grep anndata && pip list|grep pandas && pytest {args}" ] +envs.test-anndata-pandas.scripts.test-all = [ "pip list|grep anndata && pip list|grep pandas && pytest ." ] +envs.test-anndata-pandas.scripts.test-readwrite = [ + "pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py" ] -line-length = 120 -target-version = "py312" - -[tool.ruff.lint] -ignore = [ - # Do not assign a lambda expression, use a def -> lambda expression assignments are convenient - "E731", - # allow I, O, l as variable names -> I is the identity matrix, i, j, k, l is reasonable indexing notation - "E741", - # Missing docstring in public package - "D104", - # Missing docstring in public module - "D100", - # Missing docstring in __init__ - "D107", - # Missing docstring in magic method - "D105", - # Do not perform function calls in argument defaults. - "B008", - # Missing docstring in magic method - "D105", +envs.test-anndata-pandas.matrix = [ + { anndata-pandas = [ + "0.13-2", + "0.13-3", + "0.12-2", + # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13 + ] }, ] -select = [ - "D", # flake8-docstrings - "I", # isort - "E", # pycodestyle - "F", # pyflakes - "W", # pycodestyle - "Q", # flake8-quotes - "SIM", # flake8-simplify - "TID", # flake-8-tidy-imports - "NPY", # NumPy-specific rules - "PT", # flake8-pytest-style - "B", # flake8-bugbear - "UP", # pyupgrade - "C4", # flake8-comprehensions - "BLE", # flake8-blind-except - "T20", # flake8-print - "RET", # flake8-raise - "PGH", # pygrep-hooks +# every option where the if-condition is True gets included +envs.test-anndata-pandas.overrides.matrix.anndata-pandas.extra-dependencies = [ + # anndata 0.13 + { value = "anndata~=0.13", if = [ "0.13-2", "0.13-3" ] }, + # anndata 0.12 + { value = "anndata>=0.12,<0.13", if = [ "0.12-2" ] }, + # pandas 2 + { value = "pandas>=2.3,<3", if = [ "0.13-2", "0.12-2" ] }, + # pandas 3 + { value = "pandas~=3.0", if = [ "0.13-3" ] }, ] -unfixable = ["B", "C4", "UP", "BLE", "T20", "RET"] - -[tool.ruff.lint.isort] -required-imports = ["from __future__ import annotations"] - -[tool.ruff.lint.pydocstyle] -convention = "numpy" - -[tool.ruff.lint.per-file-ignores] - "tests/*" = ["D", "PT", "B024"] - "*/__init__.py" = ["F401", "D104", "D107", "E402"] - "docs/*" = ["D","B","E","A"] - "src/spatialdata/transformations/transformations.py" = ["D101","D102", "D106", "B024", "T201", "RET504", "UP006", "UP007"] - "src/spatialdata/transformations/operations.py" = ["D101","D102", "D106", "B024","D401", "T201", "RET504", "RET506", "RET505", "RET504", "UP006", "UP007"] - "src/spatialdata/transformations/ngff/*.py" = ["D101","D102", "D106", "D401", "E501","RET506", "RET505", "RET504", "UP006", "UP007"] - "src/spatialdata/transformations/*" = ["RET", "D", "UP006", "UP007"] - "src/spatialdata/models/models.py" = ["D101", "B026"] - "src/spatialdata/dataloader/datasets.py" = ["D101"] - "tests/test_models/test_models.py" = ["NPY002"] - "tests/conftest.py"= ["E402"] - "benchmarks/*" = ["ALL"] - - -# pyupgrade typing rewrite TODO: remove at some point from per-file ignore -# "UP006", "UP007" - -[tool.pixi.workspace] -channels = ["conda-forge"] -platforms = ["osx-arm64", "linux-64"] - -[tool.pixi.dependencies] -proj = "*" - -[tool.pixi.tasks] -test = "pytest" -test-parallel = "pytest -n auto --dist worksteal" -pre-commit = "pre-commit run" -pre-commit-all = "pre-commit run --all-files" - -[tool.pixi.feature.profiling.dependencies] -py-spy = "*" - -[tool.pixi.feature.profiling.pypi-dependencies] -spatialdata = { path = ".", editable = true } -profimp = "*" -memray = "*" - -[tool.pixi.feature.profiling.tasks] +envs.test-anndata-pandas.dependency-groups = [ "dev", "test" ] + +[tool.pixi] +workspace.channels = [ "conda-forge" ] +workspace.platforms = [ "linux-64", "osx-arm64" ] +dependencies.proj = "*" +tasks.pre-commit = "prek run" +tasks.pre-commit-all = "prek run --all-files" +tasks.test = "pytest" +tasks.test-parallel = "pytest -n auto --dist worksteal" +feature.profiling.dependencies.py-spy = "*" +feature.profiling.pypi-dependencies.memray = "*" +feature.profiling.pypi-dependencies.profimp = "*" +feature.profiling.pypi-dependencies.spatialdata = { path = ".", editable = true } +# Usage: pixi run -e profiling memray-flame memray-script.py..bin +feature.profiling.tasks.memray-flame = "memray flamegraph --temporal" +# Usage: pixi run -e profiling memray-run script.py +feature.profiling.tasks.memray-run = "memray run" # Usage: pixi run -e profiling pyspy script.py (prefix with sudo on macOS) -pyspy = "py-spy record --gil -o profile.speedscope.json --format speedscope -- python" +feature.profiling.tasks.pyspy = "py-spy record --gil -o profile.speedscope.json --format speedscope -- python" # Usage: pixi run -e profiling speedscope -speedscope = "npx --yes speedscope profile.speedscope.json" -# Usage: pixi run -e profiling memray-run script.py -memray-run = "memray run" -# Usage: pixi run -e profiling memray-flame memray-script.py..bin -memray-flame = "memray flamegraph --temporal" - -[tool.pixi.environments] -profiling = { features = ["profiling"], solve-group = "default" } +feature.profiling.tasks.speedscope = "npx --yes speedscope profile.speedscope.json" +environments.profiling = { features = [ "profiling" ], solve-group = "default" } -[tool.hatch.envs.test] -dependency-groups = ["test"] - -[tool.hatch.envs.test-anndata-pandas] -template = "test" -extra-dependencies = ["zarr>=3"] -scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"] -scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"] -scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."] - -[[tool.hatch.envs.test-anndata-pandas.matrix]] -anndata-pandas = [ - "0.13-2", - "0.13-3", - "0.12-2" - # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13 +[tool.ruff] +line-length = 120 +src = [ "src" ] +extend-include = [ "*.ipynb" ] +format.docstring-code-format = true +lint.select = [ + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "C4", # flake8-comprehensions + "D", # pydocstyle + "E", # Error detected by Pycodestyle + "F", # Errors detected by Pyflakes + "I", # isort + "NPY", # NumPy-specific rules + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "Q", # flake8-quotes + "RET", # flake8-return + "RUF100", # Report unused noqa directives + "SIM", # flake8-simplify + "T20", # flake8-print + "TID", # flake8-tidy-imports + "UP", # pyupgrade + "W", # Warning detected by Pycodestyle ] +lint.ignore = [ + "B008", # Errors from function calls in argument defaults. These are fine when the result is immutable. + "D100", # Missing docstring in public module + "D104", # Missing docstring in public package + "D105", # __magic__ methods are often self-explanatory, allow missing docstrings + "D107", # Missing docstring in __init__ + # Disable one in each pair of mutually incompatible rules + "D203", # We don’t want a blank line before a class docstring + "D213", # <> We want docstrings to start immediately after the opening triple quote + "D400", # first line should end with a period [Bug: doesn’t work with single-line docstrings] + "D401", # First line should be in imperative mood; try rephrasing + "E501", # line too long -> we accept long comment lines; formatter gets rid of long code lines + "E731", # Do not assign a lambda expression, use a def -> lambda expression assignments are convenient + "E741", # allow I, O, l as variable names -> I is the identity matrix +] +lint.per-file-ignores."*/__init__.py" = [ "D104", "D107", "E402", "F401" ] +lint.per-file-ignores."benchmarks/*" = [ "ALL" ] +lint.per-file-ignores."docs/*" = [ "A", "B", "D", "E", "I" ] +lint.per-file-ignores."src/spatialdata/dataloader/datasets.py" = [ "D101" ] +lint.per-file-ignores."src/spatialdata/models/models.py" = [ "B026", "D101" ] +lint.per-file-ignores."src/spatialdata/transformations/*" = [ "D", "RET", "UP006", "UP007" ] +lint.per-file-ignores."src/spatialdata/transformations/ngff/*.py" = [ + "D101", + "D102", + "D106", + "RET504", + "RET505", + "RET506", + "UP006", + "UP007" +] +lint.per-file-ignores."src/spatialdata/transformations/operations.py" = [ + "B024", + "D101", + "D102", + "D106", + "RET504", + "RET505", + "RET506", + "T201", + "UP006", + "UP007" +] +lint.per-file-ignores."src/spatialdata/transformations/transformations.py" = [ + "B024", + "D101", + "D102", + "D106", + "RET504", + "T201", + "UP006", + "UP007" +] +lint.per-file-ignores."tests/*" = [ "B024", "D", "PT" ] +lint.per-file-ignores."tests/conftest.py" = [ "E402" ] +lint.per-file-ignores."tests/test_models/test_models.py" = [ "NPY002" ] +lint.unfixable = [ "B", "BLE", "C4", "RET", "T20", "UP" ] +lint.isort.required-imports = [ "from __future__ import annotations" ] +lint.pydocstyle.convention = "numpy" + +[tool.mypy] +# TODO: extend to "tests" once the test suite carries type annotations. +files = [ "src" ] +python_version = "3.12" +disallow_any_generics = true +disallow_untyped_calls = false +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +strict_optional = true +warn_redundant_casts = true +warn_unused_ignores = false +warn_return_any = true +warn_unreachable = false +ignore_errors = false +strict_equality = true +show_column_numbers = true +error_summary = true +warn_unused_configs = true +# because of docrep +allow_untyped_decorators = true +no_implicit_optional = true +no_implicit_reexport = true +no_warn_no_return = true +show_error_codes = true + +# Installed, but shipping no `py.typed` marker. +# Infer types from their source instead of falling back to `Any`. +[[tool.mypy.overrides]] +module = [ + "annsel.*", + "dask_image.*", + "datashader.*", + "multiscale_spatial_image.*", + "numcodecs.*", + "ome_zarr.*", + "pooch.*", + "pyarrow.*", + "spatial_image.*", + "xrspatial.*", +] +follow_untyped_imports = true +# `no_implicit_reexport` is meant to discipline our own public API. +# Third-party packages re-export freely, so applying it to them only produces false positives. +implicit_reexport = true + +# Optional dependencies that are not installed when the type check runs. +[[tool.mypy.overrides]] +module = [ + "matplotlib.*", + "spatialdata_io.*", +] +ignore_missing_imports = true -[tool.hatch.envs.test-anndata-pandas.overrides] -matrix.anndata-pandas.extra-dependencies = [ - # every option where the if-condition is True gets included - - # anndata 0.13 - {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]}, - - # anndata 0.12 - {value="anndata>=0.12,<0.13", if = ["0.12-2"]}, - - # pandas 2 - {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]}, +[tool.pytest] +addopts = [ + "--import-mode=importlib", # allow using test files with same name + "-s", # print output from tests + # napari registers a pytest plugin via its entry point; disable it here since spatialdata tests don't need it + "-p no:napari", +] +# info on how to use this https://stackoverflow.com/questions/57925071/how-do-i-avoid-getting-deprecationwarning-from-inside-dependencies-with-pytest +filterwarnings = [ + # "error", # if 3rd party libs raise DeprecationWarnings, TODO: filter them individually below + # "ignore:.*U.*mode is deprecated:DeprecationWarning", +] +# These are all markers coming from xarray, dask or anndata. Added here to silence warnings. +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "network: marks tests that require network access; skipped by default, run with '--run-network'", + "gpu: run test on GPU using CuPY.", + "array_api: used by anndata.tests.helpers, not us", + "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on", +] +strict = true +testpaths = [ "tests" ] - # pandas 3 - {value="pandas~=3.0", if = ["0.13-3"]}, +[tool.coverage] +run.source = [ "spatialdata" ] +run.omit = [ + "**/test_*.py", +] +run.patch = [ "subprocess" ] + +[tool.cruft] +skip = [ + ".git", + "tests", + "src/**/__init__.py", + "src/**/basic.py", + "docs/api.md", + "docs/changelog.md", + "docs/references.bib", + "docs/references.md", + "docs/notebooks/example.ipynb", ] + +[tool.jupytext] +formats = "ipynb,md" diff --git a/src/spatialdata/__init__.py b/src/spatialdata/__init__.py index 7ba66e710..96e7032dd 100644 --- a/src/spatialdata/__init__.py +++ b/src/spatialdata/__init__.py @@ -4,7 +4,7 @@ from importlib.metadata import version from typing import TYPE_CHECKING, Any -import spatialdata.models._accessor # noqa: F401 +import spatialdata.models._accessor __version__ = version("spatialdata") diff --git a/src/spatialdata/_core/_deepcopy.py b/src/spatialdata/_core/_deepcopy.py index 9e2e7f00c..2a68ab7a2 100644 --- a/src/spatialdata/_core/_deepcopy.py +++ b/src/spatialdata/_core/_deepcopy.py @@ -12,7 +12,15 @@ from spatialdata._core.spatialdata import SpatialData from spatialdata.models._utils import SpatialElement -from spatialdata.models.models import Image2DModel, Image3DModel, Labels2DModel, Labels3DModel, PointsModel, get_model +from spatialdata.models.models import ( + Image2DModel, + Image3DModel, + Labels2DModel, + Labels3DModel, + PointsModel, + RasterSchema, + get_model, +) @singledispatch @@ -44,9 +52,11 @@ def deepcopy(element: SpatialData | SpatialElement | AnnData) -> SpatialData | S # This leads to double copying the data, but since we expect the data to be small, this is acceptable. @deepcopy.register(SpatialData) def _(sdata: SpatialData) -> SpatialData: - elements_dict = {} + elements_dict: dict[str, SpatialElement | AnnData] = {} for _, element_name, element in sdata.gen_elements(): - elements_dict[element_name] = deepcopy(element) + copied = deepcopy(element) + assert not isinstance(copied, SpatialData) + elements_dict[element_name] = copied deepcopied_attrs = _deepcopy(sdata.attrs) return SpatialData.init_from_elements(elements_dict, attrs=deepcopied_attrs) @@ -56,8 +66,9 @@ def _(element: DataArray) -> DataArray: model = get_model(element) if isinstance(element.data, DaskArray): element = element.compute() + assert issubclass(model, RasterSchema) if model in [Image2DModel, Image3DModel]: - return model.parse(element.copy(deep=True), c_coords=element["c"]) # type: ignore[call-arg] + return model.parse(element.copy(deep=True), c_coords=list(element["c"].to_numpy())) assert model in [Labels2DModel, Labels3DModel] return model.parse(element.copy(deep=True)) @@ -69,17 +80,22 @@ def _(element: DataTree) -> DataTree: # to understand the original motivation. model = get_model(element) for key in element: - ds = element[key].ds + node = element[key] + assert isinstance(node, DataTree) + ds = node.ds assert len(ds) == 1 - variable = ds.__iter__().__next__() - if isinstance(element[key][variable].data, DaskArray): - element[key][variable] = element[key][variable].compute() + variable = str(next(iter(ds))) + if isinstance(node[variable].data, DaskArray): + node[variable] = node[variable].compute() msi = element.copy(deep=True) for key in msi: - ds = msi[key].ds - variable = ds.__iter__().__next__() - msi[key][variable].data = from_array(msi[key][variable].data) - element[key][variable].data = from_array(element[key][variable].data) + copied_node = msi[key] + assert isinstance(copied_node, DataTree) + original_node = element[key] + assert isinstance(original_node, DataTree) + variable = str(next(iter(copied_node.ds))) + copied_node[variable].data = from_array(copied_node[variable].data) + original_node[variable].data = from_array(original_node[variable].data) assert model in [Image2DModel, Image3DModel, Labels2DModel, Labels3DModel] model.validate(msi) return msi diff --git a/src/spatialdata/_core/centroids.py b/src/spatialdata/_core/centroids.py index cde583e15..9979b9f20 100644 --- a/src/spatialdata/_core/centroids.py +++ b/src/spatialdata/_core/centroids.py @@ -71,9 +71,9 @@ def _get_centroids_for_labels(xdata: xr.DataArray) -> pd.DataFrame: # indexing="ij" (matrix convention) ensures the i-th grid varies along the i-th # dimension of the output, correctly aligning with xdata.dims for any number of axes. - coord_grids = np.meshgrid(*[xdata[ax].values for ax in axes], indexing="ij") + coord_grids = np.meshgrid(*[xdata[ax].to_numpy() for ax in axes], indexing="ij") data: dict[str, np.ndarray] = {} - for ax, grid in zip(axes, coord_grids, strict=True): + for ax, grid in zip((str(ax) for ax in axes), coord_grids, strict=True): coord_sums = np.bincount(flat_inverse, weights=grid.ravel().astype(float)) data[ax] = coord_sums / counts # counts > 0 by construction (unique guarantees this) @@ -94,15 +94,20 @@ def _( _validate_coordinate_system(e, coordinate_system) if isinstance(e, DataTree): - assert len(e["scale0"]) == 1 - e = next(iter(e["scale0"].values())) + scale0 = e["scale0"] + assert isinstance(scale0, DataTree) + assert len(scale0) == 1 + variable = next(iter(scale0.values())) + assert isinstance(variable, DataArray) + e = variable df = _get_centroids_for_labels(e) if not return_background and 0 in df.index: df = df.drop(index=0) # drop the background label t = get_transformation(e, coordinate_system) centroids = PointsModel.parse(df, transformations={coordinate_system: t}) - return transform(centroids, to_coordinate_system=coordinate_system) + transformed: DaskDataFrame = transform(centroids, to_coordinate_system=coordinate_system) + return transformed @get_centroids.register(GeoDataFrame) @@ -120,10 +125,11 @@ def _(e: GeoDataFrame, coordinate_system: str = "global") -> DaskDataFrame: f"Expected a GeoDataFrame either composed entirely of circles (Points with the `radius` column) or" f" Polygons/MultiPolygons. Found {type(first_geometry)} instead." ) - xy = e.centroid.get_coordinates().values + xy = e.centroid.get_coordinates().to_numpy() xy_df = pd.DataFrame(xy, columns=["x", "y"], index=e.index.copy()) points = PointsModel.parse(xy_df, transformations={coordinate_system: t}) - return transform(points, to_coordinate_system=coordinate_system) + transformed_points: DaskDataFrame = transform(points, to_coordinate_system=coordinate_system) + return transformed_points @get_centroids.register(DaskDataFrame) @@ -136,7 +142,8 @@ def _(e: DaskDataFrame, coordinate_system: str = "global") -> DaskDataFrame: t = get_transformation(e, coordinate_system) assert isinstance(t, BaseTransformation) centroids = PointsModel.parse(coords, transformations={coordinate_system: t}) - return transform(centroids, to_coordinate_system=coordinate_system) + transformed_centroids: DaskDataFrame = transform(centroids, to_coordinate_system=coordinate_system) + return transformed_centroids ## diff --git a/src/spatialdata/_core/concatenate.py b/src/spatialdata/_core/concatenate.py index cfe5447a5..2bdb68b55 100644 --- a/src/spatialdata/_core/concatenate.py +++ b/src/spatialdata/_core/concatenate.py @@ -8,6 +8,7 @@ from warnings import warn import numpy as np +import pandas as pd from anndata import AnnData from anndata._core.merge import StrategiesLiteral, resolve_merge_strategy @@ -65,7 +66,10 @@ def _concatenate_tables( rename_dict[table_instance_key] = instance_key if len(rename_dict) > 0: table = copy(table) # Shallow copy - table.obs = table.obs.rename(columns=rename_dict, copy=False) + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + table.obs = obs.rename(columns=rename_dict) tables_l.append(table) merged_table = ad.concat(tables_l, **kwargs) @@ -202,8 +206,8 @@ def concatenate( else: merged_tables[k] = v - attrs_merge = resolve_merge_strategy(attrs_merge) - attrs = attrs_merge([sdata.attrs for sdata in sdatas]) + resolved_attrs_merge = resolve_merge_strategy(attrs_merge) + attrs = resolved_attrs_merge([sdata.attrs for sdata in sdatas]) sdata = SpatialData( images=merged_images, @@ -252,7 +256,10 @@ def _fix_ensure_unique_element_names( # fix the region_key column region, region_key, _ = get_table_keys(table) - table.obs[region_key] = (table.obs[region_key].astype("str") + f"-{suffix}").astype("category") + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + obs[region_key] = (obs[region_key].astype("str") + f"-{suffix}").astype("category") new_region: str | list[str] if isinstance(region, str): new_region = f"{region}-{suffix}" diff --git a/src/spatialdata/_core/data_extent.py b/src/spatialdata/_core/data_extent.py index 6504bb082..32f05834b 100644 --- a/src/spatialdata/_core/data_extent.py +++ b/src/spatialdata/_core/data_extent.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from anndata import AnnData from dask.dataframe import DataFrame as DaskDataFrame from geopandas import GeoDataFrame from shapely import MultiPolygon, Point, Polygon @@ -106,7 +107,7 @@ def get_extent( has_labels: bool = True, has_points: bool = True, has_shapes: bool = True, - elements: list[str] | None = None, # noqa: UP007 # https://github.com/scverse/spatialdata/pull/318#issuecomment-1755714287 + elements: list[str] | None = None, # https://github.com/scverse/spatialdata/pull/318#issuecomment-1755714287 ) -> BoundingBoxDescription: """ Get the extent (bounding box) of a SpatialData object or a SpatialElement. @@ -207,6 +208,7 @@ def _( consider_element = (len(elements) == 0) or (element_name in elements) consider_element = consider_element and (element_type in include_spatial_elements) if consider_element: + assert not isinstance(element_obj, AnnData) transformations = get_transformation(element_obj, get_all=True) assert isinstance(transformations, dict) coordinate_systems = list(transformations.keys()) @@ -302,7 +304,10 @@ def _(e: DataArray, coordinate_system: str = "global") -> BoundingBoxDescription @get_extent.register def _(e: DataTree, coordinate_system: str = "global") -> BoundingBoxDescription: _check_element_has_coordinate_system(element=e, coordinate_system=coordinate_system) - xdata = next(iter(e["scale0"].values())) + scale0 = e["scale0"] + assert isinstance(scale0, DataTree) + xdata = next(iter(scale0.values())) + assert isinstance(xdata, DataArray) return _get_extent_of_data_array(xdata, coordinate_system=coordinate_system) diff --git a/src/spatialdata/_core/operations/_utils.py b/src/spatialdata/_core/operations/_utils.py index d3c438abb..600dbed75 100644 --- a/src/spatialdata/_core/operations/_utils.py +++ b/src/spatialdata/_core/operations/_utils.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING +from anndata import AnnData from xarray import DataArray, DataTree from spatialdata.models import SpatialElement, get_axes_names, get_spatial_axes @@ -108,7 +109,7 @@ def transform_to_data_extent( coordinate_system, maintain_positioning=True ) - sdata_to_return_elements = { + sdata_to_return_elements: dict[str, SpatialElement | AnnData] = { **sdata_vector_transformed_inplace.shapes, **sdata_vector_transformed_inplace.points, } @@ -128,12 +129,16 @@ def transform_to_data_extent( target_depth=None, return_regions_as_labels=True, ) + assert isinstance(rasterized, DataArray | DataTree) sdata_to_return_elements[element_name] = rasterized else: sdata_to_return_elements[element_name] = element if not maintain_positioning: - for el in sdata_to_return_elements.values(): - set_transformation(el, transformation={coordinate_system: Identity()}, set_all=True) + for element_value in sdata_to_return_elements.values(): + # tables carry no transformations + if isinstance(element_value, AnnData): + continue + set_transformation(element_value, transformation={coordinate_system: Identity()}, set_all=True) for k, v in sdata.tables.items(): sdata_to_return_elements[k] = v.copy() return SpatialData.init_from_elements(sdata_to_return_elements, attrs=sdata.attrs) @@ -152,6 +157,9 @@ def _parse_element( ) if sdata is not None: assert isinstance(element, str) - return sdata[element] - assert element is not None + looked_up = sdata[element] + if isinstance(looked_up, AnnData): + raise TypeError(f"Element {element!r} is a table, not a spatial element.") + return looked_up + assert not isinstance(element, str) return element diff --git a/src/spatialdata/_core/operations/aggregate.py b/src/spatialdata/_core/operations/aggregate.py index d0c4741e7..9c1349062 100644 --- a/src/spatialdata/_core/operations/aggregate.py +++ b/src/spatialdata/_core/operations/aggregate.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from collections.abc import Callable from typing import Any import anndata as ad @@ -177,6 +178,8 @@ def aggregate( values_[ONES_KEY] = 1 value_key = ONES_KEY + assert isinstance(by_, GeoDataFrame) + assert isinstance(values_, GeoDataFrame | DaskDataFrame) adata = _aggregate_shapes( values=values_, by=by_, @@ -195,6 +198,8 @@ def aggregate( if by_type is Labels2DModel and values_type is Image2DModel: if fractions is True: raise NotImplementedError("fractions = True is not yet supported for raster aggregation") + assert isinstance(values_, DataArray | DataTree) + assert isinstance(by_, DataArray | DataTree) adata = _aggregate_image_by_labels(values=values_, by=by_, agg_func=agg_func, **kwargs) if adata is None: @@ -202,6 +207,7 @@ def aggregate( # create a SpatialData object with the aggregated table and the "by" shapes shapes_name = by if isinstance(by, str) else "by" + assert isinstance(by_, GeoDataFrame | DataArray | DataTree) return _create_sdata_from_table_and_shapes( table=adata, table_name=table_name, @@ -237,10 +243,15 @@ def _create_sdata_from_table_and_shapes( # labels case, needs conversion from str to int if isinstance(shapes, DataArray | DataTree): - table.obs[instance_key] = table.obs[instance_key].astype(int) + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + obs[instance_key] = obs[instance_key].astype(int) if deepcopy: - shapes = _deepcopy(shapes) + copied = _deepcopy(shapes) + assert isinstance(copied, GeoDataFrame | DataArray | DataTree) + shapes = copied return SpatialData.init_from_elements({shapes_name: shapes, table_name: table}) @@ -271,28 +282,39 @@ def _aggregate_image_by_labels( AnnData of shape `(by.shape[0], len(agg_func)]`. """ from scipy import sparse - from xrspatial import zonal_stats + from xrspatial.zonal import stats as zonal_stats if isinstance(by, DataTree): - assert len(by["scale0"]) == 1 - by = next(iter(by["scale0"].values())) + by_scale0 = by["scale0"] + assert isinstance(by_scale0, DataTree) + assert len(by_scale0) == 1 + by_variable = next(iter(by_scale0.values())) + assert isinstance(by_variable, DataArray) + by = by_variable if isinstance(values, DataTree): - assert len(values["scale0"]) == 1 - values = next(iter(values["scale0"].values())) + values_scale0 = values["scale0"] + assert isinstance(values_scale0, DataTree) + assert len(values_scale0) == 1 + values_variable = next(iter(values_scale0.values())) + assert isinstance(values_variable, DataArray) + values = values_variable agg_func = [agg_func] if isinstance(agg_func, str) else agg_func outs = [] - for i, c in enumerate(values.coords["c"].values): + for i, c in enumerate(values.coords["c"].to_numpy()): with warnings.catch_warnings(): # ideally fix upstream warnings.filterwarnings( "ignore", message=".*unknown divisions.*", ) - out = zonal_stats(by, values[i, ...], stats_funcs=agg_func, **kwargs).compute() + zonal = zonal_stats(by, values[i, ...], stats_funcs=list(agg_func), **kwargs) + out = zonal.compute() if isinstance(zonal, ddf.DataFrame) else zonal + if not isinstance(out, pd.DataFrame): + raise TypeError(f"Expected the zonal statistics to be a data frame, got {type(out).__name__}.") out.columns = [f"channel_{c}_{col}" if col != "zone" else col for col in out.columns] out = out.loc[out["zone"] != 0].copy() - zones: ArrayLike = out["zone"].values + zones: ArrayLike = out["zone"].to_numpy() outs.append(out.drop(columns=["zone"])) # remove the 0 (background) df = pd.concat(outs, axis=1) @@ -310,7 +332,7 @@ def _aggregate_image_by_labels( def _aggregate_shapes( - values: gpd.GeoDataFrame, + values: gpd.GeoDataFrame | ddf.DataFrame, by: gpd.GeoDataFrame, values_sdata: SpatialData | None = None, values_element_name: str | None = None, @@ -436,6 +458,7 @@ def _aggregate_shapes( if fractions: fractions_of_values = joined.geometry.area / joined[AREAS_COLUMN] + aggregated_values: ArrayLike if categorical: # we only allow the aggregation of one categorical column at the time, because each categorical column would # give a different table as result of the aggregation, and we only support single tables @@ -443,13 +466,18 @@ def _aggregate_shapes( vk = value_key[0] if fractions_of_values is not None: joined[ONES_COLUMN] = fractions_of_values - aggregated = joined.groupby([INDEX, vk], observed=False)[ONES_COLUMN].agg(agg_func).reset_index() - aggregated_values = aggregated[ONES_COLUMN].values + grouped = joined.groupby([INDEX, vk], observed=False)[ONES_COLUMN] + if isinstance(agg_func, str): + aggregated = grouped.agg(agg_func).reset_index() + else: + agg_funcs: list[Callable[..., Any] | str | np.ufunc] = list(agg_func) + aggregated = grouped.agg(agg_funcs).reset_index() + aggregated_values = aggregated[ONES_COLUMN].to_numpy() else: if fractions_of_values is not None: joined[value_key] = joined[value_key].to_numpy() * fractions_of_values.to_numpy().reshape(-1, 1) aggregated = joined.groupby([INDEX])[value_key].agg(agg_func).reset_index() - aggregated_values = aggregated[value_key].values + aggregated_values = aggregated[value_key].to_numpy() # Here we prepare some variables to construct a sparse matrix in the coo format (edges + nodes) rows_categories = by.index.tolist() diff --git a/src/spatialdata/_core/operations/map.py b/src/spatialdata/_core/operations/map.py index 0f5a380a9..b0738dbff 100644 --- a/src/spatialdata/_core/operations/map.py +++ b/src/spatialdata/_core/operations/map.py @@ -92,7 +92,11 @@ def map_raster( if isinstance(data, DataArray): arr = data.data elif isinstance(data, DataTree): - arr = data["scale0"].values().__iter__().__next__().data + scale0 = data["scale0"] + assert isinstance(scale0, DataTree) + first_variable = next(iter(scale0.values())) + assert isinstance(first_variable, DataArray) + arr = first_variable.data else: raise ValueError("Only 'DataArray' and 'DataTree' are supported.") @@ -153,7 +157,9 @@ def map_raster( "transformations": transformations, } model = get_raster_model_from_data_dims(dims) - return model.parse(arr, **model_kwargs) + parsed = model.parse(arr, **model_kwargs) + assert isinstance(parsed, DataArray) + return parsed def _relabel(arr: da.Array) -> da.Array: @@ -206,7 +212,7 @@ def _calculate_block_num(block_id: tuple[int, ...], num_blocks: tuple[int, ...]) return block - return da.map_blocks( + relabeled: da.Array = da.map_blocks( _relabel_block, arr, dtype=arr.dtype, @@ -214,6 +220,7 @@ def _calculate_block_num(block_id: tuple[int, ...], num_blocks: tuple[int, ...]) shift=shift, meta=meta, ) + return relabeled def relabel_sequential(arr: da.Array) -> da.Array: @@ -250,4 +257,5 @@ def relabel_sequential(arr: da.Array) -> da.Array: # Note that both sides are ordered as da.unique returns an ordered array. new_labeling[unique_labels] = da.arange(len(unique_labels), dtype=arr.dtype) - return da.map_blocks(operator.getitem, new_labeling, arr, dtype=arr.dtype, chunks=arr.chunks) + consecutive: da.Array = da.map_blocks(operator.getitem, new_labeling, arr, dtype=arr.dtype, chunks=arr.chunks) + return consecutive diff --git a/src/spatialdata/_core/operations/rasterize.py b/src/spatialdata/_core/operations/rasterize.py index 4f2ea0664..113d26fc7 100644 --- a/src/spatialdata/_core/operations/rasterize.py +++ b/src/spatialdata/_core/operations/rasterize.py @@ -17,7 +17,7 @@ from spatialdata._core.operations.vectorize import to_polygons from spatialdata._core.query.relational_query import get_values from spatialdata._core.spatialdata import SpatialData -from spatialdata._types import ListOrNDArrayFloating +from spatialdata._types import ListOrNDArrayFloating, Raster_T from spatialdata._utils import _parse_list_into_array from spatialdata.models import ( Image2DModel, @@ -285,8 +285,8 @@ def rasterize( raise ValueError("When data is a SpatialData object, table_name must be None.") if agg_func is not None: raise ValueError("When data is a SpatialData object, agg_func must be None.") - new_images = {} - new_labels = {} + new_images: dict[str, Raster_T] = {} + new_labels: dict[str, Raster_T] = {} for element_type in ["points", "images", "labels", "shapes"]: elements = getattr(data, element_type) for name in elements: @@ -305,6 +305,7 @@ def rasterize( return_single_channel=return_single_channel if element_type in ("points", "shapes") else None, ) new_name = f"{name}_rasterized_{element_type}" + assert isinstance(rasterized, DataArray) model = get_model(rasterized) if model in (Image2DModel, Image3DModel): new_images[new_name] = rasterized @@ -321,6 +322,7 @@ def rasterize( raise ValueError("agg_func must be None when data is an image or labels.") if return_single_channel is not None: raise ValueError("return_single_channel must be None when data is an image or labels.") + assert isinstance(parsed_data, DataArray | DataTree) rasterized = rasterize_images_labels( data=parsed_data, axes=axes, @@ -351,6 +353,7 @@ def rasterize( rasterized = model.parse(assigner[rasterized], transformations=transformations) # type: ignore[call-arg] return rasterized if model in (PointsModel, ShapesModel): + assert isinstance(parsed_data, GeoDataFrame | DaskDataFrame) return rasterize_shapes_points( data=parsed_data, axes=axes, @@ -411,11 +414,14 @@ def _get_xarray_data_to_rasterize( latest_scale: str | None = None for scale in reversed(list(data.keys())): data_tree = data[scale] + assert isinstance(data_tree, DataTree) latest_scale = scale - v = data_tree.values() + v = list(data_tree.values()) assert len(v) == 1 - xdata = next(iter(v)) - assert set(get_spatial_axes(tuple(xdata.sizes.keys()))) == set(axes) + scale_variable = v[0] + assert isinstance(scale_variable, DataArray) + xdata = scale_variable + assert set(get_spatial_axes(tuple(str(dim) for dim in xdata.sizes))) == set(axes) corrected_affine, _ = _get_corrected_affine_matrix( data=xdata, @@ -450,7 +456,11 @@ def _get_xarray_data_to_rasterize( # when this code is reached, latest_scale is selected break assert latest_scale is not None - xdata = next(iter(data[latest_scale].values())) + latest_scale_tree = data[latest_scale] + assert isinstance(latest_scale_tree, DataTree) + latest_variable = next(iter(latest_scale_tree.values())) + assert isinstance(latest_variable, DataArray) + xdata = latest_variable if latest_scale != "scale0": transformations = xdata.attrs["transform"] pyramid_scale = _get_scale(transformations) @@ -500,7 +510,7 @@ def _get_corrected_affine_matrix( # TODO: rename this function to an internatl function and invoke this function from a function that has arguments # values, values_sdata def rasterize_images_labels( - data: SpatialElement, + data: DataArray | DataTree, axes: tuple[str, ...], min_coordinate: ListOrNDArrayFloating, max_coordinate: ListOrNDArrayFloating, @@ -600,7 +610,8 @@ def rasterize_images_labels( ) assert isinstance(transformed_dask, DaskArray) channels = xdata.coords["c"].values if schema in (Image2DModel, Image3DModel) else None - transformed_data = schema.parse(transformed_dask, dims=xdata.dims, c_coords=channels) # type: ignore[call-arg] + transformed_dims = tuple(str(dim) for dim in xdata.dims) + transformed_data = schema.parse(transformed_dask, dims=transformed_dims, c_coords=channels) # type: ignore[call-arg] if target_coordinate_system != "global": remove_transformation(transformed_data, "global") @@ -608,9 +619,10 @@ def rasterize_images_labels( sequence = Sequence([half_pixel_offset.inverse(), scale, translation, half_pixel_offset]) set_transformation(transformed_data, sequence, target_coordinate_system) - transformed_data = compute_coordinates(transformed_data) - schema.validate(transformed_data) - return transformed_data + computed = compute_coordinates(transformed_data) + assert isinstance(computed, DataArray) + schema.validate(computed) + return computed def rasterize_shapes_points( @@ -632,6 +644,8 @@ def rasterize_shapes_points( return_single_channel: bool | None = None, ) -> DataArray: import datashader as ds + from datashader.reductions import count_cat as datashader_count_cat + from datashader.reductions import first as datashader_first min_coordinate = _parse_list_into_array(min_coordinate) max_coordinate = _parse_list_into_array(max_coordinate) @@ -709,14 +723,14 @@ def rasterize_shapes_points( else: agg = cnv.points(data, x="x", y="y", agg=agg_func) - if label_index_to_category is not None and isinstance(agg_func, ds.first): + if label_index_to_category is not None and isinstance(agg_func, datashader_first): agg.attrs["label_index_to_category"] = label_index_to_category scale = Scale([(y_range[1] - y_range[0]) / plot_height, (x_range[1] - x_range[0]) / plot_width], axes=("y", "x")) translation = Translation([y_range[0], x_range[0]], axes=("y", "x")) transformations: dict[str, BaseTransformation] = {target_coordinate_system: Sequence([scale, translation])} - if isinstance(agg_func, ds.count_cat): + if isinstance(agg_func, datashader_count_cat): if return_single_channel: raise ValueError("Cannot return single channel when using count_cat aggregation") if return_regions_as_labels: @@ -746,16 +760,17 @@ def rasterize_shapes_points( def _default_agg_func( data: DaskDataFrame | GeoDataFrame, value_key: str | None, return_single_channel: bool ) -> ds.reductions.Reduction: - import datashader as ds + from datashader.reductions import count, count_cat, first + from datashader.reductions import sum as ds_sum if value_key is None: - return ds.count() + return count() if data[VALUES_COLUMN].dtype != "category": - return ds.sum(VALUES_COLUMN) + return ds_sum(VALUES_COLUMN) if return_single_channel: data[VALUES_COLUMN] = data[VALUES_COLUMN].cat.codes + 1 - return ds.first(VALUES_COLUMN) + return first(VALUES_COLUMN) - return ds.count_cat(VALUES_COLUMN) + return count_cat(VALUES_COLUMN) diff --git a/src/spatialdata/_core/operations/rasterize_bins.py b/src/spatialdata/_core/operations/rasterize_bins.py index 7c4914ce7..f3ff32fbf 100644 --- a/src/spatialdata/_core/operations/rasterize_bins.py +++ b/src/spatialdata/_core/operations/rasterize_bins.py @@ -113,8 +113,8 @@ def rasterize_bins( min_row, min_col = table.obs[row_key].min(), table.obs[col_key].min() n_rows, n_cols = table.obs[row_key].max() - min_row + 1, table.obs[col_key].max() - min_col + 1 - y = (table.obs[row_key] - min_row).values - x = (table.obs[col_key] - min_col).values + y = (table.obs[row_key] - min_row).to_numpy() + x = (table.obs[col_key] - min_col).to_numpy() if isinstance(element, DataArray): transformations = get_transformation(element, get_all=True) @@ -127,23 +127,23 @@ def rasterize_bins( raise ValueError("At least 6 bins are needed to estimate the transformation.") random_indices = RNG.choice(table.n_obs, min(20, table.n_obs), replace=True) - location_ids = table.obs[instance_key].iloc[random_indices].values + location_ids = table.obs[instance_key].iloc[random_indices].to_numpy() sub_df = element.loc[location_ids] sub_table = table[random_indices] src = np.stack([sub_table.obs[col_key] - min_col, sub_table.obs[row_key] - min_row], axis=1) if isinstance(sub_df, GeoDataFrame): if isinstance(sub_df.iloc[0].geometry, Point): - sub_x = sub_df.geometry.x.values - sub_y = sub_df.geometry.y.values + sub_x = sub_df.geometry.x.to_numpy() + sub_y = sub_df.geometry.y.to_numpy() else: assert isinstance(sub_df.iloc[0].geometry, Polygon | MultiPolygon) - sub_x = sub_df.centroid.x - sub_y = sub_df.centroid.y + sub_x = sub_df.centroid.x.to_numpy() + sub_y = sub_df.centroid.y.to_numpy() else: assert isinstance(sub_df, DaskDataFrame) - sub_x = sub_df.x.compute().values - sub_y = sub_df.y.compute().values + sub_x = sub_df.x.compute().to_numpy() + sub_y = sub_df.y.compute().to_numpy() dst = np.stack([sub_x, sub_y], axis=1) to_bins = Sequence( @@ -164,10 +164,10 @@ def rasterize_bins( if return_region_as_labels: new_instance_key = _get_relabeled_column_name(instance_key) table.obs[new_instance_key] = _relabel_labels(table=table, instance_key=instance_key) - dtype = table.obs[new_instance_key].dtype - labels_element = np.zeros((n_rows, n_cols), dtype=dtype) + relabeled = table.obs[new_instance_key].to_numpy() + labels_element = np.zeros((n_rows, n_cols), dtype=relabeled.dtype) # make labels layer that can visualy represent the cells - labels_element[y, x] = table.obs[new_instance_key].values.T + labels_element[y, x] = relabeled.T return Labels2DModel.parse(data=labels_element, dims=("y", "x"), transformations=transformations) @@ -175,39 +175,44 @@ def rasterize_bins( from scipy.sparse import csc_matrix + x_matrix = table.X if (value_key is None or any(key in table.var_names for key in keys)) and not isinstance( - table.X, csc_matrix | np.ndarray + x_matrix, csc_matrix | np.ndarray ): raise ValueError( "To speed up bins rasterization, the X matrix in the table, when sparse, should be a csc_matrix matrix. " "This can be done by calling `table.X = table.X.tocsc()`.", ) - sparse_matrix = isinstance(table.X, csc_matrix) if isinstance(value_key, str): value_key = [value_key] if value_key is None: - dtype = table.X.dtype + # Guaranteed by the check above, which always runs when `value_key` is None. + assert isinstance(x_matrix, csc_matrix | np.ndarray) + dtype = x_matrix.dtype else: values = get_values(value_key=value_key, element=table) assert isinstance(values, pd.DataFrame) - dtype = values[value_key[0]].dtype + dtype = values[value_key[0]].to_numpy().dtype if value_key is None: shape = (n_rows, n_cols) + # Guaranteed by the check above, which always runs when `value_key` is None. + assert isinstance(x_matrix, csc_matrix | np.ndarray) + values_matrix = x_matrix + def channel_rasterization(block_id: tuple[int, int, int] | None) -> ArrayLike: image: ArrayLike = np.zeros((1, *shape), dtype=dtype) if block_id is None: return image - col = table.X[:, block_id[0]] - if sparse_matrix: - bins_indices, data = col.indices, col.data - image[0, y[bins_indices], x[bins_indices]] = data + if isinstance(values_matrix, csc_matrix): + col = values_matrix[:, [block_id[0]]] + image[0, y[col.indices], x[col.indices]] = col.data else: - image[0, y, x] = col + image[0, y, x] = values_matrix[:, block_id[0]] return image image = da.map_blocks( @@ -218,22 +223,28 @@ def channel_rasterization(block_id: tuple[int, int, int] | None) -> ArrayLike: else: image = np.zeros((len(value_key), n_rows, n_cols)) - if keys[0] in table.obs: - image[:, y, x] = table.obs[keys].values.T + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + if keys[0] in obs: + image[:, y, x] = obs[list(keys)].to_numpy().T else: + # Guaranteed by the check above, which runs when a key refers to a variable. + assert isinstance(x_matrix, csc_matrix | np.ndarray) for i, key in enumerate(keys): key_index = table.var_names.get_loc(key) - if sparse_matrix: - bins_indices = table.X[:, key_index].indices - image[i, y[bins_indices], x[bins_indices]] = table.X[:, key_index].data + assert isinstance(key_index, int) + if isinstance(x_matrix, csc_matrix): + column = x_matrix[:, [key_index]] + image[i, y[column.indices], x[column.indices]] = column.data else: - image[i, y, x] = table.X[:, key_index] + image[i, y, x] = x_matrix[:, key_index] return Image2DModel.parse( data=image, dims=("c", "y", "x"), transformations=transformations, - c_coords=keys, + c_coords=[str(key) for key in keys], ) @@ -260,7 +271,8 @@ def _relabel_labels(table: AnnData, instance_key: str) -> pd.Series: relabeled_instance_key_column = table.obs[instance_key].astype("category").cat.codes + int(zero_in_instance_key) # uses only allowed dtypes that passes our model validations, in particuar no uint8 dtype = _get_uint_dtype(value=relabeled_instance_key_column.max()) - return relabeled_instance_key_column.astype(dtype) + relabeled: pd.Series = relabeled_instance_key_column.astype(np.dtype(dtype)) + return relabeled def rasterize_bins_link_table_to_labels(sdata: SpatialData, table_name: str, rasterized_labels_name: str) -> None: @@ -279,8 +291,9 @@ def rasterize_bins_link_table_to_labels(sdata: SpatialData, table_name: str, ras rasterized_labels_name The name of the rasterized labels in the spatial data object. """ - _, region_key, instance_key = get_table_keys(sdata[table_name]) - sdata[table_name].obs[region_key] = pd.Categorical([rasterized_labels_name] * sdata[table_name].n_obs) + table = sdata.tables[table_name] + _, region_key, instance_key = get_table_keys(table) + table.obs[region_key] = pd.Categorical([rasterized_labels_name] * table.n_obs) relabled_instance_key = _get_relabeled_column_name(instance_key) sdata.set_table_annotates_spatialelement( table_name=table_name, region=rasterized_labels_name, region_key=region_key, instance_key=relabled_instance_key diff --git a/src/spatialdata/_core/operations/transform.py b/src/spatialdata/_core/operations/transform.py index 49b9ef722..410e92fc3 100644 --- a/src/spatialdata/_core/operations/transform.py +++ b/src/spatialdata/_core/operations/transform.py @@ -6,13 +6,13 @@ from functools import singledispatch from typing import TYPE_CHECKING, Any, cast -import dask import dask.array as da import dask.dataframe as dd import dask_image.ndinterp import numpy as np from dask.array.core import Array as DaskArray from dask.dataframe import DataFrame as DaskDataFrame +from dask.delayed import delayed from geopandas import GeoDataFrame from shapely import Point from xarray import DataArray, Dataset, DataTree @@ -22,6 +22,7 @@ from spatialdata._utils import disable_dask_tune_optimization from spatialdata.models import SpatialElement, get_axes_names, get_model from spatialdata.models._utils import DEFAULT_COORDINATE_SYSTEM, get_channel_names +from spatialdata.models.models import RasterSchema from spatialdata.transformations._utils import _get_scale, compute_coordinates, scale_radii if TYPE_CHECKING: @@ -148,7 +149,7 @@ def _set_transformation_for_transformed_elements( to_coordinate_system The coordinate system to which the data is to be transformed. This value must be None if maintain_positioning is True. - """ # noqa: D401 + """ from spatialdata.transformations import ( BaseTransformation, get_transformation, @@ -314,6 +315,7 @@ def _( data, transformation, maintain_positioning, to_coordinate_system ) schema = get_model(data) + assert issubclass(schema, RasterSchema) from spatialdata.transformations import get_transformation kwargs = {"prefilter": False, "order": 0} @@ -335,9 +337,10 @@ def _( maintain_positioning=maintain_positioning, to_coordinate_system=to_coordinate_system, ) - transformed_data = compute_coordinates(transformed_data) - schema.validate(transformed_data) - return transformed_data + computed = compute_coordinates(transformed_data) + assert isinstance(computed, DataArray) + schema.validate(computed) + return computed @transform.register(DataTree) @@ -351,6 +354,7 @@ def _( data, transformation, maintain_positioning, to_coordinate_system ) schema = get_model(data) + assert issubclass(schema, RasterSchema) from spatialdata.models import Image2DModel, Image3DModel, Labels2DModel, Labels3DModel from spatialdata.models._utils import TRANSFORM_KEY from spatialdata.transformations import get_transformation, set_transformation @@ -371,8 +375,10 @@ def _( transformed_dict = {} raster_translation: Translation | None = None for k, v in data.items(): + assert isinstance(v, DataTree) assert len(v) == 1 - xdata = v.values().__iter__().__next__() + xdata = next(iter(v.values())) + assert isinstance(xdata, DataArray) composed: BaseTransformation if k == "scale0": @@ -382,7 +388,7 @@ def _( composed = Sequence([scale, transformation, scale.inverse()]) transformed_dask, raster_translation_single_scale = _transform_raster( - data=xdata.data, axes=xdata.dims, transformation=composed, **kwargs + data=xdata.data, axes=tuple(str(dim) for dim in xdata.dims), transformation=composed, **kwargs ) # if a scale in the transformed data has zero shape, we skip it @@ -421,9 +427,10 @@ def _( maintain_positioning=maintain_positioning, to_coordinate_system=to_coordinate_system, ) - transformed_data = compute_coordinates(transformed_data) - schema.validate(transformed_data) - return transformed_data + computed_tree = compute_coordinates(transformed_data) + assert isinstance(computed_tree, DataTree) + schema.validate(computed_tree) + return computed_tree @transform.register(DaskDataFrame) @@ -478,7 +485,7 @@ def _( # non-monotonic or duplicate indices such as those produced by multi-file parquet reads), and # (b) the original index is preserved exactly. offsets = np.cumsum([0] + lengths) - delayed_parts = [dask.delayed(transformed_pd.iloc[offsets[i] : offsets[i + 1]]) for i in range(len(lengths))] + delayed_parts = [delayed(transformed_pd.iloc[offsets[i] : offsets[i + 1]]) for i in range(len(lengths))] transformed = dd.from_delayed(delayed_parts, meta=transformed_pd.iloc[:0]) # Preserve spatialdata_attrs (feature_key, instance_key, …) from the original element; # dd.from_delayed starts with empty attrs so we must copy them explicitly. @@ -500,7 +507,8 @@ def _( to_coordinate_system=to_coordinate_system, ) PointsModel.validate(transformed) - return transformed + validated: DaskDataFrame = transformed + return validated @transform.register(GeoDataFrame) diff --git a/src/spatialdata/_core/operations/vectorize.py b/src/spatialdata/_core/operations/vectorize.py index 414584589..431bdbd17 100644 --- a/src/spatialdata/_core/operations/vectorize.py +++ b/src/spatialdata/_core/operations/vectorize.py @@ -3,13 +3,16 @@ from functools import singledispatch from typing import TYPE_CHECKING, Any -import dask import numpy as np import pandas as pd import shapely +from dask.base import compute as dask_compute from dask.dataframe import DataFrame as DaskDataFrame +from dask.delayed import delayed from geopandas import GeoDataFrame +from scipy.sparse import csc_array, csc_matrix, csr_array, csr_matrix from shapely import MultiPolygon, Point, Polygon +from shapely.geometry.base import BaseGeometry from xarray import DataArray, DataTree if TYPE_CHECKING: @@ -75,7 +78,10 @@ def _(element: DataArray | DataTree, **kwargs: Any) -> GeoDataFrame: # reduce to the single scale case if isinstance(element, DataTree): - element_single_scale = element["scale0"].values().__iter__().__next__() + scale0 = element["scale0"] + assert isinstance(scale0, DataTree) + element_single_scale = next(iter(scale0.values())) + assert isinstance(element_single_scale, DataArray) else: element_single_scale = element shape = element_single_scale.shape @@ -85,8 +91,12 @@ def _(element: DataArray | DataTree, **kwargs: Any) -> GeoDataFrame: model = Image3DModel if "z" in axes else Image2DModel ones = model.parse(np.ones((1,) + shape), dims=("c",) + axes) aggregated = aggregate(values=ones, by=element_single_scale, agg_func="sum")["table"] - areas = aggregated.X.todense().A1.reshape(-1) + x = aggregated.X + assert x is not None + areas = np.asarray(x.todense() if isinstance(x, csr_matrix | csc_matrix | csr_array | csc_array) else x).reshape(-1) aobs = aggregated.obs + if not isinstance(aobs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(aobs).__name__}.") aobs["areas"] = areas aobs["radius"] = np.sqrt(areas / np.pi) @@ -139,7 +149,8 @@ def _get_centroids(element: SpatialElement) -> pd.DataFrame: d[INTRINSIC_COORDINATE_SYSTEM] = Identity() centroids = get_centroids(element, coordinate_system=INTRINSIC_COORDINATE_SYSTEM).compute() del d[INTRINSIC_COORDINATE_SYSTEM] - return centroids + computed_centroids: pd.DataFrame = centroids + return computed_centroids def _make_circles(element: DataArray | DataTree | GeoDataFrame, obs: pd.DataFrame) -> GeoDataFrame: @@ -166,7 +177,7 @@ def to_polygons(data: SpatialElement, buffer_resolution: int | None = None) -> G For example, you can set this configuration with: >>> import dask - >>> dask.config.set(scheduler='processes') + >>> dask.config.set(scheduler="processes") Parameters ---------- @@ -198,7 +209,10 @@ def _( # reduce to the single scale case if isinstance(element, DataTree): - element_single_scale = element["scale0"].values().__iter__().__next__() + scale0 = element["scale0"] + assert isinstance(scale0, DataTree) + element_single_scale = next(iter(scale0.values())) + assert isinstance(element_single_scale, DataArray) else: element_single_scale = element @@ -211,12 +225,12 @@ def _vectorize_chunk(chunk: np.ndarray, yoff: int, xoff: int) -> GeoDataFrame: return gdf tasks = [ - dask.delayed(_vectorize_chunk)(chunk, sum(chunk_sizes[0][:iy]), sum(chunk_sizes[1][:ix])) + delayed(_vectorize_chunk)(chunk, sum(chunk_sizes[0][:iy]), sum(chunk_sizes[1][:ix])) for iy, row in enumerate(element_single_scale.data.to_delayed()) for ix, chunk in enumerate(row) ] - results = dask.compute(*tasks) + results = dask_compute(*tasks) gdf = pd.concat(results) gdf = GeoDataFrame([_dissolve_on_overlaps(*item) for item in gdf.groupby("label")], columns=["label", "geometry"]) gdf.index = gdf["label"] @@ -262,11 +276,11 @@ def _vectorize_mask( ) -def _dissolve_on_overlaps(label: int, group: GeoDataFrame) -> GeoDataFrame: +def _dissolve_on_overlaps(label: int, group: GeoDataFrame) -> tuple[int, BaseGeometry]: if len(group) == 1: return (label, group.geometry.iloc[0]) if len(np.unique(group["chunk-location"])) == 1: - return (label, MultiPolygon(list(group.geometry))) + return (label, MultiPolygon(list(group.geometry.array))) return (label, group.dissolve().geometry.iloc[0]) diff --git a/src/spatialdata/_core/query/_utils.py b/src/spatialdata/_core/query/_utils.py index ca8888b3a..1d85ee8b8 100644 --- a/src/spatialdata/_core/query/_utils.py +++ b/src/spatialdata/_core/query/_utils.py @@ -112,9 +112,11 @@ def _create_slices_and_translation( def _process_data_tree_query_result(query_result: DataTree) -> DataTree | None: d = {} for k, data_tree in query_result.items(): - v = data_tree.values() + assert isinstance(data_tree, DataTree) + v = list(data_tree.values()) assert len(v) == 1 - xdata = v.__iter__().__next__() + xdata = v[0] + assert isinstance(xdata, DataArray) if 0 in xdata.shape: if k == "scale0": return None @@ -133,8 +135,8 @@ def _process_data_tree_query_result(query_result: DataTree) -> DataTree | None: if len(scales_to_keep) == 0: return None - d = {k: Dataset({"image": d[k]}) for k in scales_to_keep} - result = DataTree.from_dict(d) + datasets = {k: Dataset({"image": d[k]}) for k in scales_to_keep} + result = DataTree.from_dict(datasets) from dask.array.core import _check_regular_chunks @@ -168,9 +170,10 @@ def _process_query_result( if not _check_regular_chunks(result.data.chunks): result.data = result.data.rechunk(result.data.chunksize) elif isinstance(result, DataTree): - result = _process_data_tree_query_result(result) - if result is None: + processed_tree = _process_data_tree_query_result(result) + if processed_tree is None: return None + result = processed_tree result = compute_coordinates(result) diff --git a/src/spatialdata/_core/query/relational_query.py b/src/spatialdata/_core/query/relational_query.py index 98d92a0a5..1a672eefc 100644 --- a/src/spatialdata/_core/query/relational_query.py +++ b/src/spatialdata/_core/query/relational_query.py @@ -13,9 +13,11 @@ import pandas as pd import xarray as xr from anndata import AnnData +from annsel import AnnselAccessor from annsel.core.typing import Predicates from dask.dataframe import DataFrame as DaskDataFrame from geopandas import GeoDataFrame +from numpy.typing import NDArray from xarray import DataArray, DataTree from spatialdata._core.spatialdata import SpatialData @@ -95,14 +97,17 @@ def _( instances = da.unique(element.data).compute() else: assert isinstance(element, DataTree) - v = element["scale0"].values() + scale0 = element["scale0"] + assert isinstance(scale0, DataTree) + v = list(scale0.values()) assert len(v) == 1 - xdata = next(iter(v)) + xdata = v[0] # can be slow instances = da.unique(xdata.data).compute() - index = pd.Index(np.sort(instances)) + index: pd.Index = pd.Index(np.sort(instances)) if not return_background and 0 in index: - return index.drop(0) # drop the background label + without_background: pd.Index = index.drop([0]) # drop the background label + return without_background return index @@ -110,14 +115,16 @@ def _( def _( element: GeoDataFrame, ) -> pd.Index: - return element.index + index: pd.Index = element.index + return index @get_element_instances.register(DaskDataFrame) def _( element: DaskDataFrame, ) -> pd.Index: - return element.index + index: pd.Index = element.index + return index def _filter_table_by_elements(table: AnnData | None, elements_dict: dict[str, dict[str, Any]]) -> AnnData | None: @@ -161,7 +168,7 @@ def _filter_table_by_elements(table: AnnData | None, elements_dict: dict[str, di def _get_joined_table_indices( joined_indices: pd.Index | None, - element_indices: pd.RangeIndex, + element_indices: pd.Index, table_instance_key_column: pd.Series, match_rows: Literal["left", "no", "right"], ) -> pd.Index: @@ -184,7 +191,7 @@ def _get_joined_table_indices( ------- The indices that of the table that match the SpatialElement indices. """ - mask = np.isin(table_instance_key_column.values, element_indices) + mask = np.isin(table_instance_key_column.to_numpy(), element_indices) if joined_indices is None: if match_rows == "left": _, joined_indices = _match_rows(table_instance_key_column, mask, element_indices, match_rows) @@ -201,7 +208,7 @@ def _get_joined_table_indices( def _get_masked_element( - element_indices: pd.RangeIndex, + element_indices: pd.Index, element: SpatialElement, table_instance_key_column: pd.Series, match_rows: Literal["left", "no", "right"], @@ -225,9 +232,11 @@ def _get_masked_element( ------- The masked spatial element based on the provided indices and match rows. """ - mask = np.isin(table_instance_key_column.values, element_indices) + mask = np.isin(table_instance_key_column.to_numpy(), element_indices) masked_table_instance_key_column = table_instance_key_column[mask] - mask_values = mask_values if len(mask_values := masked_table_instance_key_column.values) != 0 else None + mask_values: NDArray[Any] | pd.Index | None = ( + masked_values if len(masked_values := masked_table_instance_key_column.to_numpy()) != 0 else None + ) if match_rows in ["left", "right"]: left_index, _ = _match_rows(table_instance_key_column, mask, element_indices, match_rows) @@ -241,8 +250,12 @@ def _get_masked_element( mask_values = np.asarray(element_indices)[order_mask] if isinstance(element, DaskDataFrame): - return element.map_partitions(lambda df: df.loc[mask_values], meta=element) - return element.loc[mask_values, :] + masked: SpatialElement = element.map_partitions(lambda df: df.loc[mask_values], meta=element) + return masked + if not isinstance(element, GeoDataFrame): + raise TypeError(f"Only points and shapes elements can be masked, got {type(element).__name__}.") + masked_shapes: GeoDataFrame = element.loc[mask_values if mask_values is not None else [], :] + return masked_shapes def _region_as_str_if_list_of_len_one(region: list[str]) -> str | list[str]: @@ -265,8 +278,11 @@ def _right_exclusive_join_spatialelement_table( regions, region_column_name, instance_key = get_table_keys(table) if isinstance(regions, str): regions = [regions] + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") # reset_index so group_df.index gives integer positions — safe with duplicate obs names - obs = table.obs.reset_index() + obs = obs.reset_index() groups_df = obs.groupby(by=region_column_name, observed=False) keep = np.zeros(len(table), dtype=bool) has_match = False @@ -280,7 +296,7 @@ def _right_exclusive_join_spatialelement_table( else: element_indices = get_element_instances(element) submask = ~table_instance_key_column.isin(element_indices) - keep[group_df.index[submask.values]] = True + keep[group_df.index[submask.to_numpy()]] = True has_match = True element_dict[element_type][name] = None else: @@ -305,7 +321,7 @@ def _right_join_spatialelement_table( table: AnnData, match_rows: Literal["left", "no", "right"], filter_label_pixels: bool | None = None, -) -> tuple[dict[str, Any], AnnData]: +) -> tuple[dict[str, Any], AnnData | None]: if match_rows == "left": warnings.warn( "Matching rows 'left' is not supported for 'right' join; it will be treated as 'no'.", @@ -316,7 +332,10 @@ def _right_join_spatialelement_table( regions, region_column_name, instance_key = get_table_keys(table) if isinstance(regions, str): regions = [regions] - groups_df = table.obs.groupby(by=region_column_name, observed=False) + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + groups_df = obs.groupby(by=region_column_name, observed=False) for element_type, name_element in element_dict.items(): for name, element in name_element.items(): if name in regions: @@ -354,11 +373,14 @@ def _inner_join_spatialelement_table( table: AnnData, match_rows: Literal["left", "no", "right"], filter_label_pixels: bool | None = None, -) -> tuple[dict[str, Any], AnnData]: +) -> tuple[dict[str, Any], AnnData | None]: regions, region_column_name, instance_key = get_table_keys(table) if isinstance(regions, str): regions = [regions] - obs = table.obs.reset_index() + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + obs = obs.reset_index() groups_df = obs.groupby(by=region_column_name, observed=False) joined_indices = None for element_type, name_element in element_dict.items(): @@ -434,14 +456,17 @@ def _left_exclusive_join_spatialelement_table( regions, region_column_name, instance_key = get_table_keys(table) if isinstance(regions, str): regions = [regions] - groups_df = table.obs.groupby(by=region_column_name, observed=False) + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + groups_df = obs.groupby(by=region_column_name, observed=False) for element_type, name_element in element_dict.items(): for name, element in name_element.items(): if name in regions: group_df = groups_df.get_group(name) table_instance_key_column = group_df[instance_key] if element_type in ["points", "shapes"]: - mask = ~np.isin(element.index, table_instance_key_column.values) + mask = ~np.isin(element.index, table_instance_key_column.to_numpy()) masked_element = element.loc[mask, :] if mask.sum() != 0 else None element_dict[element_type][name] = masked_element else: @@ -465,7 +490,7 @@ def _left_join_spatialelement_table( table: AnnData, match_rows: Literal["left", "no", "right"], filter_label_pixels: bool | None = None, -) -> tuple[dict[str, Any], AnnData]: +) -> tuple[dict[str, Any], AnnData | None]: if match_rows == "right": warnings.warn( "Matching rows 'right' is not supported for 'left' join; it will be treated as 'no'.", @@ -476,7 +501,10 @@ def _left_join_spatialelement_table( regions, region_column_name, instance_key = get_table_keys(table) if isinstance(regions, str): regions = [regions] - obs = table.obs.reset_index() + obs = table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + obs = obs.reset_index() groups_df = obs.groupby(by=region_column_name, observed=False) joined_indices = None for element_type, name_element in element_dict.items(): @@ -521,12 +549,15 @@ def _left_join_spatialelement_table( def _match_rows( table_instance_key_column: pd.Series, - mask: pd.Series, - element_indices: pd.RangeIndex, - match_rows: str, + mask: NDArray[np.bool_], + element_indices: pd.Index, + match_rows: Literal["left", "right"], ) -> tuple[pd.Index, pd.Index]: instance_id_df = pd.DataFrame( - {"instance_id": table_instance_key_column[mask].values, "index_right": table_instance_key_column[mask].index} + { + "instance_id": table_instance_key_column[mask].to_numpy(), + "index_right": table_instance_key_column[mask].index, + } ) element_index_df = pd.DataFrame({"index_left": element_indices}) @@ -553,7 +584,7 @@ class JoinTypes(Enum): right = member(partial(_right_join_spatialelement_table)) right_exclusive = member(partial(_right_exclusive_join_spatialelement_table)) - def __call__(self, *args: Any) -> tuple[dict[str, Any], AnnData]: + def __call__(self, *args: Any) -> tuple[dict[str, Any], AnnData | None]: return self.value(*args) @@ -581,13 +612,13 @@ def _validate_element_types_for_join( spatial_elements: list[SpatialElement] | None, table: AnnData | None, ) -> None: + elements_to_check: list[SpatialElement | AnnData] = [] if sdata is not None: - elements_to_check = [] for name in spatial_element_names: elements_to_check.append(sdata[name]) else: assert spatial_elements is not None - elements_to_check = spatial_elements + elements_to_check.extend(spatial_elements) for element in elements_to_check: model = get_model(element) @@ -604,7 +635,7 @@ def join_spatialelement_table( how: Literal["left", "left_exclusive", "inner", "right", "right_exclusive"] = "left", match_rows: Literal["no", "left", "right"] = "no", filter_label_pixels: bool | None = None, -) -> tuple[dict[str, Any], AnnData]: +) -> tuple[dict[str, Any], AnnData | None]: """ Join SpatialElement(s) and table together in SQL like manner. @@ -699,12 +730,18 @@ def join_spatialelement_table( if sdata is not None and table_name is not None: if table_name not in sdata.tables: raise ValueError(f"No table with name `{table_name}` found in the SpatialData object.") - table = sdata[table_name] + table = sdata.tables[table_name] spatial_element_names = ( spatial_element_names if isinstance(spatial_element_names, list) else [spatial_element_names] ) - spatial_elements = spatial_elements if isinstance(spatial_elements, list) else [spatial_elements] - _validate_element_types_for_join(sdata, spatial_element_names, spatial_elements, table) + spatial_elements_list: list[SpatialElement] | None + if spatial_elements is None: + spatial_elements_list = None + elif isinstance(spatial_elements, list): + spatial_elements_list = spatial_elements + else: + spatial_elements_list = [spatial_elements] + _validate_element_types_for_join(sdata, spatial_element_names, spatial_elements_list, table) elements_dict: dict[str, dict[str, Any]] if sdata is not None: @@ -717,11 +754,13 @@ def join_spatialelement_table( PointsModel: "points", } elements_dict = defaultdict(lambda: defaultdict(dict)) - for name, element in zip(spatial_element_names, spatial_elements, strict=True): + assert spatial_elements_list is not None + for name, element in zip(spatial_element_names, spatial_elements_list, strict=True): element_type = _model_to_type.get(get_model(element)) if element_type is not None: elements_dict[element_type][name] = element + assert table is not None elements_dict_joined, table = _call_join(elements_dict, table, how, match_rows, filter_label_pixels) return elements_dict_joined, table @@ -732,7 +771,7 @@ def _call_join( how: str, match_rows: Literal["no", "left", "right"], filter_label_pixels: bool | None = None, -) -> tuple[dict[str, Any], AnnData]: +) -> tuple[dict[str, Any], AnnData | None]: assert any(key in elements_dict for key in ["labels", "shapes", "points"]), ( "No valid element to join in spatial_element_name. Must provide at least one of either `labels`, `points` or " "`shapes`." @@ -783,15 +822,17 @@ def match_table_to_element(sdata: SpatialData, element_name: str, table_name: st match_element_to_table : Function to match a spatial element to a table. join_spatialelement_table : General function, to join spatial elements with a table with more control. """ - _, table = join_spatialelement_table( + _, matched_table = join_spatialelement_table( sdata=sdata, spatial_element_names=element_name, table_name=table_name, how="left", match_rows="left" ) - return table + if matched_table is None: + raise ValueError(f"No rows of table {table_name!r} match element {element_name!r}.") + return matched_table def match_element_to_table( sdata: SpatialData, element_name: str | list[str], table_name: str -) -> tuple[dict[str, Any], AnnData]: +) -> tuple[dict[str, Any], AnnData | None]: """ Filter the elements and make the indices match those in the table. @@ -856,12 +897,16 @@ def match_sdata_to_table( `Tables tutorial `_. """ if table is None: - table = sdata[table_name] + if table_name is None: + raise ValueError("Exactly one of `table_name` and `table` must be provided.") + table = sdata.tables[table_name] _, region_key, instance_key = get_table_keys(table) annotated_regions = SpatialData.get_annotated_regions(table) filtered_elements, filtered_table = join_spatialelement_table( sdata, spatial_element_names=annotated_regions, table=table, how=how, filter_label_pixels=filter_label_pixels ) + if filtered_table is None: + raise ValueError(f"No rows of the table match the elements {annotated_regions}.") filtered_table = TableModel.parse( filtered_table, region=annotated_regions, @@ -937,7 +982,8 @@ def filter_by_table_query( sdata.subset(element_names=element_names, filter_tables=filter_tables) if element_names else sdata ) - filtered_table: AnnData = sdata_subset.tables[table_name].an.filter( + # `.an` is registered on AnnData at import time by annsel, so it is reached through the accessor class here. + filtered_table: AnnData = AnnselAccessor(sdata_subset.tables[table_name]).filter( obs=obs_expr, var=var_expr, x=x_expr, obs_names=obs_names_expr, var_names=var_names_expr, layer=layer ) @@ -993,15 +1039,17 @@ def _locate_value( table_name: str | None = None, ) -> list[_ValueOrigin]: el = _get_element(element=element, sdata=sdata, element_name=element_name) - origins = [] + origins: list[_ValueOrigin] = [] model = get_model(el) if model not in [PointsModel, ShapesModel, Labels2DModel, Labels3DModel, TableModel]: raise ValueError(f"Cannot get value from {model}") # adding from the dataframe columns - if model in [PointsModel, ShapesModel] and value_key in el.columns: - value = el[value_key] - is_categorical = isinstance(value.dtype, pd.CategoricalDtype) - origins.append(_ValueOrigin(origin="df", is_categorical=is_categorical, value_key=value_key)) + if model in [PointsModel, ShapesModel]: + assert isinstance(el, GeoDataFrame | DaskDataFrame) + if value_key in el.columns: + value = el[value_key] + is_categorical = isinstance(value.dtype, pd.CategoricalDtype) + origins.append(_ValueOrigin(origin="df", is_categorical=is_categorical, value_key=value_key)) if model == TableModel: origins = _get_table_origins(element=el, value_key=value_key, origins=origins) @@ -1091,9 +1139,11 @@ def get_values( ) origin = origin_values.__iter__().__next__() if origin == "df": + assert isinstance(el, GeoDataFrame | DaskDataFrame) df = el[value_key_values] - if isinstance(el, DaskDataFrame): + if isinstance(df, DaskDataFrame): df = df.compute() + assert isinstance(df, pd.DataFrame) return df if (sdata is not None and table_name is not None) or isinstance(element, AnnData): if sdata is not None and table_name is not None: @@ -1112,6 +1162,8 @@ def get_values( if element_name is not None: matched_table = matched_table[matched_table.obs[region_key] == element_name] obs = matched_table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") if origin == "obs": df = obs[value_key_values].copy() @@ -1127,12 +1179,19 @@ def get_values( if isinstance(x, scipy.sparse.csr_matrix | scipy.sparse.csc_matrix | scipy.sparse.coo_matrix): x = x.todense() + if not isinstance(x, np.ndarray): + raise TypeError(f"Expected a dense array of values, got {type(x).__name__}.") df = pd.DataFrame(x, columns=value_key_values) if origin == "obsm": data = {} for key in value_key_values: data_values = matched_table.obsm[key] if len(value_key_values) == 1 and return_obsm_as_is: + if not isinstance(data_values, pd.DataFrame | np.ndarray): + raise TypeError( + f"`obsm[{key!r}]` must be a data frame or a dense array to be returned as is, " + f"got {type(data_values).__name__}." + ) return data_values if len(value_key_values) > 1 and return_obsm_as_is: warnings.warn( diff --git a/src/spatialdata/_core/query/spatial_query.py b/src/spatialdata/_core/query/spatial_query.py index bc9ca39b7..64ffed8cd 100644 --- a/src/spatialdata/_core/query/spatial_query.py +++ b/src/spatialdata/_core/query/spatial_query.py @@ -18,7 +18,7 @@ from spatialdata._core.query._utils import _get_filtered_or_unfiltered_tables, get_bounding_box_corners from spatialdata._core.spatialdata import SpatialData from spatialdata._docs import docstring_parameter -from spatialdata._types import ArrayLike, ListOrNDArrayFloating +from spatialdata._types import ArrayLike, ListOrNDArrayFloating, Raster_T from spatialdata._utils import _parse_list_into_array from spatialdata.models import ( PointsModel, @@ -80,7 +80,7 @@ def _get_bounding_box_corners_in_intrinsic_coordinates( The transformation from the element's intrinsic coordinate system (without c) to the query coordinate system (without c and adding missing axes) - """ # noqa: E501 + """ min_coordinate = _parse_list_into_array(min_coordinate) max_coordinate = _parse_list_into_array(max_coordinate) @@ -192,7 +192,8 @@ def _get_polygon_in_intrinsic_coordinates( assert isinstance(inverse, Affine) set_transformation(polygon_gdf, inverse, "inverse") - return transform(polygon_gdf, to_coordinate_system="inverse") + transformed: GeoDataFrame = transform(polygon_gdf, to_coordinate_system="inverse") + return transformed def _get_axes_of_transformation( @@ -284,7 +285,7 @@ def _get_case_of_bounding_box_query( See https://github.com/scverse/spatialdata/pull/151#issuecomment-1444609101 for a detailed overview of the logic of this code, or see the comments below for an overview of the cases we consider. - """ # noqa: D401 + """ transform_dimension = np.linalg.matrix_rank(m_without_c_linear) transform_coordinate_length = len(output_axes_without_c) data_dim = len(input_axes_without_c) @@ -442,11 +443,22 @@ def _bounding_box_mask_points( def _dict_query_dispatcher( - elements: dict[str, SpatialElement], query_function: Callable[[SpatialElement], SpatialElement], **kwargs: Any + elements: dict[str, SpatialElement], + query_function: Callable[ + ..., + SpatialElement + | SpatialData + | Mapping[str, slice] + | list[Mapping[str, slice]] + | list[DataArray] + | list[DataTree] + | None, + ], + **kwargs: Any, ) -> dict[str, SpatialElement]: from spatialdata.transformations import get_transformation - queried_elements = {} + queried_elements: dict[str, SpatialElement] = {} for key, element in elements.items(): target_coordinate_system = kwargs["target_coordinate_system"] d = get_transformation(element, get_all=True) @@ -455,6 +467,7 @@ def _dict_query_dispatcher( result = query_function(element, **kwargs) if result is not None: # query returns None if it is empty + assert isinstance(result, DataArray | DataTree | GeoDataFrame | DaskDataFrame) queried_elements[key] = result return queried_elements @@ -470,7 +483,15 @@ def bounding_box_query( return_request_only: bool = False, filter_table: bool = True, **kwargs: Any, -) -> SpatialElement | SpatialData | None: +) -> ( + SpatialElement + | SpatialData + | Mapping[str, slice] + | list[Mapping[str, slice]] + | list[DataArray] + | list[DataTree] + | None +): """ Query a SpatialData object or SpatialElement within a bounding box. @@ -520,7 +541,8 @@ def _( ) -> SpatialData: min_coordinate = _parse_list_into_array(min_coordinate) max_coordinate = _parse_list_into_array(max_coordinate) - new_elements = {} + new_elements: dict[str, dict[str, SpatialElement]] = {} + queried_by_type: dict[str, SpatialElement] = {} for element_type in ["points", "images", "labels", "shapes"]: elements = getattr(sdata, element_type) queried_elements = _dict_query_dispatcher( @@ -532,10 +554,25 @@ def _( target_coordinate_system=target_coordinate_system, ) new_elements[element_type] = queried_elements + queried_by_type.update(queried_elements) tables = _get_filtered_or_unfiltered_tables(filter_table, new_elements, sdata) - return SpatialData(**new_elements, tables=tables, attrs=sdata.attrs) + images: dict[str, Raster_T] = {} + labels: dict[str, Raster_T] = {} + points: dict[str, DaskDataFrame] = {} + shapes: dict[str, GeoDataFrame] = {} + for name, element in queried_by_type.items(): + if isinstance(element, GeoDataFrame): + shapes[name] = element + elif isinstance(element, DaskDataFrame): + points[name] = element + elif name in sdata.images: + images[name] = element + else: + labels[name] = element + + return SpatialData(images=images, labels=labels, points=points, shapes=shapes, tables=tables, attrs=sdata.attrs) @bounding_box_query.register(DataArray) @@ -547,7 +584,7 @@ def _( max_coordinate: ListOrNDArrayFloating, target_coordinate_system: str, return_request_only: bool = False, -) -> DataArray | DataTree | Mapping[str, slice] | list[DataArray] | list[DataTree] | None: +) -> DataArray | DataTree | Mapping[str, slice] | list[Mapping[str, slice]] | list[DataArray] | list[DataTree] | None: """Implement bounding box query for Spatialdata supported DataArray. Notes @@ -587,7 +624,7 @@ def _( slices, translation_vectors = _create_slices_and_translation(min_values_np, max_values_np) if min_values.ndim == 2: # Multiple boxes - selection: list[dict[str, Any]] | dict[str, Any] = [ + selection: list[dict[str, slice]] | dict[str, slice] = [ { axis: slice(slices[box_idx, axis_idx, 0], slices[box_idx, axis_idx, 1]) for axis_idx, axis in enumerate(axes) @@ -600,23 +637,30 @@ def _( translation_vectors = translation_vectors[0].tolist() if return_request_only: - return selection + if isinstance(selection, dict): + selected: Mapping[str, slice] | list[Mapping[str, slice]] = selection + else: + selected = list(selection) + return selected # query the data - query_result: DataArray | DataTree | list[DataArray] | list[DataTree] | None = ( - image.sel(selection) if isinstance(selection, dict) else [image.sel(sel) for sel in selection] - ) - - if isinstance(query_result, list): - processed_results = [] - for result, translation_vector in zip(query_result, translation_vectors, strict=True): - processed_result = _process_query_result(result, translation_vector, axes) - if processed_result is not None: - processed_results.append(processed_result) - query_result = processed_results if processed_results else None - else: - query_result = _process_query_result(query_result, translation_vectors, axes) - return query_result + if isinstance(selection, dict): + single = image.sel(selection) + assert isinstance(single, DataArray | DataTree) + return _process_query_result(single, translation_vectors, axes) + + processed_results: list[DataArray | DataTree] = [] + for sel, translation_vector in zip(selection, translation_vectors, strict=True): + result = image.sel(sel) + assert isinstance(result, DataArray | DataTree) + processed_result = _process_query_result(result, translation_vector, axes) + if processed_result is not None: + processed_results.append(processed_result) + if not processed_results: + return None + if all(isinstance(r, DataArray) for r in processed_results): + return [r for r in processed_results if isinstance(r, DataArray)] + return [r for r in processed_results if isinstance(r, DataTree)] @bounding_box_query.register(DaskDataFrame) @@ -626,7 +670,7 @@ def _( min_coordinate: ListOrNDArrayFloating, max_coordinate: ListOrNDArrayFloating, target_coordinate_system: str, -) -> DaskDataFrame | list[DaskDataFrame] | None: +) -> DaskDataFrame | list[DaskDataFrame | None] | None: from spatialdata import transform from spatialdata.transformations import get_transformation @@ -766,7 +810,7 @@ def _( min_coordinate: ListOrNDArrayFloating, max_coordinate: ListOrNDArrayFloating, target_coordinate_system: str, -) -> GeoDataFrame | list[GeoDataFrame] | None: +) -> GeoDataFrame | list[GeoDataFrame | None] | None: from spatialdata.transformations import get_transformation min_coordinate = _parse_list_into_array(min_coordinate) @@ -870,7 +914,8 @@ def _( filter_table: bool = True, clip: bool = False, ) -> SpatialData: - new_elements = {} + new_elements: dict[str, dict[str, SpatialElement]] = {} + queried_by_type: dict[str, SpatialElement] = {} for element_type in ["points", "images", "labels", "shapes"]: elements = getattr(sdata, element_type) queried_elements = _dict_query_dispatcher( @@ -881,10 +926,25 @@ def _( clip=clip, ) new_elements[element_type] = queried_elements + queried_by_type.update(queried_elements) tables = _get_filtered_or_unfiltered_tables(filter_table, new_elements, sdata) - return SpatialData(**new_elements, tables=tables, attrs=sdata.attrs) + images: dict[str, Raster_T] = {} + labels: dict[str, Raster_T] = {} + points: dict[str, DaskDataFrame] = {} + shapes: dict[str, GeoDataFrame] = {} + for name, element in queried_by_type.items(): + if isinstance(element, GeoDataFrame): + shapes[name] = element + elif isinstance(element, DaskDataFrame): + points[name] = element + elif name in sdata.images: + images[name] = element + else: + labels[name] = element + + return SpatialData(images=images, labels=labels, points=points, shapes=shapes, tables=tables, attrs=sdata.attrs) @polygon_query.register(DataArray) @@ -895,9 +955,20 @@ def _( target_coordinate_system: str, return_request_only: bool = False, **kwargs: Any, -) -> DataArray | DataTree | None: +) -> ( + DataArray + | DataTree + | GeoDataFrame + | DaskDataFrame + | SpatialData + | Mapping[str, slice] + | list[Mapping[str, slice]] + | list[DataArray] + | list[DataTree] + | None +): gdf = GeoDataFrame(geometry=[polygon]) - min_x, min_y, max_x, max_y = gdf.bounds.values.flatten().tolist() + min_x, min_y, max_x, max_y = gdf.bounds.to_numpy().flatten().tolist() return bounding_box_query( image, min_coordinate=[min_x, min_y], diff --git a/src/spatialdata/_core/spatialdata.py b/src/spatialdata/_core/spatialdata.py index 6ee2296c8..d7941db2f 100644 --- a/src/spatialdata/_core/spatialdata.py +++ b/src/spatialdata/_core/spatialdata.py @@ -51,6 +51,8 @@ ) if TYPE_CHECKING: + from pandas._typing import DtypeObj + from spatialdata._core.query.spatial_query import BaseSpatialRequest from spatialdata._io.format import ( SpatialDataContainerFormatType, @@ -145,30 +147,30 @@ def __init__( exc_type=(ValueError, KeyError), ) as collect_error: if images is not None: - for k, v in images.items(): + for k, image in images.items(): with collect_error(location=("images", k)): - self.images[k] = v + self.images[k] = image if labels is not None: - for k, v in labels.items(): + for k, labels_element in labels.items(): with collect_error(location=("labels", k)): - self.labels[k] = v + self.labels[k] = labels_element if shapes is not None: - for k, v in shapes.items(): + for k, shapes_element in shapes.items(): with collect_error(location=("shapes", k)): - self.shapes[k] = v + self.shapes[k] = shapes_element if points is not None: - for k, v in points.items(): + for k, points_element in points.items(): with collect_error(location=("points", k)): - self.points[k] = v + self.points[k] = points_element if tables is not None: - for k, v in tables.items(): + for k, table in tables.items(): with collect_error(location=("tables", k)): - self.validate_table_in_spatialdata(v) - self.tables[k] = v + self.validate_table_in_spatialdata(table) + self.tables[k] = table def validate_table_in_spatialdata(self, table: AnnData) -> None: """ @@ -202,19 +204,23 @@ def validate_table_in_spatialdata(self, table: AnnData) -> None: UserWarning, stacklevel=2, ) + elif isinstance(element, AnnData): + raise TypeError(f"The table is annotating {r!r}, which is itself a table.") else: + dtype: DtypeObj if isinstance(element, DataArray): dtype = element.dtype elif isinstance(element, DataTree): dtype = element.scale0.ds.dtypes["image"] else: dtype = element.index.dtype - if dtype != table.obs[instance_key].dtype and ( - dtype is str or table.obs[instance_key].dtype is str + instance_dtype = self.get_instance_key_column(table).dtype + if dtype != instance_dtype and ( + pd.api.types.is_string_dtype(dtype) or pd.api.types.is_string_dtype(instance_dtype) ): raise TypeError( f"Table instance_key column ({instance_key}) has a dtype " - f"({table.obs[instance_key].dtype}) that does not match the dtype of the indices of " + f"({instance_dtype}) that does not match the dtype of the indices of " f"the annotated element ({dtype})." ) @@ -257,9 +263,12 @@ def get_region_key_column(table: AnnData) -> pd.Series: If the region key column is not found in table.obs. """ _, region_key, _ = get_table_keys(table) - if table.obs.get(region_key) is not None: - return table.obs[region_key] - raise KeyError(f"{region_key} is set as region key column. However the column is not found in table.obs.") + column = table.obs.get(region_key) + if column is None: + raise KeyError(f"{region_key} is set as region key column. However the column is not found in table.obs.") + if not isinstance(column, pd.Series): + raise TypeError(f"`table.obs[{region_key!r}]` must be a pandas Series, got {type(column).__name__}.") + return column @staticmethod def get_instance_key_column(table: AnnData) -> pd.Series: @@ -282,9 +291,14 @@ def get_instance_key_column(table: AnnData) -> pd.Series: """ _, _, instance_key = get_table_keys(table) - if table.obs.get(instance_key) is not None: - return table.obs[instance_key] - raise KeyError(f"{instance_key} is set as instance key column. However the column is not found in table.obs.") + column = table.obs.get(instance_key) + if column is None: + raise KeyError( + f"{instance_key} is set as instance key column. However the column is not found in table.obs." + ) + if not isinstance(column, pd.Series): + raise TypeError(f"`table.obs[{instance_key!r}]` must be a pandas Series, got {type(column).__name__}.") + return column def set_channel_names(self, element_name: str, channel_names: str | list[str], write: bool = False) -> None: """Set the channel names for an image `SpatialElement` in the `SpatialData` object. @@ -311,7 +325,7 @@ def set_channel_names(self, element_name: str, channel_names: str | list[str], w @staticmethod def _set_table_annotation_target( table: AnnData, - region: str | pd.Series, + region: str | pd.Series | list[str], region_key: str, instance_key: str, ) -> None: @@ -358,7 +372,7 @@ def _set_table_annotation_target( @staticmethod def _change_table_annotation_target( table: AnnData, - region: str | pd.Series, + region: str | pd.Series | list[str], region_key: None | str = None, instance_key: None | str = None, ) -> None: @@ -522,6 +536,10 @@ def aggregate( """ from spatialdata._core.operations.aggregate import aggregate + if values is None: + raise ValueError("`values` must be specified.") + if by is None: + raise ValueError("`by` must be specified.") if isinstance(values, str) and values_sdata is None: values_sdata = self if isinstance(by, str) and by_sdata is None: @@ -619,20 +637,21 @@ def filter_by_coordinate_system( from spatialdata.transformations.operations import get_transformation - elements: dict[str, dict[str, SpatialElement]] = {} - element_names_in_coordinate_system = [] - if isinstance(coordinate_system, str): - coordinate_system = [coordinate_system] - for element_type, element_name, element in self._gen_elements(): - if element_type != "tables": - transformations = get_transformation(element, get_all=True) - assert isinstance(transformations, dict) - for cs in coordinate_system: - if cs in transformations: - if element_type not in elements: - elements[element_type] = {} - elements[element_type][element_name] = element - element_names_in_coordinate_system.append(element_name) + coordinate_systems = [coordinate_system] if isinstance(coordinate_system, str) else coordinate_system + element_names_in_coordinate_system: list[str] = [] + + def _is_in_coordinate_system(element_name: str, element: SpatialElement) -> bool: + transformations = get_transformation(element, get_all=True) + assert isinstance(transformations, dict) + if any(cs in transformations for cs in coordinate_systems): + element_names_in_coordinate_system.append(element_name) + return True + return False + + images = {k: v for k, v in self.images.items() if _is_in_coordinate_system(k, v)} + labels = {k: v for k, v in self.labels.items() if _is_in_coordinate_system(k, v)} + points = {k: v for k, v in self.points.items() if _is_in_coordinate_system(k, v)} + shapes = {k: v for k, v in self.shapes.items() if _is_in_coordinate_system(k, v)} tables = self._filter_tables( set(), filter_tables, @@ -641,7 +660,7 @@ def filter_by_coordinate_system( element_names=element_names_in_coordinate_system, ) - return SpatialData(**elements, tables=tables, attrs=self.attrs) + return SpatialData(images=images, labels=labels, points=points, shapes=shapes, tables=tables, attrs=self.attrs) # TODO: move to relational query with refactor def _filter_tables( @@ -701,18 +720,18 @@ def _filter_tables( elements_dict[element_type] = { name: elements[name] for name in element_names if name in elements } - table = _filter_table_by_elements(table, elements_dict=elements_dict) - if table is not None and len(table) != 0: - tables[table_name] = table + filtered_table = _filter_table_by_elements(table, elements_dict=elements_dict) + if filtered_table is not None and len(filtered_table) != 0: + tables[table_name] = filtered_table elif by == "elements": from spatialdata._core.query.relational_query import ( _filter_table_by_elements, ) assert elements_dict is not None - table = _filter_table_by_elements(table, elements_dict=elements_dict) - if table is not None and len(table) != 0: - tables[table_name] = table + filtered_table = _filter_table_by_elements(table, elements_dict=elements_dict) + if filtered_table is not None and len(filtered_table) != 0: + tables[table_name] = filtered_table else: tables = self.tables @@ -811,6 +830,10 @@ def transform_element_to_coordinate_system( ) element = self.get(element_name) + if element is None: + raise KeyError(f"Element {element_name!r} not found in the SpatialData object.") + if isinstance(element, AnnData): + raise TypeError(f"Element {element_name!r} is a table, which has no coordinate system.") t = get_transformation_between_coordinate_systems(self, element, target_coordinate_system) if maintain_positioning: transformed = transform(element, transformation=t, maintain_positioning=maintain_positioning) @@ -855,6 +878,7 @@ def transform_element_to_coordinate_system( assert seq.transformations[1] is t.transformations[0] new_tt = seq.transformations[0] set_transformation(transformed, new_tt, target_coordinate_system) + assert isinstance(transformed, DataArray | DataTree | GeoDataFrame | DaskDataFrame) return transformed def transform_to_coordinate_system( @@ -879,18 +903,32 @@ def transform_to_coordinate_system( The transformed SpatialData. """ sdata = self.filter_by_coordinate_system(target_coordinate_system, filter_tables=False) - elements: dict[str, dict[str, SpatialElement]] = {} - for element_type, element_name, _ in sdata.gen_elements(): - if element_type != "tables": - transformed = sdata.transform_element_to_coordinate_system( - element_name, - target_coordinate_system, - maintain_positioning=maintain_positioning, - ) - if element_type not in elements: - elements[element_type] = {} - elements[element_type][element_name] = transformed - return SpatialData(**elements, tables=sdata.tables, attrs=self.attrs) + images: dict[str, Raster_T] = {} + labels: dict[str, Raster_T] = {} + points: dict[str, DaskDataFrame] = {} + shapes: dict[str, GeoDataFrame] = {} + for element_type, element_name, _ in sdata.gen_spatial_elements(): + transformed = sdata.transform_element_to_coordinate_system( + element_name, + target_coordinate_system, + maintain_positioning=maintain_positioning, + ) + # Transforming an element does not change which kind of element it is. + if element_type == "images": + assert isinstance(transformed, DataArray | DataTree) + images[element_name] = transformed + elif element_type == "labels": + assert isinstance(transformed, DataArray | DataTree) + labels[element_name] = transformed + elif element_type == "points": + assert isinstance(transformed, DaskDataFrame) + points[element_name] = transformed + else: + assert isinstance(transformed, GeoDataFrame) + shapes[element_name] = transformed + return SpatialData( + images=images, labels=labels, points=points, shapes=shapes, tables=sdata.tables, attrs=self.attrs + ) def elements_are_self_contained(self) -> dict[str, bool]: """ @@ -1004,7 +1042,13 @@ def find_groups(obj: zarr.Group, path: str) -> None: for element_type in root: if element_type in ["images", "labels", "points", "shapes", "tables"]: - for element_name in root[element_type]: + element_type_group = root[element_type] + if not isinstance(element_type_group, zarr.Group): + raise TypeError( + f"Expected a zarr group holding the {element_type!r} elements, " + f"got {type(element_type_group).__name__}." + ) + for element_name in element_type_group: path = f"{element_type}/{element_name}" elements_in_zarr.append(path) # root.visit(lambda path: find_groups(root[path], path)) @@ -1101,6 +1145,7 @@ def _validate_all_elements(self) -> None: with collect_error(location=element_path): check_valid_name(element_name) if element_type == "tables": + assert isinstance(element, AnnData) with collect_error(location=element_path): validate_table_attr_keys(element, location=element_path) @@ -1183,7 +1228,14 @@ def write( self._validate_all_elements() store = _resolve_zarr_store(file_path) - zarr_format = parsed["SpatialData"].zarr_format + container_zarr_format = parsed["SpatialData"].zarr_format + zarr_format: Literal[2, 3] + if container_zarr_format == 2: + zarr_format = 2 + elif container_zarr_format == 3: + zarr_format = 3 + else: + raise ValueError(f"Unsupported zarr format {container_zarr_format}; expected 2 or 3.") zarr_group = zarr.create_group(store=store, overwrite=overwrite, zarr_format=zarr_format) self.write_attrs(zarr_group=zarr_group, sdata_format=parsed["SpatialData"]) store.close() @@ -1240,7 +1292,19 @@ def _write_element( write_shapes, write_table, ) - from spatialdata._io.format import _parse_formats + from spatialdata._io.format import ( + PointsFormatV01, + PointsFormatV02, + RasterFormatV01, + RasterFormatV02, + RasterFormatV03, + ShapesFormatV01, + ShapesFormatV02, + ShapesFormatV03, + TablesFormatV01, + TablesFormatV02, + _parse_formats, + ) if parsed_formats is None: parsed_formats = _parse_formats(formats=parsed_formats) @@ -1248,43 +1312,64 @@ def _write_element( if element_type != "tables": from spatialdata.models import validate_element + assert not isinstance(element, AnnData) validate_element(element) if element_type == "images": + if not isinstance(element, DataArray | DataTree): + raise TypeError(f"Element {element_name!r} of type {element_type!r} is not a raster element.") + raster_format = parsed_formats["raster"] + assert isinstance(raster_format, RasterFormatV01 | RasterFormatV02 | RasterFormatV03) write_image( image=element, group=element_group, name=element_name, - element_format=parsed_formats["raster"], + element_format=raster_format, raster_compressor=raster_compressor, ) elif element_type == "labels": + if not isinstance(element, DataArray | DataTree): + raise TypeError(f"Element {element_name!r} of type {element_type!r} is not a raster element.") + raster_format = parsed_formats["raster"] + assert isinstance(raster_format, RasterFormatV01 | RasterFormatV02 | RasterFormatV03) write_labels( labels=element, group=root_group, name=element_name, - element_format=parsed_formats["raster"], + element_format=raster_format, raster_compressor=raster_compressor, ) elif element_type == "points": + if not isinstance(element, DaskDataFrame): + raise TypeError(f"Element {element_name!r} of type {element_type!r} is not a points element.") + points_format = parsed_formats["points"] + assert isinstance(points_format, PointsFormatV01 | PointsFormatV02) write_points( points=element, group=element_group, - element_format=parsed_formats["points"], + element_format=points_format, ) elif element_type == "shapes": + if not isinstance(element, GeoDataFrame): + raise TypeError(f"Element {element_name!r} of type {element_type!r} is not a shapes element.") + shapes_format = parsed_formats["shapes"] + assert isinstance(shapes_format, ShapesFormatV01 | ShapesFormatV02 | ShapesFormatV03) write_shapes( shapes=element, group=element_group, - element_format=parsed_formats["shapes"], + element_format=shapes_format, geometry_encoding=shapes_geometry_encoding, ) elif element_type == "tables": + if not isinstance(element, AnnData): + raise TypeError(f"Element {element_name!r} of type {element_type!r} is not a table.") + tables_format = parsed_formats["tables"] + assert isinstance(tables_format, TablesFormatV01 | TablesFormatV02) write_table( table=element, group=element_type_group, name=element_name, - element_format=parsed_formats["tables"], + element_format=tables_format, convert_strings_to_categoricals=convert_table_strings_to_categoricals, ) else: @@ -1367,6 +1452,7 @@ def write_element( if element_type is None: raise ValueError(f"Element with name {element_name} not found in SpatialData object.") if element_type == "tables": + assert isinstance(element, AnnData) validate_table_attr_keys(element) self._check_element_not_on_disk_with_different_type(element_type=element_type, element_name=element_name) @@ -1470,7 +1556,12 @@ def delete_element_from_disk(self, element_name: str | list[str]) -> None: # delete the element store = _resolve_zarr_store(self.path) root = zarr.open_group(store=store, mode="r+", use_consolidated=False) - del root[element_type][element_name] + element_type_group = root[element_type] + if not isinstance(element_type_group, zarr.Group): + raise TypeError( + f"Expected a zarr group holding the {element_type!r} elements, got {type(element_type_group).__name__}." + ) + del element_type_group[element_name] store.close() if self.has_consolidated_metadata(): @@ -1591,6 +1682,7 @@ def write_channel_names(self, element_name: str | None = None) -> None: from spatialdata._io._utils import overwrite_channel_names + assert isinstance(element, DataArray | DataTree) overwrite_channel_names(element_group, element) else: raise ValueError(f"Can't set channel names for element of type '{element_type}'.") @@ -1621,6 +1713,8 @@ def write_transformations(self, element_name: str | None = None) -> None: if validation_result is None: return element_type, element = validation_result + if isinstance(element, AnnData): + raise TypeError(f"Element {element_name!r} is a table, which has no transformations.") from spatialdata.transformations.operations import get_transformation @@ -1664,6 +1758,8 @@ def _element_type_from_element_name(self, element_name: str) -> str: element = self.get(element_name) if element is None: raise ValueError(f"Element with name {element_name} not found in SpatialData object.") + if isinstance(element, AnnData): + return "tables" located = self.locate_element(element) element_type = None @@ -1813,7 +1909,7 @@ def _flatten_mapping(m: Mapping[str, Any], parent_key: str = "", sep: str = "_") if key not in self.attrs: raise KeyError(f"The key '{key}' was not found in sdata.attrs.") - data = self.attrs[key] + data: dict[str, Any] | str | pd.DataFrame = self.attrs[key] # If the data is a mapping, flatten it if flatten and isinstance(data, Mapping): @@ -1890,7 +1986,9 @@ def read( if reconsolidate_metadata: from spatialdata._io.io_zarr import _write_consolidated_metadata - _write_consolidated_metadata(file_path) + if isinstance(file_path, zarr.Group): + raise TypeError("Consolidating metadata requires a path, not an already-open zarr group.") + _write_consolidated_metadata(str(file_path)) return read_zarr(file_path, selection=selection) @@ -2042,7 +2140,8 @@ def h(s: str) -> str: descr += f"{h(attr + 'level1.1')}{k!r}: {descr_class} {v.shape}" else: if isinstance(v, DataArray): - descr += f"{h(attr + 'level1.1')}{k!r}: {descr_class}[{''.join(v.dims)}] {v.shape}" + dim_names = "".join(str(dim) for dim in v.dims) + descr += f"{h(attr + 'level1.1')}{k!r}: {descr_class}[{dim_names}] {v.shape}" elif isinstance(v, DataTree): shapes = [] dims: str | None = None @@ -2053,7 +2152,7 @@ def h(s: str) -> str: vv = v[pyramid_level][dataset_name] shape = vv.shape if dims is None: - dims = "".join(vv.dims) + dims = "".join(str(dim) for dim in vv.dims) shapes.append(shape) descr += f"{h(attr + 'level1.1')}{k!r}: {descr_class}[{dims}] {', '.join(map(str, shapes))}" else: @@ -2139,9 +2238,25 @@ def _gen_spatial_element_values(self) -> Generator[SpatialElement, None, None]: A generator that yields spatial element objects contained in the SpatialData instance. """ - for element_type in ["images", "labels", "points", "shapes"]: - d = getattr(SpatialData, element_type).fget(self) - yield from d.values() + for _, _, element in self._gen_spatial_elements(): + yield element + + def _gen_spatial_elements(self) -> Generator[tuple[str, str, SpatialElement], None, None]: + """ + Generate the images, labels, points and shapes contained in the SpatialData instance. + + Returns + ------- + A generator object that returns a tuple containing the type of the element, its name, and the element itself. + """ + for element_type, elements in ( + ("images", self.images), + ("labels", self.labels), + ("points", self.points), + ("shapes", self.shapes), + ): + for name, element in elements.items(): + yield element_type, name, element def _gen_elements( self, include_tables: bool = False @@ -2159,13 +2274,10 @@ def _gen_elements( A generator object that returns a tuple containing the type of the element, its name, and the element itself. """ - element_types = ["images", "labels", "points", "shapes"] + yield from self._gen_spatial_elements() if include_tables: - element_types.append("tables") - for element_type in element_types: - d = getattr(SpatialData, element_type).fget(self) - for k, v in d.items(): - yield element_type, k, v + for name, table in self.tables.items(): + yield "tables", name, table def gen_spatial_elements( self, @@ -2180,7 +2292,7 @@ def gen_spatial_elements( A generator that yields tuples containing the element_type (string), name, and SpatialElement objects themselves. """ - return self._gen_elements() + return self._gen_spatial_elements() def gen_elements( self, @@ -2262,22 +2374,31 @@ def init_from_elements( ------- The SpatialData object. """ - elements_dict: dict[str, SpatialElement | AnnData] = {} + images: dict[str, Raster_T] = {} + labels: dict[str, Raster_T] = {} + points: dict[str, DaskDataFrame] = {} + shapes: dict[str, GeoDataFrame] = {} + tables: dict[str, AnnData] = {} for name, element in elements.items(): + # get_model() returns a schema only for the element type it matched, so the element has that type. model = get_model(element) if model in [Image2DModel, Image3DModel]: - element_type = "images" + assert isinstance(element, DataArray | DataTree) + images[name] = element elif model in [Labels2DModel, Labels3DModel]: - element_type = "labels" + assert isinstance(element, DataArray | DataTree) + labels[name] = element elif model == PointsModel: - element_type = "points" + assert isinstance(element, DaskDataFrame) + points[name] = element elif model == TableModel: - element_type = "tables" + assert isinstance(element, AnnData) + tables[name] = element else: assert model == ShapesModel - element_type = "shapes" - elements_dict.setdefault(element_type, {})[name] = element - return cls(**elements_dict, attrs=attrs) + assert isinstance(element, GeoDataFrame) + shapes[name] = element + return cls(images=images, labels=labels, points=points, shapes=shapes, tables=tables, attrs=attrs) def subset( self, @@ -2304,14 +2425,35 @@ def subset( ------- The subsetted SpatialData object. """ - elements_dict: dict[str, SpatialElement] = {} + elements_dict: dict[str, dict[str, SpatialElement]] = {} + images: dict[str, Raster_T] = {} + labels: dict[str, Raster_T] = {} + points: dict[str, DaskDataFrame] = {} + shapes: dict[str, GeoDataFrame] = {} names_tables_to_keep: set[str] = set() - for element_type, element_name, element in self._gen_elements(include_tables=True): - if element_name in element_names: - if element_type != "tables": - elements_dict.setdefault(element_type, {})[element_name] = element - else: - names_tables_to_keep.add(element_name) + for name, image in self.images.items(): + if name in element_names: + images[name] = image + for name, labels_element in self.labels.items(): + if name in element_names: + labels[name] = labels_element + for name, points_element in self.points.items(): + if name in element_names: + points[name] = points_element + for name, shapes_element in self.shapes.items(): + if name in element_names: + shapes[name] = shapes_element + for name in self.tables: + if name in element_names: + names_tables_to_keep.add(name) + for element_type, kept in ( + ("images", images), + ("labels", labels), + ("points", points), + ("shapes", shapes), + ): + if kept: + elements_dict[element_type] = dict(kept) tables = self._filter_tables( names_tables_to_keep, filter_tables, @@ -2319,7 +2461,7 @@ def subset( include_orphan_tables, elements_dict=elements_dict, ) - return SpatialData(**elements_dict, tables=tables, attrs=self.attrs) + return SpatialData(images=images, labels=labels, points=points, shapes=shapes, tables=tables, attrs=self.attrs) def __getitem__(self, item: str) -> SpatialElement | AnnData: """ @@ -2375,16 +2517,22 @@ def __setitem__(self, key: str, value: SpatialElement | AnnData) -> None: value The element. """ + # get_model() returns a schema only for the element type it matched, so the value has that type. schema = get_model(value) if schema in (Image2DModel, Image3DModel): + assert isinstance(value, DataArray | DataTree) self.images[key] = value elif schema in (Labels2DModel, Labels3DModel): + assert isinstance(value, DataArray | DataTree) self.labels[key] = value elif schema == PointsModel: + assert isinstance(value, DaskDataFrame) self.points[key] = value elif schema == ShapesModel: + assert isinstance(value, GeoDataFrame) self.shapes[key] = value elif schema == TableModel: + assert isinstance(value, AnnData) self.tables[key] = value else: raise TypeError(f"Unknown element type with schema: {schema!r}.") diff --git a/src/spatialdata/_core/validation.py b/src/spatialdata/_core/validation.py index b6c2eebfe..dc2e4bf65 100644 --- a/src/spatialdata/_core/validation.py +++ b/src/spatialdata/_core/validation.py @@ -36,7 +36,7 @@ def __str__(self) -> str: ) -def check_target_region_column_symmetry(table: AnnData, region_key: str, target: str | pd.Series) -> None: +def check_target_region_column_symmetry(table: AnnData, region_key: str, target: str | pd.Series | list[str]) -> None: """ Check region and region_key column symmetry. diff --git a/src/spatialdata/_io/_utils.py b/src/spatialdata/_io/_utils.py index fa5af1dd7..ba10405bd 100644 --- a/src/spatialdata/_io/_utils.py +++ b/src/spatialdata/_io/_utils.py @@ -23,6 +23,7 @@ from upath import UPath from upath.implementations.local import PosixUPath, WindowsUPath from xarray import DataArray, DataTree +from zarr.abc.store import Store from zarr.storage import FsspecStore, LocalStore from spatialdata._core.spatialdata import SpatialData @@ -121,14 +122,20 @@ def overwrite_coordinate_transformations_raster( ) coordinate_transformations = [t.to_dict() for t in ngff_transformations] # replace the metadata storage - if group.metadata.zarr_format == 3 and len(multiscales := group.metadata.attributes["ome"]["multiscales"]) != 1: - len_scales = len(multiscales) - raise ValueError(f"The length of multiscales metadata should be 1, found the length to be {len_scales}") - if group.metadata.zarr_format == 2: + if group.metadata.zarr_format == 3: + ome_attrs = group.metadata.attributes["ome"] + if not isinstance(ome_attrs, Mapping): + raise TypeError(f"Expected the `ome` attributes to be a JSON object, got {type(ome_attrs).__name__}.") + multiscales = ome_attrs["multiscales"] + else: multiscales = group.attrs["multiscales"] - if (len_scales := len(multiscales)) != 1: - raise ValueError(f"The length of multiscales metadata should be 1, found length of {len_scales}") + if not isinstance(multiscales, list): + raise TypeError(f"Expected `multiscales` to be a JSON array, got {type(multiscales).__name__}.") + if (len_scales := len(multiscales)) != 1: + raise ValueError(f"The length of multiscales metadata should be 1, found the length to be {len_scales}") multiscale = multiscales[0] + if not isinstance(multiscale, dict): + raise TypeError(f"Expected the multiscale entry to be a JSON object, got {type(multiscale).__name__}.") # Previously, there was CoordinateTransformations key present at the level of multiscale and datasets in multiscale. # This is not the case anymore so we are creating a new key here and keeping the one in datasets intact. @@ -138,7 +145,10 @@ def overwrite_coordinate_transformations_raster( multiscale["version"] = raster_format.version group.attrs["multiscales"] = multiscales elif isinstance(raster_format, RasterFormatV03): - ome = group.metadata.attributes["ome"] + stored_ome = group.metadata.attributes["ome"] + if not isinstance(stored_ome, Mapping): + raise TypeError(f"Expected the `ome` attributes to be a JSON object, got {type(stored_ome).__name__}.") + ome = dict(stored_ome) ome["version"] = raster_format.version ome["multiscales"] = multiscales group.attrs["ome"] = ome @@ -151,13 +161,20 @@ def overwrite_channel_names(group: zarr.Group, element: DataArray | DataTree) -> if isinstance(element, DataArray): channel_names = element.coords["c"].data.tolist() else: - channel_names = element["scale0"]["image"].coords["c"].data.tolist() + scale0 = element["scale0"] + assert isinstance(scale0, DataTree) + image = scale0["image"] + assert isinstance(image, DataArray) + channel_names = image.coords["c"].data.tolist() channel_metadata = [{"label": name} for name in channel_names] # We don't use the ome-zarr load node API, and ome-zarr-py >= 0.18 emits no `omero` block, so default to empty. - omero_meta = group.attrs.get("omero") or group.attrs.get("ome", {}).get("omero") or {} + stored_ome = group.attrs.get("ome") + ome_meta = dict(stored_ome) if isinstance(stored_ome, Mapping) else None + stored_omero = group.attrs.get("omero") or (ome_meta or {}).get("omero") + omero_meta = dict(stored_omero) if isinstance(stored_omero, Mapping) else {} omero_meta["channels"] = channel_metadata - if ome_meta := group.attrs.get("ome", None): + if ome_meta: ome_meta["omero"] = omero_meta group.attrs["ome"] = ome_meta else: @@ -285,6 +302,7 @@ def _(element: DataArray) -> list[str]: @get_dask_backing_files.register(DataTree) def _(element: DataTree) -> list[str]: dask_data_scale0 = get_pyramid_levels(element, attr="data", n=0) + assert isinstance(dask_data_scale0, DaskArray) return _get_backing_files(dask_data_scale0) @@ -341,7 +359,9 @@ def _search_for_backing_files_recursively(subgraph: Any, files: list[str]) -> No if name is not None: if name.startswith("original-from-zarr"): # LocalStore.store does not have an attribute path, but we keep it like this for backward compat. - path = getattr(v.store, "path", None) if getattr(v.store, "path", None) else v.store.root + path = getattr(v.store, "path", None) or getattr(v.store, "root", None) + if path is None: + raise TypeError(f"Cannot determine the path backing a store of type {type(v.store).__name__}.") files.append(str(UPath(path).resolve())) elif name.startswith("read-parquet") or name.startswith("read_parquet"): # Here v is a read_parquet task with arguments and the only value is a dictionary. @@ -457,9 +477,7 @@ def _is_element_self_contained( return all(_backed_elements_contained_in_path(path=element_path, object=element)) -def _resolve_zarr_store( - path: str | Path | UPath | zarr.storage.StoreLike | zarr.Group, **kwargs: Any -) -> zarr.storage.StoreLike: +def _resolve_zarr_store(path: str | Path | UPath | zarr.storage.StoreLike | zarr.Group, **kwargs: Any) -> Store: """ Normalize different Zarr store inputs into a usable store instance. @@ -506,17 +524,13 @@ def _resolve_zarr_store( if isinstance(path.store, FsspecStore): # if the store within the zarr.Group is an FSStore, return it # but extend the path of the store with that of the zarr.Group - return FsspecStore(path.store.path + "/" + path.path, fs=path.store.fs, **kwargs) - if isinstance(path.store, zarr.storage.ConsolidatedMetadataStore): - # if the store is a ConsolidatedMetadataStore, just return the underlying FSSpec store - return path.store.store + return FsspecStore(path.store.path + "/" + path.path, **{**kwargs, "fs": path.store.fs}) raise ValueError(f"Unsupported store type or zarr.Group: {type(path.store)}") - if isinstance(path, zarr.storage.StoreLike): - # if the input already a store, wrap it in an FSStore - return FsspecStore(path, **kwargs) + if isinstance(path, Store): + return path if isinstance(path, UPath): # if input is a remote UPath, map it to an FSStore - return FsspecStore(path.path, fs=path.fs, **kwargs) + return FsspecStore(path.path, **{**kwargs, "fs": path.fs}) raise TypeError(f"Unsupported type: {type(path)}") diff --git a/src/spatialdata/_io/format.py b/src/spatialdata/_io/format.py index cfb5be1b6..a9aa21ad3 100644 --- a/src/spatialdata/_io/format.py +++ b/src/spatialdata/_io/format.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Iterator, Mapping from typing import Any import ome_zarr.format @@ -45,6 +45,8 @@ def _parse_version(group: zarr.Group, expect_attrs_key: bool) -> str | None: if expect_attrs_key and ATTRS_KEY not in group.attrs: return None attrs_key_group = group.attrs[ATTRS_KEY] if expect_attrs_key else group.attrs + if not isinstance(attrs_key_group, Mapping): + raise TypeError(f"Expected a mapping of attributes, got {type(attrs_key_group)!r}.") version_found = "version" in attrs_key_group if not version_found: return None diff --git a/src/spatialdata/_io/io_points.py b/src/spatialdata/_io/io_points.py index bb203cad2..b7cc4ba3a 100644 --- a/src/spatialdata/_io/io_points.py +++ b/src/spatialdata/_io/io_points.py @@ -2,11 +2,12 @@ import warnings from pathlib import Path +from typing import Any import zarr from dask.dataframe import DataFrame as DaskDataFrame from dask.dataframe import read_parquet -from ome_zarr.format import Format +from zarr.storage import LocalStore from spatialdata._io._utils import ( _get_transformations_from_ngff_dict, @@ -14,7 +15,7 @@ overwrite_coordinate_transformations_non_raster, ) from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning -from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version +from spatialdata._io.format import CurrentPointsFormat, PointsFormats, PointsFormatType, _parse_version from spatialdata.models import get_axes_names from spatialdata.transformations._utils import ( _get_transformations, @@ -27,19 +28,27 @@ def _read_points( ) -> DaskDataFrame: """Read points from a zarr store.""" f = zarr.open(Path(store), mode="r") # Path avoids zarr v3 URL-parsing special chars (e.g. #) in names + if not isinstance(f, zarr.Group): + raise TypeError(f"Expected a zarr group holding a points element, got {type(f).__name__}.") version = _parse_version(f, expect_attrs_key=True) assert version is not None points_format = PointsFormats[version] - store_root = f.store_path.store.root + element_store = f.store_path.store + if not isinstance(element_store, LocalStore): + raise TypeError(f"Reading a points element requires a local zarr store, got {type(element_store).__name__}.") + store_root = element_store.root path = store_root / f.path / "points.parquet" # cache on remote file needed for parquet reader to work # TODO: allow reading in the metadata without caching all the data points = read_parquet("simplecache::" + str(path) if str(path).startswith("http") else path) assert isinstance(points, DaskDataFrame) - transformations = _get_transformations_from_ngff_dict(f.attrs.asdict()["coordinateTransformations"]) + ngff_transformations = f.attrs.asdict()["coordinateTransformations"] + if not isinstance(ngff_transformations, list): + raise TypeError(f"Expected coordinateTransformations to be a list, got {type(ngff_transformations).__name__}.") + transformations = _get_transformations_from_ngff_dict(ngff_transformations) _set_transformations(points, transformations) attrs = points_format.attrs_from_dict(f.attrs.asdict()) @@ -52,7 +61,7 @@ def write_points( points: DaskDataFrame, group: zarr.Group, group_type: str = "ngff:points", - element_format: Format = CurrentPointsFormat(), + element_format: PointsFormatType = CurrentPointsFormat(), ) -> None: """Write a points element to a zarr store. @@ -75,7 +84,10 @@ def write_points( transformations = _get_transformations(points) assert transformations is not None # mypy: validate_element() in _write_element guarantees this - store_root = group.store_path.store.root + element_store = group.store_path.store + if not isinstance(element_store, LocalStore): + raise TypeError(f"Writing a points element requires a local zarr store, got {type(element_store).__name__}.") + store_root = element_store.root path = store_root / group.path / "points.parquet" # The following code iterates through all columns in the 'points' DataFrame. If the column's datatype is @@ -93,7 +105,7 @@ def write_points( del points_without_transform.attrs["transform"] points_without_transform.to_parquet(path) - attrs = element_format.attrs_to_dict(points.attrs) + attrs: dict[str, Any] = dict(element_format.attrs_to_dict(points.attrs)) attrs["version"] = element_format.spatialdata_format_version _write_metadata( diff --git a/src/spatialdata/_io/io_raster.py b/src/spatialdata/_io/io_raster.py index b9a2964f0..dd237869c 100644 --- a/src/spatialdata/_io/io_raster.py +++ b/src/spatialdata/_io/io_raster.py @@ -1,9 +1,9 @@ from __future__ import annotations import warnings -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, Literal, TypeGuard, cast +from typing import Any, Literal, TypeGuard import dask.array as da import numpy as np @@ -73,26 +73,26 @@ def _is_regular_dask_chunk_grid(chunk_grid: Sequence[Sequence[int]]) -> bool: -------- Triggers ``continue`` on the first ``if`` (single or empty axis): - >>> _is_regular_dask_chunk_grid([(4,)]) # single chunk → True + >>> _is_regular_dask_chunk_grid([(4,)]) # single chunk → True True - >>> _is_regular_dask_chunk_grid([()]) # empty axis → True + >>> _is_regular_dask_chunk_grid([()]) # empty axis → True True Triggers the first ``return False`` (non-uniform interior chunks): - >>> _is_regular_dask_chunk_grid([(4, 4, 3, 4)]) # interior sizes differ → False + >>> _is_regular_dask_chunk_grid([(4, 4, 3, 4)]) # interior sizes differ → False False Triggers the second ``return False`` (last chunk larger than the first): - >>> _is_regular_dask_chunk_grid([(4, 4, 4, 5)]) # last > first → False + >>> _is_regular_dask_chunk_grid([(4, 4, 4, 5)]) # last > first → False False Exits with ``return True``: - >>> _is_regular_dask_chunk_grid([(4, 4, 4, 4)]) # all equal → True + >>> _is_regular_dask_chunk_grid([(4, 4, 4, 4)]) # all equal → True True - >>> _is_regular_dask_chunk_grid([(4, 4, 4, 1)]) # last < first → True + >>> _is_regular_dask_chunk_grid([(4, 4, 4, 1)]) # last < first → True True Empty grid (loop never executes) → True: @@ -193,6 +193,10 @@ def _read_multiscale( node = nodes[0] loaded_node = node.load(Multiscales) + if not isinstance(loaded_node, Multiscales): + raise TypeError( + f"Expected {image_loc.basename()} to hold a multiscales node, got {type(loaded_node).__name__}." + ) datasets, multiscales = ( loaded_node.datasets, loaded_node.zarr.root_attrs["multiscales"], @@ -200,7 +204,7 @@ def _read_multiscale( # This works for all versions as in zarr v3 the level of the 'ome' key is taken as root_attrs. omero_metadata = loaded_node.zarr.root_attrs.get("omero") # TODO: check if below is still valid - legacy_channels_metadata = node.load(Multiscales).zarr.root_attrs.get("channels_metadata", None) # legacy v0.1 + legacy_channels_metadata = loaded_node.zarr.root_attrs.get("channels_metadata", None) # legacy v0.1 assert len(multiscales) == 1 # checking for multiscales[0]["coordinateTransformations"] would make fail # something that doesn't have coordinateTransformations in top level @@ -217,12 +221,12 @@ def _read_multiscale( channels = [d["label"] for d in omero_metadata["channels"]] axes = [i["name"] for i in node.metadata["axes"]] if len(datasets) > 1: - arrays = [node.load(Multiscales).array(resolution=d) for d in datasets] + arrays = [loaded_node.array(resolution=d) for d in datasets] msi = dask_arrays_to_datatree(arrays, dims=axes, channels=channels) _set_transformations(msi, transformations) return compute_coordinates(msi) - data = node.load(Multiscales).array(resolution=datasets[0]) + data = loaded_node.array(resolution=datasets[0]) si = DataArray( data, name="image", @@ -269,7 +273,7 @@ def _write_raster( storage_options: JSONDict | list[JSONDict] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, label_metadata: JSONDict | None = None, - **metadata: str | JSONDict | list[JSONDict], + **metadata: Any, ) -> None: """Write raster data to disk. @@ -328,13 +332,23 @@ def _write_raster( else: raise ValueError("Not a valid labels object") - group = group["labels"][name] if raster_type == "labels" else group + if raster_type == "labels": + labels_group = group["labels"] + if not isinstance(labels_group, zarr.Group): + raise TypeError(f"Expected a zarr group holding the labels, got {type(labels_group).__name__}.") + label_group = labels_group[name] + if not isinstance(label_group, zarr.Group): + raise TypeError(f"Expected a zarr group holding the label {name!r}, got {type(label_group).__name__}.") + group = label_group if raster_type == "image": # ome-zarr-py >= 0.18 no longer writes the omero channel metadata, so we write it ourselves. overwrite_channel_names(group, raster_data) if ATTRS_KEY not in group.attrs: group.attrs[ATTRS_KEY] = {} - attrs = group.attrs[ATTRS_KEY] + stored_attrs = group.attrs[ATTRS_KEY] + if not isinstance(stored_attrs, Mapping): + raise TypeError(f"Expected {ATTRS_KEY} to be a JSON object, got {type(stored_attrs).__name__}.") + attrs = dict(stored_attrs) attrs["version"] = raster_format.spatialdata_format_version # triggers the write operation group.attrs[ATTRS_KEY] = attrs @@ -356,10 +370,10 @@ def _build_v3_codec( def _apply_compression( - storage_options: JSONDict | list[JSONDict], + storage_options: JSONDict | list[JSONDict] | None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None, - zarr_format: Literal[2, 3] = 3, -) -> JSONDict | list[JSONDict]: + zarr_format: int = 3, +) -> JSONDict | list[JSONDict] | None: """Apply compression settings to storage options. Parameters @@ -377,6 +391,8 @@ def _apply_compression( """ if not raster_compressor: return storage_options + if zarr_format not in (2, 3): + raise ValueError(f"Unsupported zarr format {zarr_format}; expected 2 or 3.") ((compression, compression_level),) = raster_compressor.items() @@ -427,7 +443,7 @@ def _write_raster_dataarray( raster_format: RasterFormatType, storage_options: JSONDict | list[JSONDict] | None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None, - **metadata: str | JSONDict | list[JSONDict], + **metadata: Any, ) -> None: """Write raster data of type DataArray to disk. @@ -455,13 +471,11 @@ def _write_raster_dataarray( data = raster_data.data transformations = _get_transformations(raster_data) assert transformations is not None # mypy: validate_element() in _write_element guarantees this - input_axes: tuple[str, ...] = tuple(raster_data.dims) + input_axes: tuple[str, ...] = tuple(str(dim) for dim in raster_data.dims) parsed_axes = _get_valid_axes(axes=list(input_axes), fmt=raster_format) storage_options = _prepare_storage_options(storage_options) # Apply compression if specified - storage_options = _apply_compression( - storage_options, raster_compressor, zarr_format=cast(Literal[2, 3], raster_format.zarr_format) - ) + storage_options = _apply_compression(storage_options, raster_compressor, zarr_format=raster_format.zarr_format) # Explicitly disable pyramid generation for single-scale rasters. Recent ome-zarr versions default # write_image()/write_labels() to scale_factors=(2, 4, 8, 16), which would otherwise write s0, s1, ... @@ -481,7 +495,17 @@ def _write_raster_dataarray( **metadata, ) - trans_group = group["labels"][element_name] if raster_type == "labels" else group + if raster_type == "labels": + labels_group = group["labels"] + if not isinstance(labels_group, zarr.Group): + raise TypeError(f"Expected a zarr group holding the labels, got {type(labels_group).__name__}.") + trans_group = labels_group[element_name] + if not isinstance(trans_group, zarr.Group): + raise TypeError( + f"Expected a zarr group holding the label {element_name!r}, got {type(trans_group).__name__}." + ) + else: + trans_group = group overwrite_coordinate_transformations_raster( group=trans_group, transformations=transformations, @@ -498,7 +522,7 @@ def _write_raster_datatree( raster_format: RasterFormatType, storage_options: JSONDict | list[JSONDict] | None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None, - **metadata: str | JSONDict | list[JSONDict], + **metadata: Any, ) -> zarr.Group: """Write raster data of type DataTree to disk. @@ -529,7 +553,9 @@ def _write_raster_datatree( # saving only the transformations of the first scale d = dict(raster_data["scale0"]) assert len(d) == 1 - xdata = d.values().__iter__().__next__() + xdata = next(iter(d.values())) + if not isinstance(xdata, DataArray): + raise TypeError(f"Expected the first scale to hold a DataArray, got {type(xdata).__name__}.") transformations = _get_transformations_xarray(xdata) assert transformations is not None # mypy: validate_element() in _write_element guarantees this @@ -564,7 +590,17 @@ def _write_raster_datatree( # This workaround should not be needed once https://github.com/ome/ome-zarr-py/issues/580 is fixed. group = zarr.open_group(store=group.store, path=group.path, mode="r+", use_consolidated=False) - trans_group = group["labels"][element_name] if raster_type == "labels" else group + if raster_type == "labels": + labels_group = group["labels"] + if not isinstance(labels_group, zarr.Group): + raise TypeError(f"Expected a zarr group holding the labels, got {type(labels_group).__name__}.") + trans_group = labels_group[element_name] + if not isinstance(trans_group, zarr.Group): + raise TypeError( + f"Expected a zarr group holding the label {element_name!r}, got {type(trans_group).__name__}." + ) + else: + trans_group = group overwrite_coordinate_transformations_raster( group=trans_group, transformations=transformations, @@ -581,7 +617,7 @@ def write_image( element_format: RasterFormatType = CurrentRasterFormat(), storage_options: JSONDict | list[JSONDict] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, - **metadata: str | JSONDict | list[JSONDict], + **metadata: Any, ) -> None: if element_format.zarr_format == 2: warnings.warn( diff --git a/src/spatialdata/_io/io_shapes.py b/src/spatialdata/_io/io_shapes.py index f8528868d..833406d39 100644 --- a/src/spatialdata/_io/io_shapes.py +++ b/src/spatialdata/_io/io_shapes.py @@ -8,8 +8,8 @@ import zarr from geopandas import GeoDataFrame, read_parquet from natsort import natsorted -from ome_zarr.format import Format from shapely import from_ragged_array, to_ragged_array +from zarr.storage import LocalStore from spatialdata._io._utils import ( _get_transformations_from_ngff_dict, @@ -20,6 +20,7 @@ from spatialdata._io.format import ( CurrentShapesFormat, ShapesFormats, + ShapesFormatType, ShapesFormatV01, ShapesFormatV02, ShapesFormatV03, @@ -37,6 +38,8 @@ def _read_shapes( ) -> GeoDataFrame: """Read shapes from a zarr store.""" f = zarr.open(Path(store), mode="r") # Path avoids zarr v3 URL-parsing special chars (e.g. #) in names + if not isinstance(f, zarr.Group): + raise TypeError(f"Expected a zarr group holding a shapes element, got {type(f).__name__}.") version = _parse_version(f, expect_attrs_key=True) assert version is not None shape_format = ShapesFormats[version] @@ -56,7 +59,12 @@ def _read_shapes( geometry = from_ragged_array(typ, coords, offsets) geo_df = GeoDataFrame({"geometry": geometry}, index=index) elif isinstance(shape_format, ShapesFormatV02 | ShapesFormatV03): - store_root = f.store_path.store.root + element_store = f.store_path.store + if not isinstance(element_store, LocalStore): + raise TypeError( + f"Reading a shapes element requires a local zarr store, got {type(element_store).__name__}." + ) + store_root = element_store.root path = Path(store_root) / f.path / "shapes.parquet" geo_df = read_parquet(path) else: @@ -64,7 +72,10 @@ def _read_shapes( f"Unsupported shapes format {shape_format} from version {version}. Please update the spatialdata library." ) - transformations = _get_transformations_from_ngff_dict(f.attrs.asdict()["coordinateTransformations"]) + ngff_transformations = f.attrs.asdict()["coordinateTransformations"] + if not isinstance(ngff_transformations, list): + raise TypeError(f"Expected coordinateTransformations to be a list, got {type(ngff_transformations).__name__}.") + transformations = _get_transformations_from_ngff_dict(ngff_transformations) _set_transformations(geo_df, transformations) return geo_df @@ -73,7 +84,7 @@ def write_shapes( shapes: GeoDataFrame, group: zarr.Group, group_type: str = "ngff:shapes", - element_format: Format = CurrentShapesFormat(), + element_format: ShapesFormatType = CurrentShapesFormat(), geometry_encoding: Literal["WKB", "geoarrow"] | None = None, ) -> None: """Write shapes to spatialdata zarr store. @@ -124,7 +135,7 @@ def write_shapes( overwrite_coordinate_transformations_non_raster(group=group, axes=axes, transformations=transformations) -def _write_shapes_v01(shapes: GeoDataFrame, group: zarr.Group, element_format: Format) -> Any: +def _write_shapes_v01(shapes: GeoDataFrame, group: zarr.Group, element_format: ShapesFormatV01) -> Any: """Write shapes to spatialdata zarr store using format ShapesFormatV01. Parameters @@ -136,20 +147,20 @@ def _write_shapes_v01(shapes: GeoDataFrame, group: zarr.Group, element_format: F element_format The format of the shapes element used to store it. """ - import numcodecs - # np.array() creates a writable copy, needed for pandas 3.0 CoW compatibility # https://github.com/geopandas/geopandas/issues/3697 geometry, coords, offsets = to_ragged_array(np.array(shapes.geometry)) group.create_array(name="coords", data=coords) for i, o in enumerate(offsets): group.create_array(name=f"offset{i}", data=o) + index_values = shapes.index.to_numpy() if shapes.index.dtype.kind == "U" or shapes.index.dtype.kind == "O": - group.create_array(name="Index", data=shapes.index.values, dtype=object, object_codec=numcodecs.VLenUTF8()) + index_array = group.create_array(name="Index", shape=index_values.shape, dtype="string") + index_array[:] = index_values else: - group.create_array(name="Index", data=shapes.index.values) + group.create_array(name="Index", data=index_values) if geometry.name == "POINT": - group.create_array(name=ShapesModel.RADIUS_KEY, data=shapes[ShapesModel.RADIUS_KEY].values) + group.create_array(name=ShapesModel.RADIUS_KEY, data=shapes[ShapesModel.RADIUS_KEY].to_numpy()) attrs = element_format.attrs_to_dict(geometry) attrs["version"] = element_format.spatialdata_format_version @@ -157,7 +168,10 @@ def _write_shapes_v01(shapes: GeoDataFrame, group: zarr.Group, element_format: F def _write_shapes_v02_v03( - shapes: GeoDataFrame, group: zarr.Group, element_format: Format, geometry_encoding: Literal["WKB", "geoarrow"] + shapes: GeoDataFrame, + group: zarr.Group, + element_format: ShapesFormatV02 | ShapesFormatV03, + geometry_encoding: Literal["WKB", "geoarrow"], ) -> Any: """Write shapes to spatialdata zarr store using format ShapesFormatV02 or ShapesFormatV03. @@ -175,7 +189,10 @@ def _write_shapes_v02_v03( """ from spatialdata.models._utils import TRANSFORM_KEY - store_root = group.store_path.store.root + element_store = group.store_path.store + if not isinstance(element_store, LocalStore): + raise TypeError(f"Writing a shapes element requires a local zarr store, got {type(element_store).__name__}.") + store_root = element_store.root path = store_root / group.path / "shapes.parquet" # Temporarily remove transformations from attrs to avoid serialization issues @@ -184,6 +201,6 @@ def _write_shapes_v02_v03( shapes.to_parquet(path, geometry_encoding=geometry_encoding) shapes.attrs[TRANSFORM_KEY] = transforms - attrs = element_format.attrs_to_dict(shapes.attrs) + attrs = element_format.attrs_to_dict({str(k): v for k, v in shapes.attrs.items()}) attrs["version"] = element_format.spatialdata_format_version return attrs diff --git a/src/spatialdata/_io/io_table.py b/src/spatialdata/_io/io_table.py index d7795015d..8ea535bb3 100644 --- a/src/spatialdata/_io/io_table.py +++ b/src/spatialdata/_io/io_table.py @@ -9,7 +9,6 @@ from anndata import AnnData from anndata import read_zarr as read_anndata_zarr from anndata._io.specs import write_elem as write_adata -from ome_zarr.format import Format from packaging.version import Version from spatialdata._io._utils import _resolve_zarr_store @@ -17,6 +16,7 @@ from spatialdata._io.format import ( CurrentTablesFormat, TablesFormats, + TablesFormatType, TablesFormatV01, TablesFormatV02, _parse_version, @@ -28,6 +28,8 @@ def _read_table(store: str | Path) -> AnnData: table = read_anndata_zarr(str(store)) f = zarr.open(Path(store), mode="r") # Path avoids zarr v3 URL-parsing special chars (e.g. #) in names + if not isinstance(f, zarr.Group): + raise TypeError(f"Expected a zarr group holding a table element, got {type(f).__name__}.") version = _parse_version(f, expect_attrs_key=False) assert version is not None table_format = TablesFormats[version] @@ -59,7 +61,7 @@ def write_table( group: zarr.Group, name: str, group_type: str = "ngff:regions_table", - element_format: Format = CurrentTablesFormat(), + element_format: TablesFormatType = CurrentTablesFormat(), convert_strings_to_categoricals: bool = False, ) -> None: """ @@ -123,7 +125,10 @@ def write_table( # was still empty, and zarr writes attributes as a whole document based on the handle's cached view, so writing # through the stale handle would erase the `encoding-type`/`encoding-version` metadata that anndata just wrote # (https://github.com/scverse/spatialdata/issues/1183). - table_group = group[name] + written_table_group = group[name] + if not isinstance(written_table_group, zarr.Group): + raise TypeError(f"Expected a zarr group holding the table {name!r}, got {type(written_table_group).__name__}.") + table_group = written_table_group table_group.attrs["spatialdata-encoding-type"] = group_type table_group.attrs["region"] = region diff --git a/src/spatialdata/_io/io_zarr.py b/src/spatialdata/_io/io_zarr.py index 9324f8b7f..24d476442 100644 --- a/src/spatialdata/_io/io_zarr.py +++ b/src/spatialdata/_io/io_zarr.py @@ -15,6 +15,7 @@ from pyarrow import ArrowInvalid from upath import UPath from zarr.errors import ArrayNotFoundError +from zarr.storage import LocalStore from spatialdata._core.spatialdata import SpatialData from spatialdata._io._utils import ( @@ -32,7 +33,7 @@ def _read_zarr_group_spatialdata_element( root_group: zarr.Group, - root_store_path: str, + root_store_path: Path, sdata_version: Literal["0.1", "0.2"], selector: set[str], read_func: Callable[..., Any], @@ -48,12 +49,20 @@ def _read_zarr_group_spatialdata_element( ): if group_name in selector and group_name in root_group: group = root_group[group_name] + if not isinstance(group, zarr.Group): + raise TypeError( + f"Expected a zarr group holding the {group_name!r} elements, got {type(group).__name__}." + ) count = 0 for subgroup_name in group: if Path(subgroup_name).name.startswith("."): # skip hidden files like .zgroup or .zmetadata continue elem_group = group[subgroup_name] + if not isinstance(elem_group, zarr.Group): + raise TypeError( + f"Expected a zarr group holding the element {subgroup_name!r}, got {type(elem_group).__name__}." + ) elem_group_path = os.path.join(root_store_path, elem_group.path) with handle_read_errors( on_bad_files, @@ -170,7 +179,10 @@ def read_zarr( UserWarning, stacklevel=2, ) - root_store_path = root_group.store.root + root_store = root_group.store + if not isinstance(root_store, LocalStore): + raise TypeError(f"Reading a SpatialData object requires a local zarr store, got {type(root_store).__name__}.") + root_store_path = root_store.root images: dict[str, Raster_T] = {} labels: dict[str, Raster_T] = {} @@ -215,11 +227,13 @@ def read_zarr( ) # read attrs metadata - attrs = root_group.attrs.asdict() - if "spatialdata_attrs" in attrs: + root_attrs = root_group.attrs.asdict() + attrs: dict[str, Any] | None + if "spatialdata_attrs" in root_attrs: # when refactoring the read_zarr function into reading componenets separately (and according to the version), # we can move the code below (.pop()) into attrs_from_dict() - attrs.pop("spatialdata_attrs") + root_attrs.pop("spatialdata_attrs") + attrs = root_attrs else: attrs = None @@ -231,6 +245,10 @@ def read_zarr( tables=tables, attrs=attrs, ) + if not isinstance(resolved_store, LocalStore): + raise TypeError( + f"Reading a SpatialData object requires a local zarr store, got {type(resolved_store).__name__}." + ) sdata.path = resolved_store.root return sdata @@ -314,7 +332,15 @@ def _group_for_element_exists(zarr_path: Path, element_type: str, element_name: "shapes", "tables", ] - exists = element_type in root and element_name in root[element_type] + if element_type in root: + elements_group = root[element_type] + if not isinstance(elements_group, zarr.Group): + raise TypeError( + f"Expected a zarr group holding the {element_type!r} elements, got {type(elements_group).__name__}." + ) + exists = element_name in elements_group + else: + exists = False store.close() return exists diff --git a/src/spatialdata/_utils.py b/src/spatialdata/_utils.py index 1a8cd1920..a0086b7b2 100644 --- a/src/spatialdata/_utils.py +++ b/src/spatialdata/_utils.py @@ -131,19 +131,23 @@ def _compute_paddings(data: DataArray, axis: str) -> tuple[int, int]: # always optimize later d = dict(unpadded["scale0"]) assert len(d) == 1 - xdata = d.values().__iter__().__next__() + xdata = next(iter(d.values())) + assert isinstance(xdata, DataArray) left_pad, right_pad = _compute_paddings(data=xdata, axis=ax) unpadded = unpadded.sel({ax: slice(left_pad, right_pad)}) translation_axes.append(ax) translation_values.append(left_pad) - d = {} + subtrees = {} for k, v in unpadded.items(): - assert len(v.values()) == 1 - xdata = v.values().__iter__().__next__() + assert isinstance(v, DataTree) + variables = list(v.values()) + assert len(variables) == 1 + xdata = variables[0] + assert isinstance(xdata, DataArray) if 0 not in xdata.shape: - d[k] = Dataset({"image": xdata}) - unpadded = DataTree.from_dict(d) + subtrees[k] = Dataset({"image": xdata}) + unpadded = DataTree.from_dict(subtrees) else: raise TypeError(f"Unsupported type: {type(raster)}") @@ -200,20 +204,23 @@ def iterate_pyramid_levels( ------- A generator to iterate over the pyramid levels. """ - names = data["scale0"].ds.keys() - name: str = next(iter(names)) + scale0 = data["scale0"] + assert isinstance(scale0, DataTree) + name = str(next(iter(scale0.ds.keys()))) for scale in data: - yield data[scale][name] if attr is None else getattr(data[scale][name], attr) + node = data[scale] + assert isinstance(node, DataTree) + yield node[name] if attr is None else getattr(node[name], attr) -def _inplace_fix_subset_categorical_obs(subset_adata: AnnData, original_adata: AnnData) -> None: +def _inplace_fix_subset_categorical_obs(subset_adata: AnnData | None, original_adata: AnnData) -> None: """ Fix categorical obs columns of subset_adata to match the categories of original_adata. Parameters ---------- subset_adata - The subset AnnData object + The subset AnnData object, or None when the subset is empty original_adata The original AnnData object @@ -221,9 +228,14 @@ def _inplace_fix_subset_categorical_obs(subset_adata: AnnData, original_adata: A ----- See discussion here: https://github.com/scverse/anndata/issues/997 """ + if subset_adata is None: + return if not hasattr(subset_adata, "obs") or not hasattr(original_adata, "obs"): return - obs = pd.DataFrame(subset_adata.obs) + subset_obs = subset_adata.obs + if not isinstance(subset_obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(subset_obs).__name__}.") + obs = pd.DataFrame(subset_obs) for column in obs.columns: is_categorical = isinstance(obs[column].dtype, pd.CategoricalDtype) if is_categorical: diff --git a/src/spatialdata/dataloader/datasets.py b/src/spatialdata/dataloader/datasets.py index 03879abc8..b34ad5bf5 100644 --- a/src/spatialdata/dataloader/datasets.py +++ b/src/spatialdata/dataloader/datasets.py @@ -11,9 +11,10 @@ import numpy as np import pandas as pd from anndata import AnnData +from dask.dataframe import DataFrame as DaskDataFrame from geopandas import GeoDataFrame from pandas import CategoricalDtype -from scipy.sparse import issparse +from scipy.sparse import csc_array, csc_matrix, csr_array, csr_matrix from torch.utils.data import Dataset from xarray import DataArray, DataTree @@ -38,7 +39,7 @@ __all__ = ["ImageTilesDataset"] -class ImageTilesDataset(Dataset): +class ImageTilesDataset(Dataset[Any]): """ :class:`torch.utils.data.Dataset` for loading tiles from a :class:`spatialdata.SpatialData` object. @@ -186,6 +187,8 @@ def _validate( # check that the coordinate systems are valid for the elements cs = regions_to_coordinate_systems[region_name] + assert not isinstance(region_elem, AnnData) + assert not isinstance(image_elem, AnnData) region_trans = get_transformation(region_elem, get_all=True) image_trans = get_transformation(image_elem, get_all=True) assert isinstance(region_trans, dict) @@ -203,6 +206,7 @@ def _validate( if get_model(region_elem) in [Labels2DModel, Labels3DModel]: indices = get_element_instances(region_elem).tolist() else: + assert isinstance(region_elem, GeoDataFrame | DaskDataFrame) indices = region_elem.index.tolist() table = self.sdata.tables[table_name] if not isinstance(self.sdata.tables[table_name].obs[region_key].dtype, CategoricalDtype): @@ -241,7 +245,9 @@ def _preprocess( dims_l = [] tables_l = [] for cs, region_name, image_name in self._cs_region_image: - circles = to_circles(self.sdata[region_name]) + region_element = self.sdata[region_name] + assert not isinstance(region_element, AnnData) + circles = to_circles(region_element) dims_l.append(get_axes_names(circles)) tile_coords = _get_tile_coords( @@ -255,14 +261,18 @@ def _preprocess( # Pre-compute all per-tile slice selections in a single vectorized call. # Passing 2-D min/max arrays triggers the multi-box path in bounding_box_query, # which returns a list of {axis: slice} dicts — one per tile. - tile_coords["selection"] = bounding_box_query( - self.sdata[image_name], + image_element = self.sdata[image_name] + assert not isinstance(image_element, AnnData) + selections = bounding_box_query( + image_element, ("x", "y"), - min_coordinate=tile_coords[["minx", "miny"]].values, - max_coordinate=tile_coords[["maxx", "maxy"]].values, + min_coordinate=tile_coords[["minx", "miny"]].to_numpy(), + max_coordinate=tile_coords[["maxx", "maxy"]].to_numpy(), target_coordinate_system=cs, return_request_only=True, ) + assert isinstance(selections, list) + tile_coords["selection"] = pd.Series(selections, index=tile_coords.index, dtype=object) tile_coords_df.append(tile_coords) inst = circles.index.values @@ -283,6 +293,7 @@ def _preprocess( match_rows="left", ) # get index dictionary, with `instance_id`, `cs`, `region`, and `image` + assert table is not None tables_l.append(table) # concatenate and assign to self @@ -309,7 +320,7 @@ def _ensure_single_scale(data: DataArray | DataTree) -> DataArray: def _return_function( idx: int, tile: Any, - dataset_table: AnnData, + dataset_table: AnnData | None, dataset_index: pd.DataFrame, table_name: str | None, return_annot: str | list[str] | None, @@ -320,12 +331,17 @@ def _return_function( # where return_table can be a single column or a list of columns return_annot = [return_annot] if isinstance(return_annot, str) else return_annot # return tuple of (tile, table) - if np.all([i in dataset_table.obs for i in return_annot]): - return tile, dataset_table.obs[return_annot].iloc[idx].values.reshape(1, -1) + assert dataset_table is not None + obs = dataset_table.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + if np.all([i in obs for i in return_annot]): + return tile, obs[return_annot].iloc[idx].to_numpy().reshape(1, -1) if np.all([i in dataset_table.var_names for i in return_annot]): - if issparse(dataset_table.X): - return tile, dataset_table[idx, return_annot].X.A - return tile, dataset_table[idx, return_annot].X + x = dataset_table[idx, return_annot].X + if isinstance(x, csr_matrix | csc_matrix | csr_array | csc_array): + return tile, np.asarray(x.todense()) + return tile, x raise ValueError( f"If `return_annot` is a `str`, it must be a column name in the table or a variable name in the table. " f"If it is a `list` of `str`, each element should be as above, and they should all be entirely in obs " @@ -333,6 +349,7 @@ def _return_function( ) # return spatialdata consisting of the image tile and, if available, the associated table if table_name: + assert dataset_table is not None table_row = dataset_table[idx].copy() # let's reset the target annotation metadata to avoid a warning when constructing the SpatialData object if TableModel.ATTRS_KEY in table_row.uns: @@ -370,12 +387,13 @@ def __getitem__(self, idx: int) -> Any | SpatialData: t_coords = self.tiles_coords.iloc[idx] image = self.sdata[row["image"]] + assert isinstance(image, DataArray | DataTree) if self._rasterize: tile = self._crop_image( image, axes=tuple(self.dims), - min_coordinate=t_coords[[f"min{i}" for i in self.dims]].values, - max_coordinate=t_coords[[f"max{i}" for i in self.dims]].values, + min_coordinate=t_coords[[f"min{i}" for i in self.dims]].to_numpy(), + max_coordinate=t_coords[[f"max{i}" for i in self.dims]].to_numpy(), target_coordinate_system=row["cs"], ) else: @@ -481,8 +499,10 @@ def _get_tile_coords( transform(circles, to_coordinate_system=cs) if tile_dim_in_units is not None: circles.radius = tile_dim_in_units / 2 - else: + elif tile_scale is not None: circles.radius *= tile_scale + else: + raise ValueError("One of `tile_scale` and `tile_dim_in_units` must be given.") # if rasterize is True, the tile dim is determined from the diameter of the circles in cs; else we need to # transform the circles to the intrinsic coordinate system of the element if not rasterize: diff --git a/src/spatialdata/datasets.py b/src/spatialdata/datasets.py index 51d360203..fb57a28b1 100644 --- a/src/spatialdata/datasets.py +++ b/src/spatialdata/datasets.py @@ -153,7 +153,7 @@ def raccoon( from skimage.segmentation import slic im_data = scipy.datasets.face() - im = Image2DModel.parse(im_data, dims=["y", "x", "c"]) + im = Image2DModel.parse(np.asarray(im_data), dims=["y", "x", "c"]) labels_data = slic(im_data, n_segments=100, compactness=10, sigma=1) labels = Labels2DModel.parse(labels_data, dims=["y", "x"]) coords = np.array([[610, 450], [730, 325], [575, 300], [480, 90]]) @@ -303,8 +303,8 @@ def _labels_blobs( dims = ["z", "y", "x"] model = Labels3DModel if scale_factors is None: - return model.parse(out, transformations=transformations, dims=dims) - return model.parse(out, transformations=transformations, dims=dims, scale_factors=scale_factors) + return model.parse(np.asarray(out), transformations=transformations, dims=dims) + return model.parse(np.asarray(out), transformations=transformations, dims=dims, scale_factors=scale_factors) def _generate_blobs(self, length: int = 512, seed: int | None = None, ndim: int = 2) -> ArrayLike: from scipy.ndimage import gaussian_filter @@ -384,9 +384,13 @@ def get_poly(i: int) -> Polygon: ] ) + minx: float + miny: float + maxx: float + maxy: float minx = miny = bbox[0] maxx = maxy = bbox[1] - polygons: list[Polygon] = [] + polygons: list[Polygon | MultiPolygon] = [] for i in range(n): # generate random points rng1 = default_rng(i) @@ -403,16 +407,20 @@ def get_poly(i: int) -> Polygon: # by translating it by the size of the first polygon. poly2 = get_poly(i) last = polygons.pop() + # `last` was just appended above, so it is the single polygon this loop generated. + assert isinstance(last, Polygon) # Calculate the size of the polygon + # note: this rebinds `maxx`/`maxy`, which `get_poly` reads on the next iteration (minx, miny, maxx, maxy) = poly2.bounds dx = maxx - minx dy = maxy - miny # Translate the polygon - poly2 = translate(poly2, xoff=dx, yoff=dy) + translated = translate(poly2, xoff=dx, yoff=dy) + assert isinstance(translated, Polygon) - polygons.append(MultiPolygon([last, poly2])) + polygons.append(MultiPolygon([last, translated])) return polygons # function that generates random shapely points given a bounding box @@ -451,10 +459,13 @@ def blobs_annotating_element(name: BlobsTypes) -> SpatialData: SpatialData object with the desired element annotated by the table. """ sdata = blobs(length=50) + element = sdata[name] + assert not isinstance(element, AnnData) if name in ["blobs_labels", "blobs_multiscale_labels"]: - instance_id = get_element_instances(sdata[name]).tolist() + instance_id = get_element_instances(element).tolist() else: - index = sdata[name].index + assert isinstance(element, GeoDataFrame | DaskDataFrame) + index = element.index instance_id = index.compute().tolist() if isinstance(index, dask.dataframe.Index) else index.tolist() n = len(instance_id) obs_df = pd.DataFrame( diff --git a/src/spatialdata/models/_accessor.py b/src/spatialdata/models/_accessor.py index 3174ee80d..77acf437c 100644 --- a/src/spatialdata/models/_accessor.py +++ b/src/spatialdata/models/_accessor.py @@ -19,7 +19,8 @@ class AttrsAccessor(MutableMapping[str, str | dict[str, Any]]): def __init__(self, dask_obj: DaskDataFrame | DaskSeries): self._obj = dask_obj if not hasattr(dask_obj, "_attrs"): - dask_obj._attrs = {} + # `_attrs` is attached to the dask collection at runtime, so it is set dynamically. + setattr(dask_obj, "_attrs", {}) # noqa: B010 def __getitem__(self, key: str) -> Any: return self._obj._attrs[key] diff --git a/src/spatialdata/models/_utils.py b/src/spatialdata/models/_utils.py index 4a1d122f1..1ffe50a49 100644 --- a/src/spatialdata/models/_utils.py +++ b/src/spatialdata/models/_utils.py @@ -10,7 +10,9 @@ import pandas as pd from dask.dataframe import DataFrame as DaskDataFrame from geopandas import GeoDataFrame +from shapely import get_coordinate_dimension from shapely.geometry import MultiPolygon, Point, Polygon +from shapely.geometry.base import BaseGeometry from xarray import DataArray, DataTree from spatialdata._logging import logger @@ -122,7 +124,7 @@ def get_spatial_axes(axes: tuple[ValidAxis_t, ...]) -> tuple[ValidAxis_t, ...]: @singledispatch -def get_axes_names(e: SpatialElement) -> tuple[str, ...]: +def get_axes_names(e: SpatialElement | pd.DataFrame) -> tuple[str, ...]: """ Get the dimensions of a SpatialElement. @@ -140,9 +142,9 @@ def get_axes_names(e: SpatialElement) -> tuple[str, ...]: @get_axes_names.register(DataArray) def _(e: DataArray) -> tuple[str, ...]: - dims = e.dims + dims = tuple(str(dim) for dim in e.dims) _validate_dims(dims) - return dims # type: ignore[no-any-return] + return dims @get_axes_names.register(DataTree) @@ -150,10 +152,13 @@ def _(e: DataTree) -> tuple[str, ...]: if "scale0" in e: # dims_coordinates = tuple(i for i in e["scale0"].dims.keys()) - assert len(e["scale0"].values()) == 1 - xdata = e["scale0"].values().__iter__().__next__() - dims_data = xdata.dims - assert isinstance(dims_data, tuple) + scale0 = e["scale0"] + if not isinstance(scale0, DataTree): + raise TypeError(f"Expected `scale0` to be a DataTree node, got {type(scale0).__name__}.") + variables = list(scale0.values()) + assert len(variables) == 1 + xdata = variables[0] + dims_data = tuple(str(dim) for dim in xdata.dims) _validate_dims(dims_data) return dims_data @@ -163,7 +168,7 @@ def _(e: DataTree) -> tuple[str, ...]: @get_axes_names.register(GeoDataFrame) def _(e: GeoDataFrame) -> tuple[str, ...]: all_dims = (X, Y, Z) - n = e.geometry.iloc[0]._ndim + n = get_coordinate_dimension(e.geometry.iloc[0]) dims = all_dims[:n] if Z not in dims and Z in e.columns: dims += (Z,) @@ -325,11 +330,12 @@ def force_2d(gdf: GeoDataFrame) -> None: GeoDataFrame with 2D or 3D geometries """ - new_shapes = [] + new_shapes: list[BaseGeometry] = [] any_3d = False for shape in gdf.geometry: if shape.has_z: any_3d = True + new_shape: BaseGeometry if isinstance(shape, Point): new_shape = Point(shape.x, shape.y) elif isinstance(shape, Polygon): diff --git a/src/spatialdata/models/models.py b/src/spatialdata/models/models.py index 5b44fe520..b0a6c16f6 100644 --- a/src/spatialdata/models/models.py +++ b/src/spatialdata/models/models.py @@ -6,7 +6,7 @@ from collections.abc import Mapping, Sequence from functools import singledispatchmethod from pathlib import Path -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal, overload import dask.dataframe as dd import numpy as np @@ -19,6 +19,7 @@ from multiscale_spatial_image import to_multiscale as to_multiscale_msi from multiscale_spatial_image.to_multiscale.to_multiscale import Methods from pandas import CategoricalDtype +from shapely import get_coordinate_dimension from shapely._geometry import GeometryType from shapely.geometry import MultiPolygon, Point, Polygon from shapely.geometry.collection import GeometryCollection @@ -50,6 +51,9 @@ ) from spatialdata.transformations.transformations import Identity +if TYPE_CHECKING: + from pandas._typing import DtypeObj + __all__ = ["Chunks_t", "ScaleFactors_t"] ATTRS_KEY = "spatialdata_attrs" @@ -85,6 +89,35 @@ class RasterSchema: ATTRS_KEY = ATTRS_KEY dims: tuple[str, ...] + @overload + @classmethod + def parse( + cls, + data: ArrayLike | DataArray | DaskArray, + dims: Sequence[str] | None = ..., + c_coords: str | list[str] | None = ..., + transformations: MappingToCoordinateSystem_t | None = ..., + scale_factors: None = ..., + method: Methods | None = ..., + chunks: Chunks_t | None = ..., + **kwargs: Any, + ) -> DataArray: ... + + @overload + @classmethod + def parse( + cls, + data: ArrayLike | DataArray | DaskArray, + dims: Sequence[str] | None = ..., + c_coords: str | list[str] | None = ..., + transformations: MappingToCoordinateSystem_t | None = ..., + *, + scale_factors: ScaleFactors_t, + method: Methods | None = ..., + chunks: Chunks_t | None = ..., + **kwargs: Any, + ) -> DataTree: ... + @classmethod def parse( cls, @@ -174,43 +207,48 @@ def parse( transformations = transformations.copy() if "name" in kwargs: raise ValueError("The `name` argument is not (yet) supported for raster data.") + if c_coords is not None and C not in cls.dims: + raise ValueError("`c_coords` is not supported for labels") # if dims is specified inside the data, get the value of dims from the data + array: DataArray | DaskArray + parsed_dims: tuple[str, ...] if isinstance(data, DataArray): if not isinstance(data.data, DaskArray): # numpy -> dask data.data = from_array(data.data) + data_dims = tuple(str(dim) for dim in data.dims) if dims is not None: - if set(dims).symmetric_difference(data.dims): + if set(dims).symmetric_difference(data_dims): raise ValueError( - f"`dims`: {dims} does not match `data.dims`: {data.dims}, please specify the dims only once." + f"`dims`: {dims} does not match `data.dims`: {data_dims}, please specify the dims only once." ) + parsed_dims = tuple(dims) else: - dims = data.dims + parsed_dims = data_dims # but if dims don't match the model's dims, throw error - if set(dims).symmetric_difference(cls.dims): - raise ValueError(f"Wrong `dims`: {dims}. Expected {cls.dims}.") + if set(parsed_dims).symmetric_difference(cls.dims): + raise ValueError(f"Wrong `dims`: {parsed_dims}. Expected {cls.dims}.") _reindex = lambda d: d + array = data # if there are no dims in the data, use the model's dims or provided dims elif isinstance(data, np.ndarray | DaskArray): - if not isinstance(data, DaskArray): # numpy -> dask - data = from_array(data) + array = data if isinstance(data, DaskArray) else from_array(data) # numpy -> dask if dims is None: - dims = cls.dims + parsed_dims = cls.dims else: - if len(set(dims).symmetric_difference(cls.dims)) > 0: - raise ValueError(f"Wrong `dims`: {dims}. Expected {cls.dims}.") - _reindex = lambda d: dims.index(d) + parsed_dims = tuple(dims) + if len(set(parsed_dims).symmetric_difference(cls.dims)) > 0: + raise ValueError(f"Wrong `dims`: {parsed_dims}. Expected {cls.dims}.") + _reindex = lambda d: parsed_dims.index(d) else: raise ValueError(f"Unsupported data type: {type(data)}.") # transpose if possible - if tuple(dims) != cls.dims: + if parsed_dims != cls.dims: try: - if isinstance(data, DataArray): - data = data.transpose(*list(cls.dims)) - elif isinstance(data, DaskArray): - data = data.transpose(*[_reindex(d) for d in cls.dims]) + if isinstance(array, DataArray): + array = array.transpose(*list(cls.dims)) else: - raise ValueError(f"Unsupported data type: {type(data)}.") + array = array.transpose(*[_reindex(d) for d in cls.dims]) except ValueError as e: raise ValueError( f"Cannot transpose arrays to match `dims`: {dims}.", @@ -219,59 +257,61 @@ def parse( # finally convert to spatial image if c_coords is not None: - c_coords = _check_match_length_channels_c_dim(data, c_coords, cls.dims) + c_coords = _check_match_length_channels_c_dim(array, c_coords, cls.dims) - if c_coords is not None and len(c_coords) != data.shape[cls.dims.index("c")]: + if c_coords is not None and len(c_coords) != array.shape[cls.dims.index("c")]: raise ValueError( f"The number of channel names `{len(c_coords)}` does not match the length of dimension 'c'" - f" with length {data.shape[cls.dims.index('c')]}." + f" with length {array.shape[cls.dims.index('c')]}." ) - data = to_spatial_image(array_like=data, dims=cls.dims, c_coords=c_coords, **kwargs) + image: DataArray = to_spatial_image(array_like=array, dims=cls.dims, c_coords=c_coords, **kwargs) # parse transformations - _parse_transformations(data, transformations) + _parse_transformations(image, transformations) # convert to multiscale if needed + parsed: DataArray | DataTree if scale_factors is not None: - parsed_transform = _get_transformations(data) + parsed_transform = _get_transformations(image) # delete transforms - del data.attrs["transform"] + del image.attrs["transform"] if isinstance(chunks, tuple): - chunks = {dim: chunks[index] for index, dim in enumerate(data.dims)} + chunks = {dim: chunks[index] for index, dim in enumerate(image.dims)} if isinstance(chunks, float): - chunks = {dim: chunks for index, dim in data.dims} + chunks = {dim: chunks for index, dim in image.dims} if method is not None: - data = to_multiscale_msi( - data, + parsed = to_multiscale_msi( + image, scale_factors=scale_factors, method=method, chunks=chunks, ) elif C in cls.dims: # Images: multiscale-spatial-image is faster (see https://github.com/scverse/spatialdata/issues/1079) - data = to_multiscale_msi( - data, + parsed = to_multiscale_msi( + image, scale_factors=scale_factors, method=Methods.XARRAY_COARSEN, chunks=chunks, ) else: # Labels: ome-zarr-py based implementation uses less memory - data = to_multiscale_ozp( - data, + parsed = to_multiscale_ozp( + image, scale_factors=scale_factors, chunks=chunks, ) - _parse_transformations(data, parsed_transform) + _parse_transformations(parsed, parsed_transform) else: # Chunk single scale images if chunks is not None: if isinstance(chunks, tuple): - chunks = dict(zip(data.dims, chunks, strict=True)) - data = data.chunk(chunks=chunks) + chunks = dict(zip(image.dims, chunks, strict=True)) + image = image.chunk(chunks=chunks) + parsed = image # recompute coordinates for (multiscale) spatial image - data = compute_coordinates(data) - cls.validate(data) - return data + parsed = compute_coordinates(parsed) + cls.validate(parsed) + return parsed @classmethod def validate(cls, data: Any) -> None: @@ -310,7 +350,10 @@ def _validate_datatree(cls, data: DataTree) -> None: raise ValueError(f"Expected exactly one data variable for the datatree: found `{name}`.") name = list(name)[0] for d in data: - cls._validate_dataarray(data[d][name]) + scale = data[d][name] + if not isinstance(scale, DataArray): + raise TypeError(f"Expected scale `{d}` to hold a DataArray, got {type(scale).__name__}.") + cls._validate_dataarray(scale) @classmethod def _validate_dataarray(cls, data: DataArray) -> None: @@ -398,6 +441,7 @@ def _check_chunk_size_not_too_large(cls, data: DataArray | DataTree) -> None: for d in data: cls._check_chunk_size_not_too_large(data[d][name]) + @staticmethod def _validate_labels_dtype(data: DataArray | DataTree) -> None: dtype = data.dtype if isinstance(data, DataArray) else data["scale0"]["image"].dtype if not (np.issubdtype(dtype, np.integer) or np.issubdtype(dtype, np.bool_)): @@ -409,16 +453,6 @@ def _validate_labels_dtype(data: DataArray | DataTree) -> None: class Labels2DModel(RasterSchema): dims = (Y, X) - @classmethod - def parse( # noqa: D102 - self, - *args: Any, - **kwargs: Any, - ) -> DataArray | DataTree: - if kwargs.get("c_coords") is not None: - raise ValueError("`c_coords` is not supported for labels") - return super().parse(*args, **kwargs) - @classmethod def validate(cls, data: Any) -> None: super().validate(data) @@ -428,12 +462,6 @@ def validate(cls, data: Any) -> None: class Labels3DModel(RasterSchema): dims = (Z, Y, X) - @classmethod - def parse(self, *args: Any, **kwargs: Any) -> DataArray | DataTree: # noqa: D102 - if kwargs.get("c_coords") is not None: - raise ValueError("`c_coords` is not supported for labels") - return super().parse(*args, **kwargs) - @classmethod def validate(cls, data: Any) -> None: super().validate(data) @@ -487,7 +515,7 @@ def validate(cls, data: GeoDataFrame) -> None: if isinstance(geom_, Point): if cls.RADIUS_KEY not in data.columns: raise ValueError(f"Column `{cls.RADIUS_KEY}` not found." + SUGGESTION) - radii = data[cls.RADIUS_KEY].values + radii = data[cls.RADIUS_KEY].to_numpy() if np.any(radii <= 0): raise ValueError("Radii of circles must be positive.") if np.any(np.isnan(radii)) or np.any(np.isinf(radii)): @@ -507,7 +535,7 @@ def validate(cls, data: GeoDataFrame) -> None: f"At least one transformation is required." + SUGGESTION ) if len(data) > 0: - n = data.geometry.iloc[0]._ndim + n = get_coordinate_dimension(data.geometry.iloc[0]) if n != 2: warnings.warn( f"The geometry column of the GeoDataFrame has {n} dimensions, while 2 is expected. Please consider " @@ -608,10 +636,10 @@ def _( index: ArrayLike | None = None, transformations: MappingToCoordinateSystem_t | None = None, ) -> GeoDataFrame: - geometry = GeometryType(geometry) - data = from_ragged_array(geometry_type=geometry, coords=data, offsets=offsets) - geo_df = GeoDataFrame({"geometry": data}) - if GeometryType(geometry).name == "POINT": + geometry_type = GeometryType(geometry) + geometries = from_ragged_array(geometry_type=geometry_type, coords=data, offsets=offsets) + geo_df = GeoDataFrame({"geometry": geometries}) + if geometry_type.name == "POINT": if radius is None: raise ValueError("If `geometry` is `Circles`, `radius` must be provided.") geo_df[cls.RADIUS_KEY] = radius @@ -780,8 +808,7 @@ def _( ndim = data.shape[1] axes = [X, Y, Z][:ndim] index = annotation.index if annotation is not None else None - df_dict = {ax: data[:, i] for i, ax in enumerate(axes)} - df_kwargs = {"data": df_dict, "index": index} + df_dict: dict[str, Any] = {ax: data[:, i] for i, ax in enumerate(axes)} if annotation is not None: if feature_key is not None: @@ -795,7 +822,7 @@ def _( if c not in handled_columns: df_dict[c] = annotation[c] - table: DaskDataFrame = dd.from_pandas(pd.DataFrame(**df_kwargs), **kwargs) + table: DaskDataFrame = dd.from_pandas(pd.DataFrame(data=df_dict, index=index), **kwargs) return cls._add_metadata_and_validate( table, feature_key=feature_key, @@ -1094,7 +1121,7 @@ def _validate_table_annotation_metadata(cls, data: AnnData) -> None: _INT_TYPES = [int, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64] - def _is_int_or_str_dtype(d: np.dtype) -> bool: + def _is_int_or_str_dtype(d: DtypeObj) -> bool: return d in _INT_TYPES or isinstance(d, pd.StringDtype) # First, check the top-level dtype (covers plain int and StringDtype cases) @@ -1175,7 +1202,7 @@ def validate( f"Instance key `{instance_key}` not in `adata.obs`. Please create the column and parse" f" using TableModel.parse(adata)." ) - if data.obs[instance_key].isnull().values.any(): + if bool(data.obs[instance_key].isnull().to_numpy().any()): raise ValueError("`table.obs[instance_key]` must not contain null values, but it does.") cls._validate_table_annotation_metadata(data) @@ -1253,7 +1280,10 @@ def parse( # note! this is an expensive check and therefore we skip it during validation # https://github.com/scverse/spatialdata/issues/715 - grouped = adata.obs.groupby(region_key, observed=True) + obs = adata.obs + if not isinstance(obs, pd.DataFrame): + raise TypeError(f"`table.obs` must be a pandas DataFrame, got {type(obs).__name__}.") + grouped = obs.groupby(region_key, observed=True) grouped_size = grouped.size() grouped_nunique = grouped.nunique() not_unique = grouped_size[grouped_size != grouped_nunique[instance_key]].index.tolist() @@ -1285,7 +1315,7 @@ def parse( def get_model( - e: SpatialElement, + e: SpatialElement | AnnData, validate: bool = True, ) -> Schema_t: """ @@ -1302,30 +1332,28 @@ def get_model( ------- The SpatialData model. """ - - def _validate_and_return( - schema: Schema_t, - e: SpatialElement, - ) -> Schema_t: - if validate: - schema.validate(e) - return schema - if isinstance(e, DataArray | DataTree): axes = get_axes_names(e) + raster_schema: type[Image2DModel] | type[Image3DModel] | type[Labels2DModel] | type[Labels3DModel] if "c" in axes: - if "z" in axes: - return _validate_and_return(Image3DModel, e) - return _validate_and_return(Image2DModel, e) - if "z" in axes: - return _validate_and_return(Labels3DModel, e) - return _validate_and_return(Labels2DModel, e) + raster_schema = Image3DModel if "z" in axes else Image2DModel + else: + raster_schema = Labels3DModel if "z" in axes else Labels2DModel + if validate: + raster_schema.validate(e) + return raster_schema if isinstance(e, GeoDataFrame): - return _validate_and_return(ShapesModel, e) + if validate: + ShapesModel.validate(e) + return ShapesModel if isinstance(e, DaskDataFrame): - return _validate_and_return(PointsModel, e) + if validate: + PointsModel.validate(e) + return PointsModel if isinstance(e, AnnData): - return _validate_and_return(TableModel, e) + if validate: + TableModel.validate(e) + return TableModel raise TypeError(f"Unsupported type {type(e)}") diff --git a/src/spatialdata/py.typed b/src/spatialdata/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/spatialdata/testing.py b/src/spatialdata/testing.py index 17945818a..f699d2e1c 100644 --- a/src/spatialdata/testing.py +++ b/src/spatialdata/testing.py @@ -3,7 +3,7 @@ from anndata import AnnData from anndata.tests.helpers import assert_equal as assert_anndata_equal from dask.dataframe import DataFrame as DaskDataFrame -from dask.dataframe.tests.test_dataframe import assert_eq as assert_dask_dataframe_equal +from dask.dataframe.utils import assert_eq as assert_dask_dataframe_equal from geopandas import GeoDataFrame from geopandas.testing import assert_geodataframe_equal from xarray import DataArray, DataTree @@ -107,6 +107,7 @@ def assert_elements_are_identical( # compare transformations (only for SpatialElements) if not isinstance(element0, AnnData): + assert not isinstance(element1, AnnData) transformations0 = get_transformation(element0, get_all=True) transformations1 = get_transformation(element1, get_all=True) assert isinstance(transformations0, dict) @@ -124,9 +125,11 @@ def assert_elements_are_identical( elif isinstance(element0, DataArray | DataTree): assert_equal(element0, element1) elif isinstance(element0, GeoDataFrame): + assert isinstance(element1, GeoDataFrame) assert_geodataframe_equal(element0, element1, check_less_precise=True) else: assert isinstance(element0, DaskDataFrame) + assert isinstance(element1, DaskDataFrame) assert_dask_dataframe_equal(element0, element1, check_divisions=False) if PointsModel.ATTRS_KEY in element0.attrs or PointsModel.ATTRS_KEY in element1.attrs: assert element0.attrs[PointsModel.ATTRS_KEY] == element1.attrs[PointsModel.ATTRS_KEY] diff --git a/src/spatialdata/transformations/_utils.py b/src/spatialdata/transformations/_utils.py index 6d3b2c1a4..032c89c13 100644 --- a/src/spatialdata/transformations/_utils.py +++ b/src/spatialdata/transformations/_utils.py @@ -96,8 +96,10 @@ def _(e: DataTree, transformations: MappingToCoordinateSystem_t) -> None: if scale != f"scale{i}": pass assert scale == f"scale{i}" + assert isinstance(node, DataTree) assert len(dict(node)) == 1 xdata = list(node.values())[0] + assert isinstance(xdata, DataArray) new_shape = np.array(xdata.shape) if i > 0: assert old_shape is not None @@ -139,9 +141,12 @@ def _(e: DataTree) -> MappingToCoordinateSystem_t | None: "A multiscale image must not contain a transformation in the outer level; the transformations need to be " "stored in the inner levels." ) - d = dict(e["scale0"]) + scale0 = e["scale0"] + assert isinstance(scale0, DataTree) + d = dict(scale0) assert len(d) == 1 - xdata = d.values().__iter__().__next__() + xdata = next(iter(d.values())) + assert isinstance(xdata, DataArray) return _get_transformations_xarray(xdata) diff --git a/src/spatialdata/transformations/operations.py b/src/spatialdata/transformations/operations.py index 31bd5cccb..9091cec38 100644 --- a/src/spatialdata/transformations/operations.py +++ b/src/spatialdata/transformations/operations.py @@ -1,7 +1,7 @@ from __future__ import annotations import contextlib -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import numpy as np from dask.dataframe import DataFrame as DaskDataFrame @@ -193,10 +193,10 @@ def remove_transformation( write_to_sdata.write_transformations(element_name=element_name) -def _build_transformations_graph(sdata: SpatialData) -> nx.Graph: +def _build_transformations_graph(sdata: SpatialData) -> nx.DiGraph[Any]: import networkx as nx - g = nx.DiGraph() + g: nx.DiGraph[Any] = nx.DiGraph() gen = sdata._gen_spatial_element_values() for cs in sdata.coordinate_systems: g.add_node(cs) @@ -313,9 +313,10 @@ def _describe_paths(paths: list[list[int | str]]) -> str: f"coordinate system. Available paths are:{s}" ) else: + intermediate: Any = intermediate_coordinate_systems if has_type_spatial_element(intermediate_coordinate_systems): - intermediate_coordinate_systems = id(intermediate_coordinate_systems) - paths = [p for p in paths if intermediate_coordinate_systems in p] + intermediate = id(intermediate_coordinate_systems) + paths = [p for p in paths if intermediate in p] if len(paths) == 0: # error 3 raise RuntimeError("No path found between the two coordinate systems passing through the intermediate") @@ -390,8 +391,9 @@ def get_transformation_between_landmarks( references_xy = np.stack([references_coords.geometry.x, references_coords.geometry.y], axis=1) moving_xy = np.stack([moving_coords.geometry.x, moving_coords.geometry.y], axis=1) elif isinstance(references_coords, DaskDataFrame): - references_xy = references_coords[["x", "y"]].to_dask_array().compute() - moving_xy = moving_coords[["x", "y"]].to_dask_array().compute() + assert isinstance(moving_coords, DaskDataFrame) + references_xy = np.asarray(references_coords[["x", "y"]].compute()) + moving_xy = np.asarray(moving_coords[["x", "y"]].compute()) else: raise TypeError("references_coords must be either an GeoDataFrame or a DaskDataFrame") diff --git a/src/spatialdata/transformations/transformations.py b/src/spatialdata/transformations/transformations.py index 3a394b120..32b81b1be 100644 --- a/src/spatialdata/transformations/transformations.py +++ b/src/spatialdata/transformations/transformations.py @@ -114,9 +114,7 @@ def inverse(self) -> BaseTransformation: Returns ------- - BaseTransformation - A new transformation that is the inverse of this one, such that applying - both in sequence yields the identity transformation. + A new transformation that is the inverse of this one, such that applying both in sequence yields the identity transformation. """ pass @@ -138,9 +136,8 @@ def to_affine_matrix(self, input_axes: tuple[ValidAxis_t, ...], output_axes: tup Returns ------- - ArrayLike - A homogeneous affine matrix of shape ``(len(output_axes) + 1, len(input_axes) + 1)``. - The last row is always ``[0, 0, ..., 1]`` (homogeneity). + A homogeneous affine matrix of shape ``(len(output_axes) + 1, len(input_axes) + 1)``. + The last row is always ``[0, 0, ..., 1]`` (homogeneity). """ pass diff --git a/src/spatialdata/utils/points.py b/src/spatialdata/utils/points.py index bc4a59231..492cda8d2 100644 --- a/src/spatialdata/utils/points.py +++ b/src/spatialdata/utils/points.py @@ -8,7 +8,7 @@ def _make_points(coordinates: np.ndarray) -> DaskDataFrame: - """Helper function to make a Points element.""" # noqa: D401 + """Helper function to make a Points element.""" k0 = int(len(coordinates) / 3) k1 = len(coordinates) - k0 genes = np.hstack((np.repeat("a", k0), np.repeat("b", k1))) diff --git a/tests/core/query/test_spatial_query.py b/tests/core/query/test_spatial_query.py index f77fd4739..894457db9 100644 --- a/tests/core/query/test_spatial_query.py +++ b/tests/core/query/test_spatial_query.py @@ -836,7 +836,7 @@ def test_query_points_3d_bounding_box_axes_order_independent(scales): # throws a warning to print the path where the file was written. # import matplotlib.pyplot as plt - # import spatialdata_plot # noqa: F401 + # import spatialdata_plot # from matplotlib.patches import Rectangle # from spatialdata import SpatialData diff --git a/tests/data/multipolygon.json b/tests/data/multipolygon.json index a8a7bf9d5..d3d0a998f 100644 --- a/tests/data/multipolygon.json +++ b/tests/data/multipolygon.json @@ -4,14 +4,7 @@ { "type": "MultiPolygon", "coordinates": [ - [ - [ - [40.0, 40.0], - [20.0, 45.0], - [45.0, 30.0], - [40.0, 40.0] - ] - ], + [[[40.0, 40.0], [20.0, 45.0], [45.0, 30.0], [40.0, 40.0]]], [ [ [20.0, 35.0], @@ -21,34 +14,15 @@ [45.0, 20.0], [20.0, 35.0] ], - [ - [30.0, 20.0], - [20.0, 15.0], - [20.0, 25.0], - [30.0, 20.0] - ] + [[30.0, 20.0], [20.0, 15.0], [20.0, 25.0], [30.0, 20.0]] ] ] }, { "type": "MultiPolygon", "coordinates": [ - [ - [ - [40.0, 40.0], - [20.0, 45.0], - [45.0, 30.0], - [40.0, 40.0] - ] - ], - [ - [ - [30.0, 20.0], - [20.0, 15.0], - [20.0, 25.0], - [30.0, 20.0] - ] - ] + [[[40.0, 40.0], [20.0, 45.0], [45.0, 30.0], [40.0, 40.0]]], + [[[30.0, 20.0], [20.0, 15.0], [20.0, 25.0], [30.0, 20.0]]] ] } ] diff --git a/tests/data/polygon.json b/tests/data/polygon.json index 9608da3a7..84e7e598b 100644 --- a/tests/data/polygon.json +++ b/tests/data/polygon.json @@ -4,23 +4,13 @@ { "type": "Polygon", "coordinates": [ - [ - [40.0, 40.0], - [20.0, 45.0], - [45.0, 30.0], - [40.0, 40.0] - ] + [[40.0, 40.0], [20.0, 45.0], [45.0, 30.0], [40.0, 40.0]] ] }, { "type": "Polygon", "coordinates": [ - [ - [40.0, 50.0], - [20.0, 15.0], - [45.0, 50.0], - [40.0, 50.0] - ] + [[40.0, 50.0], [20.0, 15.0], [45.0, 50.0], [40.0, 50.0]] ] } ] diff --git a/tests/transformations/test_transformations.py b/tests/transformations/test_transformations.py index b0bc05ae8..dd5b52dba 100644 --- a/tests/transformations/test_transformations.py +++ b/tests/transformations/test_transformations.py @@ -767,7 +767,7 @@ def test_get_affine_for_element(images): np.array( [ # fmt: off - # c y x # noqa: E265 + # c y x [1, 0, 0, 0], # c [0, 0, 1, 1], # x [0, 1, 0, 2], # y