Optional c-blosc2 backend for Blosc that writes Blosc1-format chunks

.github/workflows/blosc1-compat.yaml

@@ -0,0 +1,97 @@
+name: Blosc1 compatibility
+
+# Property-based and exhaustive tests (tests/test_blosc1_compat.py) that chunks written by
+# numcodecs' Blosc decode with C-Blosc 1.x, and the reverse, using python-blosc (which
+# bundles C-Blosc 1.x) as the reference.
+#
+# This workflow is self-contained and optional. To turn it off, any of:
+#   * set the repository variable BLOSC1_COMPAT_CI to 'false'
+#     (Settings > Secrets and variables > Actions > Variables), without a code change;
+#   * delete this file; nothing else refers to it. tests/test_blosc1_compat.py skips itself
+#     when python-blosc or hypothesis is missing, so the other workflows are unaffected;
+#   * run pytest elsewhere with -m "not blosc1_compat".
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/blosc1-compat.yaml'
+      - 'tests/test_blosc1_compat.py'
+      - 'src/numcodecs/blosc.pyx'
+      - 'src/numcodecs/_blosc_shim.h'
+      - 'src/numcodecs/meson.build'
+      - 'subprojects/**'
+      - 'packages/c-blosc2-blosc1-compat/**'
+      - 'c-blosc'
+  pull_request:
+    paths:
+      - '.github/workflows/blosc1-compat.yaml'
+      - 'tests/test_blosc1_compat.py'
+      - 'src/numcodecs/blosc.pyx'
+      - 'src/numcodecs/_blosc_shim.h'
+      - 'src/numcodecs/meson.build'
+      - 'subprojects/**'
+      - 'packages/c-blosc2-blosc1-compat/**'
+      - 'c-blosc'
+  schedule:
+    - cron: '0 5 * * 1'  # weekly, with many more examples than a pull request
+  workflow_dispatch:
+    inputs:
+      examples:
+        description: 'Hypothesis examples per property'
+        default: '1000'
+
+permissions:
+  contents: read
+
+jobs:
+  compat:
+    # the repository variable is the off switch
+    if: vars.BLOSC1_COMPAT_CI != 'false'
+    name: ${{ matrix.backend }} backend, ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # the c-blosc 1.x backend is the control: the tests must pass on it too, which
+          # shows that they test the format and not an implementation detail
+          - {backend: blosc1, platform: ubuntu-latest}
+          - {backend: blosc2, platform: ubuntu-latest}
+          - {backend: blosc2, platform: ubuntu-22.04-arm}
+          - {backend: blosc2, platform: macos-14}
+          - {backend: blosc2, platform: windows-latest}
+    defaults:
+      run:
+        shell: bash -e {0}
+    env:
+      BLOSC1_COMPAT_EXAMPLES: ${{ inputs.examples || (github.event_name == 'schedule' && '5000') || '300' }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # pyproject.toml's license-files lists the c-blosc submodule's licenses, and the
+          # blosc1 backend builds it
+          submodules: recursive
+          fetch-depth: 0 # required for version resolution
+      - uses: actions/setup-python@v6
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+      - name: Install numcodecs with the ${{ matrix.backend }} backend
+        run: python -m pip install -v ".[test]" -Csetup-args=-Dblosc_backend=${{ matrix.backend }}
+      - name: Install the C-Blosc 1.x reference
+        # python-blosc has wheels for most platforms and builds from source (with cmake)
+        # for the rest
+        run: python -m pip install blosc hypothesis cmake
+      - name: Versions
+        run: |
+          python - <<'EOF'
+          import blosc, hypothesis, numcodecs.blosc as nb
+          print("numcodecs backend:", nb.LIBRARY, nb.VERSION_STRING, nb.list_compressors())
+          print("reference:", "python-blosc", blosc.__version__, "C-Blosc", blosc.VERSION_STRING)
+          print("hypothesis:", hypothesis.__version__)
+          assert nb.LIBRARY == ("c-blosc2" if "${{ matrix.backend }}" == "blosc2" else "c-blosc")
+          EOF
+      - name: Run the compatibility tests
+        run: pytest tests/test_blosc1_compat.py -m blosc1_compat --no-cov -v -p no:cacheprovider

.github/workflows/blosc2-compat.yaml

@@ -0,0 +1,85 @@
+name: c-blosc2 Blosc1 compatibility
+
+on:
+  push:
+    branches: [main]
+    paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h']
+  pull_request:
+    paths: ['packages/c-blosc2-blosc1-compat/**', 'subprojects/**', 'src/numcodecs/blosc.pyx', 'src/numcodecs/_blosc_shim.h']
+  workflow_dispatch:
+
+jobs:
+  wrap-in-sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: python packages/c-blosc2-blosc1-compat/tools/sync_wrap.py --check
+
+  forward-compat:
+    # c-blosc2 chunks must be readable by C-Blosc 1.x
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - run: pip install meson ninja cmake
+      - name: Build C-Blosc 1.x
+        run: |
+          git clone --depth 1 --branch v1.21.6 https://github.com/Blosc/c-blosc.git /tmp/c-blosc
+          cmake -S /tmp/c-blosc -B /tmp/b1 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
+            -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_STATIC=OFF -DCMAKE_BUILD_TYPE=Release
+          cmake --build /tmp/b1 -j
+      - name: Build c-blosc2 through the wrap
+        run: |
+          mkdir /tmp/probe && cd /tmp/probe
+          ln -s "$GITHUB_WORKSPACE/subprojects" subprojects
+          printf "project('probe','c')\nb = dependency('blosc2')\nexecutable('probe', files('p.c'), dependencies: b)\n" > meson.build
+          printf '#include <blosc2.h>\nint main(void){return 0;}\n' > p.c
+          meson setup b && meson compile -C b
+      - name: Run the compatibility check
+        run: |
+          B=/tmp/probe/b
+          # c-blosc2 uses a system zstd/lz4/zlib when it finds one, otherwise the wrap's
+          # static build; link whichever is there
+          LIBS="$(find $B/subprojects -name libblosc2.a | head -n 1)"
+          for l in zstd lz4 z; do
+            a="$(find $B/subprojects -name lib$l.a | head -n 1)"
+            LIBS="$LIBS ${a:--l$l}"
+          done
+          python packages/c-blosc2-blosc1-compat/tests/run.py \
+            --blosc1-include /tmp/c-blosc/blosc --blosc1-lib /tmp/b1/blosc \
+            --blosc2-include "/tmp/probe/subprojects/c-blosc2/include" \
+            --blosc2-ldflags "$LIBS"
+
+  test-blosc2-backend:
+    # the numcodecs test suite with Blosc built on c-blosc2 instead of c-blosc 1.x
+    runs-on: ${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {platform: ubuntu-latest, python-version: "3.13"}
+          - {platform: ubuntu-latest, python-version: "3.14t"}
+          - {platform: ubuntu-22.04-arm, python-version: "3.13"}
+          - {platform: macos-14, python-version: "3.13"}
+          - {platform: windows-latest, python-version: "3.13"}
+    defaults:
+      run:
+        shell: bash -e {0}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          # pyproject.toml's license-files still lists the c-blosc submodule's licenses
+          submodules: recursive
+          fetch-depth: 0 # required for version resolution
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Install numcodecs with the c-blosc2 backend
+        run: python -m pip install -v ".[test,test_extras]" -Csetup-args=-Dblosc_backend=blosc2
+      - name: Check the backend
+        run: python -c "import numcodecs.blosc as b; print(b.LIBRARY, b.VERSION_STRING, b.list_compressors()); assert b.LIBRARY == 'c-blosc2'"
+      - name: Run tests
+        run: pytest -v

.gitignore

@@ -99,4 +99,9 @@ ENV/
 .pixi/*
 *.egg-info
 pixi.lock
-uv.lock
+uv.lock
+# Wrapped subprojects: only the .wrap files and packagefiles are tracked
+/subprojects/*/
+!/subprojects/packagefiles/
+/subprojects/packagecache/
+/subprojects/.wraplock

docs/release.md

@@ -18,6 +18,21 @@
 
 ## 0.17.1
 
+### Enhancements
+
+* Add an optional c-blosc2 backend for the `Blosc` codec, selected at build time with
+  `-Dblosc_backend=blosc2` (default is still c-blosc 1.x). c-blosc2 is built from a meson
+  wrap (`subprojects/c-blosc2.wrap`) of c-blosc2, which can write Blosc1-format
+  chunks that C-Blosc 1.x can decode (stock c-blosc2 output is not readable by C-Blosc 1.x).
+  zstd, lz4 and zlib are resolved separately from Blosc (system libraries or WrapDB wraps,
+  `-Dsystem_*`) and can be left out of Blosc with `-Dblosc_zstd=disabled` and
+  `-Dblosc_zlib=disabled`. `numcodecs.blosc.LIBRARY` reports the library in use; c-blosc2
+  does not provide `snappy`.
+* Validate Blosc frames before decompression, allowing valid zero-length frames to round-trip
+  while rejecting truncated frames, by {user}`Igor Stadnyk <fallenmi>`, {issue}`831` (#854).
+* `Blosc` can encode arrays whose dtype has an itemsize of 0, by
+  {user}`Raashish Aggarwal <raashish1601>`, {issue}`516` (#870).
+
 ### Security
 
 * `Pickle.decode` now refuses to unpickle data unless the process opts in, because

meson.options

@@ -1,5 +1,11 @@
+option('blosc_backend', type: 'combo', choices: ['blosc1', 'blosc2'], value: 'blosc1',
+       description: 'C library behind the Blosc codec. Both write Blosc1-format chunks.')
 option('system_blosc', type: 'feature', value: 'disabled',
-       description: 'Use system-installed Blosc library')
+       description: 'Use system-installed Blosc library (c-blosc 1.x for blosc_backend=blosc1, c-blosc2 for blosc2)')
+option('blosc_zstd', type: 'feature', value: 'enabled',
+       description: 'blosc_backend=blosc2 only: build Zstandard support into Blosc')
+option('blosc_zlib', type: 'feature', value: 'enabled',
+       description: 'blosc_backend=blosc2 only: build zlib support into Blosc')
 option('system_lz4', type: 'feature', value: 'disabled',
        description: 'Use system-installed LZ4 library')
 option('system_zlib', type: 'feature', value: 'disabled',

packages/c-blosc2-blosc1-compat/README.md

@@ -0,0 +1,91 @@
+# c-blosc2-blosc1-compat
+
+A build overlay for [c-blosc2](https://github.com/Blosc/c-blosc2) that makes it usable as the
+backend of numcodecs' `Blosc` codec: a meson build (upstream only has CMake) for a c-blosc2
+that can **write chunks that C-Blosc 1.x can decode**. Stock c-blosc2 cannot, even
+through its `blosc1_*` API or with `BLOSC_BLOSC1_COMPAT` (its README says so, and
+`tests/run.py` confirms it).
+
+This directory is self-contained (it does not import anything from numcodecs) so it can be
+split out with `git subtree split` if it ever needs to live in its own repository.
+
+## Contents
+
+| Path | What |
+|---|---|
+| `UPSTREAM` | the c-blosc2 commit in use; `subprojects/c-blosc2.wrap` must match |
+| `meson/` | `meson.build` and `meson.options` for c-blosc2 |
+| `tests/` | the forward-compatibility check, see below |
+| `tools/` | `sync_wrap.py` |
+| `TODO.md` | moving the pin to a c-blosc2 release |
+
+## The c-blosc2 version
+
+`subprojects/c-blosc2.wrap` clones [Blosc/c-blosc2](https://github.com/Blosc/c-blosc2) at a
+commit on `main`. It includes the change merged in
+[Blosc/c-blosc2#815](https://github.com/Blosc/c-blosc2/pull/815):
+`blosc2_ctx_set_blosc1_format()`, which makes a compression context write format version 2
+(Blosc1) chunks. Format version 2 differs from what c-blosc2 writes in three ways that
+matter to a reader:
+
+1. the format version byte and the 16-byte header;
+2. bitshuffle: version 2 shuffles a block only if its element count is a multiple of 8, and
+   stores it unshuffled otherwise;
+3. block splitting: a C-Blosc 1.x reader recomputes it from the typesize and blocksize.
+
+Only plain shuffle/bitshuffle with blosclz, lz4, lz4hc, zlib and zstd are supported in this
+mode. The output decodes with C-Blosc 1.x but is not byte-identical to what it writes.
+
+The reader fix for bit-shuffled version 2 blocks with leftover bytes
+([#812](https://github.com/Blosc/c-blosc2/issues/812),
+[#813](https://github.com/Blosc/c-blosc2/pull/813)) is also in upstream c-blosc2.
+
+## Wiring into the build
+
+Meson only looks for `patch_directory` in `subprojects/packagefiles`, so
+`subprojects/packagefiles/c-blosc2/` holds a copy of `meson/`. After editing here run
+`python tools/sync_wrap.py`; CI runs it with `--check`.
+
+## Testing
+
+`tests/run.py` builds small C programs against C-Blosc 1.x and c-blosc2 and checks 735
+cases in three directions (see its docstring for the command line): c-blosc2 global-context
+to C-Blosc 1.x, c-blosc2 context API to C-Blosc 1.x, and C-Blosc 1.x to c-blosc2. All must
+decode. c-blosc2 has its own unit tests (`tests/test_blosc1_format.c`).
+
+## Compatibility CI
+
+`tests/test_blosc1_compat.py` (repository `tests/`, not this directory) checks that chunks
+written by numcodecs decode with C-Blosc 1.x and the reverse, with python-blosc, which
+bundles C-Blosc 1.x, as the reference. It combines property-based tests (hypothesis) over
+compressors, levels, shuffles, typesizes up to 300, buffer sizes around the edges that
+matter (32 bytes, multiples of 8 and 128 elements, bytes left over after the last item),
+block sizes and data kinds, with an exhaustive grid. Both numcodecs code paths are checked:
+the one through blosc's global context and the one through a context of its own. The
+compressed bytes are not expected to equal what C-Blosc 1.x writes (the libraries pick
+different default block sizes); the decoded bytes are.
+
+`.github/workflows/blosc1-compat.yaml` runs it on the c-blosc2 backend on Linux (x86_64 and
+arm64), macOS and Windows, and on the default c-blosc 1.x backend as a control. It runs 300
+examples per property on a pull request and 5000 weekly. It is independent of everything
+else and can be turned off by
+
+- setting the repository variable `BLOSC1_COMPAT_CI` to `false` (no code change),
+- deleting the workflow file (the test module skips itself when python-blosc or hypothesis
+  is not installed, so no other workflow changes), or
+- `pytest -m "not blosc1_compat"` anywhere else.
+
+To run it locally: `pip install blosc hypothesis`, then
+`pytest tests/test_blosc1_compat.py` (`BLOSC1_COMPAT_EXAMPLES` sets the examples).
+
+## Updating c-blosc2
+
+Edit `UPSTREAM` and the `revision` in
+`subprojects/c-blosc2.wrap` together; a test checks that they agree. Then run
+`python tools/sync_wrap.py` and the tests. If upstream changes its source file list, update
+`meson/meson.build`.
+
+## Upstreaming
+
+#815 is merged but not yet in a c-blosc2 release; the wrap uses a commit on `main` until it
+is. See `TODO.md`.

packages/c-blosc2-blosc1-compat/TODO.md

@@ -0,0 +1,18 @@
+# TODO
+
+## Move the wrap to a c-blosc2 release
+
+`subprojects/c-blosc2.wrap` clones https://github.com/Blosc/c-blosc2 at a commit on `main`
+(see `UPSTREAM`) that includes the Blosc1 format writer, merged in
+https://github.com/Blosc/c-blosc2/pull/815 (it builds on #812 / #813).
+
+Check which c-blosc2 release first contains #815. When there is one:
+
+1. point the wrap's `revision` at its tag (or switch to a `[wrap-file]` with `source_url` and
+   `source_hash`);
+2. update `UPSTREAM` to match, and check that `meson/meson.build` still lists the right
+   source files;
+3. run `python tools/sync_wrap.py` and `tests/run.py`;
+4. update `docs/release.md` and `README.md`.
+
+Do not release numcodecs with the wrap pointing at an unreleased commit of `main`.

packages/c-blosc2-blosc1-compat/UPSTREAM

@@ -0,0 +1,5 @@
+repository = https://github.com/Blosc/c-blosc2
+branch = main
+commit = d77f4b9167c72ac97bc3f4c0644bf46f77c8d71f
+version = 3.3.6.dev
+pull_request = https://github.com/Blosc/c-blosc2/pull/815