diff --git a/.cargo/config.toml b/.cargo/config.toml index 91a099a61..af951327f 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,12 +1,5 @@ [target.x86_64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] [target.aarch64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] - +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 911c536a8..8aeb86f10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,34 +15,66 @@ # specific language governing permissions and limitations # under the License. -name: Python Release Build +# Reusable workflow for running building +# This ensures the same tests run for both debug (PRs) and release (main/tags) builds + +name: Build + on: - pull_request: - branches: ["main"] - push: - tags: ["*-rc*"] - branches: ["branch-*"] + workflow_call: + inputs: + build_mode: + description: 'Build mode: debug or release' + required: true + type: string + run_wheels: + description: 'Whether to build distribution wheels' + required: false + type: boolean + default: false + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 jobs: - build: + # ============================================ + # Linting Jobs + # ============================================ + lint-rust: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "nightly" + components: rustfmt + + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + + - name: Check formatting + run: cargo +nightly fmt --all -- --check + + lint-python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Install Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: "3.12" - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@v6 with: - enable-cache: true + enable-cache: true - # Use the --no-install-package to only install the dependencies - # but do not yet build the rust library - name: Install dependencies run: uv sync --dev --no-install-package datafusion - # Update output format to enable automatic inline annotations. - name: Run Ruff run: | uv run --no-project ruff check --output-format=github python/ @@ -50,26 +82,168 @@ jobs: - name: Run codespell run: | - uv run --no-project codespell --toml pyproject.toml + uv run --no-project codespell --toml pyproject.toml + + lint-toml: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install taplo + uses: taiki-e/install-action@v2 + with: + tool: taplo-cli + + # if you encounter an error, try running 'taplo format' to fix the formatting automatically. + - name: Check Cargo.toml formatting + run: taplo format --check generate-license: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: astral-sh/setup-uv@v7 + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Install cargo-license + uses: taiki-e/install-action@v2 with: - enable-cache: true + tool: cargo-license - name: Generate license file run: uv run --no-project python ./dev/create_license.py + - uses: actions/upload-artifact@v6 with: name: python-wheel-license path: LICENSE.txt + # ============================================ + # Build - Linux x86_64 + # ============================================ + build-manylinux-x86_64: + needs: [generate-license, lint-rust, lint-python] + name: ManyLinux x86_64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - run: rm LICENSE.txt + - name: Download LICENSE.txt + uses: actions/download-artifact@v7 + with: + name: python-wheel-license + path: . + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + with: + key: ${{ inputs.build_mode }} + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Build (release mode) + uses: PyO3/maturin-action@v1 + if: inputs.build_mode == 'release' + with: + target: x86_64-unknown-linux-gnu + manylinux: "2_28" + args: --release --strip --features protoc,substrait --out dist + rustup-components: rust-std + + - name: Build (debug mode) + uses: PyO3/maturin-action@v1 + if: inputs.build_mode == 'debug' + with: + target: x86_64-unknown-linux-gnu + manylinux: "2_28" + args: --features protoc,substrait --out dist + rustup-components: rust-std + + - name: Build FFI test library + uses: PyO3/maturin-action@v1 + with: + target: x86_64-unknown-linux-gnu + manylinux: "2_28" + working-directory: examples/datafusion-ffi-example + args: --out dist + rustup-components: rust-std + + - run: cp examples/datafusion-ffi-example/dist/*.whl dist/ + + - name: Archive wheels + uses: actions/upload-artifact@v6 + with: + name: dist-manylinux-x86_64 + path: dist/* + + # ============================================ + # Build - Linux ARM64 + # ============================================ + build-manylinux-aarch64: + needs: [generate-license, lint-rust, lint-python] + name: ManyLinux arm64 + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v6 + + - run: rm LICENSE.txt + - name: Download LICENSE.txt + uses: actions/download-artifact@v7 + with: + name: python-wheel-license + path: . + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + with: + key: ${{ inputs.build_mode }} + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Build (release mode) + uses: PyO3/maturin-action@v1 + if: inputs.build_mode == 'release' + with: + target: aarch64-unknown-linux-gnu + manylinux: "2_28" + args: --release --strip --features protoc,substrait --out dist + rustup-components: rust-std + + - name: Build (debug mode) + uses: PyO3/maturin-action@v1 + if: inputs.build_mode == 'debug' + with: + target: aarch64-unknown-linux-gnu + manylinux: "2_28" + args: --features protoc,substrait --out dist + rustup-components: rust-std + + - name: Archive wheels + uses: actions/upload-artifact@v6 + if: inputs.build_mode == 'release' + with: + name: dist-manylinux-aarch64 + path: dist/* + + # ============================================ + # Build - macOS arm64 / Windows + # ============================================ build-python-mac-win: - needs: [generate-license] - name: Mac/Win + needs: [generate-license, lint-rust, lint-python] + name: macOS arm64 & Windows runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -79,10 +253,6 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - uses: dtolnay/rust-toolchain@stable - run: rm LICENSE.txt @@ -92,20 +262,38 @@ jobs: name: python-wheel-license path: . + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + with: + key: ${{ inputs.build_mode }} + + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Install Protoc uses: arduino/setup-protoc@v3 with: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true + - name: Install dependencies + run: uv sync --dev --no-install-package datafusion - - name: Build Python package - run: | - uv sync --dev --no-install-package datafusion - uv run --no-project maturin build --release --strip --features substrait + # Run clippy BEFORE maturin so we can avoid rebuilding. The features must match + # exactly the features used by maturin. Linux maturin builds need to happen in a + # container so only run this for our mac runner. + - name: Run Clippy + if: matrix.os != 'windows-latest' + run: cargo clippy --no-deps --all-targets --features substrait -- -D warnings + + - name: Build Python package (release mode) + if: inputs.build_mode == 'release' + run: uv run --no-project maturin build --release --strip --features substrait + + - name: Build Python package (debug mode) + if: inputs.build_mode != 'release' + run: uv run --no-project maturin build --features substrait - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -120,13 +308,17 @@ jobs: - name: Archive wheels uses: actions/upload-artifact@v6 + if: inputs.build_mode == 'release' with: name: dist-${{ matrix.os }} path: target/wheels/* + # ============================================ + # Build - macOS x86_64 (release only) + # ============================================ build-macos-x86_64: - needs: [generate-license] - name: Mac x86_64 + if: inputs.build_mode == 'release' + needs: [generate-license, lint-rust, lint-python] runs-on: macos-15-intel strategy: fail-fast: false @@ -135,10 +327,6 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - uses: dtolnay/rust-toolchain@stable - run: rm LICENSE.txt @@ -148,19 +336,26 @@ jobs: name: python-wheel-license path: . + - name: Cache Cargo + uses: Swatinem/rust-cache@v2 + with: + key: ${{ inputs.build_mode }} + + - uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + - name: Install Protoc uses: arduino/setup-protoc@v3 with: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: astral-sh/setup-uv@v7 - with: - enable-cache: true + - name: Install dependencies + run: uv sync --dev --no-install-package datafusion - - name: Build Python package + - name: Build (release mode) run: | - uv sync --dev --no-install-package datafusion uv run --no-project maturin build --release --strip --features substrait - name: List Mac wheels @@ -172,68 +367,14 @@ jobs: name: dist-macos-aarch64 path: target/wheels/* - build-manylinux-x86_64: - needs: [generate-license] - name: Manylinux x86_64 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - run: rm LICENSE.txt - - name: Download LICENSE.txt - uses: actions/download-artifact@v7 - with: - name: python-wheel-license - path: . - - run: cat LICENSE.txt - - name: Build wheels - uses: PyO3/maturin-action@v1 - env: - RUST_BACKTRACE: 1 - with: - rust-toolchain: nightly - target: x86_64 - manylinux: auto - rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 - args: --release --manylinux 2014 --features protoc,substrait - - name: Archive wheels - uses: actions/upload-artifact@v6 - with: - name: dist-manylinux-x86_64 - path: target/wheels/* - - build-manylinux-aarch64: - needs: [generate-license] - name: Manylinux arm64 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - run: rm LICENSE.txt - - name: Download LICENSE.txt - uses: actions/download-artifact@v7 - with: - name: python-wheel-license - path: . - - run: cat LICENSE.txt - - name: Build wheels - uses: PyO3/maturin-action@v1 - env: - RUST_BACKTRACE: 1 - with: - rust-toolchain: nightly - target: aarch64 - # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail - manylinux: 2_28 - rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 - args: --release --features protoc,substrait - - name: Archive wheels - uses: actions/upload-artifact@v6 - with: - name: dist-manylinux-aarch64 - path: target/wheels/* + # ============================================ + # Build - Source Distribution + # ============================================ build-sdist: needs: [generate-license] name: Source distribution + if: inputs.build_mode == 'release' runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -253,16 +394,22 @@ jobs: args: --release --sdist --out dist --features protoc,substrait - name: Assert sdist build does not generate wheels run: | - if [ "$(ls -A target/wheels)" ]; then - echo "Error: Sdist build generated wheels" - exit 1 - else - echo "Directory is clean" - fi + if [ "$(ls -A target/wheels)" ]; then + echo "Error: Sdist build generated wheels" + exit 1 + else + echo "Directory is clean" + fi shell: bash - + + # ============================================ + # Build - Source Distribution + # ============================================ + merge-build-artifacts: runs-on: ubuntu-latest + name: Merge build artifacts + if: inputs.build_mode == 'release' needs: - build-python-mac-win - build-macos-x86_64 @@ -276,6 +423,9 @@ jobs: name: dist pattern: dist-* + # ============================================ + # Build - Documentation + # ============================================ # Documentation build job that runs after wheels are built build-docs: name: Build docs @@ -312,7 +462,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.10" - name: Install dependencies uses: astral-sh/setup-uv@v7 @@ -326,20 +476,21 @@ jobs: name: dist-manylinux-x86_64 path: wheels/ - # Install from the pre-built wheel - - name: Install from pre-built wheel + # Install from the pre-built wheels + - name: Install from pre-built wheels run: | set -x uv venv # Install documentation dependencies uv sync --dev --no-install-package datafusion --group docs - # Install the pre-built wheel - WHEEL=$(find wheels/ -name "*.whl" | head -1) - if [ -n "$WHEEL" ]; then - echo "Installing wheel: $WHEEL" - uv pip install "$WHEEL" + # Install all pre-built wheels + WHEELS=$(find wheels/ -name "*.whl") + if [ -n "$WHEELS" ]; then + echo "Installing wheels:" + echo "$WHEELS" + uv pip install wheels/*.whl else - echo "ERROR: No wheel found!" + echo "ERROR: No wheels found!" exit 1 fi @@ -368,16 +519,3 @@ jobs: git commit -m 'Publish built docs triggered by ${{ github.sha }}' git push || git push --force fi - - # NOTE: PyPI publish needs to be done manually for now after release passed the vote - # release: - # name: Publish in PyPI - # needs: [build-manylinux, build-python-mac-win] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/download-artifact@v7 - # - name: Publish to PyPI - # uses: pypa/gh-action-pypi-publish@master - # with: - # user: __token__ - # password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..ab284b522 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# CI workflow for pull requests - runs tests in DEBUG mode for faster feedback + +name: CI + +on: + pull_request: + branches: ["main"] + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + uses: ./.github/workflows/build.yml + with: + build_mode: debug + run_wheels: false + secrets: inherit + + test: + needs: build + uses: ./.github/workflows/test.yml + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..bddc89eac --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Release workflow - runs tests in RELEASE mode and builds distribution wheels +# Triggered on: +# - Merges to main +# - Release candidate tags (*-rc*) +# - Release tags (e.g., 45.0.0) + +name: Release Build + +on: + push: + branches: + - "main" + tags: + - "*-rc*" # Release candidates (e.g., 45.0.0-rc1) + - "[0-9]+.*" # Release tags (e.g., 45.0.0) + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + build: + uses: ./.github/workflows/build.yml + with: + build_mode: release + run_wheels: true + secrets: inherit + + test: + needs: build + uses: ./.github/workflows/test.yml + secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yml similarity index 76% rename from .github/workflows/test.yaml rename to .github/workflows/test.yml index df4d8fcda..6fd6b02a4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yml @@ -15,16 +15,13 @@ # specific language governing permissions and limitations # under the License. -name: Python test -on: - push: - branches: [main] - pull_request: - branches: [main] +# Reusable workflow for running tests +# This ensures the same tests run for both debug (PRs) and release (main/tags) builds + +name: Test -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true +on: + workflow_call: jobs: test-matrix: @@ -50,7 +47,7 @@ jobs: EXAMPLE_VERSION=$(grep -A 1 "name = \"datafusion-common\"" examples/datafusion-ffi-example/Cargo.lock | grep "version = " | head -1 | sed 's/.*version = "\(.*\)"/\1/') echo "Main crate datafusion version: $MAIN_VERSION" echo "FFI example datafusion version: $EXAMPLE_VERSION" - + if [ "$MAIN_VERSION" != "$EXAMPLE_VERSION" ]; then echo "❌ Error: FFI example datafusion versions don't match!" exit 1 @@ -59,8 +56,6 @@ jobs: - name: Setup Rust Toolchain uses: dtolnay/rust-toolchain@stable id: rust-toolchain - with: - components: clippy,rustfmt - name: Install Protoc uses: arduino/setup-protoc@v3 @@ -79,28 +74,46 @@ jobs: path: ~/.cargo key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }} - - name: Run Clippy - if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }} - run: cargo clippy --all-targets --all-features -- -D clippy::all -D warnings -A clippy::redundant_closure - - - name: Install dependencies and build + - name: Install dependencies uses: astral-sh/setup-uv@v7 with: enable-cache: true + # Download the Linux wheel built in the build workflow + - name: Download pre-built Linux wheel + uses: actions/download-artifact@v7 + with: + name: dist-manylinux-x86_64 + path: wheels/ + + # Install from the pre-built wheels + - name: Install from pre-built wheels + run: | + set -x + uv venv + # Install development dependencies + uv sync --dev --no-install-package datafusion + # Install all pre-built wheels + WHEELS=$(find wheels/ -name "*.whl") + if [ -n "$WHEELS" ]; then + echo "Installing wheels:" + echo "$WHEELS" + uv pip install wheels/*.whl + else + echo "ERROR: No wheels found!" + exit 1 + fi + - name: Run tests env: RUST_BACKTRACE: 1 run: | git submodule update --init - uv sync --dev --no-install-package datafusion - uv run --no-project maturin develop --uv - uv run --no-project pytest -v . + uv run --no-project pytest -v . --import-mode=importlib - name: FFI unit tests run: | cd examples/datafusion-ffi-example - uv run --no-project maturin develop --uv uv run --no-project pytest python/tests/_test*.py - name: Cache the generated dataset @@ -121,19 +134,3 @@ jobs: cd examples/tpch uv run --no-project python convert_data_to_parquet.py uv run --no-project pytest _tests.py - - nightly-fmt: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6 - - - name: Setup Rust Toolchain - uses: dtolnay/rust-toolchain@stable - id: rust-toolchain - with: - toolchain: "nightly" - components: clippy,rustfmt - - - name: Check Formatting - run: cargo +nightly fmt -- --check diff --git a/Cargo.toml b/Cargo.toml index af2ffb012..f4e8575c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,13 +27,13 @@ license = "Apache-2.0" edition = "2021" rust-version = "1.78" include = [ - "/src", - "/datafusion", - "/LICENSE.txt", - "build.rs", - "pyproject.toml", - "Cargo.toml", - "Cargo.lock", + "/src", + "/datafusion", + "/LICENSE.txt", + "build.rs", + "pyproject.toml", + "Cargo.toml", + "Cargo.lock", ] [features] @@ -43,15 +43,15 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.47", features = [ - "macros", - "rt", - "rt-multi-thread", - "sync", + "macros", + "rt", + "rt-multi-thread", + "sync", ] } pyo3 = { version = "0.26", features = [ - "extension-module", - "abi3", - "abi3-py310", + "extension-module", + "abi3", + "abi3-py310", ] } pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] } pyo3-log = "0.13.2" @@ -64,16 +64,16 @@ datafusion-ffi = { version = "52" } prost = "0.14.1" # keep in line with `datafusion-substrait` uuid = { version = "1.18", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = [ - "local_dynamic_tls", + "local_dynamic_tls", ] } async-trait = "0.1.89" futures = "0.3" cstr = "0.2" object_store = { version = "0.12.4", features = [ - "aws", - "gcp", - "azure", - "http", + "aws", + "gcp", + "azure", + "http", ] } url = "2" log = "0.4.27" diff --git a/dev/create_license.py b/dev/create_license.py index a28a0abec..acbf8587c 100644 --- a/dev/create_license.py +++ b/dev/create_license.py @@ -22,11 +22,9 @@ import subprocess from pathlib import Path -subprocess.check_output(["cargo", "install", "cargo-license"]) data = subprocess.check_output( [ - "cargo", - "license", + "cargo-license", "--avoid-build-deps", "--avoid-dev-deps", "--do-not-bundle", diff --git a/examples/datafusion-ffi-example/.cargo/config.toml b/examples/datafusion-ffi-example/.cargo/config.toml index 91a099a61..af951327f 100644 --- a/examples/datafusion-ffi-example/.cargo/config.toml +++ b/examples/datafusion-ffi-example/.cargo/config.toml @@ -1,12 +1,5 @@ [target.x86_64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] [target.aarch64-apple-darwin] -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] - +rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"] diff --git a/examples/datafusion-ffi-example/Cargo.toml b/examples/datafusion-ffi-example/Cargo.toml index e6708fce8..d54add58b 100644 --- a/examples/datafusion-ffi-example/Cargo.toml +++ b/examples/datafusion-ffi-example/Cargo.toml @@ -21,14 +21,18 @@ version = "0.2.0" edition = "2021" [dependencies] -datafusion-catalog = { version = "52" , default-features = false } -datafusion-common = { version = "52" , default-features = false } -datafusion-functions-aggregate = { version = "52" } -datafusion-functions-window = { version = "52" } -datafusion-expr = { version = "52" } +datafusion-catalog = { version = "52", default-features = false } +datafusion-common = { version = "52", default-features = false } +datafusion-functions-aggregate = { version = "52" } +datafusion-functions-window = { version = "52" } +datafusion-expr = { version = "52" } datafusion-ffi = { version = "52" } -pyo3 = { version = "0.26", features = ["extension-module", "abi3", "abi3-py39"] } +pyo3 = { version = "0.26", features = [ + "extension-module", + "abi3", + "abi3-py39", +] } arrow = { version = "57" } arrow-array = { version = "57" } arrow-schema = { version = "57" } diff --git a/examples/datafusion-ffi-example/pyproject.toml b/examples/datafusion-ffi-example/pyproject.toml index 0c54df95c..7f85e9487 100644 --- a/examples/datafusion-ffi-example/pyproject.toml +++ b/examples/datafusion-ffi-example/pyproject.toml @@ -23,9 +23,9 @@ build-backend = "maturin" name = "datafusion_ffi_example" requires-python = ">=3.9" classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", ] dynamic = ["version"] diff --git a/pyproject.toml b/pyproject.toml index 497943a34..d315dbe19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,26 +27,26 @@ license = { file = "LICENSE.txt" } requires-python = ">=3.10" keywords = ["datafusion", "dataframe", "rust", "query-engine"] classifiers = [ - "Development Status :: 2 - Pre-Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "License :: OSI Approved", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", - "Programming Language :: Python", - "Programming Language :: Rust", + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "License :: OSI Approved", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python", + "Programming Language :: Rust", ] dependencies = [ - "pyarrow>=16.0.0;python_version<'3.14'", - "pyarrow>=22.0.0;python_version>='3.14'", - "typing-extensions;python_version<'3.13'" + "pyarrow>=16.0.0;python_version<'3.14'", + "pyarrow>=22.0.0;python_version>='3.14'", + "typing-extensions;python_version<'3.13'", ] dynamic = ["version"] @@ -73,23 +73,23 @@ asyncio_default_fixture_loop_scope = "function" # Enable docstring linting using the google style guide [tool.ruff.lint] -select = ["ALL" ] +select = ["ALL"] ignore = [ - "A001", # Allow using words like min as variable names - "A002", # Allow using words like filter as variable names - "ANN401", # Allow Any for wrapper classes - "COM812", # Recommended to ignore these rules when using with ruff-format - "FIX002", # Allow TODO lines - consider removing at some point - "FBT001", # Allow boolean positional args - "FBT002", # Allow boolean positional args - "ISC001", # Recommended to ignore these rules when using with ruff-format - "SLF001", # Allow accessing private members - "TD002", # Do not require author names in TODO statements - "TD003", # Allow TODO lines - "PLR0913", # Allow many arguments in function definition - "PD901", # Allow variable name df - "N812", # Allow importing functions as `F` - "A005", # Allow module named io + "A001", # Allow using words like min as variable names + "A002", # Allow using words like filter as variable names + "ANN401", # Allow Any for wrapper classes + "COM812", # Recommended to ignore these rules when using with ruff-format + "FIX002", # Allow TODO lines - consider removing at some point + "FBT001", # Allow boolean positional args + "FBT002", # Allow boolean positional args + "ISC001", # Recommended to ignore these rules when using with ruff-format + "SLF001", # Allow accessing private members + "TD002", # Do not require author names in TODO statements + "TD003", # Allow TODO lines + "PLR0913", # Allow many arguments in function definition + "PD901", # Allow variable name df + "N812", # Allow importing functions as `F` + "A005", # Allow module named io ] [tool.ruff.lint.pydocstyle] @@ -104,61 +104,96 @@ extend-allowed-calls = ["lit", "datafusion.lit"] # Disable docstring checking for these directories [tool.ruff.lint.per-file-ignores] "python/tests/*" = [ - "ANN", - "ARG", - "BLE001", - "D", - "S101", - "SLF", - "PD", - "PLR2004", - "PT011", - "RUF015", - "S608", - "PLR0913", - "PT004", + "ANN", + "ARG", + "BLE001", + "D", + "S101", + "SLF", + "PD", + "PLR2004", + "PT011", + "RUF015", + "S608", + "PLR0913", + "PT004", +] +"examples/*" = [ + "D", + "W505", + "E501", + "T201", + "S101", + "PLR2004", + "ANN001", + "ANN202", + "INP001", + "DTZ007", + "RUF015", +] +"dev/*" = [ + "D", + "E", + "T", + "S", + "PLR", + "C", + "SIM", + "UP", + "EXE", + "N817", + "ERA001", + "ANN001", +] +"benchmarks/*" = [ + "D", + "F", + "T", + "BLE", + "FURB", + "PLR", + "E", + "TD", + "TRY", + "S", + "SIM", + "EXE", + "UP", + "ERA001", + "ANN001", + "INP001", ] -"examples/*" = ["D", "W505", "E501", "T201", "S101", "PLR2004", "ANN001", "ANN202", "INP001", "DTZ007", "RUF015"] -"dev/*" = ["D", "E", "T", "S", "PLR", "C", "SIM", "UP", "EXE", "N817", "ERA001", "ANN001"] -"benchmarks/*" = ["D", "F", "T", "BLE", "FURB", "PLR", "E", "TD", "TRY", "S", "SIM", "EXE", "UP", "ERA001", "ANN001", "INP001"] "docs/*" = ["D"] "docs/source/conf.py" = ["ERA001", "ANN001", "INP001"] [tool.codespell] -skip = [ - "./target", - "uv.lock", - "./python/tests/test_functions.py" -] +skip = ["./target", "uv.lock", "./python/tests/test_functions.py"] count = true -ignore-words-list = [ - "ans", - "IST" -] +ignore-words-list = ["ans", "IST"] [dependency-groups] dev = [ - "maturin>=1.8.1", - "numpy>1.25.0;python_version<'3.14'", - "numpy>=2.3.2;python_version>='3.14'", - "pyarrow>=19.0.0", - "pre-commit>=4.3.0", - "pyyaml>=6.0.3", - "pytest>=7.4.4", - "pytest-asyncio>=0.23.3", - "ruff>=0.9.1", - "toml>=0.10.2", - "pygithub==2.5.0", - "codespell==2.4.1", + "maturin>=1.8.1", + "numpy>1.25.0;python_version<'3.14'", + "numpy>=2.3.2;python_version>='3.14'", + "pyarrow>=19.0.0", + "pre-commit>=4.3.0", + "pyyaml>=6.0.3", + "pytest>=7.4.4", + "pytest-asyncio>=0.23.3", + "ruff>=0.9.1", + "toml>=0.10.2", + "pygithub==2.5.0", + "codespell==2.4.1", ] docs = [ - "sphinx>=7.1.2", - "pydata-sphinx-theme==0.8.0", - "myst-parser>=3.0.1", - "jinja2>=3.1.5", - "ipython>=8.12.3", - "pandas>=2.0.3", - "pickleshare>=0.7.5", - "sphinx-autoapi>=3.4.0", - "setuptools>=75.3.0", + "sphinx>=7.1.2", + "pydata-sphinx-theme==0.8.0", + "myst-parser>=3.0.1", + "jinja2>=3.1.5", + "ipython>=8.12.3", + "pandas>=2.0.3", + "pickleshare>=0.7.5", + "sphinx-autoapi>=3.4.0", + "setuptools>=75.3.0", ]