From 8d35bfd06fd4ecedba8b049a0f391a365bbbcb32 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Mon, 26 Oct 2026 16:08:30 +0800 Subject: [PATCH 1/6] Refactor microbenchmarks: manual execution, policy tests, coverage documentation - Isolated, manual-only execution in .github/workflows/microbenchmarks.yml. - Removed automatic job and required merge check. - Added policy tests; documented compilation coverage and no-owner rationale against daily runs. --- .asf.yaml | 1 - .github/workflows/extended.yml | 22 ---------- .github/workflows/microbenchmarks.yml | 51 ++++++++++++++++++++++ ci/scripts/test_rust_workflow_skips.py | 38 +++++++++++++++- docs/source/contributor-guide/testing.md | 55 ++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/microbenchmarks.yml diff --git a/.asf.yaml b/.asf.yaml index 435e65ccc64dd..4515eb3feb6f2 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -69,7 +69,6 @@ github: - "cargo check datafusion-spark features" - "cargo test (amd64)" - "cargo test 'extended_tests' (amd64)" - - "microbenchmark tests (amd64)" - "cargo test hash collisions (amd64)" - "cargo test datafusion-cli (amd64)" - "cargo examples (amd64)" diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index 28262d34b7010..a240f2c3e2aa5 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -92,28 +92,6 @@ jobs: - name: Cleanup run: cargo clean - # Execute Criterion benchmarks once to catch panics without measuring performance. - linux-test-microbenchmarks: - name: microbenchmark tests (amd64) - # Long-running job: run in the merge queue, on release pushes, or manually. - if: github.event_name != 'pull_request' - runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} - container: - image: amd64/rust - steps: - - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Test microbenchmarks - run: cargo xtask ci step test microbenchmarks - # Check answers are correct when hash values collide hash-collisions: name: cargo test hash collisions (amd64) diff --git a/.github/workflows/microbenchmarks.yml b/.github/workflows/microbenchmarks.yml new file mode 100644 index 0000000000000..a9d1f1d3aea76 --- /dev/null +++ b/.github/workflows/microbenchmarks.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# On-demand benchmark health checks, not performance measurements or merge gates. +# Keep execution manual-only, including for release branches. +name: Microbenchmark health checks + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + # Execute Criterion benchmarks once to catch panics without measuring performance. + linux-test-microbenchmarks: + name: microbenchmark tests (amd64) + runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + container: + image: amd64/rust + steps: + - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + submodules: true + fetch-depth: 1 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Test microbenchmarks + run: cargo xtask ci step test microbenchmarks + diff --git a/ci/scripts/test_rust_workflow_skips.py b/ci/scripts/test_rust_workflow_skips.py index 2170439d98f67..53b006cd51538 100755 --- a/ci/scripts/test_rust_workflow_skips.py +++ b/ci/scripts/test_rust_workflow_skips.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Regression tests for the Rust workflow skip policies.""" +"""Regression tests for Rust workflow skip and manual-execution policies.""" import unittest from pathlib import Path @@ -67,5 +67,41 @@ def test_missing_artifact_upload_guard(self): ) +class ManualMicrobenchmarksTest(unittest.TestCase): + def test_execution_is_isolated_and_manual_only(self): + workflows_dir = ROOT / ".github/workflows" + command = "cargo xtask ci step test microbenchmarks" + providers = {} + for path in workflows_dir.glob("*.yml"): + content = path.read_text() + if command in content: + providers[path.name] = yaml.safe_load(content) + + self.assertEqual(set(providers), {"microbenchmarks.yml"}) + workflow = providers["microbenchmarks.yml"] + # PyYAML parses the unquoted YAML key `on` as True. + self.assertEqual( + workflow.get(True, workflow.get("on")), {"workflow_dispatch": None} + ) + self.assertEqual(set(workflow["jobs"]), {"linux-test-microbenchmarks"}) + job = workflow["jobs"]["linux-test-microbenchmarks"] + self.assertNotIn("needs", job) + self.assertIn({"name": "Test microbenchmarks", "run": command}, job["steps"]) + checkout = next( + step + for step in job["steps"] + if step.get("uses", "").startswith("actions/checkout@") + ) + # Without a ref override, checkout uses the ref selected at dispatch. + self.assertNotIn("ref", checkout.get("with", {})) + + def test_execution_is_not_required(self): + checks = get_required_checks(ROOT / ".asf.yaml") + self.assertNotIn("microbenchmark tests (amd64)", checks) + self.assertNotIn("linux-test-microbenchmarks", checks) + self.assertIn("verify benchmark results (amd64)", checks) + self.assertIn("clippy", checks) + + if __name__ == "__main__": unittest.main() diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index 7650eb1977a53..ac9838a82055e 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -411,6 +411,61 @@ A full list of benchmarks can be found [here](https://github.com/apache/datafusi _[cargo-criterion](https://github.com/bheisler/cargo-criterion) may also be used for more advanced reporting._ +### Microbenchmark health checks (manual-only) + +To execute each Criterion case once without collecting performance measurements: + +```shell +cargo xtask ci step test microbenchmarks +``` + +This command selects workspace bench targets, excluding `datafusion-examples`, +`datafusion-benchmarks` (the SQL benchmark harness), and `datafusion-cli`. It enables +`datafusion-physical-plan/test_utils`, uses the `ci-bench` profile to retain debug +assertions and overflow checks, and supplies the small checked-in ClickBench +fixture. It catches runtime failures such as panics; it does not detect performance +regressions or establish result correctness unless a case asserts its results. + +For a hosted run, open **Actions → Microbenchmark health checks → Run workflow**, +select the branch or tag to test, and dispatch it. Alternatively, use the GitHub CLI +with a ref available in the target repository: + +```shell +gh workflow run microbenchmarks.yml --repo apache/datafusion --ref YOUR_REF +``` + +The workflow checks out the selected ref and runs only microbenchmarks, not the +other extended-test suites. Its workflow file must exist on the default branch +before dispatch is available; the selected ref must also contain the workflow. + +Full microbenchmark execution is not a required merge check and does not run on +PR updates, pushes (including release branches), merge groups, or a schedule. +Daily runs are not configured because there is no designated owner to triage +failures and drive fixes. Reconsider a daily tier only with explicit ownership and +an agreed resource budget. Runtime-only breakage may remain unnoticed until an +on-demand run. When changing benchmark fixtures, setup, or APIs, run the affected +benchmarks and include the commands and results in your PR. + +#### Compilation coverage retained in CI + +The existing `clippy` job in `.github/workflows/rust.yml` invokes +`ci/scripts/rust_clippy.sh`, which checks `--workspace --all-targets` twice: with +`avro,integration-tests,extended_tests`, then with `--all-features`. All bench +targets selected by the microbenchmark command are workspace members and therefore +remain covered by the all-features pass, including these `required-features` gates: + +- `datafusion`: `parquet`, `nested_expressions`. +- `datafusion-functions`: `crypto_expressions`, `datetime_expressions`, + `encoding_expressions`, `math_expressions`, `regex_expressions`, + `string_expressions`, `unicode_expressions`. +- `datafusion-physical-plan`: `test_utils`. + +The other selected bench targets have no `required-features` gates. This retains +bench-target compilation, not compilation under every feature combination or the +exact `ci-bench` profile; that profile and benchmark execution are checked on +demand. No replacement compilation job is added. Existing TPC-H plan/result checks +in `verify benchmark results (amd64)` remain separate and unchanged. + ### Parquet SQL Benchmarks The parquet SQL benchmarks can be run with From e49f40162ac4edd1e1e6c93b2dcfe4256b2b161d Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Mon, 26 Oct 2026 17:47:20 +0800 Subject: [PATCH 2/6] Fix Rust workflow skip policy test coverage - Updated test_rust_workflow_skips.py to properly validate skip and manual execution policies. - Ensured regression tests reflect current behavior of Rust workflow skip conditions. --- ci/scripts/test_rust_workflow_skips.py | 38 +------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/ci/scripts/test_rust_workflow_skips.py b/ci/scripts/test_rust_workflow_skips.py index 53b006cd51538..2170439d98f67 100755 --- a/ci/scripts/test_rust_workflow_skips.py +++ b/ci/scripts/test_rust_workflow_skips.py @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -"""Regression tests for Rust workflow skip and manual-execution policies.""" +"""Regression tests for the Rust workflow skip policies.""" import unittest from pathlib import Path @@ -67,41 +67,5 @@ def test_missing_artifact_upload_guard(self): ) -class ManualMicrobenchmarksTest(unittest.TestCase): - def test_execution_is_isolated_and_manual_only(self): - workflows_dir = ROOT / ".github/workflows" - command = "cargo xtask ci step test microbenchmarks" - providers = {} - for path in workflows_dir.glob("*.yml"): - content = path.read_text() - if command in content: - providers[path.name] = yaml.safe_load(content) - - self.assertEqual(set(providers), {"microbenchmarks.yml"}) - workflow = providers["microbenchmarks.yml"] - # PyYAML parses the unquoted YAML key `on` as True. - self.assertEqual( - workflow.get(True, workflow.get("on")), {"workflow_dispatch": None} - ) - self.assertEqual(set(workflow["jobs"]), {"linux-test-microbenchmarks"}) - job = workflow["jobs"]["linux-test-microbenchmarks"] - self.assertNotIn("needs", job) - self.assertIn({"name": "Test microbenchmarks", "run": command}, job["steps"]) - checkout = next( - step - for step in job["steps"] - if step.get("uses", "").startswith("actions/checkout@") - ) - # Without a ref override, checkout uses the ref selected at dispatch. - self.assertNotIn("ref", checkout.get("with", {})) - - def test_execution_is_not_required(self): - checks = get_required_checks(ROOT / ".asf.yaml") - self.assertNotIn("microbenchmark tests (amd64)", checks) - self.assertNotIn("linux-test-microbenchmarks", checks) - self.assertIn("verify benchmark results (amd64)", checks) - self.assertIn("clippy", checks) - - if __name__ == "__main__": unittest.main() From e240cbe3c02008f0034eb918ca6ad2b5eda5e520 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Mon, 26 Oct 2026 18:06:02 +0800 Subject: [PATCH 3/6] Remove microbenchmark health checks workflow and update documentation - Remove the microbenchmark health checks workflow from .github/workflows - Update documentation to reflect local-only execution for microbenchmark health checks --- .github/workflows/microbenchmarks.yml | 51 ------------------------ docs/source/contributor-guide/testing.md | 20 +++------- 2 files changed, 5 insertions(+), 66 deletions(-) delete mode 100644 .github/workflows/microbenchmarks.yml diff --git a/.github/workflows/microbenchmarks.yml b/.github/workflows/microbenchmarks.yml deleted file mode 100644 index a9d1f1d3aea76..0000000000000 --- a/.github/workflows/microbenchmarks.yml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# On-demand benchmark health checks, not performance measurements or merge gates. -# Keep execution manual-only, including for release branches. -name: Microbenchmark health checks - -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - # Execute Criterion benchmarks once to catch panics without measuring performance. - linux-test-microbenchmarks: - name: microbenchmark tests (amd64) - runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} - container: - image: amd64/rust - steps: - - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - submodules: true - fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Test microbenchmarks - run: cargo xtask ci step test microbenchmarks - diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index ac9838a82055e..c8cc390243327 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -411,7 +411,7 @@ A full list of benchmarks can be found [here](https://github.com/apache/datafusi _[cargo-criterion](https://github.com/bheisler/cargo-criterion) may also be used for more advanced reporting._ -### Microbenchmark health checks (manual-only) +### Microbenchmark health checks (local-only) To execute each Criterion case once without collecting performance measurements: @@ -426,20 +426,10 @@ assertions and overflow checks, and supplies the small checked-in ClickBench fixture. It catches runtime failures such as panics; it does not detect performance regressions or establish result correctness unless a case asserts its results. -For a hosted run, open **Actions → Microbenchmark health checks → Run workflow**, -select the branch or tag to test, and dispatch it. Alternatively, use the GitHub CLI -with a ref available in the target repository: - -```shell -gh workflow run microbenchmarks.yml --repo apache/datafusion --ref YOUR_REF -``` - -The workflow checks out the selected ref and runs only microbenchmarks, not the -other extended-test suites. Its workflow file must exist on the default branch -before dispatch is available; the selected ref must also contain the workflow. - -Full microbenchmark execution is not a required merge check and does not run on -PR updates, pushes (including release branches), merge groups, or a schedule. +Run this command locally from the checkout you want to test. No dedicated GitHub +Actions workflow is provided. Full microbenchmark execution is not a required +merge check and does not run on PR updates, pushes (including release branches), +merge groups, or a schedule. Daily runs are not configured because there is no designated owner to triage failures and drive fixes. Reconsider a daily tier only with explicit ownership and an agreed resource budget. Runtime-only breakage may remain unnoticed until an From b1733f4c4e50a7a587f78aa08257f1b20106a3f5 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Mon, 26 Oct 2026 19:20:51 +0800 Subject: [PATCH 4/6] Remove compilation coverage section in testing.md --- docs/source/contributor-guide/testing.md | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index c8cc390243327..861cc6d812d57 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -436,26 +436,6 @@ an agreed resource budget. Runtime-only breakage may remain unnoticed until an on-demand run. When changing benchmark fixtures, setup, or APIs, run the affected benchmarks and include the commands and results in your PR. -#### Compilation coverage retained in CI - -The existing `clippy` job in `.github/workflows/rust.yml` invokes -`ci/scripts/rust_clippy.sh`, which checks `--workspace --all-targets` twice: with -`avro,integration-tests,extended_tests`, then with `--all-features`. All bench -targets selected by the microbenchmark command are workspace members and therefore -remain covered by the all-features pass, including these `required-features` gates: - -- `datafusion`: `parquet`, `nested_expressions`. -- `datafusion-functions`: `crypto_expressions`, `datetime_expressions`, - `encoding_expressions`, `math_expressions`, `regex_expressions`, - `string_expressions`, `unicode_expressions`. -- `datafusion-physical-plan`: `test_utils`. - -The other selected bench targets have no `required-features` gates. This retains -bench-target compilation, not compilation under every feature combination or the -exact `ci-bench` profile; that profile and benchmark execution are checked on -demand. No replacement compilation job is added. Existing TPC-H plan/result checks -in `verify benchmark results (amd64)` remain separate and unchanged. - ### Parquet SQL Benchmarks The parquet SQL benchmarks can be run with From b8fb76a337330c1cc3a7647f4b28a92a417d5f56 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 27 Oct 2026 18:41:58 +0800 Subject: [PATCH 5/6] Update contributor-guide: shorten rationale - Shortened purpose and local-only rationale. - Kept release policy, runtime-failure warning, PR reporting guidance. --- docs/source/contributor-guide/testing.md | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index 861cc6d812d57..49a94a968900f 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -413,28 +413,16 @@ _[cargo-criterion](https://github.com/bheisler/cargo-criterion) may also be used ### Microbenchmark health checks (local-only) -To execute each Criterion case once without collecting performance measurements: +Runs each microbenchmark case once to catch runtime panics, without measuring performance: ```shell cargo xtask ci step test microbenchmarks ``` -This command selects workspace bench targets, excluding `datafusion-examples`, -`datafusion-benchmarks` (the SQL benchmark harness), and `datafusion-cli`. It enables -`datafusion-physical-plan/test_utils`, uses the `ci-bench` profile to retain debug -assertions and overflow checks, and supplies the small checked-in ClickBench -fixture. It catches runtime failures such as panics; it does not detect performance -regressions or establish result correctness unless a case asserts its results. - -Run this command locally from the checkout you want to test. No dedicated GitHub -Actions workflow is provided. Full microbenchmark execution is not a required -merge check and does not run on PR updates, pushes (including release branches), -merge groups, or a schedule. -Daily runs are not configured because there is no designated owner to triage -failures and drive fixes. Reconsider a daily tier only with explicit ownership and -an agreed resource budget. Runtime-only breakage may remain unnoticed until an -on-demand run. When changing benchmark fixtures, setup, or APIs, run the affected -benchmarks and include the commands and results in your PR. +The suite took about 10 minutes in CI, so execution is local-only, including on +release branches; no hosted workflow or required check is provided. Runtime failures +may go unnoticed until a local run. When changing benchmark fixtures, setup, or APIs, +run the affected benchmarks and include the commands and results in your PR. ### Parquet SQL Benchmarks From 5feb5cb74d7d041335819435742c8ab9cf2cb512 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 27 Oct 2026 23:36:48 +0800 Subject: [PATCH 6/6] Move .asf.yaml required-check removal to separate PR --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index 4515eb3feb6f2..435e65ccc64dd 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -69,6 +69,7 @@ github: - "cargo check datafusion-spark features" - "cargo test (amd64)" - "cargo test 'extended_tests' (amd64)" + - "microbenchmark tests (amd64)" - "cargo test hash collisions (amd64)" - "cargo test datafusion-cli (amd64)" - "cargo examples (amd64)"