From 51db3b2fc7b705c5d6da87e1cff68c7d137d821c Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 15 Jul 2026 15:45:50 -0400 Subject: [PATCH] refactor(ci): share Rust micro-benchmark workflow Declare the two Rust micro-benchmark binaries in a small dedicated resolver and expose them through the existing matrix command. Replace the duplicated develop and PR jobs with one reusable workflow while preserving their mode-specific result handling. Keep this separate from the SQL profile migration so each change can be reviewed and landed independently. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Connor Tsui --- .github/workflows/bench-core.yml | 204 ++++++++++++++++++ .github/workflows/bench-pr.yml | 121 +---------- .github/workflows/bench.yml | 139 +----------- bench-orchestrator/README.md | 8 +- .../bench_orchestrator/benchmarks.py | 28 ++- bench-orchestrator/bench_orchestrator/cli.py | 12 +- .../bench_orchestrator/matrix.py | 25 +++ bench-orchestrator/tests/test_matrix.py | 27 ++- 8 files changed, 307 insertions(+), 257 deletions(-) create mode 100644 .github/workflows/bench-core.yml diff --git a/.github/workflows/bench-core.yml b/.github/workflows/bench-core.yml new file mode 100644 index 00000000000..6e7cda5972d --- /dev/null +++ b/.github/workflows/bench-core.yml @@ -0,0 +1,204 @@ +name: Rust micro-benchmarks + +# Reusable workflow for the Rust micro-benchmarks (random access, compression). Called by bench.yml +# (mode=develop) and bench-pr.yml (mode=pr). The matrix is resolved declaratively from +# bench-orchestrator; see bench-orchestrator/bench_orchestrator/matrix.py. + +on: + workflow_call: + inputs: + mode: + required: true + type: string + profile: + required: false + type: string + default: "micro" + +jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.resolve.outputs.matrix }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode == 'pr' + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode != 'pr' + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false + - name: Resolve benchmark matrix + id: resolve + shell: bash + env: + PROFILE: ${{ inputs.profile }} + run: | + echo "matrix=$(uv run --project bench-orchestrator vx-bench matrix "$PROFILE")" >> "$GITHUB_OUTPUT" + + bench: + needs: plan + timeout-minutes: 120 + env: + RUST_BACKTRACE: full + VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" + FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.matrix) }} + + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + with: + sccache: s3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode == 'pr' + with: + ref: ${{ github.event.pull_request.head.sha }} + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode != 'pr' + - name: Setup benchmark environment + run: sudo bash scripts/setup-benchmark.sh + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }} + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false + + - name: Install DuckDB + run: | + wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb + chmod +x duckdb + echo "$PWD" >> "$GITHUB_PATH" + + - uses: ./.github/actions/system-info + + - name: Build binary + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native" + run: | + cargo build --bin ${{ matrix.id }} --profile release_debug ${{ matrix.build_args }} --features unstable_encodings + + - name: Setup Polar Signals + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 + with: + polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" + project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + profiling_frequency: 199 + extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + + - name: Run ${{ matrix.name }} benchmark (per-combination) + if: ${{ matrix.split }} + shell: bash + run: | + python3 scripts/random-access-split.py ${{ inputs.mode != 'pr' && '--v3' || '' }} + + - name: Run ${{ matrix.name }} benchmark + if: ${{ !matrix.split }} + shell: bash + run: | + bash scripts/bench-taskset.sh target/release_debug/${{ matrix.id }} \ + --formats ${{ matrix.formats }} \ + -d gh-json \ + -o results.json \ + ${{ inputs.mode != 'pr' && '--gh-json-v3 results.v3.jsonl' || '' }} + + - name: Setup AWS CLI + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole + aws-region: us-east-1 + + - name: Compare results + if: inputs.mode == 'pr' + shell: bash + run: | + set -Eeu -o pipefail -x + + python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request + gzip -d -c data.json.gz > base.json + + echo "# Benchmarks: ${{ matrix.name }}" > comment.md + echo '' >> comment.md + uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \ + >> comment.md + cat comment.md >> "$GITHUB_STEP_SUMMARY" + + - name: Comment PR + if: inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + with: + file-path: comment.md + comment-tag: bench-pr-comment-${{ matrix.id }} + + - name: Comment PR on failure + if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + with: + message: | + # BENCHMARK FAILED + + Benchmark `${{ matrix.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + comment-tag: bench-pr-comment-${{ matrix.id }} + + - name: Upload Benchmark Results + if: inputs.mode == 'develop' + shell: bash + run: | + bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json + + # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live benchmarks + # website (see bench.yml for the full rationale). Gated on inputs.mode == 'develop' + the + # ingest-role ARN var. post-ingest.py mints the RDS IAM token (boto3) from the assumed + # GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert. + - name: Configure AWS credentials for v4 ingest (OIDC) + if: inputs.mode == 'develop' && vars.GH_BENCH_INGEST_ROLE_ARN != '' + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + aws-region: ${{ vars.RDS_BENCH_REGION }} + - name: Ingest results to v4 Postgres + if: inputs.mode == 'develop' && vars.GH_BENCH_INGEST_ROLE_ARN != '' + shell: bash + env: + RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }} + AWS_REGION: ${{ vars.RDS_BENCH_REGION }} + BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + run: | + set -Eeuo pipefail + curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ + -o "${RUNNER_TEMP}/rds-global-bundle.pem" + DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" + uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \ + scripts/post-ingest.py results.v3.jsonl \ + --postgres "${DSN}" \ + --commit-sha "${{ github.sha }}" \ + --region "${AWS_REGION}" + + - name: Alert incident.io + if: failure() && inputs.mode == 'develop' + uses: ./.github/actions/alert-incident-io + with: + api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} + alert-title: "${{ matrix.name }} benchmark failed on develop" + deduplication-key: ci-bench-${{ matrix.id }}-failure diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 663d23a4d21..9fa327a9f8d 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -19,122 +19,13 @@ permissions: id-token: write # enables AWS-GitHub OIDC jobs: + # Rust micro-benchmarks (random access, compression). Matrix resolved from the `micro` profile. bench: - timeout-minutes: 120 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') - || 'ubuntu-latest' }} - strategy: - matrix: - benchmark: - - id: random-access-bench - name: Random Access - build_args: "--features lance" - - id: compress-bench - name: Compression - steps: - - uses: runs-on/action@v2 - if: github.event.pull_request.head.repo.fork == false - with: - sccache: s3 - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup benchmark environment - run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: ${{ github.event.pull_request.head.repo.fork == false && 'true' || 'false' }} - - - name: Install DuckDB - run: | - wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb - chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH - - - uses: ./.github/actions/system-info - - - name: Build binary - shell: bash - env: - RUSTFLAGS: "-C target-cpu=native" - run: | - cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - - - name: Setup Polar Signals - if: github.event.pull_request.head.repo.fork == false - uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 - with: - polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" - project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" - profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - - - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) - if: matrix.benchmark.id == 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - python3 scripts/random-access-split.py - - - name: Run ${{ matrix.benchmark.name }} benchmark - if: matrix.benchmark.id != 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json - - - name: Setup AWS CLI - if: github.event.pull_request.head.repo.fork == false - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole - aws-region: us-east-1 - - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false - - - name: Compare results - shell: bash - run: | - set -Eeu -o pipefail -x - - python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request - gzip -d -c data.json.gz > base.json - - echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md - echo '' >> comment.md - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ - >> comment.md - cat comment.md >> $GITHUB_STEP_SUMMARY - - - name: Comment PR - if: github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 - with: - file-path: comment.md - comment-tag: bench-pr-comment-${{ matrix.benchmark.id }} - - - name: Comment PR on failure - if: failure() && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 - with: - message: | - # BENCHMARK FAILED - - Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. - comment-tag: bench-pr-comment-${{ matrix.benchmark.id }} + uses: ./.github/workflows/bench-core.yml + secrets: inherit + with: + mode: "pr" + profile: "micro" sql: uses: ./.github/workflows/sql-benchmarks.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f05722a3bf8..454d1ce7783 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -31,140 +31,13 @@ jobs: bash scripts/commit-json.sh > new-commit.json bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json + # Rust micro-benchmarks (random access, compression). Matrix resolved from the `micro` profile. bench: - timeout-minutes: 120 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=bench-dedicated/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id) - || 'ubuntu-latest' }} - strategy: - fail-fast: false - matrix: - benchmark: - - id: random-access-bench - name: Random Access - build_args: "--features lance" - formats: "parquet,lance,vortex" - - id: compress-bench - name: Compression - build_args: "--features lance" - formats: "parquet,lance,vortex" - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - - name: Setup benchmark environment - run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - - - name: Install DuckDB - run: | - wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb - chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH - - - uses: ./.github/actions/system-info - - - name: Build binary - shell: bash - env: - RUSTFLAGS: "-C target-cpu=native" - run: | - cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - - - name: Setup Polar Signals - uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 - with: - polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" - project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" - profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - - - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) - if: matrix.benchmark.id == 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - python3 scripts/random-access-split.py --v3 - - - name: Run ${{ matrix.benchmark.name }} benchmark - if: matrix.benchmark.id != 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json --gh-json-v3 results.v3.jsonl - - - name: Setup AWS CLI - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole - aws-region: us-east-1 - - - name: Upload Benchmark Results - shell: bash - run: | - bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json - - # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live - # benchmarks website; a failure here fails the job. Gated on the ingest-role ARN var - # (the assume-role input that MUST exist for OIDC to succeed). post-ingest.py mints - # the RDS IAM token internally (boto3) from the assumed GitHubBenchmarkIngestRole; - # sslmode=verify-full validates the cert. - # - # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only - # the uv binary, never the synced workspace. A full sync would rebuild - # vortex-python via sccache->S3, which fails under the ingest-role creds - # (rds-db:connect only) and is pure waste here. - - name: Install uv for v4 ingest - if: vars.GH_BENCH_INGEST_ROLE_ARN != '' - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false - - name: Configure AWS credentials for v4 ingest (OIDC) - if: vars.GH_BENCH_INGEST_ROLE_ARN != '' - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} - aws-region: ${{ vars.RDS_BENCH_REGION }} - - name: Ingest results to v4 Postgres - if: vars.GH_BENCH_INGEST_ROLE_ARN != '' - shell: bash - env: - RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} - RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }} - AWS_REGION: ${{ vars.RDS_BENCH_REGION }} - BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }} - BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} - run: | - set -Eeuo pipefail - curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ - -o "${RUNNER_TEMP}/rds-global-bundle.pem" - DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" - uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \ - scripts/post-ingest.py results.v3.jsonl \ - --postgres "${DSN}" \ - --commit-sha "${{ github.sha }}" \ - --region "${AWS_REGION}" - - - name: Alert incident.io - if: failure() - uses: ./.github/actions/alert-incident-io - with: - api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} - alert-title: "${{ matrix.benchmark.name }} benchmark failed on develop" - deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure + uses: ./.github/workflows/bench-core.yml + secrets: inherit + with: + mode: "develop" + profile: "micro" sql: uses: ./.github/workflows/sql-benchmarks.yml diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index a954b9c3268..e83db450096 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -156,8 +156,9 @@ vx-bench clean --older-than "30 days" [options] ## Declarative benchmark matrix To change what CI benchmarks, edit **`bench_orchestrator/benchmarks.py`**. `BENCHMARKS` is the list -of benchmark suites and their supported targets; `PROFILES` says how much coverage each workflow -runs. Matrix rendering is kept separately in `bench_orchestrator/matrix.py`. +of SQL suites and their supported targets, `PROFILES` says how much coverage each SQL workflow +runs, and `MICRO_BENCHMARKS` lists the Rust benchmark binaries. Matrix rendering is kept +separately in `bench_orchestrator/matrix.py`. This replaces the parallel `pr_targets`/`develop_targets` arrays and duplicated `full`/`base` matrices that previously lived in workflow YAML. @@ -184,7 +185,8 @@ support rules that validate a run. ### Built-in profiles `develop` runs full coverage on merge, `pr` runs the default targets for every SQL benchmark, and -`nightly` runs SF=100 TPC-H on NVMe and S3. List them with `vx-bench matrix`. +`nightly` runs SF=100 TPC-H on NVMe and S3. The `micro` profile describes the two Rust benchmark +binaries used by the shared develop/PR workflow. List them with `vx-bench matrix`. ### Adding a benchmark diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py index 6857a3b4a15..cf4b29f3f59 100644 --- a/bench-orchestrator/bench_orchestrator/benchmarks.py +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -1,19 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -"""The SQL benchmarks and profiles run by CI. +"""The benchmarks and profiles run by CI. This is the file to edit when changing benchmark coverage: * ``BENCHMARKS`` declares which benchmark suites exist and every target they support. * ``PROFILES`` declares how much of that coverage each CI workflow runs. +* ``MICRO_BENCHMARKS`` declares the Rust benchmark binaries run by the shared workflow. Matrix rendering lives in :mod:`bench_orchestrator.matrix` and should not need to change when a benchmark is added or removed. """ from .config import Benchmark, Engine, Format -from .matrix import STANDARD, BenchmarkDef, Profile, Storage, all_targets, defaults, df, duck +from .matrix import STANDARD, BenchmarkDef, MicroDef, Profile, Storage, all_targets, defaults, df, duck def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef: @@ -162,3 +163,26 @@ def _fineweb(storage: Storage) -> BenchmarkDef: description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", ), } + + +# ================================================================================================== +# RUST MICRO-BENCHMARK DEFINITIONS — ADD OR REMOVE MICRO-BENCHMARKS HERE +# ================================================================================================== + +MICRO_BENCHMARKS: list[MicroDef] = [ + MicroDef( + id="random-access-bench", + name="Random Access", + build_args="--features lance", + formats=(Format.PARQUET, Format.LANCE, Format.VORTEX), + split=True, + ), + MicroDef( + id="compress-bench", + name="Compression", + build_args="--features lance", + formats=(Format.PARQUET, Format.LANCE, Format.VORTEX), + ), +] + +MICRO_PROFILE_DESCRIPTION = "Rust micro-benchmarks: random access and compression." diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index ef8308389d3..afef5bc0320 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -16,7 +16,7 @@ from rich.console import Console from rich.table import Table -from .benchmarks import BENCHMARKS, PROFILES +from .benchmarks import BENCHMARKS, MICRO_BENCHMARKS, MICRO_PROFILE_DESCRIPTION, PROFILES from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( @@ -31,7 +31,7 @@ parse_targets_json, resolve_axis_targets, ) -from .matrix import resolve_matrix +from .matrix import resolve_matrix, resolve_micro_matrix from .runner.builder import BenchmarkBuilder from .runner.executor import BenchmarkExecutor from .storage.store import ResultStore @@ -234,11 +234,17 @@ def matrix( if profile is None or list_profiles: for name, prof in PROFILES.items(): console.print(f"[bold cyan]{name}[/bold cyan]: {prof.description}") + console.print(f"[bold cyan]micro[/bold cyan]: {MICRO_PROFILE_DESCRIPTION}") + return + + if profile == "micro": + entries = resolve_micro_matrix(MICRO_BENCHMARKS) + typer.echo(json.dumps(entries, indent=2 if pretty else None)) return prof = PROFILES.get(profile) if prof is None: - known = ", ".join(PROFILES) + known = ", ".join((*PROFILES, "micro")) console.print(f"[red]Unknown profile '{profile}'. Available: {known}[/red]") raise typer.Exit(1) diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py index 9f1a4bbd989..c716fd78614 100644 --- a/bench-orchestrator/bench_orchestrator/matrix.py +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -156,6 +156,17 @@ def subcommand(self) -> str: return self.benchmark.value +@dataclass(frozen=True) +class MicroDef: + """A Rust micro-benchmark binary and the arguments needed to build and run it.""" + + id: str + name: str + build_args: str + formats: tuple[Format, ...] + split: bool = False + + # A target policy maps a benchmark's declared targets to the targets a profile actually runs. TargetPolicy = Callable[[BenchmarkDef], TargetSet] @@ -234,3 +245,17 @@ def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list continue entries.append(_matrix_entry(benchmark, run_targets)) return entries + + +def resolve_micro_matrix(benchmarks: Iterable[MicroDef]) -> list[dict[str, object]]: + """Resolve Rust micro-benchmark declarations into GitHub Actions matrix entries.""" + return [ + { + "id": benchmark.id, + "name": benchmark.name, + "build_args": benchmark.build_args, + "formats": ",".join(fmt.value for fmt in benchmark.formats), + "split": benchmark.split, + } + for benchmark in benchmarks + ] diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index 01691ceea22..1cb57415cbc 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -7,7 +7,7 @@ from typing import cast from bench_orchestrator import cli as cli_module -from bench_orchestrator.benchmarks import BENCHMARKS, PROFILES +from bench_orchestrator.benchmarks import BENCHMARKS, MICRO_BENCHMARKS, PROFILES from bench_orchestrator.config import Benchmark, Engine, Format from bench_orchestrator.matrix import ( DEFAULTS, @@ -19,6 +19,7 @@ df, duck, resolve_matrix, + resolve_micro_matrix, ) from typer.testing import CliRunner @@ -127,3 +128,27 @@ def test_matrix_command_emits_json_and_rejects_unknown_profiles() -> None: result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) assert result.exit_code == 1 + + +def test_micro_matrix_matches_the_two_shared_workflow_jobs() -> None: + entries = resolve_micro_matrix(MICRO_BENCHMARKS) + assert entries == [ + { + "id": "random-access-bench", + "name": "Random Access", + "build_args": "--features lance", + "formats": "parquet,lance,vortex", + "split": True, + }, + { + "id": "compress-bench", + "name": "Compression", + "build_args": "--features lance", + "formats": "parquet,lance,vortex", + "split": False, + }, + ] + + result = runner.invoke(cli_module.app, ["matrix", "micro"]) + assert result.exit_code == 0 + assert json.loads(result.stdout) == entries