From 9f41fc32c011cdf8d507f417fa64fb1f25f4bc18 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Sat, 4 Apr 2026 18:51:18 +0100 Subject: [PATCH] Unify benchmark build Signed-off-by: Adam Gutglick --- .github/workflows/bench-pr.yml | 54 ++++++++++++++++---- .github/workflows/bench.yml | 54 +++++++++++++++----- .github/workflows/nightly-bench.yml | 1 + .github/workflows/sql-benchmarks.yml | 74 +++++++++++++++++++++++----- 4 files changed, 149 insertions(+), 34 deletions(-) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 1fab5eb28bc..4a559e3d58c 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -19,7 +19,43 @@ permissions: id-token: write # enables AWS-GitHub OIDC jobs: + build: + timeout-minutes: 60 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=bench-dedicated/tag=bench-build{1}', github.run_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.event.pull_request.head.repo.fork == false + with: + sccache: s3 + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build binaries + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + run: | + cargo build --package random-access-bench --package compress-bench --profile release_debug + - name: Stage binaries for upload + shell: bash + run: | + mkdir -p staging/bin + cp target/release_debug/random-access-bench staging/bin/ + cp target/release_debug/compress-bench staging/bin/ + - uses: actions/upload-artifact@v7 + with: + name: bench-binaries + path: staging/ + retention-days: 1 + bench: + needs: build timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -30,22 +66,16 @@ jobs: benchmark: - id: random-access-bench name: Random Access - build_args: "--features lance" - id: compress-bench name: Compression steps: - uses: runs-on/action@v2 if: github.event.pull_request.head.repo.fork == false - with: - sccache: s3 - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha }} - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install DuckDB run: | @@ -55,12 +85,16 @@ jobs: - uses: ./.github/actions/system-info - - name: Build binary + - uses: actions/download-artifact@v8 + with: + name: bench-binaries + path: staging + - name: Install binaries shell: bash - env: - RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | - cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} + mkdir -p target/release_debug + cp staging/bin/* target/release_debug/ + chmod +x target/release_debug/* - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ebeb429849b..e2d6ba95a6e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -31,7 +31,41 @@ jobs: bash scripts/commit-json.sh > new-commit.json bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json + build: + timeout-minutes: 60 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=bench-dedicated/extras=s3-cache/tag=bench-build', github.run_id) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@v6 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build binaries + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + run: | + cargo build --bin random-access-bench --bin compress-bench --profile release_debug --features lance + - name: Stage binaries for upload + shell: bash + run: | + mkdir -p staging/bin + cp target/release_debug/random-access-bench staging/bin/ + cp target/release_debug/compress-bench staging/bin/ + - uses: actions/upload-artifact@v7 + with: + name: bench-binaries + path: staging/ + retention-days: 1 + bench: + needs: build timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -42,23 +76,16 @@ jobs: benchmark: - id: random-access-bench name: Random Access - build_args: "--features lance" formats: "parquet,lance,vortex" - id: compress-bench name: Compression - build_args: "--features lance" formats: "parquet,lance,vortex" steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - uses: actions/checkout@v6 - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install DuckDB run: | @@ -68,12 +95,16 @@ jobs: - uses: ./.github/actions/system-info - - name: Build binary + - uses: actions/download-artifact@v8 + with: + name: bench-binaries + path: staging + - name: Install binaries shell: bash - env: - RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | - cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} + mkdir -p target/release_debug + cp staging/bin/* target/release_debug/ + chmod +x target/release_debug/* - name: Setup Polar Signals uses: polarsignals/gh-actions-ps-profiling@v0.8.1 @@ -115,6 +146,7 @@ jobs: secrets: inherit with: mode: "develop" + build_lance: true benchmark_matrix: | [ { diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index 7646ffc8338..bf9ca25a52c 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -23,6 +23,7 @@ jobs: with: mode: "develop" machine_type: ${{ matrix.machine_type.instance_name }} + build_lance: true # datafusion:vortex uses a lot of memory benchmark_matrix: | [ diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 08afe175341..3b112a55ff2 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -10,6 +10,10 @@ on: required: false type: string default: c6id.8xlarge + build_lance: + required: false + type: boolean + default: false benchmark_matrix: required: false type: string @@ -98,7 +102,55 @@ on: ] jobs: + build: + timeout-minutes: 60 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=bench-dedicated/tag=sql-build{1}', github.run_id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + with: + sccache: s3 + - uses: actions/checkout@v6 + if: inputs.mode == 'pr' + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/checkout@v6 + if: inputs.mode != 'pr' + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build binaries + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + run: | + packages="--bin data-gen --bin datafusion-bench --bin duckdb-bench" + if [ "${{ inputs.build_lance }}" = "true" ]; then + packages="$packages --bin lance-bench" + fi + cargo build $packages --profile release_debug + - name: Stage binaries for upload + shell: bash + run: | + mkdir -p staging/bin staging/lib + cp target/release_debug/data-gen staging/bin/ + cp target/release_debug/datafusion-bench staging/bin/ + cp target/release_debug/duckdb-bench staging/bin/ + if [ -f target/release_debug/lance-bench ]; then + cp target/release_debug/lance-bench staging/bin/ + fi + cp target/duckdb-lib-*/libduckdb.so staging/lib/ 2>/dev/null || true + - uses: actions/upload-artifact@v7 + with: + name: sql-bench-binaries + path: staging/ + retention-days: 1 + bench: + needs: build timeout-minutes: 120 strategy: fail-fast: false @@ -112,8 +164,6 @@ jobs: steps: - uses: runs-on/action@v2 if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false - with: - sccache: s3 - uses: actions/checkout@v6 if: inputs.mode == 'pr' with: @@ -123,9 +173,6 @@ jobs: if: inputs.mode != 'pr' - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install DuckDB run: | @@ -135,16 +182,17 @@ jobs: - uses: ./.github/actions/system-info - - name: Build binaries + - uses: actions/download-artifact@v8 + with: + name: sql-bench-binaries + path: staging + - name: Install binaries shell: bash - env: - RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | - packages="--bin data-gen --bin datafusion-bench --bin duckdb-bench" - if [ "${{ matrix.build_lance }}" = "true" ]; then - packages="$packages --bin lance-bench" - fi - cargo build $packages --profile release_debug + mkdir -p target/release_debug + cp staging/bin/* target/release_debug/ + chmod +x target/release_debug/* + echo "LD_LIBRARY_PATH=${{ github.workspace }}/staging/lib" >> "$GITHUB_ENV" - name: Generate data shell: bash