Compare PR benchmarks against the benchmarks database instead of S3

.github/actions/fetch-bench-baseline/action.yml

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+name: "Fetch benchmark baseline"
+description: >-
+  Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres: assume the
+  read-only benchmarks role over OIDC and sign in as bench_ci_read with an RDS IAM token. If the
+  baseline cannot be read, it writes no baseline file and the report says the baseline could not be
+  read. The calling job needs `id-token: write`. Composite actions cannot read `vars`, so callers
+  pass the repository variables in.
+
+inputs:
+  pr-records:
+    description: "The PR run's --ingest-jsonl output"
+    default: results.ingest.jsonl
+  output:
+    description: "Where to write the baseline records"
+    default: base.ingest.jsonl
+  role-arn:
+    description: "The read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN)"
+    required: true
+  region:
+    description: "AWS region of the benchmarks RDS instance (vars.RDS_BENCH_REGION)"
+    required: true
+  endpoint:
+    description: "Hostname of the benchmarks RDS instance (vars.RDS_BENCH_INSTANCE_ENDPOINT)"
+    required: true
+  database:
+    description: "Name of the benchmarks database (vars.RDS_BENCH_DB_NAME)"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    # The credentials stay in step outputs so the job's existing AWS credentials, which later
+    # steps such as the S3 cleanup still use, are not replaced.
+    - name: Configure AWS credentials for the benchmarks database (OIDC)
+      id: aws
+      if: inputs.role-arn != ''
+      continue-on-error: true
+      uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd  # v6
+      with:
+        role-to-assume: ${{ inputs.role-arn }}
+        aws-region: ${{ inputs.region }}
+        retry-max-attempts: 1
+        output-credentials: true
+        output-env-credentials: false
+    - name: Fetch baseline
+      shell: bash
+      env:
+        PR_RECORDS: ${{ inputs.pr-records }}
+        OUTPUT: ${{ inputs.output }}
+        AWS_OUTCOME: ${{ steps.aws.outcome }}
+        AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+        AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+        AWS_REGION: ${{ inputs.region }}
+        RDS_BENCH_INSTANCE_ENDPOINT: ${{ inputs.endpoint }}
+        RDS_BENCH_DB_NAME: ${{ inputs.database }}
+      run: |
+        set -Eeu -o pipefail
+        if [[ "$AWS_OUTCOME" != "success" ]]; then
+          # No baseline file tells the report that the baseline could not be read.
+          echo "::warning::Could not assume the read-only benchmarks role; reporting without a baseline."
+          exit 0
+        fi
+        curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
+          -o "${RUNNER_TEMP}/rds-global-bundle.pem"
+        DSN="postgresql://bench_ci_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem"
+        if ! uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \
+            --postgres "$DSN" --region "$AWS_REGION"; then
+          echo "::warning::Could not read the baseline from the benchmarks database; reporting without a baseline."
+          rm -f "$OUTPUT"
+        fi

.github/workflows/commit-metadata.yml

@@ -59,4 +59,5 @@ jobs:
             scripts/post-ingest.py empty.jsonl \
             --postgres "${DSN}" \
             --commit-sha "${{ github.sha }}" \
+            --branch "${{ github.ref_name }}" \
             --region "${AWS_REGION}"

.github/workflows/develop-bench.yml

@@ -263,6 +263,7 @@ jobs:
             scripts/post-ingest.py results.ingest.jsonl \
             --postgres "${DSN}" \
             --commit-sha "${{ github.sha }}" \
+            --branch "${{ github.ref_name }}" \
             --region "${AWS_REGION}"
 
       - name: Alert incident.io

.github/workflows/pr-bench-gpu-compress.yml

@@ -14,6 +14,7 @@ on:
 permissions:
   contents: read
   pull-requests: write
+  id-token: write  # enables AWS-GitHub OIDC
 
 jobs:
   gpu-compress-bench:
@@ -48,9 +49,11 @@ jobs:
       - name: Install cuDF
         # The GPU Parquet number is a cuDF `read_parquet`. cuDF ships prebuilt manylinux wheels
         # on NVIDIA's index, so it stays a runtime dependency and never enters the Rust build.
+        # `--no-config` keeps the workspace's `exclude-newer` off this standalone venv: NVIDIA's
+        # index publishes no upload dates, so that cutoff would filter out every cuDF wheel.
         run: |
           uv venv --python 3.12 .venv-cudf
-          uv pip install --python .venv-cudf \
+          uv pip install --no-config --python .venv-cudf \
             --extra-index-url https://pypi.nvidia.com \
             cudf-cu12 pandas pyarrow
           echo "$PWD/.venv-cudf/bin" >> "$GITHUB_PATH"
@@ -78,18 +81,22 @@ jobs:
           # (cuda.yaml), which also sets only FLAT_LAYOUT_INLINE_ARRAY_NODE.
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          python3 scripts/compress-split.py --gpu-decompress
+          python3 scripts/compress-split.py --gpu-decompress --emit-ingest-records
+      - name: Fetch baseline from the benchmarks database
+        uses: ./.github/actions/fetch-bench-baseline
+        with:
+          role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }}
+          region: ${{ vars.RDS_BENCH_REGION }}
+          endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
+          database: ${{ vars.RDS_BENCH_DB_NAME }}
+
       - name: Publish results
         shell: bash
         run: |
           set -Eeu -o pipefail -x
 
-          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz \
-            data.json.gz --no-sign-request
-          gzip -d -c data.json.gz > base.json
-
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json \
-            "GPU Compression" > comment.md
+          uv run --no-project scripts/compare-benchmark-jsons.py base.ingest.jsonl results.json \
+            "GPU Compression" --pr-records results.ingest.jsonl > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
       - name: Comment PR
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false

.github/workflows/pr-bench-runner.yml

@@ -189,7 +189,7 @@ jobs:
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
           AWS_REGION: "us-east-1"
         run: |
-          python3 scripts/random-access-split.py \
+          python3 scripts/random-access-split.py --emit-ingest-records \
             ${{ inputs.remote_data_dir != '' && format('--remote-data-dir {0}', inputs.remote_data_dir) || '' }}
 
       - name: Run ${{ inputs.benchmark_name }} benchmark (per-dataset)
@@ -199,7 +199,7 @@ jobs:
           RUST_BACKTRACE: full
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          python3 scripts/compress-split.py
+          python3 scripts/compress-split.py --emit-ingest-records
 
       - name: Run ${{ inputs.benchmark_name }} benchmark (per-column-encoder)
         if: inputs.benchmark_id == 'string-bench'
@@ -210,16 +210,21 @@ jobs:
         run: |
           python3 scripts/string-split.py
 
+      - name: Fetch baseline from the benchmarks database
+        uses: ./.github/actions/fetch-bench-baseline
+        with:
+          role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }}
+          region: ${{ vars.RDS_BENCH_REGION }}
+          endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
+          database: ${{ vars.RDS_BENCH_DB_NAME }}
+
       - name: Compare results
         shell: bash
         run: |
           set -Eeu -o pipefail -x
 
-          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
-          gzip -d -c data.json.gz > base.json
-
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \
-            > comment.md
+          uv run --no-project scripts/compare-benchmark-jsons.py base.ingest.jsonl results.json \
+            "${{ inputs.benchmark_name }}" --pr-records results.ingest.jsonl > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR

.github/workflows/sql-bench-matrix.yml

@@ -242,17 +242,23 @@ jobs:
             -o sizes.json
           cat sizes.json >> results.json
 
+      - name: Fetch baseline from the benchmarks database
+        if: inputs.mode == 'pr'
+        uses: ./.github/actions/fetch-bench-baseline
+        with:
+          role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }}
+          region: ${{ vars.RDS_BENCH_REGION }}
+          endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
+          database: ${{ vars.RDS_BENCH_DB_NAME }}
+
       - name: Compare results
         if: inputs.mode == 'pr'
         shell: bash
         run: |
           set -Eeu -o pipefail -x
 
-          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
-          gzip -d -c data.json.gz > base.json
-
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \
-            > comment.md
+          uv run --no-project scripts/compare-benchmark-jsons.py base.ingest.jsonl results.json \
+            "${{ matrix.name }}" --pr-records results.ingest.jsonl > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR
@@ -314,6 +320,7 @@ jobs:
             scripts/post-ingest.py results.ingest.jsonl \
             --postgres "${DSN}" \
             --commit-sha "${{ github.sha }}" \
+            --branch "${{ github.ref_name }}" \
             --region "${AWS_REGION}"
 
       - name: Alert incident.io