Spark FileFormat connector (upstream PR 9658), integration branch

.config/nextest.toml

@@ -1,6 +1,14 @@
 [profile.default]
 slow-timeout = { period = "30s", terminate-after = 5 }
 
+# Spatial tests concurrently call INSTALL spatial which fails
+[test-groups]
+duckdb-extension-install = { max-threads = 1 }
+
 [[profile.default.overrides]]
 filter = 'test(compress_large_int | fsst_compress_offsets_overflow_i32)'
 priority = 100
+
+[[profile.default.overrides]]
+filter = 'package(vortex-duckdb) & (test(spatial_) | test(test_geometry))'
+test-group = 'duckdb-extension-install'

.github/actions/assert-cpu/action.yml

@@ -0,0 +1,30 @@
+name: "Assert CPU"
+description: "Fail unless the runner has the expected CPU, so `-C target-cpu=native` builds match the benchmark machine"
+
+inputs:
+  implementer:
+    description: "Expected `CPU implementer` from /proc/cpuinfo"
+    required: false
+    default: "0x41"  # Arm
+  part:
+    description: "Expected `CPU part` from /proc/cpuinfo"
+    required: false
+    default: "0xd4f"  # Neoverse V2 (Graviton4)
+
+runs:
+  using: "composite"
+  steps:
+    - name: Assert CPU
+      shell: bash
+      env:
+        EXPECTED_IMPLEMENTER: ${{ inputs.implementer }}
+        EXPECTED_PART: ${{ inputs.part }}
+      run: |
+        implementer=$(awk -F': *' '/^CPU implementer/ {print $2; exit}' /proc/cpuinfo)
+        part=$(awk -F': *' '/^CPU part/ {print $2; exit}' /proc/cpuinfo)
+        echo "CPU implementer=${implementer:-unknown} part=${part:-unknown}"
+        lscpu | grep -E "^(Model name|Vendor ID)" || true
+        if [[ "$implementer" != "$EXPECTED_IMPLEMENTER" || "$part" != "$EXPECTED_PART" ]]; then
+          echo "::error::Expected CPU implementer=$EXPECTED_IMPLEMENTER part=$EXPECTED_PART, got implementer=${implementer:-unknown} part=${part:-unknown}"
+          exit 1
+        fi

.github/actions/fetch-bench-baseline/action.yml

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+name: "Fetch benchmark baseline"
+description: >-
+  Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres: assume the
+  read-only benchmarks role over OIDC and sign in as bench_ci_read with an RDS IAM token. If the
+  baseline cannot be read, it writes no baseline file and the report says the baseline could not be
+  read. The calling job needs `id-token: write`. Composite actions cannot read `vars`, so callers
+  pass the repository variables in.
+
+inputs:
+  pr-records:
+    description: "The PR run's --ingest-jsonl output"
+    default: results.ingest.jsonl
+  output:
+    description: "Where to write the baseline records"
+    default: base.ingest.jsonl
+  role-arn:
+    description: "The read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN)"
+    required: true
+  region:
+    description: "AWS region of the benchmarks RDS instance (vars.RDS_BENCH_REGION)"
+    required: true
+  endpoint:
+    description: "Hostname of the benchmarks RDS instance (vars.RDS_BENCH_INSTANCE_ENDPOINT)"
+    required: true
+  database:
+    description: "Name of the benchmarks database (vars.RDS_BENCH_DB_NAME)"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    # The credentials stay in step outputs so the job's existing AWS credentials, which later
+    # steps such as the S3 cleanup still use, are not replaced.
+    - name: Configure AWS credentials for the benchmarks database (OIDC)
+      id: aws
+      if: inputs.role-arn != ''
+      continue-on-error: true
+      uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd  # v6
+      with:
+        role-to-assume: ${{ inputs.role-arn }}
+        aws-region: ${{ inputs.region }}
+        retry-max-attempts: 1
+        output-credentials: true
+        output-env-credentials: false
+    - name: Fetch baseline
+      shell: bash
+      env:
+        PR_RECORDS: ${{ inputs.pr-records }}
+        OUTPUT: ${{ inputs.output }}
+        AWS_OUTCOME: ${{ steps.aws.outcome }}
+        AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
+        AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
+        AWS_REGION: ${{ inputs.region }}
+        RDS_BENCH_INSTANCE_ENDPOINT: ${{ inputs.endpoint }}
+        RDS_BENCH_DB_NAME: ${{ inputs.database }}
+      run: |
+        set -Eeu -o pipefail
+        if [[ "$AWS_OUTCOME" != "success" ]]; then
+          # No baseline file tells the report that the baseline could not be read.
+          echo "::warning::Could not assume the read-only benchmarks role; reporting without a baseline."
+          exit 0
+        fi
+        curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
+          -o "${RUNNER_TEMP}/rds-global-bundle.pem"
+        DSN="postgresql://bench_ci_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem"
+        if ! uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \
+            --postgres "$DSN" --region "$AWS_REGION"; then
+          echo "::warning::Could not read the baseline from the benchmarks database; reporting without a baseline."
+          rm -f "$OUTPUT"
+        fi