From 8cc24be6c469e3149eeeb617152df08cfa5987cb Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 29 Oct 2026 19:13:50 +0000 Subject: [PATCH 01/10] Compare PR benchmarks against the benchmarks database instead of S3 PR benchmark comments read their develop baseline from the 353 MB `data.json.gz` S3 dump. Read it from the benchmarks Postgres instead, which develop runs already write to. - `scripts/fetch-benchmark-baseline.py` reads the PR run's ingest records, finds the newest commit reachable from HEAD that recorded the same measurements, and writes that commit's rows. It connects as the read-only `bench_read` role. - `compare-benchmark-jsons.py` compares ingest records on both sides. File sizes and suite metadata still come from the PR's results JSON. - Random access, compression and GPU compression PR runs now emit ingest records. String Encoding and SQL file sizes are not in the database, so their PR comments show the PR measurements without a baseline. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .../actions/fetch-bench-baseline/action.yml | 49 +++ .github/workflows/pr-bench-gpu-compress.yml | 19 +- .github/workflows/pr-bench-runner.yml | 20 +- .github/workflows/sql-bench-matrix.yml | 17 +- scripts/compare-benchmark-jsons.py | 208 +++++-------- scripts/fetch-benchmark-baseline.py | 197 ++++++++++++ scripts/tests/test_benchmark_reporting.py | 291 ++++++++---------- 7 files changed, 488 insertions(+), 313 deletions(-) create mode 100644 .github/actions/fetch-bench-baseline/action.yml create mode 100644 scripts/fetch-benchmark-baseline.py diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml new file mode 100644 index 00000000000..66bbbb7b455 --- /dev/null +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +name: "Fetch benchmark baseline" +description: >- + Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres as the + read-only bench_read role. Without a password (fork PRs get no secrets), it writes an empty + baseline and the report shows the PR measurements alone. + +inputs: + pr-records: + description: "The PR run's --ingest-jsonl output" + required: true + output: + description: "Where to write the baseline records" + required: true + password: + description: "Password of the bench_read Postgres role" + required: false + default: "" + endpoint: + description: "Hostname of the benchmarks RDS instance" + required: true + database: + description: "Name of the benchmarks database" + required: true + +runs: + using: "composite" + steps: + - name: Fetch baseline + shell: bash + env: + PR_RECORDS: ${{ inputs.pr-records }} + OUTPUT: ${{ inputs.output }} + PGPASSWORD: ${{ inputs.password }} + RDS_BENCH_INSTANCE_ENDPOINT: ${{ inputs.endpoint }} + RDS_BENCH_DB_NAME: ${{ inputs.database }} + run: | + set -Eeu -o pipefail + if [[ -z "$PGPASSWORD" ]]; then + echo "::warning::No benchmarks database credential; reporting without a baseline." + : > "$OUTPUT" + exit 0 + fi + curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ + -o "${RUNNER_TEMP}/rds-global-bundle.pem" + DSN="postgresql://bench_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" + uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" --postgres "$DSN" diff --git a/.github/workflows/pr-bench-gpu-compress.yml b/.github/workflows/pr-bench-gpu-compress.yml index 4c9676b59a1..14fe3946a6c 100644 --- a/.github/workflows/pr-bench-gpu-compress.yml +++ b/.github/workflows/pr-bench-gpu-compress.yml @@ -78,18 +78,23 @@ jobs: # (cuda.yaml), which also sets only FLAT_LAYOUT_INLINE_ARRAY_NODE. FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" run: | - python3 scripts/compress-split.py --gpu-decompress + python3 scripts/compress-split.py --gpu-decompress --emit-ingest-records + - name: Fetch baseline from the benchmarks database + uses: ./.github/actions/fetch-bench-baseline + with: + pr-records: results.ingest.jsonl + output: base.ingest.jsonl + password: ${{ secrets.BENCH_READ_PASSWORD }} + endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + database: ${{ vars.RDS_BENCH_DB_NAME }} + - name: Publish results shell: bash run: | set -Eeu -o pipefail -x - python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz \ - data.json.gz --no-sign-request - gzip -d -c data.json.gz > base.json - - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json \ - "GPU Compression" > comment.md + uv run --no-project scripts/compare-benchmark-jsons.py base.ingest.jsonl results.json \ + "GPU Compression" --pr-records results.ingest.jsonl > comment.md cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index 3bb5a4c9845..df98a55b664 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -187,7 +187,7 @@ jobs: FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" AWS_REGION: "us-east-1" run: | - python3 scripts/random-access-split.py \ + python3 scripts/random-access-split.py --emit-ingest-records \ ${{ inputs.remote_data_dir != '' && format('--remote-data-dir {0}', inputs.remote_data_dir) || '' }} - name: Run ${{ inputs.benchmark_name }} benchmark (per-dataset) @@ -197,7 +197,7 @@ jobs: RUST_BACKTRACE: full FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" run: | - python3 scripts/compress-split.py + python3 scripts/compress-split.py --emit-ingest-records - name: Run ${{ inputs.benchmark_name }} benchmark (per-column-encoder) if: inputs.benchmark_id == 'string-bench' @@ -208,16 +208,22 @@ jobs: run: | python3 scripts/string-split.py + - name: Fetch baseline from the benchmarks database + uses: ./.github/actions/fetch-bench-baseline + with: + pr-records: results.ingest.jsonl + output: base.ingest.jsonl + password: ${{ secrets.BENCH_READ_PASSWORD }} + endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + database: ${{ vars.RDS_BENCH_DB_NAME }} + - name: Compare results shell: bash run: | set -Eeu -o pipefail -x - python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request - gzip -d -c data.json.gz > base.json - - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \ - > comment.md + uv run --no-project scripts/compare-benchmark-jsons.py base.ingest.jsonl results.json \ + "${{ inputs.benchmark_name }}" --pr-records results.ingest.jsonl > comment.md cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index 07795d62420..8e811e55ec4 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -240,17 +240,24 @@ jobs: -o sizes.json cat sizes.json >> results.json + - name: Fetch baseline from the benchmarks database + if: inputs.mode == 'pr' + uses: ./.github/actions/fetch-bench-baseline + with: + pr-records: results.ingest.jsonl + output: base.ingest.jsonl + password: ${{ secrets.BENCH_READ_PASSWORD }} + endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + database: ${{ vars.RDS_BENCH_DB_NAME }} + - name: Compare results if: inputs.mode == 'pr' shell: bash run: | set -Eeu -o pipefail -x - python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request - gzip -d -c data.json.gz > base.json - - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \ - > comment.md + uv run --no-project scripts/compare-benchmark-jsons.py base.ingest.jsonl results.json \ + "${{ matrix.name }}" --pr-records results.ingest.jsonl > comment.md cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index 4e2fe966b2f..7b95d478e7e 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -11,11 +11,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors +import argparse import math import os import re -import subprocess -import sys from collections.abc import Callable from dataclasses import dataclass from io import StringIO @@ -165,137 +164,74 @@ def extract_target_fields(name: str, target: object = None) -> pd.Series: return pd.Series({"engine": engine, "file_format": file_format, "query": query}) -def benchmark_identity(row: object) -> tuple[Any, Any, Any, str, str, Any] | None: - """Return the measurement identity used to find a matching baseline.""" +def ingest_dataset_label(record: dict[str, Any]) -> str: + """Join a record's dataset dims into one label, skipping the absent ones.""" - if row.get("metric") == FILE_SIZE_METRIC or isinstance(row.get("file_size"), dict): - return None - - name = row.get("name") - if name is None: - return None - - engine, file_format, _query = comparison_target(name, row.get("target")) - return ( - identity_value(name), - identity_value(row.get("storage")), - dataset_key(row.get("dataset")), - engine, - file_format, - identity_value(row.get("unit")), - ) - - -def benchmark_identity_rows(df: pd.DataFrame) -> pd.DataFrame: - """Return timing rows with the identity used to match a PR benchmark.""" - - _file_size_rows, timing_rows = split_file_size_rows(df) - if timing_rows.empty or "name" not in timing_rows.columns: - return pd.DataFrame(columns=["commit_id", "benchmark_identity"]) - - timing_rows = timing_rows.copy() - if "commit_id" not in timing_rows.columns: - timing_rows["commit_id"] = pd.NA - - timing_rows["benchmark_identity"] = [benchmark_identity(row) for _, row in timing_rows.iterrows()] - - return timing_rows[["commit_id", "benchmark_identity"]] + label = str(record["dataset"]) + if record.get("dataset_variant") is not None: + label += f"/{record['dataset_variant']}" + if record.get("scale_factor") is not None: + label += f"/sf={record['scale_factor']}" + return label -def read_jsonl_rows_for_commit(path: str, commit_id: str) -> pd.DataFrame: - """Read the latest copy of each row matching a history commit. +def ingest_record_row(record: dict[str, Any]) -> dict[str, Any] | None: + """Convert one `--ingest-jsonl` record into a comparison row. - Re-running a develop workflow appends a second result block for the same - commit. Keeping the last copy of each logical row prevents a many-to-one - merge from weighting that baseline multiple times. + The names keep the shapes `comparison_target` parses: `_qNN/engine:format` for SQL queries + and `random-access/.../ext-variant` for random access, so both sides of the comparison get + the same engine, format, and query attribution. """ - rows_by_identity: dict[tuple[Any, ...], dict[str, Any]] = {} - with open(path, encoding="utf-8") as lines: - for line in lines: - if '"commit_id"' not in line or f'"{commit_id}"' not in line: - continue - record = orjson.loads(line) - if record.get("commit_id") != commit_id: - continue - - file_size = record.get("file_size") - if isinstance(file_size, dict): - identity = ( - FILE_SIZE_METRIC, - file_size.get("benchmark"), - file_size.get("scale_factor"), - file_size.get("format"), - file_size.get("file"), - ) - else: - identity = ("timing", benchmark_identity(record)) - rows_by_identity[identity] = record - return pd.DataFrame(rows_by_identity.values()) - - -def git_tree_commit_ids() -> set[str]: - """Return every commit reachable from the checked-out branch head. + kind = record.get("kind") + file_format = record.get("format") + row: dict[str, Any] = {"commit_id": record.get("commit_sha"), "storage": None, "unit": "ns"} + if kind == "query_measurement": + engine = record["engine"] + row["name"] = f"{ingest_dataset_label(record)}_q{record['query_idx']:02d}/{engine}:{file_format}" + row["storage"] = record["storage"] + row["target"] = {"engine": engine, "format": file_format} + elif kind == "random_access_time": + ext = "vortex" if str(file_format).startswith("vortex") else file_format + row["name"] = f"random-access/{record['dataset']}/{ext}-{record['open_mode']}" + row["target"] = {"engine": "vortex", "format": file_format} + elif kind == "compression_time": + op = {"encode": "compress", "decode": "decompress"}.get(record["op"], record["op"]) + row["name"] = f"{op} time/{ingest_dataset_label(record)}" + row["target"] = {"engine": "vortex", "format": file_format} + elif kind == "compression_size": + row["name"] = f"{file_format} size/{ingest_dataset_label(record)}" + row["target"] = {"engine": "vortex", "format": file_format} + row["unit"] = "bytes" + row["value"] = record["value_bytes"] + return row + else: + return None + row["value"] = record["value_ns"] + row["all_runtimes"] = record["all_runtimes_ns"] + return row - A shallow checkout reaches only its own tip, so every recorded baseline - commit looks unreachable and the report silently degrades into "no - baseline is available for this benchmark yet" — indistinguishable from a - genuinely new benchmark, and wrong. Refuse the guess and say what to fix. - """ - is_shallow = subprocess.run( - ["git", "rev-parse", "--is-shallow-repository"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - if is_shallow == "true": - raise SystemExit( - "Cannot resolve a baseline from a shallow checkout: the baseline is the newest " - "recorded commit reachable from HEAD, and a shallow clone reaches only its own " - "tip. Check the repository out with `fetch-depth: 0`." - ) +def read_benchmark_rows(path: str) -> pd.DataFrame: + """Read comparison rows from a JSONL file; a missing file reads as no rows. - commits = subprocess.run( - ["git", "rev-list", "HEAD"], - check=True, - capture_output=True, - text=True, - ).stdout - return set(commits.splitlines()) - - -def read_latest_baseline_rows( - path: str, - pr: pd.DataFrame, - reachable_commit_ids: set[str], -) -> pd.DataFrame: - """Read rows from the latest reachable commit matching the PR benchmark. - - A benchmark can be new to the PR workflow and therefore have no baseline - yet. Return an empty frame with the PR schema in that case so the report - can show the measurements without comparison. + Lines carrying a `kind` are `--ingest-jsonl` records and are converted; any other line is + already a results-JSON row. """ - pr_identities = set(benchmark_identity_rows(pr)["benchmark_identity"]) - if not pr_identities: - return pd.read_json(path, lines=True) - - baseline_commit_id = None - with open(path, encoding="utf-8") as lines: - for line in lines: - if '"name"' not in line or '"commit_id"' not in line: - continue - record = orjson.loads(line) - if benchmark_identity(record) in pr_identities: - commit_id = record.get("commit_id") - if commit_id is not None and commit_id in reachable_commit_ids: - baseline_commit_id = commit_id - - if baseline_commit_id is None: - return pr.iloc[0:0].copy() - - return read_jsonl_rows_for_commit(path, baseline_commit_id) + rows = [] + if os.path.exists(path): + with open(path, encoding="utf-8") as lines: + for line in lines: + if not line.strip(): + continue + record = orjson.loads(line) + row = ingest_record_row(record) if "kind" in record else record + if row is not None: + rows.append(row) + if not rows: + return pd.DataFrame(columns=["commit_id", "name", "storage", "unit", "value", "all_runtimes", "target"]) + return pd.DataFrame(rows) def normalize_measurement_rows(df: pd.DataFrame) -> pd.DataFrame: @@ -1126,11 +1062,26 @@ def group_sort_key(group_key: tuple[str, str, str]) -> tuple[int, int, int, str, def main() -> None: """Render the benchmark comparison markdown used in CI PR comments.""" - benchmark_name = sys.argv[3] if len(sys.argv) > 3 else "" - - pr = pd.read_json(sys.argv[2], lines=True) - title = format_title(benchmark_name, pr) - base = read_latest_baseline_rows(sys.argv[1], pr, git_tree_commit_ids()) + parser = argparse.ArgumentParser(description=main.__doc__) + parser.add_argument("base_records", help="Baseline records written by fetch-benchmark-baseline.py.") + parser.add_argument("pr_results", help="The PR run's results JSON, for file sizes and suite metadata.") + parser.add_argument("benchmark_name", nargs="?", default="") + parser.add_argument( + "--pr-records", + help="The PR run's --ingest-jsonl records. Without them, PR rows come from the results JSON " + "and have no baseline.", + ) + args = parser.parse_args() + benchmark_name = args.benchmark_name + + pr_results = pd.read_json(args.pr_results, lines=True) + title = format_title(benchmark_name, pr_results) + pr_file_sizes, pr = split_file_size_rows(pr_results) + if args.pr_records: + pr_records = read_benchmark_rows(args.pr_records) + if not pr_records.empty: + pr = pr_records + base_file_sizes, base = split_file_size_rows(read_benchmark_rows(args.base_records)) base_commit_ids = set(base["commit_id"].unique()) pr_commit_id = set(pr["commit_id"].unique()) @@ -1139,9 +1090,6 @@ def main() -> None: base_commit_id = next(iter(base_commit_ids), None) pr_commit_id = next(iter(pr_commit_id)) - base_file_sizes, base = split_file_size_rows(base) - pr_file_sizes, pr = split_file_size_rows(pr) - base = normalize_measurement_rows(base) pr = normalize_measurement_rows(pr) diff --git a/scripts/fetch-benchmark-baseline.py b/scripts/fetch-benchmark-baseline.py new file mode 100644 index 00000000000..0b3aed879cb --- /dev/null +++ b/scripts/fetch-benchmark-baseline.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = ["psycopg[binary]"] +# /// + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Fetch the develop baseline for a PR benchmark run from the benchmarks Postgres. + +Reads the PR's `--ingest-jsonl` records, finds the newest commit reachable from +`HEAD` that recorded any of the same measurements, and writes that commit's +matching rows in the same JSONL record shape. `compare-benchmark-jsons.py` then +compares the two record files. + +The connection uses the read-only `bench_read` role. Its password comes from +`$PGPASSWORD`, so it never appears in the process arguments. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from psycopg import Connection + +# Per record kind: the fact table and the natural-key columns other than +# `commit_sha`. Rows from different commits match when these columns are equal. +# `None` compares equal to `None`, as the ingest path stores absent dims as NULL. +_KINDS: dict[str, tuple[str, tuple[str, ...], tuple[str, ...]]] = { + "query_measurement": ( + "query_measurements", + ("dataset", "dataset_variant", "scale_factor", "query_idx", "storage", "engine", "format"), + ("value_ns", "all_runtimes_ns"), + ), + "compression_time": ( + "compression_times", + ("dataset", "dataset_variant", "format", "op"), + ("value_ns", "all_runtimes_ns"), + ), + "compression_size": ( + "compression_sizes", + ("dataset", "dataset_variant", "format"), + ("value_bytes", "uncompressed_bytes"), + ), + "random_access_time": ( + "random_access_times", + ("dataset", "format", "open_mode"), + ("value_ns", "all_runtimes_ns"), + ), +} + +_INTEGER_DIMS = frozenset({"query_idx"}) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("pr_records", type=Path, help="The PR run's --ingest-jsonl output.") + parser.add_argument("output", type=Path, help="Where to write the baseline records.") + parser.add_argument( + "--postgres", + metavar="DSN", + required=True, + help="Libpq DSN for the read-only bench_read role, with sslmode=verify-full.", + ) + return parser.parse_args() + + +def read_records(path: Path) -> list[dict]: + if not path.exists(): + return [] + with path.open(encoding="utf-8") as lines: + return [json.loads(line) for line in lines if line.strip()] + + +def identities_by_kind(records: list[dict]) -> dict[str, list[tuple]]: + """Return the distinct natural keys of the PR records, grouped by kind.""" + + identities: dict[str, set[tuple]] = {} + for record in records: + kind = record.get("kind") + if kind not in _KINDS: + continue + _table, dims, _values = _KINDS[kind] + identities.setdefault(kind, set()).add(tuple(record.get(dim) for dim in dims)) + return {kind: sorted(keys, key=repr) for kind, keys in identities.items()} + + +def reachable_commits() -> list[str]: + """Return every commit reachable from `HEAD`; a shallow checkout cannot pick a baseline.""" + + is_shallow = subprocess.run( + ["git", "rev-parse", "--is-shallow-repository"], check=True, capture_output=True, text=True + ).stdout.strip() + if is_shallow == "true": + raise SystemExit( + "Cannot resolve a baseline from a shallow checkout: the baseline is the newest " + "recorded commit reachable from HEAD. Check the repository out with `fetch-depth: 0`." + ) + commits = subprocess.run(["git", "rev-list", "HEAD"], check=True, capture_output=True, text=True).stdout + return commits.split() + + +def _identity_join(kind: str, identities: list[tuple]) -> tuple[str, list[list]]: + """Return a join of `t` against the PR identities, and its array parameters.""" + + _table, dims, _values = _KINDS[kind] + arrays = [[identity[i] for identity in identities] for i in range(len(dims))] + casts = ", ".join("%s::integer[]" if dim in _INTEGER_DIMS else "%s::text[]" for dim in dims) + conditions = " AND ".join(f"t.{dim} IS NOT DISTINCT FROM p.{dim}" for dim in dims) + return f"JOIN unnest({casts}) AS p({', '.join(dims)}) ON {conditions}", arrays + + +def newest_baseline_commit(conn: Connection, identities: dict[str, list[tuple]], commits: list[str]) -> str | None: + """Return the newest reachable commit that recorded any PR identity.""" + + newest = None + for kind, keys in identities.items(): + table, _dims, _values = _KINDS[kind] + join, arrays = _identity_join(kind, keys) + row = conn.execute( + f""" + SELECT t.commit_sha, c.timestamp + FROM {table} t + JOIN commits c ON c.commit_sha = t.commit_sha + {join} + WHERE t.commit_sha = ANY(%s::text[]) + ORDER BY c.timestamp DESC, t.commit_sha DESC + LIMIT 1 + """, + [*arrays, commits], + ).fetchone() + if row is not None and (newest is None or (row[1], row[0]) > (newest[1], newest[0])): + newest = row + return None if newest is None else newest[0] + + +def baseline_records(conn: Connection, identities: dict[str, list[tuple]], commit_sha: str) -> list[dict]: + """Return the rows `commit_sha` recorded for the PR identities, as ingest records.""" + + records = [] + for kind, keys in identities.items(): + table, dims, values = _KINDS[kind] + join, arrays = _identity_join(kind, keys) + columns = ("commit_sha", *dims, *values) + rows = conn.execute( + f""" + SELECT {", ".join(f"t.{column}" for column in columns)} + FROM {table} t + {join} + WHERE t.commit_sha = %s + """, + [*arrays, commit_sha], + ).fetchall() + records.extend({"kind": kind, **dict(zip(columns, row))} for row in rows) + return records + + +def connect(dsn: str) -> Connection: + import psycopg + from psycopg import conninfo + + params = conninfo.conninfo_to_dict(dsn) + if params.get("sslmode", "verify-full") != "verify-full": + raise SystemExit("--postgres requires sslmode=verify-full") + params["sslmode"] = "verify-full" + params["options"] = f"{params.get('options') or ''} -c search_path=public -c statement_timeout=60000".strip() + params["connect_timeout"] = 10 + return psycopg.connect(**params) + + +def main() -> None: + args = parse_args() + identities = identities_by_kind(read_records(args.pr_records)) + records: list[dict] = [] + if identities: + commits = reachable_commits() + with connect(args.postgres) as conn: + conn.read_only = True + commit_sha = newest_baseline_commit(conn, identities, commits) + if commit_sha is not None: + records = baseline_records(conn, identities, commit_sha) + print(f"Baseline commit {commit_sha}: {len(records)} records") + if not records: + print("No baseline found; the report shows PR measurements without comparison.") + with args.output.open("w", encoding="utf-8") as output: + for record in records: + output.write(json.dumps(record) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index 87a8f1370e9..4b4c8afb07d 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -15,6 +15,16 @@ COMPARE_SCRIPT = REPO_ROOT / "scripts" / "compare-benchmark-jsons.py" CAPTURE_SCRIPT = REPO_ROOT / "scripts" / "capture-file-sizes.py" RANDOM_ACCESS_SPLIT_SCRIPT = REPO_ROOT / "scripts" / "random-access-split.py" +FETCH_BASELINE_SCRIPT = REPO_ROOT / "scripts" / "fetch-benchmark-baseline.py" + + +def load_module(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module def load_compare_module() -> ModuleType: @@ -100,6 +110,34 @@ def stored_timing_row( return row +def query_record(commit: str, query_idx: int, engine: str, file_format: str, value: int) -> dict[str, object]: + return { + "kind": "query_measurement", + "commit_sha": commit, + "dataset": "tpch", + "dataset_variant": None, + "scale_factor": "1", + "query_idx": query_idx, + "storage": "nvme", + "engine": engine, + "format": file_format, + "value_ns": value, + "all_runtimes_ns": [value, value, value], + } + + +def random_access_record(commit: str, file_format: str, open_mode: str, value: int) -> dict[str, object]: + return { + "kind": "random_access_time", + "commit_sha": commit, + "dataset": "taxi/uniform", + "format": file_format, + "open_mode": open_mode, + "value_ns": value, + "all_runtimes_ns": [value, value, value], + } + + def stored_custom_row( commit: str, name: str, @@ -139,28 +177,19 @@ def render_report( base_rows: list[dict[str, object]], pr_rows: list[dict[str, object]], benchmark_name: str, + pr_records: list[dict[str, object]] | None = None, ) -> str: - repo = git_repo_with_one_commit(tmp_path / "repo") - head_commit = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=repo, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - base_rows = [row | {"commit_id": head_commit} for row in base_rows] base_path = tmp_path / "base.jsonl" pr_path = tmp_path / "pr.jsonl" base_path.write_text("".join(f"{json.dumps(row)}\n" for row in base_rows), encoding="utf-8") pr_path.write_text("".join(f"{json.dumps(row)}\n" for row in pr_rows), encoding="utf-8") + command = [sys.executable, str(COMPARE_SCRIPT), str(base_path), str(pr_path), benchmark_name] + if pr_records is not None: + records_path = tmp_path / "pr.ingest.jsonl" + records_path.write_text("".join(f"{json.dumps(row)}\n" for row in pr_records), encoding="utf-8") + command += ["--pr-records", str(records_path)] - result = subprocess.run( - [sys.executable, str(COMPARE_SCRIPT), str(base_path), str(pr_path), benchmark_name], - cwd=repo, - check=False, - capture_output=True, - text=True, - ) + result = subprocess.run(command, check=False, capture_output=True, text=True) assert result.returncode == 0, result.stderr return result.stdout @@ -171,139 +200,6 @@ def markdown_row(report: str, name: str) -> list[str]: return [cell.strip() for cell in line.strip("|").split("|")] -def test_read_latest_baseline_rows_streams_latest_matching_benchmark_commit(tmp_path: Path) -> None: - compare = load_compare_module() - history_path = tmp_path / "history.jsonl" - history_rows = [ - stored_timing_row( - "base-old", - "tpch_q01/datafusion:parquet", - 100, - "nvme", - {"scale_factor": "1.0"}, - ), - file_size_record_for("base-old", 100, "tpch", "1.0", "vortex-file-compressed", "part-0.vortex"), - stored_timing_row( - "base-current", - "tpch_q01/datafusion:parquet", - 110, - "nvme", - {"scale_factor": "1.0"}, - ), - file_size_record_for("base-current", 120, "tpch", "1.0", "vortex-file-compressed", "part-0.vortex"), - stored_timing_row( - "base-wrong-unit", - "tpch_q01/datafusion:parquet", - 111, - "nvme", - {"scale_factor": "1.0"}, - ) - | {"unit": "ms"}, - stored_timing_row( - "base-wrong-target", - "tpch_q01/datafusion:parquet", - 112, - "nvme", - {"scale_factor": "1.0"}, - file_format="vortex", - ), - stored_timing_row("base-other", "clickbench_q01/datafusion:parquet", 200, "nvme"), - ] - history_path.write_text( - "".join(f"{json.dumps(row)}\n" for row in history_rows), - encoding="utf-8", - ) - pr = pd.DataFrame( - [ - stored_timing_row( - "pr-sha", - "tpch_q01/datafusion:parquet", - 115, - "nvme", - {"scale_factor": "1.0"}, - ), - ] - ) - - selected = compare.read_latest_baseline_rows( - history_path, - pr, - {"base-old", "base-current", "base-wrong-unit", "base-wrong-target", "base-other"}, - ) - - pd.testing.assert_frame_equal(selected, pd.DataFrame(history_rows[2:4])) - - -def test_read_latest_baseline_rows_skips_commits_outside_git_tree(tmp_path: Path) -> None: - compare = load_compare_module() - history_path = tmp_path / "history.jsonl" - history_rows = [ - stored_timing_row("base-reachable", "tpch_q01/datafusion:parquet", 100), - file_size_record_for( - "base-reachable", - 100, - "tpch", - "1.0", - "vortex-file-compressed", - "part-0.vortex", - ), - stored_timing_row("base-off-tree", "tpch_q01/datafusion:parquet", 110), - file_size_record_for( - "base-off-tree", - 120, - "tpch", - "1.0", - "vortex-file-compressed", - "part-0.vortex", - ), - ] - history_path.write_text( - "".join(f"{json.dumps(row)}\n" for row in history_rows), - encoding="utf-8", - ) - pr = pd.DataFrame([stored_timing_row("pr", "tpch_q01/datafusion:parquet", 105)]) - - selected = compare.read_latest_baseline_rows(history_path, pr, {"base-reachable"}) - - pd.testing.assert_frame_equal(selected, pd.DataFrame(history_rows[:2])) - - -def test_read_latest_baseline_rows_returns_empty_without_reachable_match(tmp_path: Path) -> None: - compare = load_compare_module() - history_path = tmp_path / "history.jsonl" - history_path.write_text( - f"{json.dumps(stored_timing_row('base-off-tree', 'tpch_q01/datafusion:parquet', 100))}\n", - encoding="utf-8", - ) - pr = pd.DataFrame([stored_timing_row("pr", "tpch_q01/datafusion:parquet", 105)]) - - selected = compare.read_latest_baseline_rows(history_path, pr, set()) - - assert selected.empty - - -def test_read_latest_baseline_rows_uses_last_result_from_rerun(tmp_path: Path) -> None: - compare = load_compare_module() - history_path = tmp_path / "history.jsonl" - history_path.write_text( - "".join( - f"{json.dumps(row)}\n" - for row in [ - stored_timing_row("base", "tpch_q01/datafusion:parquet", 100), - stored_timing_row("base", "tpch_q01/datafusion:parquet", 110), - ] - ), - encoding="utf-8", - ) - pr = pd.DataFrame([stored_timing_row("pr", "tpch_q01/datafusion:parquet", 105)]) - - selected = compare.read_latest_baseline_rows(history_path, pr, {"base"}) - - pd.testing.assert_frame_equal( - selected, pd.DataFrame([stored_timing_row("base", "tpch_q01/datafusion:parquet", 110)]) - ) - - def test_within_engine_analysis_uses_each_engines_own_parquet_control() -> None: compare = load_compare_module() rows = [ @@ -405,10 +301,9 @@ def test_comparison_report_groups_by_target_and_unit(tmp_path: Path) -> None: def test_comparison_report_handles_missing_benchmark_baseline(tmp_path: Path) -> None: - base_rows = [stored_custom_row("base-sha", "other timing/fixture", "ms", 12.5)] pr_rows = [stored_custom_row("pr-sha", "new timing/fixture", "ms", 3.125)] - report = render_report(tmp_path, base_rows, pr_rows, "New benchmark") + report = render_report(tmp_path, [], pr_rows, "New benchmark") assert "No baseline is available for this benchmark yet" in report assert "vs base `none`" in report @@ -731,7 +626,7 @@ def test_capture_file_sizes_emits_shared_benchmark_rows(tmp_path: Path) -> None: ] -def test_shallow_checkout_fails_instead_of_reporting_no_baseline(tmp_path: Path) -> None: +def test_baseline_fetch_refuses_a_shallow_checkout(tmp_path: Path) -> None: """A shallow clone must fail loudly, not render a baseline-free report. `git rev-list HEAD` reaches only the tip of a shallow clone, so every @@ -746,19 +641,11 @@ def test_shallow_checkout_fails_instead_of_reporting_no_baseline(tmp_path: Path) check=True, capture_output=True, ) - base_path = tmp_path / "base.jsonl" - pr_path = tmp_path / "pr.jsonl" - base_path.write_text( - f"{json.dumps(stored_timing_row('recorded', 'tpch_q01/datafusion:parquet', 100))}\n", - encoding="utf-8", - ) - pr_path.write_text( - f"{json.dumps(stored_timing_row('pr', 'tpch_q01/datafusion:parquet', 105))}\n", - encoding="utf-8", - ) + pr_path = tmp_path / "pr.ingest.jsonl" + pr_path.write_text(f"{json.dumps(query_record('pr', 1, 'datafusion', 'parquet', 105))}\n", encoding="utf-8") result = subprocess.run( - [sys.executable, str(COMPARE_SCRIPT), str(base_path), str(pr_path), "TPC-H"], + [sys.executable, str(FETCH_BASELINE_SCRIPT), str(pr_path), str(tmp_path / "base.jsonl"), "--postgres", ""], cwd=shallow, check=False, capture_output=True, @@ -768,7 +655,83 @@ def test_shallow_checkout_fails_instead_of_reporting_no_baseline(tmp_path: Path) assert result.returncode != 0 assert "shallow checkout" in result.stderr assert "fetch-depth: 0" in result.stderr - assert "No baseline is available" not in result.stdout + + +def test_baseline_fetch_groups_distinct_pr_identities_by_kind() -> None: + fetch = load_module("fetch_benchmark_baseline", FETCH_BASELINE_SCRIPT) + records = [ + query_record("pr", 1, "datafusion", "parquet", 100), + query_record("pr", 1, "datafusion", "parquet", 100), + random_access_record("pr", "parquet", "cached", 100), + {"kind": "vector_search_run", "dataset": "cohere"}, + ] + + assert fetch.identities_by_kind(records) == { + "query_measurement": [("tpch", None, "1", 1, "nvme", "datafusion", "parquet")], + "random_access_time": [("taxi/uniform", "parquet", "cached")], + } + + +def test_baseline_fetch_matches_absent_dims_as_equal() -> None: + fetch = load_module("fetch_benchmark_baseline", FETCH_BASELINE_SCRIPT) + + join, arrays = fetch._identity_join("compression_size", [("taxi", None, "parquet"), ("taxi", "gpu", "lance")]) + + assert "unnest(%s::text[], %s::text[], %s::text[]) AS p(dataset, dataset_variant, format)" in join + assert "t.dataset_variant IS NOT DISTINCT FROM p.dataset_variant" in join + assert arrays == [["taxi", "taxi"], [None, "gpu"], ["parquet", "lance"]] + + +def test_report_compares_ingest_records(tmp_path: Path) -> None: + targets = [("parquet", 100, 105), ("vortex-file-compressed", 80, 70)] + base_records = [query_record("base-sha", 1, "datafusion", target, base) for target, base, _pr in targets] + base_records += [random_access_record("base-sha", target, "cached", base) for target, base, _pr in targets] + pr_records = [query_record("pr-sha", 1, "datafusion", target, pr) for target, _base, pr in targets] + pr_records += [random_access_record("pr-sha", target, "cached", pr) for target, _base, pr in targets] + pr_records.append(random_access_record("pr-sha", "lance", "cached", 300)) + pr_results = [ + stored_timing_row("pr-sha", "ignored when records exist", 1) | {"doc": "benchmarks/tpch.md"}, + file_size_record("pr-sha", 125), + ] + + report = render_report(tmp_path, base_records, pr_results, "TPC-H", pr_records) + + assert "vs base `base-sha`" in report + assert "**Verdict**:" in report + assert "**Engines**: DataFusion " in report + assert "random-access " in report + assert "ignored when records exist" not in report + assert markdown_row(report, "tpch/sf=1_q01/datafusion:vortex-file-compressed")[1] == "70 / 80 / -12.5% 🟢" + assert markdown_row(report, "random-access/taxi/uniform/parquet-cached")[1] == "105 / 100 / +5.0%" + assert markdown_row(report, "random-access/taxi/uniform/lance-cached")[1] == "300 / — / no baseline" + assert "random-access / lance / ns " in report + assert "_No baseline file sizes found for base commit._" in report + + +def test_report_converts_compression_records() -> None: + compare = load_compare_module() + common = {"commit_sha": "sha", "dataset": "taxi", "dataset_variant": None, "format": "parquet"} + + encode = compare.ingest_record_row( + common | {"kind": "compression_time", "op": "encode", "value_ns": 5, "all_runtimes_ns": [6, 5]} + ) + decode = compare.ingest_record_row( + common + | { + "kind": "compression_time", + "dataset_variant": "gpu", + "op": "decode", + "value_ns": 5, + "all_runtimes_ns": [6, 5], + } + ) + size = compare.ingest_record_row(common | {"kind": "compression_size", "value_bytes": 10, "uncompressed_bytes": 20}) + + assert encode is not None and decode is not None and size is not None + assert encode["name"] == "compress time/taxi" + assert decode["name"] == "decompress time/taxi/gpu" + assert (size["name"], size["unit"], size["value"]) == ("parquet size/taxi", "bytes", 10) + assert compare.ingest_record_row({"kind": "vector_search_run"}) is None def test_summary_reports_hot_alone_when_no_row_recorded_its_runs(tmp_path: Path) -> None: From d540c8f8d91f01116809dd54a9a0e2c3229cb60a Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 29 Oct 2026 19:49:38 +0000 Subject: [PATCH 02/10] Keep benchmark comment names and say why a baseline is missing - SQL rows keep the `tpch_q01/engine:format` names of the results JSON; the scale factor and variant move to the dataset join key. - When the baseline cannot be read (no database credential), the comment says so instead of claiming the benchmark has no baseline yet. - File sizes have no baseline in the database, so the comment no longer prints a file-size section that can never compare anything. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .../actions/fetch-bench-baseline/action.yml | 6 +-- scripts/compare-benchmark-jsons.py | 23 ++++++++--- scripts/tests/test_benchmark_reporting.py | 40 ++++++++++++++++++- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml index 66bbbb7b455..c4239e50a10 100644 --- a/.github/actions/fetch-bench-baseline/action.yml +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -4,8 +4,8 @@ name: "Fetch benchmark baseline" description: >- Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres as the - read-only bench_read role. Without a password (fork PRs get no secrets), it writes an empty - baseline and the report shows the PR measurements alone. + read-only bench_read role. Without a password (fork PRs get no secrets), it writes no + baseline and the report says the baseline could not be read. inputs: pr-records: @@ -39,8 +39,8 @@ runs: run: | set -Eeu -o pipefail if [[ -z "$PGPASSWORD" ]]; then + # No baseline file tells the report that the baseline could not be read. echo "::warning::No benchmarks database credential; reporting without a baseline." - : > "$OUTPUT" exit 0 fi curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index 7b95d478e7e..a519be616f5 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -165,13 +165,11 @@ def extract_target_fields(name: str, target: object = None) -> pd.Series: def ingest_dataset_label(record: dict[str, Any]) -> str: - """Join a record's dataset dims into one label, skipping the absent ones.""" + """Join a record's dataset and variant into one label, skipping an absent variant.""" label = str(record["dataset"]) if record.get("dataset_variant") is not None: label += f"/{record['dataset_variant']}" - if record.get("scale_factor") is not None: - label += f"/sf={record['scale_factor']}" return label @@ -188,8 +186,13 @@ def ingest_record_row(record: dict[str, Any]) -> dict[str, Any] | None: row: dict[str, Any] = {"commit_id": record.get("commit_sha"), "storage": None, "unit": "ns"} if kind == "query_measurement": engine = record["engine"] - row["name"] = f"{ingest_dataset_label(record)}_q{record['query_idx']:02d}/{engine}:{file_format}" + row["name"] = f"{record['dataset']}_q{record['query_idx']:02d}/{engine}:{file_format}" row["storage"] = record["storage"] + # Like the results JSON, the scale factor and variant stay out of the name and in the + # dataset join key. + dims = {dim: record.get(dim) for dim in ("dataset_variant", "scale_factor") if record.get(dim) is not None} + if dims: + row["dataset"] = dims row["target"] = {"engine": engine, "format": file_format} elif kind == "random_access_time": ext = "vortex" if str(file_format).startswith("vortex") else file_format @@ -1081,6 +1084,7 @@ def main() -> None: pr_records = read_benchmark_rows(args.pr_records) if not pr_records.empty: pr = pr_records + base_fetched = os.path.exists(args.base_records) base_file_sizes, base = split_file_size_rows(read_benchmark_rows(args.base_records)) base_commit_ids = set(base["commit_id"].unique()) @@ -1161,7 +1165,13 @@ def main() -> None: if summary_fields: print("
".join(summary_fields)) print("") - if base_commit_id is None: + if not base_fetched: + print( + "_The baseline could not be read from the benchmarks database; " + "PR measurements are shown without comparison._" + ) + print("") + elif base_commit_id is None: print("_No baseline is available for this benchmark yet; PR measurements are shown without comparison._") print("") if verdict is not None or engine_summary is not None: @@ -1235,7 +1245,8 @@ def delta_cell( print("") print("") - file_size_report = format_file_size_report(base_file_sizes, pr_file_sizes) + # File sizes are not stored in the benchmarks database, so they have no baseline to compare. + file_size_report = "" if base_file_sizes.empty else format_file_size_report(base_file_sizes, pr_file_sizes) if file_size_report: print("") print("---") diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index 4b4c8afb07d..d54f46c4ce3 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -701,11 +701,47 @@ def test_report_compares_ingest_records(tmp_path: Path) -> None: assert "**Engines**: DataFusion " in report assert "random-access " in report assert "ignored when records exist" not in report - assert markdown_row(report, "tpch/sf=1_q01/datafusion:vortex-file-compressed")[1] == "70 / 80 / -12.5% 🟢" + assert markdown_row(report, "tpch_q01/datafusion:vortex-file-compressed")[1] == "70 / 80 / -12.5% 🟢" assert markdown_row(report, "random-access/taxi/uniform/parquet-cached")[1] == "105 / 100 / +5.0%" assert markdown_row(report, "random-access/taxi/uniform/lance-cached")[1] == "300 / — / no baseline" assert "random-access / lance / ns " in report - assert "_No baseline file sizes found for base commit._" in report + assert "File Size" not in report + + +def test_report_says_when_the_baseline_could_not_be_read(tmp_path: Path) -> None: + pr_records = [query_record("pr-sha", 1, "datafusion", "parquet", 100)] + records_path = tmp_path / "pr.ingest.jsonl" + records_path.write_text(f"{json.dumps(pr_records[0])}\n", encoding="utf-8") + results_path = tmp_path / "results.json" + results_path.write_text(f"{json.dumps(stored_timing_row('pr-sha', 'unused', 1))}\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(COMPARE_SCRIPT), + str(tmp_path / "missing.jsonl"), + str(results_path), + "TPC-H", + "--pr-records", + str(records_path), + ], + check=True, + capture_output=True, + text=True, + ) + + assert "The baseline could not be read from the benchmarks database" in result.stdout + assert "No baseline is available for this benchmark yet" not in result.stdout + + +def test_query_records_keep_scale_factors_apart() -> None: + compare = load_compare_module() + sf1 = compare.ingest_record_row(query_record("sha", 1, "duckdb", "parquet", 5)) + sf10 = compare.ingest_record_row(query_record("sha", 1, "duckdb", "parquet", 5) | {"scale_factor": "10"}) + + assert sf1 is not None and sf10 is not None + assert sf1["name"] == sf10["name"] == "tpch_q01/duckdb:parquet" + assert compare.dataset_key(sf1["dataset"]) != compare.dataset_key(sf10["dataset"]) def test_report_converts_compression_records() -> None: From c3a54a8eda697940e0ded223f65054eab80e011f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 10:32:41 +0000 Subject: [PATCH 03/10] Read the PR benchmark baseline with the ingest role, like post-ingest.py Instead of a new `bench_read` password secret, the baseline fetch assumes `vars.GH_BENCH_INGEST_ROLE_ARN` over OIDC and connects through `post-ingest.py`'s `connect_postgres`, as `bench_ingest` with an RDS IAM token. The transaction is read-only. The role's credentials stay in step outputs so the job's own AWS credentials, which the S3 cleanup step uses, are not replaced. If the role cannot be assumed, the comment says the baseline could not be read. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .../actions/fetch-bench-baseline/action.yml | 41 ++++++++++++++----- .github/workflows/pr-bench-gpu-compress.yml | 4 +- .github/workflows/pr-bench-runner.yml | 3 +- .github/workflows/sql-bench-matrix.yml | 3 +- scripts/fetch-benchmark-baseline.py | 32 ++++++++------- 5 files changed, 55 insertions(+), 28 deletions(-) diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml index c4239e50a10..d055dd8b69e 100644 --- a/.github/actions/fetch-bench-baseline/action.yml +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -3,9 +3,10 @@ name: "Fetch benchmark baseline" description: >- - Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres as the - read-only bench_read role. Without a password (fork PRs get no secrets), it writes no - baseline and the report says the baseline could not be read. + Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres the same + way post-ingest.py writes it: assume the ingest role over OIDC and sign in as bench_ingest with + an RDS IAM token. If the role cannot be assumed, it writes no baseline and the report says the + baseline could not be read. inputs: pr-records: @@ -14,10 +15,13 @@ inputs: output: description: "Where to write the baseline records" required: true - password: - description: "Password of the bench_read Postgres role" + role-arn: + description: "The benchmarks ingest role (vars.GH_BENCH_INGEST_ROLE_ARN)" required: false default: "" + region: + description: "AWS region of the benchmarks RDS instance" + required: true endpoint: description: "Hostname of the benchmarks RDS instance" required: true @@ -28,22 +32,39 @@ inputs: runs: using: "composite" steps: + # The credentials stay in step outputs so the job's existing AWS credentials, which later + # steps such as the S3 cleanup still use, are not replaced. + - name: Configure AWS credentials for the benchmarks database (OIDC) + id: aws + if: inputs.role-arn != '' + continue-on-error: true + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ inputs.role-arn }} + aws-region: ${{ inputs.region }} + output-credentials: true + output-env-credentials: false - name: Fetch baseline shell: bash env: PR_RECORDS: ${{ inputs.pr-records }} OUTPUT: ${{ inputs.output }} - PGPASSWORD: ${{ inputs.password }} + AWS_OUTCOME: ${{ steps.aws.outcome }} + AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }} + AWS_REGION: ${{ inputs.region }} RDS_BENCH_INSTANCE_ENDPOINT: ${{ inputs.endpoint }} RDS_BENCH_DB_NAME: ${{ inputs.database }} run: | set -Eeu -o pipefail - if [[ -z "$PGPASSWORD" ]]; then + if [[ "$AWS_OUTCOME" != "success" ]]; then # No baseline file tells the report that the baseline could not be read. - echo "::warning::No benchmarks database credential; reporting without a baseline." + echo "::warning::Could not assume the benchmarks ingest role; reporting without a baseline." exit 0 fi curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ -o "${RUNNER_TEMP}/rds-global-bundle.pem" - DSN="postgresql://bench_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" - uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" --postgres "$DSN" + DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" + uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \ + --postgres "$DSN" --region "$AWS_REGION" diff --git a/.github/workflows/pr-bench-gpu-compress.yml b/.github/workflows/pr-bench-gpu-compress.yml index 14fe3946a6c..772e09737ad 100644 --- a/.github/workflows/pr-bench-gpu-compress.yml +++ b/.github/workflows/pr-bench-gpu-compress.yml @@ -14,6 +14,7 @@ on: permissions: contents: read pull-requests: write + id-token: write # enables AWS-GitHub OIDC jobs: gpu-compress-bench: @@ -84,7 +85,8 @@ jobs: with: pr-records: results.ingest.jsonl output: base.ingest.jsonl - password: ${{ secrets.BENCH_READ_PASSWORD }} + role-arn: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + region: ${{ vars.RDS_BENCH_REGION }} endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} database: ${{ vars.RDS_BENCH_DB_NAME }} diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index df98a55b664..0228164f153 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -213,7 +213,8 @@ jobs: with: pr-records: results.ingest.jsonl output: base.ingest.jsonl - password: ${{ secrets.BENCH_READ_PASSWORD }} + role-arn: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + region: ${{ vars.RDS_BENCH_REGION }} endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} database: ${{ vars.RDS_BENCH_DB_NAME }} diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index 8e811e55ec4..fdf6f118194 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -246,7 +246,8 @@ jobs: with: pr-records: results.ingest.jsonl output: base.ingest.jsonl - password: ${{ secrets.BENCH_READ_PASSWORD }} + role-arn: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + region: ${{ vars.RDS_BENCH_REGION }} endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} database: ${{ vars.RDS_BENCH_DB_NAME }} diff --git a/scripts/fetch-benchmark-baseline.py b/scripts/fetch-benchmark-baseline.py index 0b3aed879cb..54238e09e2a 100644 --- a/scripts/fetch-benchmark-baseline.py +++ b/scripts/fetch-benchmark-baseline.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # /// script # requires-python = ">=3.11" -# dependencies = ["psycopg[binary]"] +# dependencies = ["psycopg[binary]", "boto3"] # /// # SPDX-License-Identifier: Apache-2.0 @@ -14,16 +14,19 @@ matching rows in the same JSONL record shape. `compare-benchmark-jsons.py` then compares the two record files. -The connection uses the read-only `bench_read` role. Its password comes from -`$PGPASSWORD`, so it never appears in the process arguments. +It connects the same way `post-ingest.py` does: as `bench_ingest`, with an RDS IAM token +minted from the AWS credentials of the assumed `GH_BENCH_INGEST_ROLE_ARN` role. The +transaction is read-only. """ from __future__ import annotations import argparse +import importlib.util import json import subprocess from pathlib import Path +from types import ModuleType from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -66,8 +69,9 @@ def parse_args() -> argparse.Namespace: "--postgres", metavar="DSN", required=True, - help="Libpq DSN for the read-only bench_read role, with sslmode=verify-full.", + help="Libpq DSN of the benchmarks database, as accepted by post-ingest.py --postgres.", ) + parser.add_argument("--region", default=None, help="AWS region for RDS IAM token minting.") return parser.parse_args() @@ -161,17 +165,15 @@ def baseline_records(conn: Connection, identities: dict[str, list[tuple]], commi return records -def connect(dsn: str) -> Connection: - import psycopg - from psycopg import conninfo +def post_ingest_module() -> ModuleType: + """Load `scripts/post-ingest.py`, whose database connector this script reuses.""" - params = conninfo.conninfo_to_dict(dsn) - if params.get("sslmode", "verify-full") != "verify-full": - raise SystemExit("--postgres requires sslmode=verify-full") - params["sslmode"] = "verify-full" - params["options"] = f"{params.get('options') or ''} -c search_path=public -c statement_timeout=60000".strip() - params["connect_timeout"] = 10 - return psycopg.connect(**params) + path = Path(__file__).resolve().parent / "post-ingest.py" + spec = importlib.util.spec_from_file_location("post_ingest", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def main() -> None: @@ -180,7 +182,7 @@ def main() -> None: records: list[dict] = [] if identities: commits = reachable_commits() - with connect(args.postgres) as conn: + with post_ingest_module().connect_postgres(args.postgres, args.region) as conn: conn.read_only = True commit_sha = newest_baseline_commit(conn, identities, commits) if commit_sha is not None: From 1ede4e8da86cad316298b0eeeba50959517bb854 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 12:33:11 +0000 Subject: [PATCH 04/10] Read the PR benchmark baseline with the read-only benchmarks role Assume vars.GH_BENCH_READ_ROLE_ARN and sign in as the SELECT-only bench_pr_read user with an RDS IAM token, instead of the develop-only ingest role. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Claude --- .../actions/fetch-bench-baseline/action.yml | 13 ++++---- .github/workflows/pr-bench-gpu-compress.yml | 2 +- .github/workflows/pr-bench-runner.yml | 2 +- .github/workflows/sql-bench-matrix.yml | 2 +- scripts/fetch-benchmark-baseline.py | 33 +++++++++++++++---- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml index d055dd8b69e..bfde4177590 100644 --- a/.github/actions/fetch-bench-baseline/action.yml +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -3,10 +3,9 @@ name: "Fetch benchmark baseline" description: >- - Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres the same - way post-ingest.py writes it: assume the ingest role over OIDC and sign in as bench_ingest with - an RDS IAM token. If the role cannot be assumed, it writes no baseline and the report says the - baseline could not be read. + Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres: assume the + read-only benchmarks role over OIDC and sign in as bench_pr_read with an RDS IAM token. If the + role cannot be assumed, it writes no baseline and the report says the baseline could not be read. inputs: pr-records: @@ -16,7 +15,7 @@ inputs: description: "Where to write the baseline records" required: true role-arn: - description: "The benchmarks ingest role (vars.GH_BENCH_INGEST_ROLE_ARN)" + description: "The read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN)" required: false default: "" region: @@ -60,11 +59,11 @@ runs: set -Eeu -o pipefail if [[ "$AWS_OUTCOME" != "success" ]]; then # No baseline file tells the report that the baseline could not be read. - echo "::warning::Could not assume the benchmarks ingest role; reporting without a baseline." + echo "::warning::Could not assume the read-only benchmarks role; reporting without a baseline." exit 0 fi curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ -o "${RUNNER_TEMP}/rds-global-bundle.pem" - DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" + DSN="postgresql://bench_pr_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \ --postgres "$DSN" --region "$AWS_REGION" diff --git a/.github/workflows/pr-bench-gpu-compress.yml b/.github/workflows/pr-bench-gpu-compress.yml index 772e09737ad..2b668c6a8cf 100644 --- a/.github/workflows/pr-bench-gpu-compress.yml +++ b/.github/workflows/pr-bench-gpu-compress.yml @@ -85,7 +85,7 @@ jobs: with: pr-records: results.ingest.jsonl output: base.ingest.jsonl - role-arn: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} region: ${{ vars.RDS_BENCH_REGION }} endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} database: ${{ vars.RDS_BENCH_DB_NAME }} diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index 0228164f153..8b7c3a5b5eb 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -213,7 +213,7 @@ jobs: with: pr-records: results.ingest.jsonl output: base.ingest.jsonl - role-arn: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} region: ${{ vars.RDS_BENCH_REGION }} endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} database: ${{ vars.RDS_BENCH_DB_NAME }} diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index fdf6f118194..e08e33652e3 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -246,7 +246,7 @@ jobs: with: pr-records: results.ingest.jsonl output: base.ingest.jsonl - role-arn: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} region: ${{ vars.RDS_BENCH_REGION }} endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} database: ${{ vars.RDS_BENCH_DB_NAME }} diff --git a/scripts/fetch-benchmark-baseline.py b/scripts/fetch-benchmark-baseline.py index 54238e09e2a..1fbca4e4c91 100644 --- a/scripts/fetch-benchmark-baseline.py +++ b/scripts/fetch-benchmark-baseline.py @@ -14,9 +14,9 @@ matching rows in the same JSONL record shape. `compare-benchmark-jsons.py` then compares the two record files. -It connects the same way `post-ingest.py` does: as `bench_ingest`, with an RDS IAM token -minted from the AWS credentials of the assumed `GH_BENCH_INGEST_ROLE_ARN` role. The -transaction is read-only. +It connects as the read-only `bench_pr_read` Postgres user, with an RDS IAM token minted +(by `post-ingest.py`'s token helper) from the AWS credentials of the assumed +`GH_BENCH_READ_ROLE_ARN` role, over verify-full TLS. The session is read-only. """ from __future__ import annotations @@ -69,7 +69,7 @@ def parse_args() -> argparse.Namespace: "--postgres", metavar="DSN", required=True, - help="Libpq DSN of the benchmarks database, as accepted by post-ingest.py --postgres.", + help="Libpq DSN of the benchmarks database. Without a password, an RDS IAM token is minted.", ) parser.add_argument("--region", default=None, help="AWS region for RDS IAM token minting.") return parser.parse_args() @@ -166,7 +166,7 @@ def baseline_records(conn: Connection, identities: dict[str, list[tuple]], commi def post_ingest_module() -> ModuleType: - """Load `scripts/post-ingest.py`, whose database connector this script reuses.""" + """Load `scripts/post-ingest.py`, whose RDS IAM token helper this script reuses.""" path = Path(__file__).resolve().parent / "post-ingest.py" spec = importlib.util.spec_from_file_location("post_ingest", path) @@ -176,14 +176,33 @@ def post_ingest_module() -> ModuleType: return module +def connect_postgres(dsn: str, region: str | None) -> Connection: + """Open a read-only, verify-full TLS connection, minting an RDS IAM token if the DSN has no password.""" + + import psycopg + from psycopg import conninfo + + params = conninfo.conninfo_to_dict(dsn) + if params.setdefault("sslmode", "verify-full") != "verify-full": + raise SystemExit(f"--postgres requires sslmode=verify-full; DSN specified {params['sslmode']!r}.") + if not params.get("password"): + params["password"] = post_ingest_module()._rds_iam_token( + host=params["host"], port=int(params.get("port", 5432)), user=params["user"], region=region + ) + params["connect_timeout"] = 10 + params["options"] = "-c statement_timeout=30000" + conn = psycopg.connect(**params) + conn.read_only = True + return conn + + def main() -> None: args = parse_args() identities = identities_by_kind(read_records(args.pr_records)) records: list[dict] = [] if identities: commits = reachable_commits() - with post_ingest_module().connect_postgres(args.postgres, args.region) as conn: - conn.read_only = True + with connect_postgres(args.postgres, args.region) as conn: commit_sha = newest_baseline_commit(conn, identities, commits) if commit_sha is not None: records = baseline_records(conn, identities, commit_sha) From 8bfccbd879cab99166329f05aa7748acb90d0bcb Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 12:40:41 +0000 Subject: [PATCH 05/10] Report without a baseline when the benchmarks database rejects the login A failed baseline read now degrades to the "baseline could not be read" comment instead of failing the benchmark job. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .github/actions/fetch-bench-baseline/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml index bfde4177590..a033c92e278 100644 --- a/.github/actions/fetch-bench-baseline/action.yml +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -65,5 +65,8 @@ runs: curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ -o "${RUNNER_TEMP}/rds-global-bundle.pem" DSN="postgresql://bench_pr_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" - uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \ - --postgres "$DSN" --region "$AWS_REGION" + if ! uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \ + --postgres "$DSN" --region "$AWS_REGION"; then + echo "::warning::Could not read the baseline from the benchmarks database; reporting without a baseline." + rm -f "$OUTPUT" + fi From 2e69732a41de0fc63d933dbcb7e88057dba6adac Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 12:41:23 +0000 Subject: [PATCH 06/10] Read the PR benchmark baseline as bench_ci_read vortex.dev migration 011 names the read-only IAM user bench_ci_read. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .github/actions/fetch-bench-baseline/action.yml | 4 ++-- scripts/fetch-benchmark-baseline.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml index a033c92e278..68dbb3678bd 100644 --- a/.github/actions/fetch-bench-baseline/action.yml +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -4,7 +4,7 @@ name: "Fetch benchmark baseline" description: >- Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres: assume the - read-only benchmarks role over OIDC and sign in as bench_pr_read with an RDS IAM token. If the + read-only benchmarks role over OIDC and sign in as bench_ci_read with an RDS IAM token. If the role cannot be assumed, it writes no baseline and the report says the baseline could not be read. inputs: @@ -64,7 +64,7 @@ runs: fi curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ -o "${RUNNER_TEMP}/rds-global-bundle.pem" - DSN="postgresql://bench_pr_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" + DSN="postgresql://bench_ci_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" if ! uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \ --postgres "$DSN" --region "$AWS_REGION"; then echo "::warning::Could not read the baseline from the benchmarks database; reporting without a baseline." diff --git a/scripts/fetch-benchmark-baseline.py b/scripts/fetch-benchmark-baseline.py index 1fbca4e4c91..59fcde1ea33 100644 --- a/scripts/fetch-benchmark-baseline.py +++ b/scripts/fetch-benchmark-baseline.py @@ -14,7 +14,7 @@ matching rows in the same JSONL record shape. `compare-benchmark-jsons.py` then compares the two record files. -It connects as the read-only `bench_pr_read` Postgres user, with an RDS IAM token minted +It connects as the read-only `bench_ci_read` Postgres user, with an RDS IAM token minted (by `post-ingest.py`'s token helper) from the AWS credentials of the assumed `GH_BENCH_READ_ROLE_ARN` role, over verify-full TLS. The session is read-only. """ From eeae6158b97a951c779e8e575b843bdc81e21cad Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 12:43:15 +0000 Subject: [PATCH 07/10] Record the branch explicitly when ingesting benchmark results post-ingest.py takes a required --branch and writes it to commits.branch (vortex.dev migration 012). The ingest workflows pass github.ref_name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .github/workflows/commit-metadata.yml | 1 + .github/workflows/develop-bench.yml | 1 + .github/workflows/sql-bench-matrix.yml | 1 + scripts/post-ingest.py | 18 ++++++++++++++---- scripts/tests/test_post_ingest.py | 1 + 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/commit-metadata.yml b/.github/workflows/commit-metadata.yml index 6e3ed76c3c8..a412b22d6fc 100644 --- a/.github/workflows/commit-metadata.yml +++ b/.github/workflows/commit-metadata.yml @@ -59,4 +59,5 @@ jobs: scripts/post-ingest.py empty.jsonl \ --postgres "${DSN}" \ --commit-sha "${{ github.sha }}" \ + --branch "${{ github.ref_name }}" \ --region "${AWS_REGION}" diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index 2cb49a6a7b0..3aeeede5bb3 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -261,6 +261,7 @@ jobs: scripts/post-ingest.py results.ingest.jsonl \ --postgres "${DSN}" \ --commit-sha "${{ github.sha }}" \ + --branch "${{ github.ref_name }}" \ --region "${AWS_REGION}" - name: Alert incident.io diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index e08e33652e3..488f05117cc 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -320,6 +320,7 @@ jobs: scripts/post-ingest.py results.ingest.jsonl \ --postgres "${DSN}" \ --commit-sha "${{ github.sha }}" \ + --branch "${{ github.ref_name }}" \ --region "${AWS_REGION}" - name: Alert incident.io diff --git a/scripts/post-ingest.py b/scripts/post-ingest.py index 5dbd260e068..3c4115e7b7a 100755 --- a/scripts/post-ingest.py +++ b/scripts/post-ingest.py @@ -74,6 +74,11 @@ def parse_args() -> argparse.Namespace: required=True, help="40-hex commit SHA. Usually ${{ github.sha }} in CI.", ) + parser.add_argument( + "--branch", + required=True, + help="Branch the results were measured on, stored as commits.branch. Usually ${{ github.ref_name }} in CI.", + ) parser.add_argument( "--repo-url", default="https://github.com/vortex-data/vortex", @@ -125,10 +130,12 @@ def git_show_field(sha: str, fmt: str, cwd: Path | None) -> str: return result.stdout.strip() -def build_commit(sha: str, repo_url: str, git_dir: Path | None) -> dict: +def build_commit(sha: str, branch: str, repo_url: str, git_dir: Path | None) -> dict: sha = sha.strip().lower() if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha): raise SystemExit(f"commit SHA must be 40-hex lowercase, got: {sha!r}") + if not branch: + raise SystemExit("--branch must not be empty") timestamp = git_show_field(sha, "%cI", git_dir) message = git_show_field(sha, "%s", git_dir) @@ -140,6 +147,7 @@ def build_commit(sha: str, repo_url: str, git_dir: Path | None) -> dict: return { "sha": sha, + "branch": branch, "timestamp": timestamp, "message": message, "author_name": author_name, @@ -739,10 +747,11 @@ def _upsert_commit(conn, commit: dict) -> None: conn.execute( """ INSERT INTO commits ( - commit_sha, timestamp, message, author_name, author_email, + commit_sha, branch, timestamp, message, author_name, author_email, committer_name, committer_email, tree_sha, url - ) VALUES (%s, %s::timestamptz, %s, %s, %s, %s, %s, %s, %s) + ) VALUES (%s, %s, %s::timestamptz, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (commit_sha) DO UPDATE SET + branch = excluded.branch, timestamp = excluded.timestamp, message = excluded.message, author_name = excluded.author_name, @@ -754,6 +763,7 @@ def _upsert_commit(conn, commit: dict) -> None: """, ( commit["sha"], + commit["branch"], commit["timestamp"], commit["message"], commit["author_name"], @@ -981,7 +991,7 @@ def _main_postgres(args: argparse.Namespace) -> int: records = read_records(args.jsonl_path) # `build_commit` runs `git show `, so the SHA must be in the runner's local git # history. A shallow checkout missing the SHA fails before opening a database connection. - commit = build_commit(args.commit_sha, args.repo_url, args.git_dir) + commit = build_commit(args.commit_sha, args.branch, args.repo_url, args.git_dir) conn = connect_postgres(args.postgres, args.region) try: inserted, updated = ingest_postgres(conn, commit, records) diff --git a/scripts/tests/test_post_ingest.py b/scripts/tests/test_post_ingest.py index 05d2fbf6990..203f606e38a 100644 --- a/scripts/tests/test_post_ingest.py +++ b/scripts/tests/test_post_ingest.py @@ -28,6 +28,7 @@ def writer() -> ModuleType: def commit() -> dict[str, str]: return { "sha": "a" * 40, + "branch": "develop", "timestamp": "2026-09-08T12:00:00Z", "message": "benchmark", "author_name": "Author", From 86d3c3e486389b260c1b251f07551c42938c9c16 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 12:58:12 +0000 Subject: [PATCH 08/10] Make the PR baseline query fast and keep its inputs in the action The baseline lookup matched PR identities with IS NOT DISTINCT FROM against every reachable commit, which no index can serve: TPC-H hit the 30 s statement timeout. Each identity is now its own query, with absent dims as IS NULL, reading its rows newest first from the chart or summary index in pages until one is from a commit reachable from HEAD. On 4.2M synthetic query rows it returns the same records in 0.3 s instead of 31 s. The action reads the role, region, endpoint and database from repo variables and defaults the record paths, so the three workflows only name it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .../actions/fetch-bench-baseline/action.yml | 35 ++---- .github/workflows/pr-bench-gpu-compress.yml | 7 -- .github/workflows/pr-bench-runner.yml | 7 -- .github/workflows/sql-bench-matrix.yml | 7 -- scripts/fetch-benchmark-baseline.py | 115 +++++++++++------- scripts/tests/test_benchmark_reporting.py | 11 +- 6 files changed, 90 insertions(+), 92 deletions(-) diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml index 68dbb3678bd..bc73bbbad9e 100644 --- a/.github/actions/fetch-bench-baseline/action.yml +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -4,29 +4,17 @@ name: "Fetch benchmark baseline" description: >- Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres: assume the - read-only benchmarks role over OIDC and sign in as bench_ci_read with an RDS IAM token. If the - role cannot be assumed, it writes no baseline and the report says the baseline could not be read. + read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN) over OIDC and sign in as bench_ci_read + with an RDS IAM token. If the baseline cannot be read, it writes no baseline file and the report + says the baseline could not be read. The calling job needs `id-token: write`. inputs: pr-records: description: "The PR run's --ingest-jsonl output" - required: true + default: results.ingest.jsonl output: description: "Where to write the baseline records" - required: true - role-arn: - description: "The read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN)" - required: false - default: "" - region: - description: "AWS region of the benchmarks RDS instance" - required: true - endpoint: - description: "Hostname of the benchmarks RDS instance" - required: true - database: - description: "Name of the benchmarks database" - required: true + default: base.ingest.jsonl runs: using: "composite" @@ -35,12 +23,13 @@ runs: # steps such as the S3 cleanup still use, are not replaced. - name: Configure AWS credentials for the benchmarks database (OIDC) id: aws - if: inputs.role-arn != '' + if: vars.GH_BENCH_READ_ROLE_ARN != '' continue-on-error: true uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: - role-to-assume: ${{ inputs.role-arn }} - aws-region: ${{ inputs.region }} + role-to-assume: ${{ vars.GH_BENCH_READ_ROLE_ARN }} + aws-region: ${{ vars.RDS_BENCH_REGION }} + retry-max-attempts: 1 output-credentials: true output-env-credentials: false - name: Fetch baseline @@ -52,9 +41,9 @@ runs: AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }} - AWS_REGION: ${{ inputs.region }} - RDS_BENCH_INSTANCE_ENDPOINT: ${{ inputs.endpoint }} - RDS_BENCH_DB_NAME: ${{ inputs.database }} + AWS_REGION: ${{ vars.RDS_BENCH_REGION }} + RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }} run: | set -Eeu -o pipefail if [[ "$AWS_OUTCOME" != "success" ]]; then diff --git a/.github/workflows/pr-bench-gpu-compress.yml b/.github/workflows/pr-bench-gpu-compress.yml index 2b668c6a8cf..902cd570910 100644 --- a/.github/workflows/pr-bench-gpu-compress.yml +++ b/.github/workflows/pr-bench-gpu-compress.yml @@ -82,13 +82,6 @@ jobs: python3 scripts/compress-split.py --gpu-decompress --emit-ingest-records - name: Fetch baseline from the benchmarks database uses: ./.github/actions/fetch-bench-baseline - with: - pr-records: results.ingest.jsonl - output: base.ingest.jsonl - role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} - region: ${{ vars.RDS_BENCH_REGION }} - endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} - database: ${{ vars.RDS_BENCH_DB_NAME }} - name: Publish results shell: bash diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index 8b7c3a5b5eb..df44b572048 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -210,13 +210,6 @@ jobs: - name: Fetch baseline from the benchmarks database uses: ./.github/actions/fetch-bench-baseline - with: - pr-records: results.ingest.jsonl - output: base.ingest.jsonl - role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} - region: ${{ vars.RDS_BENCH_REGION }} - endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} - database: ${{ vars.RDS_BENCH_DB_NAME }} - name: Compare results shell: bash diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index 488f05117cc..0490aa95243 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -243,13 +243,6 @@ jobs: - name: Fetch baseline from the benchmarks database if: inputs.mode == 'pr' uses: ./.github/actions/fetch-bench-baseline - with: - pr-records: results.ingest.jsonl - output: base.ingest.jsonl - role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} - region: ${{ vars.RDS_BENCH_REGION }} - endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} - database: ${{ vars.RDS_BENCH_DB_NAME }} - name: Compare results if: inputs.mode == 'pr' diff --git a/scripts/fetch-benchmark-baseline.py b/scripts/fetch-benchmark-baseline.py index 59fcde1ea33..613a80459f0 100644 --- a/scripts/fetch-benchmark-baseline.py +++ b/scripts/fetch-benchmark-baseline.py @@ -58,7 +58,8 @@ ), } -_INTEGER_DIMS = frozenset({"query_idx"}) +# Rows read per query while looking for an identity's newest reachable commit. +_PAGE_SIZE = 200 def parse_args() -> argparse.Namespace: @@ -110,58 +111,83 @@ def reachable_commits() -> list[str]: return commits.split() -def _identity_join(kind: str, identities: list[tuple]) -> tuple[str, list[list]]: - """Return a join of `t` against the PR identities, and its array parameters.""" - - _table, dims, _values = _KINDS[kind] - arrays = [[identity[i] for identity in identities] for i in range(len(dims))] - casts = ", ".join("%s::integer[]" if dim in _INTEGER_DIMS else "%s::text[]" for dim in dims) - conditions = " AND ".join(f"t.{dim} IS NOT DISTINCT FROM p.{dim}" for dim in dims) - return f"JOIN unnest({casts}) AS p({', '.join(dims)}) ON {conditions}", arrays +def identity_condition(kind: str, identity: tuple) -> tuple[str, list]: + """Return a `WHERE` condition on `t` matching one identity, and its parameters. + Absent dims match with `IS NULL` rather than `IS NOT DISTINCT FROM`, which + no index can serve, so each lookup is a range scan of the table's chart or + summary index instead of a scan of the whole table. + """ -def newest_baseline_commit(conn: Connection, identities: dict[str, list[tuple]], commits: list[str]) -> str | None: - """Return the newest reachable commit that recorded any PR identity.""" + _table, dims, _values = _KINDS[kind] + conditions = [] + params = [] + for dim, value in zip(dims, identity): + if value is None: + conditions.append(f"t.{dim} IS NULL") + else: + conditions.append(f"t.{dim} = %s") + params.append(value) + return " AND ".join(conditions), params + + +def newest_baseline_commit( + conn: Connection, identities: dict[str, list[tuple]], reachable: set[str] +) -> tuple[str, object] | None: + """Return the newest reachable commit that recorded any PR identity, with its timestamp. + + Each identity's rows are read newest first, a page at a time, until one is from a commit + reachable from `HEAD`. That is usually the first page: the baseline is the PR's merge base + or close to it. One query per identity, as Postgres plans a `UNION ALL` of them far worse. + """ newest = None for kind, keys in identities.items(): table, _dims, _values = _KINDS[kind] - join, arrays = _identity_join(kind, keys) - row = conn.execute( - f""" - SELECT t.commit_sha, c.timestamp - FROM {table} t - JOIN commits c ON c.commit_sha = t.commit_sha - {join} - WHERE t.commit_sha = ANY(%s::text[]) - ORDER BY c.timestamp DESC, t.commit_sha DESC - LIMIT 1 - """, - [*arrays, commits], - ).fetchone() - if row is not None and (newest is None or (row[1], row[0]) > (newest[1], newest[0])): - newest = row - return None if newest is None else newest[0] - - -def baseline_records(conn: Connection, identities: dict[str, list[tuple]], commit_sha: str) -> list[dict]: + # `query_measurements` carries its commit's timestamp at the end of its summary index, + # so its rows come back newest first straight from an index range scan. + if table == "query_measurements": + source, timestamp = "query_measurements t", "t.commit_timestamp" + else: + source, timestamp = f"{table} t JOIN commits c ON c.commit_sha = t.commit_sha", "c.timestamp" + for identity in keys: + where, params = identity_condition(kind, identity) + before = None + while True: + page = conn.execute( + f"SELECT t.commit_sha, {timestamp} FROM {source}" + f" WHERE {where} AND {timestamp} IS NOT NULL" + + (f" AND {timestamp} < %s" if before is not None else "") + + f" ORDER BY {timestamp} DESC LIMIT {_PAGE_SIZE}", + [*params, before] if before is not None else params, + ).fetchall() + found = next((row for row in page if row[0] in reachable), None) + if found is not None or len(page) < _PAGE_SIZE: + break + before = page[-1][1] + if found is not None and (newest is None or (found[1], found[0]) > (newest[1], newest[0])): + newest = (found[0], found[1]) + return newest + + +def baseline_records( + conn: Connection, identities: dict[str, list[tuple]], commit_sha: str, commit_timestamp: object +) -> list[dict]: """Return the rows `commit_sha` recorded for the PR identities, as ingest records.""" records = [] for kind, keys in identities.items(): table, dims, values = _KINDS[kind] - join, arrays = _identity_join(kind, keys) columns = ("commit_sha", *dims, *values) - rows = conn.execute( - f""" - SELECT {", ".join(f"t.{column}" for column in columns)} - FROM {table} t - {join} - WHERE t.commit_sha = %s - """, - [*arrays, commit_sha], - ).fetchall() - records.extend({"kind": kind, **dict(zip(columns, row))} for row in rows) + for identity in keys: + where, params = identity_condition(kind, identity) + where += " AND t.commit_sha = %s" + params.append(commit_sha) + if table == "query_measurements": + where += " AND t.commit_timestamp = %s" + params.append(commit_timestamp) + rows = conn.execute(f"SELECT {', '.join(f't.{c}' for c in columns)} FROM {table} t WHERE {where}", params) + records.extend({"kind": kind, **dict(zip(columns, row))} for row in rows.fetchall()) return records @@ -203,9 +229,10 @@ def main() -> None: if identities: commits = reachable_commits() with connect_postgres(args.postgres, args.region) as conn: - commit_sha = newest_baseline_commit(conn, identities, commits) - if commit_sha is not None: - records = baseline_records(conn, identities, commit_sha) + newest = newest_baseline_commit(conn, identities, set(commits)) + if newest is not None: + commit_sha, commit_timestamp = newest + records = baseline_records(conn, identities, commit_sha, commit_timestamp) print(f"Baseline commit {commit_sha}: {len(records)} records") if not records: print("No baseline found; the report shows PR measurements without comparison.") diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index d54f46c4ce3..0b55fe1e3ca 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -675,11 +675,14 @@ def test_baseline_fetch_groups_distinct_pr_identities_by_kind() -> None: def test_baseline_fetch_matches_absent_dims_as_equal() -> None: fetch = load_module("fetch_benchmark_baseline", FETCH_BASELINE_SCRIPT) - join, arrays = fetch._identity_join("compression_size", [("taxi", None, "parquet"), ("taxi", "gpu", "lance")]) + absent = fetch.identity_condition("compression_size", ("taxi", None, "parquet")) + present = fetch.identity_condition("compression_size", ("taxi", "gpu", "lance")) - assert "unnest(%s::text[], %s::text[], %s::text[]) AS p(dataset, dataset_variant, format)" in join - assert "t.dataset_variant IS NOT DISTINCT FROM p.dataset_variant" in join - assert arrays == [["taxi", "taxi"], [None, "gpu"], ["parquet", "lance"]] + assert absent == ("t.dataset = %s AND t.dataset_variant IS NULL AND t.format = %s", ["taxi", "parquet"]) + assert present == ( + "t.dataset = %s AND t.dataset_variant = %s AND t.format = %s", + ["taxi", "gpu", "lance"], + ) def test_report_compares_ingest_records(tmp_path: Path) -> None: From 62656cd6af3ec865c40da4d65c06ce91c3cb45bb Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 13:05:41 +0000 Subject: [PATCH 09/10] Install cuDF for the GPU benchmark without the workspace's exclude-newer #10997 set exclude-newer = "7 days" for the workspace. NVIDIA's index publishes no upload dates, so uv filtered out every cudf-cu12 wheel and the GPU compression benchmark failed to install cuDF. The cuDF venv is standalone, so install into it with --no-config. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .github/workflows/pr-bench-gpu-compress.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-bench-gpu-compress.yml b/.github/workflows/pr-bench-gpu-compress.yml index 902cd570910..19c8c2ea1f1 100644 --- a/.github/workflows/pr-bench-gpu-compress.yml +++ b/.github/workflows/pr-bench-gpu-compress.yml @@ -49,9 +49,11 @@ jobs: - name: Install cuDF # The GPU Parquet number is a cuDF `read_parquet`. cuDF ships prebuilt manylinux wheels # on NVIDIA's index, so it stays a runtime dependency and never enters the Rust build. + # `--no-config` keeps the workspace's `exclude-newer` off this standalone venv: NVIDIA's + # index publishes no upload dates, so that cutoff would filter out every cuDF wheel. run: | uv venv --python 3.12 .venv-cudf - uv pip install --python .venv-cudf \ + uv pip install --no-config --python .venv-cudf \ --extra-index-url https://pypi.nvidia.com \ cudf-cu12 pandas pyarrow echo "$PWD/.venv-cudf/bin" >> "$GITHUB_PATH" From 98f50a38909d5b31023fd60e71c5492e010756d7 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 30 Oct 2026 13:08:15 +0000 Subject: [PATCH 10/10] Pass the repository variables into the baseline action Composite actions cannot read the vars context, so GitHub rejected the action file. Callers pass the role, region, endpoint and database in; the record paths keep their defaults. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UeniBBtX1DUban9Fda2Edh Signed-off-by: Joe Isaacs --- .../actions/fetch-bench-baseline/action.yml | 31 +++++++++++++------ .github/workflows/pr-bench-gpu-compress.yml | 5 +++ .github/workflows/pr-bench-runner.yml | 5 +++ .github/workflows/sql-bench-matrix.yml | 5 +++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.github/actions/fetch-bench-baseline/action.yml b/.github/actions/fetch-bench-baseline/action.yml index bc73bbbad9e..7141c6f655b 100644 --- a/.github/actions/fetch-bench-baseline/action.yml +++ b/.github/actions/fetch-bench-baseline/action.yml @@ -4,9 +4,10 @@ name: "Fetch benchmark baseline" description: >- Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres: assume the - read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN) over OIDC and sign in as bench_ci_read - with an RDS IAM token. If the baseline cannot be read, it writes no baseline file and the report - says the baseline could not be read. The calling job needs `id-token: write`. + read-only benchmarks role over OIDC and sign in as bench_ci_read with an RDS IAM token. If the + baseline cannot be read, it writes no baseline file and the report says the baseline could not be + read. The calling job needs `id-token: write`. Composite actions cannot read `vars`, so callers + pass the repository variables in. inputs: pr-records: @@ -15,6 +16,18 @@ inputs: output: description: "Where to write the baseline records" default: base.ingest.jsonl + role-arn: + description: "The read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN)" + required: true + region: + description: "AWS region of the benchmarks RDS instance (vars.RDS_BENCH_REGION)" + required: true + endpoint: + description: "Hostname of the benchmarks RDS instance (vars.RDS_BENCH_INSTANCE_ENDPOINT)" + required: true + database: + description: "Name of the benchmarks database (vars.RDS_BENCH_DB_NAME)" + required: true runs: using: "composite" @@ -23,12 +36,12 @@ runs: # steps such as the S3 cleanup still use, are not replaced. - name: Configure AWS credentials for the benchmarks database (OIDC) id: aws - if: vars.GH_BENCH_READ_ROLE_ARN != '' + if: inputs.role-arn != '' continue-on-error: true uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: - role-to-assume: ${{ vars.GH_BENCH_READ_ROLE_ARN }} - aws-region: ${{ vars.RDS_BENCH_REGION }} + role-to-assume: ${{ inputs.role-arn }} + aws-region: ${{ inputs.region }} retry-max-attempts: 1 output-credentials: true output-env-credentials: false @@ -41,9 +54,9 @@ runs: AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }} - AWS_REGION: ${{ vars.RDS_BENCH_REGION }} - RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} - RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }} + AWS_REGION: ${{ inputs.region }} + RDS_BENCH_INSTANCE_ENDPOINT: ${{ inputs.endpoint }} + RDS_BENCH_DB_NAME: ${{ inputs.database }} run: | set -Eeu -o pipefail if [[ "$AWS_OUTCOME" != "success" ]]; then diff --git a/.github/workflows/pr-bench-gpu-compress.yml b/.github/workflows/pr-bench-gpu-compress.yml index 19c8c2ea1f1..3f8b8d0f4f7 100644 --- a/.github/workflows/pr-bench-gpu-compress.yml +++ b/.github/workflows/pr-bench-gpu-compress.yml @@ -84,6 +84,11 @@ jobs: python3 scripts/compress-split.py --gpu-decompress --emit-ingest-records - name: Fetch baseline from the benchmarks database uses: ./.github/actions/fetch-bench-baseline + with: + role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} + region: ${{ vars.RDS_BENCH_REGION }} + endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + database: ${{ vars.RDS_BENCH_DB_NAME }} - name: Publish results shell: bash diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index 7f7cd8e7bd7..9a97c65682e 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -212,6 +212,11 @@ jobs: - name: Fetch baseline from the benchmarks database uses: ./.github/actions/fetch-bench-baseline + with: + role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} + region: ${{ vars.RDS_BENCH_REGION }} + endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + database: ${{ vars.RDS_BENCH_DB_NAME }} - name: Compare results shell: bash diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index 19201b9d4dd..f1661284475 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -245,6 +245,11 @@ jobs: - name: Fetch baseline from the benchmarks database if: inputs.mode == 'pr' uses: ./.github/actions/fetch-bench-baseline + with: + role-arn: ${{ vars.GH_BENCH_READ_ROLE_ARN }} + region: ${{ vars.RDS_BENCH_REGION }} + endpoint: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + database: ${{ vars.RDS_BENCH_DB_NAME }} - name: Compare results if: inputs.mode == 'pr'