From 07d7db581742cca17a01497025ccefc1e20f50e8 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 15 Jul 2026 14:46:37 -0400 Subject: [PATCH 1/3] feat(bench-orchestrator): declare benchmark matrix profiles Define SQL benchmarks once, then resolve named profiles into the GitHub Actions matrix shape. Add the vx-bench matrix command, documentation, and focused regression coverage for selection, targets, storage fields, and preserved develop/nightly behavior. This establishes the source of truth independently of the workflow migration and addresses #4935. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Connor Tsui --- bench-orchestrator/README.md | 93 +++++++ .../bench_orchestrator/benchmarks.py | 164 ++++++++++++ bench-orchestrator/bench_orchestrator/cli.py | 32 +++ .../bench_orchestrator/matrix.py | 236 ++++++++++++++++++ bench-orchestrator/tests/test_matrix.py | 129 ++++++++++ 5 files changed, 654 insertions(+) create mode 100644 bench-orchestrator/bench_orchestrator/benchmarks.py create mode 100644 bench-orchestrator/bench_orchestrator/matrix.py create mode 100644 bench-orchestrator/tests/test_matrix.py diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index 0b267008a85..a954b9c3268 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -70,6 +70,22 @@ vx-bench prepare-data [options] - `--formats-json`: Exact data formats as JSON, e.g. `'["parquet","vortex"]'` - `--opt`: Benchmark-specific options such as `scale-factor=10.0` +### `matrix` - Resolve CI Benchmark Matrices + +Emit the GitHub Actions `include:` array for a named [profile](#declarative-benchmark-matrix). The +benchmark workflows use this to decide what runs on pull requests, `develop`, and nightly. + +```bash +vx-bench matrix # list available profiles +vx-bench matrix develop # emit the develop matrix as JSON (one line) +vx-bench matrix nightly --pretty +``` + +**Options:** + +- `--list`: List available profiles and exit +- `--pretty`: Pretty-print the JSON output + ### `compare` - Compare Results Compare benchmark results within a run or across multiple runs. Results are displayed in a pivot table format. @@ -137,6 +153,83 @@ vx-bench clean --older-than "30 days" [options] - `--keep-labeled`: Don't delete labeled runs (default: true) - `--dry-run, -n`: Show what would be deleted +## Declarative benchmark matrix + +To change what CI benchmarks, edit **`bench_orchestrator/benchmarks.py`**. `BENCHMARKS` is the list +of benchmark suites and their supported targets; `PROFILES` says how much coverage each workflow +runs. Matrix rendering is kept separately in `bench_orchestrator/matrix.py`. + +This replaces the parallel `pr_targets`/`develop_targets` arrays and duplicated `full`/`base` +matrices that previously lived in workflow YAML. + +### Model + +The model has three small pieces: + +- **Target composition** — each `BenchmarkDef` declares its canonical target set once, as a delta + from the shared `STANDARD` core (both engines × parquet/vortex/vortex-compact): + - opt in to extras: `STANDARD | df(Format.LANCE) | duck(Format.DUCKDB)` + - restrict an engine: `STANDARD.only(Engine.DUCKDB)` +- **Selection** — a `Profile` chooses either the regular benchmark set or the explicitly nightly + benchmark set. +- **Target policy** — a `Profile` narrows each selected benchmark's declared targets: + `all_targets` for full coverage, `defaults` for the cheap lane + (datafusion,duckdb × parquet,vortex). + +`data_formats` is *derived* from the resolved targets (arrow and lance excluded, since data-gen +does not produce them), so it can never drift from what actually runs. Targets invalid for a +benchmark's storage (lance has no remote reader) are dropped by the resolver, reusing the same +support rules that validate a run. + +### Built-in profiles + +`develop` runs full coverage on merge, `pr` runs the default targets for every SQL benchmark, and +`nightly` runs SF=100 TPC-H on NVMe and S3. List them with `vx-bench matrix`. + +### Adding a benchmark + +Add one `BenchmarkDef` (or a factory call) to `BENCHMARKS` in +`bench_orchestrator/benchmarks.py`. Its `targets` are the superset it supports; `PROFILES` decides +how much of that coverage each workflow runs. + +### How a workflow consumes it + +Generate the matrix in a `plan` job, then fan out in the `bench` job: + +```yaml +jobs: + plan: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.p.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - id: p + run: echo "matrix=$(uv run --project bench-orchestrator vx-bench matrix ${{ inputs.profile }})" >> "$GITHUB_OUTPUT" + bench: + needs: plan + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - name: Run ${{ matrix.name }} + run: | + uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ + --targets-json '${{ toJSON(matrix.targets) }}' \ + --formats-json '${{ toJSON(matrix.data_formats) }}' \ + ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} +``` + +### Tests + +`tests/test_matrix.py` covers each profile with targeted assertions on its load-bearing fields -- +which benchmarks are selected, their engine/format targets, and the secondary fields CI depends on +(`remote_key`, `scale_factor`, `data_formats`) -- plus a structural smoke check that every profile +resolves to a well-formed matrix. The `develop` and `nightly` profiles additionally assert they +reproduce the target sets the old inline workflow matrices used, proving the registry is +behavior-preserving. + ## Example Workflows ### 1. Basic Performance Comparison diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py new file mode 100644 index 00000000000..6857a3b4a15 --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""The SQL benchmarks and profiles run by CI. + +This is the file to edit when changing benchmark coverage: + +* ``BENCHMARKS`` declares which benchmark suites exist and every target they support. +* ``PROFILES`` declares how much of that coverage each CI workflow runs. + +Matrix rendering lives in :mod:`bench_orchestrator.matrix` and should not need to change when a +benchmark is added or removed. +""" + +from .config import Benchmark, Engine, Format +from .matrix import STANDARD, BenchmarkDef, Profile, Storage, all_targets, defaults, df, duck + + +def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef: + # Nightly historically uses the same matrix IDs as SF=1. The profiles are disjoint, so the + # duplicate IDs never occur in one resolved matrix and historical result series stay intact. + suffix = "" if scale_factor in {1, 100} else f"-{int(scale_factor)}" + if storage is Storage.NVME: + target_set = df( + Format.ARROW, + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.LANCE, + ) | duck( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.DUCKDB, + ) + local_dir = None + remote_key = None + else: + target_set = STANDARD + local_dir = f"vortex-bench/data/tpch/{scale_factor:.1f}" + remote_key = f"tpch/{scale_factor:.1f}/" + + name = f"TPC-H on {storage.label}" if scale_factor == 100 else f"TPC-H SF={scale_factor:g} on {storage.label}" + return BenchmarkDef( + id=f"tpch-{storage.value}{suffix}", + benchmark=Benchmark.TPCH, + name=name, + targets=target_set, + storage=storage, + scale_factor=scale_factor, + iterations=iterations, + nightly=scale_factor == 100, + local_dir=local_dir, + remote_key=remote_key, + ) + + +def _clickbench(benchmark: Benchmark, name: str) -> BenchmarkDef: + return BenchmarkDef( + id=f"{benchmark.value}-nvme", + benchmark=benchmark, + name=name, + targets=df( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.LANCE, + ) + | duck( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.DUCKDB, + ), + ) + + +def _fineweb(storage: Storage) -> BenchmarkDef: + if storage is Storage.NVME: + return BenchmarkDef( + id="fineweb", + benchmark=Benchmark.FINEWEB, + name="FineWeb NVMe", + targets=STANDARD, + scale_factor=100, + ) + return BenchmarkDef( + id="fineweb-s3", + benchmark=Benchmark.FINEWEB, + name="FineWeb S3", + targets=STANDARD, + storage=Storage.S3, + scale_factor=100, + local_dir="vortex-bench/data/fineweb", + remote_key="fineweb/", + ) + + +# ================================================================================================== +# BENCHMARK DEFINITIONS — ADD OR REMOVE SQL BENCHMARKS HERE +# ================================================================================================== + +BENCHMARKS: list[BenchmarkDef] = [ + _clickbench(Benchmark.CLICKBENCH, "Clickbench on NVME"), + _clickbench(Benchmark.CLICKBENCH_SORTED, "Clickbench Sorted on NVME"), + _tpch(1.0, Storage.NVME), + _tpch(1.0, Storage.S3), + _tpch(10.0, Storage.NVME), + _tpch(10.0, Storage.S3), + _tpch(100, Storage.NVME, iterations=None), # nightly only + _tpch(100.0, Storage.S3, iterations=None), # nightly only + BenchmarkDef( + id="tpcds-nvme", + benchmark=Benchmark.TPCDS, + name="TPC-DS SF=1 on NVME", + targets=STANDARD | duck(Format.DUCKDB), + scale_factor=1.0, + ), + BenchmarkDef( + id="statpopgen", + benchmark=Benchmark.STATPOPGEN, + name="Statistical and Population Genetics", + targets=STANDARD.only(Engine.DUCKDB), + scale_factor=100, + local_dir="vortex-bench/data/statpopgen", + ), + _fineweb(Storage.NVME), + _fineweb(Storage.S3), + BenchmarkDef( + id="polarsignals", + benchmark=Benchmark.POLARSIGNALS, + name="PolarSignals Profiling", + targets=df(Format.VORTEX), + scale_factor=1, + ), + BenchmarkDef( + id="appian-nvme", + benchmark=Benchmark.APPIAN, + name="Appian on NVME", + targets=STANDARD | duck(Format.DUCKDB), + iterations=10, + ), +] + + +# ================================================================================================== +# CI PROFILES — CHOOSE WHICH DECLARED COVERAGE EACH WORKFLOW RUNS HERE +# ================================================================================================== + +PROFILES: dict[str, Profile] = { + "develop": Profile( + targets=all_targets, + description="Every SQL benchmark at full target coverage; runs on each merge to develop.", + ), + "pr": Profile( + targets=defaults, + description="Every SQL benchmark at default targets (datafusion,duckdb × parquet,vortex).", + ), + "nightly": Profile( + nightly=True, + targets=defaults, + description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", + ), +} diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index b9d7f9bb2ef..ef8308389d3 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -3,6 +3,7 @@ """CLI for benchmark orchestration.""" +import json import subprocess from contextlib import contextmanager from datetime import datetime, timedelta @@ -15,6 +16,7 @@ from rich.console import Console from rich.table import Table +from .benchmarks import BENCHMARKS, PROFILES from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( @@ -29,6 +31,7 @@ parse_targets_json, resolve_axis_targets, ) +from .matrix import resolve_matrix from .runner.builder import BenchmarkBuilder from .runner.executor import BenchmarkExecutor from .storage.store import ResultStore @@ -214,6 +217,35 @@ def prepare_data( raise typer.Exit(1) from exc +@app.command("matrix") +def matrix( + profile: Annotated[ + str | None, + typer.Argument(help="Profile to resolve (omit to list available profiles)"), + ] = None, + list_profiles: Annotated[bool, typer.Option("--list", help="List available profiles and exit")] = False, + pretty: Annotated[bool, typer.Option("--pretty", help="Pretty-print the JSON output")] = False, +) -> None: + """Emit the GitHub Actions benchmark matrix (the `include:` array) for a profile. + + With no profile (or --list) it prints the available profiles. Otherwise it prints the resolved + matrix as JSON on stdout, suitable for `matrix=$(vx-bench matrix ) >> $GITHUB_OUTPUT`. + """ + if profile is None or list_profiles: + for name, prof in PROFILES.items(): + console.print(f"[bold cyan]{name}[/bold cyan]: {prof.description}") + return + + prof = PROFILES.get(profile) + if prof is None: + known = ", ".join(PROFILES) + console.print(f"[red]Unknown profile '{profile}'. Available: {known}[/red]") + raise typer.Exit(1) + + entries = resolve_matrix(prof, BENCHMARKS) + typer.echo(json.dumps(entries, indent=2 if pretty else None)) + + @app.command() def run( benchmark: Annotated[Benchmark, typer.Argument(help="Benchmark suite to run")], diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py new file mode 100644 index 00000000000..9f1a4bbd989 --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -0,0 +1,236 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Resolve declarative benchmark definitions into GitHub Actions matrices. + +The benchmarks and profiles that CI runs are intentionally kept out of this implementation module. +Edit :mod:`bench_orchestrator.benchmarks` to change them. + +There are three independent axes, and every difference between CI jobs lives in exactly one place: + +* **Target composition** -- each :class:`BenchmarkDef` declares its canonical target set once, as a + delta from the shared :data:`STANDARD` core (opt in with ``| df(...)`` or restrict with + ``.only(...)``). +* **Selection** -- a :class:`Profile` picks the benchmarks for one CI schedule. +* **Target policy** -- a :class:`Profile` narrows each selected benchmark's declared targets + (:func:`all_targets` for full coverage, :func:`defaults` for the cheap PR/nightly lane). + +The resolver turns ``(profile)`` into the GitHub Actions ``include:`` array. A workflow consumes it +with the standard dynamic-matrix pattern:: + + plan: + outputs: {matrix: "${{ steps.p.outputs.matrix }}"} + steps: + - id: p + run: echo "matrix=$(uv run vx-bench matrix ${{ inputs.profile }})" >> "$GITHUB_OUTPUT" + bench: + needs: plan + strategy: + matrix: + include: ${{ fromJSON(needs.plan.outputs.matrix) }} + +Run ``vx-bench matrix --pretty`` locally to preview exactly what a job will run. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from enum import Enum + +from .config import Benchmark, BenchmarkTarget, Engine, Format + + +class Storage(Enum): + """Where a benchmark's data lives when it runs.""" + + NVME = "nvme" + S3 = "s3" + + @property + def label(self) -> str: + """Human-facing name used in benchmark display names (e.g. ``TPC-H SF=1 on NVME``).""" + return "NVME" if self is Storage.NVME else "S3" + + +def _dedupe(targets: Iterable[BenchmarkTarget]) -> tuple[BenchmarkTarget, ...]: + """Normalize and de-duplicate targets, preserving first-seen order.""" + return tuple(dict.fromkeys(target.normalized() for target in targets)) + + +@dataclass(frozen=True) +class TargetSet: + """An ordered, de-duplicated set of engine/format targets with small algebra. + + Supports ``|`` (union / opt-in) and :meth:`only` (restrict to engines) so a benchmark can + express its targets as a delta from a shared default. + """ + + targets: tuple[BenchmarkTarget, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "targets", _dedupe(self.targets)) + + def __or__(self, other: TargetSet) -> TargetSet: + """Return the union of two target sets (opt in to more targets).""" + return TargetSet(self.targets + other.targets) + + def only(self, *engines: Engine) -> TargetSet: + """Restrict to targets whose engine is one of ``engines``.""" + keep = set(engines) + return TargetSet(tuple(target for target in self.targets if target.engine in keep)) + + def formats(self) -> list[Format]: + """Return the distinct formats referenced by these targets, in first-seen order.""" + return list(dict.fromkeys(target.format for target in self.targets)) + + def __iter__(self): + return iter(self.targets) + + def __len__(self) -> int: + return len(self.targets) + + +def targets(engine: Engine, *formats: Format) -> TargetSet: + """Build a :class:`TargetSet` for one engine across several formats.""" + return TargetSet(tuple(BenchmarkTarget(engine=engine, format=fmt) for fmt in formats)) + + +def df(*formats: Format) -> TargetSet: + """Shortcut for DataFusion targets across ``formats``.""" + return targets(Engine.DATAFUSION, *formats) + + +def duck(*formats: Format) -> TargetSet: + """Shortcut for DuckDB targets across ``formats``.""" + return targets(Engine.DUCKDB, *formats) + + +#: The columnar comparison core shared by almost every SQL benchmark: both engines across the +#: three headline Vortex/Parquet formats. Benchmarks opt in to extras (arrow, lance, duckdb). +STANDARD = df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT) | duck( + Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT +) + +#: The cheap default lane used for pull requests and nightly: two engines, two headline formats. +DEFAULTS = df(Format.PARQUET, Format.VORTEX) | duck(Format.PARQUET, Format.VORTEX) + +# arrow is an in-memory format and lance data is produced by the lance backend itself, so neither +# is generated by the data-gen step. Everything else in a benchmark's targets needs data prepared. +_NOT_GENERATED = frozenset({Format.ARROW, Format.LANCE}) + +# Canonical ordering for the generated `data_formats` list, so emitted matrices are stable. +_FORMAT_ORDER = ( + Format.ARROW, + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.VORTEX_NATIVE, + Format.DUCKDB, + Format.LANCE, +) + + +@dataclass(frozen=True) +class BenchmarkDef: + """A single benchmark and the canonical set of targets it runs. + + The declared ``targets`` are the *superset* this benchmark cares about (its intent). Profiles + narrow this set via a target policy; they never add targets a benchmark did not declare. + """ + + id: str + benchmark: Benchmark + name: str + targets: TargetSet + storage: Storage = Storage.NVME + scale_factor: float | int | None = None + iterations: int | None = None + nightly: bool = False + local_dir: str | None = None + remote_key: str | None = None + + @property + def subcommand(self) -> str: + """The ``vx-bench`` subcommand for this benchmark (its benchmark enum value).""" + return self.benchmark.value + + +# A target policy maps a benchmark's declared targets to the targets a profile actually runs. +TargetPolicy = Callable[[BenchmarkDef], TargetSet] + + +def all_targets(benchmark: BenchmarkDef) -> TargetSet: + """Target policy: run every target the benchmark declared.""" + return benchmark.targets + + +def defaults(benchmark: BenchmarkDef) -> TargetSet: + """Target policy: keep only the default lane (datafusion,duckdb × parquet,vortex). + + This intersects the default lane with the benchmark's declared targets, so a benchmark that + only supports a subset (e.g. duckdb-only) still runs a sensible default. + """ + engines = {Engine.DATAFUSION, Engine.DUCKDB} + formats = {Format.PARQUET, Format.VORTEX} + return TargetSet(tuple(t for t in benchmark.targets if t.engine in engines and t.format in formats)) + + +@dataclass(frozen=True) +class Profile: + """A named CI benchmark configuration: which benchmarks, and how many targets each runs.""" + + nightly: bool = False + targets: TargetPolicy = all_targets + description: str = "" + + +def _valid_for_storage(target_set: TargetSet, storage: Storage) -> TargetSet: + """Drop targets that are invalid for the given storage (lance has no remote reader).""" + if storage is Storage.S3: + return TargetSet(tuple(t for t in target_set if t.format is not Format.LANCE)) + return target_set + + +def _data_formats(target_set: TargetSet) -> list[Format]: + """Formats the data-gen step must produce for these targets, in canonical order.""" + present = set(target_set.formats()) + return [fmt for fmt in _FORMAT_ORDER if fmt in present and fmt not in _NOT_GENERATED] + + +def _scale_string(scale_factor: float | int) -> str: + """Render a scale factor exactly as declared for ``--opt scale-factor``.""" + return str(scale_factor) + + +def _matrix_entry(benchmark: BenchmarkDef, run_targets: TargetSet) -> dict[str, object]: + """Build one GitHub Actions ``include:`` entry for a resolved benchmark.""" + entry: dict[str, object] = { + "id": benchmark.id, + "subcommand": benchmark.subcommand, + "name": benchmark.name, + "targets": [t.to_dict() for t in run_targets], + "data_formats": [fmt.value for fmt in _data_formats(run_targets)], + } + if benchmark.scale_factor is not None: + entry["scale_factor"] = _scale_string(benchmark.scale_factor) + if benchmark.iterations is not None: + entry["iterations"] = str(benchmark.iterations) + if benchmark.local_dir is not None: + entry["local_dir"] = benchmark.local_dir + if benchmark.remote_key is not None: + entry["remote_key"] = benchmark.remote_key + return entry + + +def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list[dict[str, object]]: + """Resolve ``profile`` and ``benchmarks`` into GitHub Actions matrix entries.""" + entries: list[dict[str, object]] = [] + for benchmark in benchmarks: + if benchmark.nightly != profile.nightly: + continue + run_targets = _valid_for_storage(profile.targets(benchmark), benchmark.storage) + if not run_targets: + continue + entries.append(_matrix_entry(benchmark, run_targets)) + return entries diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py new file mode 100644 index 00000000000..01691ceea22 --- /dev/null +++ b/bench-orchestrator/tests/test_matrix.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Contract tests for the declarative CI benchmark matrix.""" + +import json +from typing import cast + +from bench_orchestrator import cli as cli_module +from bench_orchestrator.benchmarks import BENCHMARKS, PROFILES +from bench_orchestrator.config import Benchmark, Engine, Format +from bench_orchestrator.matrix import ( + DEFAULTS, + BenchmarkDef, + Profile, + Storage, + all_targets, + defaults, + df, + duck, + resolve_matrix, +) +from typer.testing import CliRunner + +runner = CliRunner() + + +def _targets(entry: dict[str, object]) -> list[dict[str, str]]: + return cast("list[dict[str, str]]", entry["targets"]) + + +def test_default_policy_only_narrows_declared_targets() -> None: + benchmark = BenchmarkDef( + id="duckdb-only", + benchmark=Benchmark.TPCH, + name="DuckDB only", + targets=duck(Format.VORTEX, Format.DUCKDB), + ) + + assert set(defaults(benchmark)) == set(duck(Format.VORTEX)) + + +def test_resolver_emits_the_fields_consumed_by_the_workflow() -> None: + benchmark = BenchmarkDef( + id="remote", + benchmark=Benchmark.TPCH, + name="Remote", + targets=df(Format.ARROW, Format.PARQUET, Format.LANCE, Format.VORTEX) | duck(Format.DUCKDB), + storage=Storage.S3, + scale_factor=1, + iterations=10, + local_dir="data/tpch", + remote_key="tpch/1.0/", + ) + + [entry] = resolve_matrix(Profile(targets=all_targets), [benchmark]) + + assert entry == { + "id": "remote", + "subcommand": "tpch", + "name": "Remote", + "targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "duckdb"}, + ], + "data_formats": ["parquet", "vortex", "duckdb"], + "scale_factor": "1", + "iterations": "10", + "local_dir": "data/tpch", + "remote_key": "tpch/1.0/", + } + + +def test_ci_profiles_have_distinct_and_consistent_roles() -> None: + assert set(PROFILES) == {"develop", "pr", "nightly"} + regular_ids = {benchmark.id for benchmark in BENCHMARKS if not benchmark.nightly} + nightly_ids = {benchmark.id for benchmark in BENCHMARKS if benchmark.nightly} + develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} + pr = {entry["id"]: entry for entry in resolve_matrix(PROFILES["pr"], BENCHMARKS)} + nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + + assert set(develop) == regular_ids + assert set(pr) == regular_ids + assert set(nightly) == nightly_ids + assert len(develop) == len([benchmark for benchmark in BENCHMARKS if not benchmark.nightly]) + assert len(nightly) == len([benchmark for benchmark in BENCHMARKS if benchmark.nightly]) + + default_targets = set(DEFAULTS) + for entry in (*pr.values(), *nightly.values()): + targets = {(Engine(target["engine"]), Format(target["format"])) for target in _targets(entry)} + assert targets + assert targets <= {(target.engine, target.format) for target in default_targets} + + tpch = develop["tpch-nvme"] + assert [(target["engine"], target["format"]) for target in _targets(tpch)] == [ + ("datafusion", "arrow"), + ("datafusion", "parquet"), + ("datafusion", "vortex"), + ("datafusion", "vortex-compact"), + ("datafusion", "lance"), + ("duckdb", "parquet"), + ("duckdb", "vortex"), + ("duckdb", "vortex-compact"), + ("duckdb", "duckdb"), + ] + + +def test_existing_display_and_scale_values_are_preserved() -> None: + develop = {entry["id"]: entry for entry in resolve_matrix(PROFILES["develop"], BENCHMARKS)} + nightly = {entry["id"]: entry for entry in resolve_matrix(PROFILES["nightly"], BENCHMARKS)} + + assert develop["tpch-nvme"]["scale_factor"] == "1.0" + assert develop["statpopgen"]["scale_factor"] == "100" + assert develop["polarsignals"]["scale_factor"] == "1" + assert nightly["tpch-nvme"]["name"] == "TPC-H on NVME" + assert nightly["tpch-nvme"]["scale_factor"] == "100" + assert nightly["tpch-s3"]["name"] == "TPC-H on S3" + assert nightly["tpch-s3"]["scale_factor"] == "100.0" + + +def test_matrix_command_emits_json_and_rejects_unknown_profiles() -> None: + result = runner.invoke(cli_module.app, ["matrix", "develop"]) + assert result.exit_code == 0 + assert json.loads(result.stdout) + + result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) + assert result.exit_code == 1 From 80f1d97908518a05536bb64d1da52259f3787e6d Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 15 Jul 2026 14:47:33 -0400 Subject: [PATCH 2/3] refactor(ci): resolve SQL benchmark matrices from profiles Replace the duplicated full/base YAML matrices with a plan job that resolves the requested bench-orchestrator profile. Wire develop, PR, full-PR, and nightly callers to explicit profiles, derive remote paths from matrix metadata, and preserve the previous nightly iteration count. Build lance-bench whenever a resolved entry contains a Lance target so the full PR profile and its built backends stay consistent. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Connor Tsui --- .github/workflows/bench-pr.yml | 1 + .github/workflows/bench.yml | 1 + .github/workflows/nightly-bench.yml | 44 +-- .github/workflows/sql-benchmarks.yml | 570 +++------------------------ .github/workflows/sql-pr.yml | 3 +- 5 files changed, 52 insertions(+), 567 deletions(-) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 12137726122..663d23a4d21 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -141,3 +141,4 @@ jobs: secrets: inherit with: mode: "pr" + profile: "pr" diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 4459726348b..f05722a3bf8 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -171,3 +171,4 @@ jobs: secrets: inherit with: mode: "develop" + profile: "develop" diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index b89395516be..680023ea1cb 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -23,49 +23,7 @@ jobs: with: mode: "develop" machine_type: ${{ matrix.machine_type.instance_name }} - benchmark_matrix: | - [ - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H on S3", - "local_dir": "vortex-bench/data/tpch/100.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100.0" - } - ] + profile: "nightly" strategy: # A single run not should kill the others fail-fast: false diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 243db83aff8..8471aa1844d 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -1,4 +1,4 @@ -name: "SQL-related benchmarks" +name: SQL benchmarks on: workflow_call: @@ -10,505 +10,41 @@ on: required: false type: string default: i7i.metal-24xl - benchmark_matrix: - required: false - type: string - description: "JSON string containing the full matrix configuration" - default: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ] - }, - { - "id": "clickbench-sorted-nvme", - "subcommand": "clickbench-sorted", - "name": "Clickbench Sorted on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ] - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpch-s3-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on S3", - "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "local_dir": "vortex-bench/data/statpopgen", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "data_formats": ["vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "scale_factor": "1" - }, - { - "id": "appian-nvme", - "subcommand": "appian", - "name": "Appian on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "iterations": "10" - } - ] - base_benchmark_matrix: - required: false - type: string - description: "JSON string containing the base matrix configuration" - default: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ] - }, - { - "id": "clickbench-sorted-nvme", - "subcommand": "clickbench-sorted", - "name": "Clickbench Sorted on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ] - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "arrow"}, - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "local_dir": "vortex-bench/data/statpopgen", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "data_formats": ["vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "scale_factor": "1" - } - ] - benchmark_profile: - required: false + profile: + required: true type: string - description: "Benchmark profile to run: full or base" - default: "full" + description: "Benchmark matrix profile resolved by `vx-bench matrix` (e.g. develop, pr, nightly)" jobs: + # Resolve the benchmark matrix declaratively from bench-orchestrator instead of maintaining + # inline JSON matrices in this file. The definitions are in + # bench-orchestrator/bench_orchestrator/benchmarks.py. + plan: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.resolve.outputs.matrix }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode == 'pr' + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode != 'pr' + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false + - name: Resolve benchmark matrix + id: resolve + shell: bash + env: + PROFILE: ${{ inputs.profile }} + run: | + echo "matrix=$(uv run --project bench-orchestrator vx-bench matrix "$PROFILE")" >> "$GITHUB_OUTPUT" + bench: + needs: plan timeout-minutes: 120 env: VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" @@ -518,7 +54,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(inputs.benchmark_profile == 'base' && inputs.base_benchmark_matrix || inputs.benchmark_matrix) }} + include: ${{ fromJSON(needs.plan.outputs.matrix) }} runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -555,28 +91,14 @@ jobs: - uses: ./.github/actions/system-info - - name: Resolve benchmark targets - id: targets - shell: bash - run: | - if [ "${{ inputs.mode }}" = "pr" ]; then - targets='${{ toJSON(matrix.pr_targets) }}' - else - targets='${{ toJSON(matrix.develop_targets) }}' - fi - { - echo 'targets_json<<__TARGETS_JSON__' - echo "$targets" - echo '__TARGETS_JSON__' - } >> "$GITHUB_OUTPUT" - - name: Build binaries shell: bash env: RUSTFLAGS: "-C target-cpu=native" run: | packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench) - if [ "${{ inputs.mode }}" != "pr" ]; then + if [ "${{ inputs.mode }}" != "pr" ] || \ + [ "${{ contains(toJSON(matrix.targets), 'lance') }}" = "true" ]; then packages+=(--bin lance-bench) fi cargo build "${packages[@]}" --profile release_debug --features unstable_encodings @@ -600,13 +122,14 @@ jobs: role-duration-seconds: 7200 - name: Upload data - if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) + if: matrix.remote_key != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: AWS_REGION: "us-east-1" + REMOTE_DIR: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}" run: | - aws s3 rm --recursive ${{ matrix.remote_storage }} - aws s3 cp --recursive ${{matrix.local_dir}} ${{ matrix.remote_storage }} + aws s3 rm --recursive "$REMOTE_DIR" + aws s3 cp --recursive ${{ matrix.local_dir }} "$REMOTE_DIR" - name: Setup Polar Signals if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false @@ -619,7 +142,7 @@ jobs: extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - name: Run ${{ matrix.name }} benchmark - if: matrix.remote_storage == null || github.event.pull_request.head.repo.fork == true + if: matrix.remote_key == null || github.event.pull_request.head.repo.fork == true shell: bash env: OTEL_SERVICE_NAME: "vortex-bench" @@ -629,7 +152,7 @@ jobs: OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ - --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --targets-json '${{ toJSON(matrix.targets) }}' \ --output results.json \ --gh-json-v3 results.v3.jsonl \ --no-build \ @@ -638,10 +161,11 @@ jobs: ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Run ${{ matrix.name }} benchmark (remote) - if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) + if: matrix.remote_key != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: AWS_REGION: "us-east-1" + REMOTE_DIR: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}" OTEL_SERVICE_NAME: "vortex-bench" OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf" OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}" @@ -649,17 +173,17 @@ jobs: OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ - --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --targets-json '${{ toJSON(matrix.targets) }}' \ --output results.json \ --gh-json-v3 results.v3.jsonl \ --no-build \ --runner "ec2_${{ inputs.machine_type }}" \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ - --opt remote-data-dir=${{ matrix.remote_storage }} \ + --opt remote-data-dir="$REMOTE_DIR" \ ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Capture file sizes - if: matrix.remote_storage == null + if: matrix.remote_key == null shell: bash run: | uv run --no-project scripts/capture-file-sizes.py \ @@ -700,7 +224,7 @@ jobs: message: | # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨 - Benchmark `${{ matrix.name }}` (${{ inputs.benchmark_profile }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + Benchmark `${{ matrix.name }}` (${{ inputs.profile }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. comment-tag: bench-pr-comment-${{ matrix.id }} - name: Upload Benchmark Results diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/sql-pr.yml index 45b0ed1d675..db8d1d61aab 100644 --- a/.github/workflows/sql-pr.yml +++ b/.github/workflows/sql-pr.yml @@ -39,4 +39,5 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_profile: ${{ inputs.benchmark_profile || 'base' }} + # base -> the cheap default-target profile; full -> develop's full target coverage on the PR. + profile: ${{ inputs.benchmark_profile == 'full' && 'develop' || 'pr' }} From 51db3b2fc7b705c5d6da87e1cff68c7d137d821c Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Wed, 15 Jul 2026 15:45:50 -0400 Subject: [PATCH 3/3] refactor(ci): share Rust micro-benchmark workflow Declare the two Rust micro-benchmark binaries in a small dedicated resolver and expose them through the existing matrix command. Replace the duplicated develop and PR jobs with one reusable workflow while preserving their mode-specific result handling. Keep this separate from the SQL profile migration so each change can be reviewed and landed independently. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Connor Tsui --- .github/workflows/bench-core.yml | 204 ++++++++++++++++++ .github/workflows/bench-pr.yml | 121 +---------- .github/workflows/bench.yml | 139 +----------- bench-orchestrator/README.md | 8 +- .../bench_orchestrator/benchmarks.py | 28 ++- bench-orchestrator/bench_orchestrator/cli.py | 12 +- .../bench_orchestrator/matrix.py | 25 +++ bench-orchestrator/tests/test_matrix.py | 27 ++- 8 files changed, 307 insertions(+), 257 deletions(-) create mode 100644 .github/workflows/bench-core.yml diff --git a/.github/workflows/bench-core.yml b/.github/workflows/bench-core.yml new file mode 100644 index 00000000000..6e7cda5972d --- /dev/null +++ b/.github/workflows/bench-core.yml @@ -0,0 +1,204 @@ +name: Rust micro-benchmarks + +# Reusable workflow for the Rust micro-benchmarks (random access, compression). Called by bench.yml +# (mode=develop) and bench-pr.yml (mode=pr). The matrix is resolved declaratively from +# bench-orchestrator; see bench-orchestrator/bench_orchestrator/matrix.py. + +on: + workflow_call: + inputs: + mode: + required: true + type: string + profile: + required: false + type: string + default: "micro" + +jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + matrix: ${{ steps.resolve.outputs.matrix }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode == 'pr' + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode != 'pr' + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false + - name: Resolve benchmark matrix + id: resolve + shell: bash + env: + PROFILE: ${{ inputs.profile }} + run: | + echo "matrix=$(uv run --project bench-orchestrator vx-bench matrix "$PROFILE")" >> "$GITHUB_OUTPUT" + + bench: + needs: plan + timeout-minutes: 120 + env: + RUST_BACKTRACE: full + VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" + FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.matrix) }} + + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + with: + sccache: s3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode == 'pr' + with: + ref: ${{ github.event.pull_request.head.sha }} + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: inputs.mode != 'pr' + - name: Setup benchmark environment + run: sudo bash scripts/setup-benchmark.sh + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }} + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false + + - name: Install DuckDB + run: | + wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb + chmod +x duckdb + echo "$PWD" >> "$GITHUB_PATH" + + - uses: ./.github/actions/system-info + + - name: Build binary + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native" + run: | + cargo build --bin ${{ matrix.id }} --profile release_debug ${{ matrix.build_args }} --features unstable_encodings + + - name: Setup Polar Signals + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 + with: + polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" + project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + profiling_frequency: 199 + extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + + - name: Run ${{ matrix.name }} benchmark (per-combination) + if: ${{ matrix.split }} + shell: bash + run: | + python3 scripts/random-access-split.py ${{ inputs.mode != 'pr' && '--v3' || '' }} + + - name: Run ${{ matrix.name }} benchmark + if: ${{ !matrix.split }} + shell: bash + run: | + bash scripts/bench-taskset.sh target/release_debug/${{ matrix.id }} \ + --formats ${{ matrix.formats }} \ + -d gh-json \ + -o results.json \ + ${{ inputs.mode != 'pr' && '--gh-json-v3 results.v3.jsonl' || '' }} + + - name: Setup AWS CLI + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole + aws-region: us-east-1 + + - name: Compare results + if: inputs.mode == 'pr' + shell: bash + run: | + set -Eeu -o pipefail -x + + python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request + gzip -d -c data.json.gz > base.json + + echo "# Benchmarks: ${{ matrix.name }}" > comment.md + echo '' >> comment.md + uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \ + >> comment.md + cat comment.md >> "$GITHUB_STEP_SUMMARY" + + - name: Comment PR + if: inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + with: + file-path: comment.md + comment-tag: bench-pr-comment-${{ matrix.id }} + + - name: Comment PR on failure + if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + with: + message: | + # BENCHMARK FAILED + + Benchmark `${{ matrix.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + comment-tag: bench-pr-comment-${{ matrix.id }} + + - name: Upload Benchmark Results + if: inputs.mode == 'develop' + shell: bash + run: | + bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json + + # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live benchmarks + # website (see bench.yml for the full rationale). Gated on inputs.mode == 'develop' + the + # ingest-role ARN var. post-ingest.py mints the RDS IAM token (boto3) from the assumed + # GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert. + - name: Configure AWS credentials for v4 ingest (OIDC) + if: inputs.mode == 'develop' && vars.GH_BENCH_INGEST_ROLE_ARN != '' + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} + aws-region: ${{ vars.RDS_BENCH_REGION }} + - name: Ingest results to v4 Postgres + if: inputs.mode == 'develop' && vars.GH_BENCH_INGEST_ROLE_ARN != '' + shell: bash + env: + RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} + RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }} + AWS_REGION: ${{ vars.RDS_BENCH_REGION }} + BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + run: | + set -Eeuo pipefail + curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ + -o "${RUNNER_TEMP}/rds-global-bundle.pem" + DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" + uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \ + scripts/post-ingest.py results.v3.jsonl \ + --postgres "${DSN}" \ + --commit-sha "${{ github.sha }}" \ + --region "${AWS_REGION}" + + - name: Alert incident.io + if: failure() && inputs.mode == 'develop' + uses: ./.github/actions/alert-incident-io + with: + api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} + alert-title: "${{ matrix.name }} benchmark failed on develop" + deduplication-key: ci-bench-${{ matrix.id }}-failure diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 663d23a4d21..9fa327a9f8d 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -19,122 +19,13 @@ permissions: id-token: write # enables AWS-GitHub OIDC jobs: + # Rust micro-benchmarks (random access, compression). Matrix resolved from the `micro` profile. bench: - timeout-minutes: 120 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') - || 'ubuntu-latest' }} - strategy: - matrix: - benchmark: - - id: random-access-bench - name: Random Access - build_args: "--features lance" - - id: compress-bench - name: Compression - steps: - - uses: runs-on/action@v2 - if: github.event.pull_request.head.repo.fork == false - with: - sccache: s3 - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup benchmark environment - run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: ${{ github.event.pull_request.head.repo.fork == false && 'true' || 'false' }} - - - name: Install DuckDB - run: | - wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb - chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH - - - uses: ./.github/actions/system-info - - - name: Build binary - shell: bash - env: - RUSTFLAGS: "-C target-cpu=native" - run: | - cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - - - name: Setup Polar Signals - if: github.event.pull_request.head.repo.fork == false - uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 - with: - polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" - project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" - profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - - - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) - if: matrix.benchmark.id == 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - python3 scripts/random-access-split.py - - - name: Run ${{ matrix.benchmark.name }} benchmark - if: matrix.benchmark.id != 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json - - - name: Setup AWS CLI - if: github.event.pull_request.head.repo.fork == false - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole - aws-region: us-east-1 - - - name: Install uv - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false - - - name: Compare results - shell: bash - run: | - set -Eeu -o pipefail -x - - python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request - gzip -d -c data.json.gz > base.json - - echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md - echo '' >> comment.md - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ - >> comment.md - cat comment.md >> $GITHUB_STEP_SUMMARY - - - name: Comment PR - if: github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 - with: - file-path: comment.md - comment-tag: bench-pr-comment-${{ matrix.benchmark.id }} - - - name: Comment PR on failure - if: failure() && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 - with: - message: | - # BENCHMARK FAILED - - Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. - comment-tag: bench-pr-comment-${{ matrix.benchmark.id }} + uses: ./.github/workflows/bench-core.yml + secrets: inherit + with: + mode: "pr" + profile: "micro" sql: uses: ./.github/workflows/sql-benchmarks.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f05722a3bf8..454d1ce7783 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -31,140 +31,13 @@ jobs: bash scripts/commit-json.sh > new-commit.json bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json + # Rust micro-benchmarks (random access, compression). Matrix resolved from the `micro` profile. bench: - timeout-minutes: 120 - runs-on: >- - ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=bench-dedicated/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id) - || 'ubuntu-latest' }} - strategy: - fail-fast: false - matrix: - benchmark: - - id: random-access-bench - name: Random Access - build_args: "--features lance" - formats: "parquet,lance,vortex" - - id: compress-bench - name: Compression - build_args: "--features lance" - formats: "parquet,lance,vortex" - steps: - - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' - with: - sccache: s3 - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - - name: Setup benchmark environment - run: sudo bash scripts/setup-benchmark.sh - - uses: ./.github/actions/setup-rust - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - - - name: Install DuckDB - run: | - wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb - chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH - - - uses: ./.github/actions/system-info - - - name: Build binary - shell: bash - env: - RUSTFLAGS: "-C target-cpu=native" - run: | - cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - - - name: Setup Polar Signals - uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 - with: - polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" - project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" - profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - - - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) - if: matrix.benchmark.id == 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - python3 scripts/random-access-split.py --v3 - - - name: Run ${{ matrix.benchmark.name }} benchmark - if: matrix.benchmark.id != 'random-access-bench' - shell: bash - env: - RUST_BACKTRACE: full - VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" - FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" - run: | - bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} --formats ${{ matrix.benchmark.formats }} -d gh-json -o results.json --gh-json-v3 results.v3.jsonl - - - name: Setup AWS CLI - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole - aws-region: us-east-1 - - - name: Upload Benchmark Results - shell: bash - run: | - bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json - - # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live - # benchmarks website; a failure here fails the job. Gated on the ingest-role ARN var - # (the assume-role input that MUST exist for OIDC to succeed). post-ingest.py mints - # the RDS IAM token internally (boto3) from the assumed GitHubBenchmarkIngestRole; - # sslmode=verify-full validates the cert. - # - # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only - # the uv binary, never the synced workspace. A full sync would rebuild - # vortex-python via sccache->S3, which fails under the ingest-role creds - # (rds-db:connect only) and is pure waste here. - - name: Install uv for v4 ingest - if: vars.GH_BENCH_INGEST_ROLE_ARN != '' - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - with: - sync: false - - name: Configure AWS credentials for v4 ingest (OIDC) - if: vars.GH_BENCH_INGEST_ROLE_ARN != '' - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }} - aws-region: ${{ vars.RDS_BENCH_REGION }} - - name: Ingest results to v4 Postgres - if: vars.GH_BENCH_INGEST_ROLE_ARN != '' - shell: bash - env: - RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }} - RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }} - AWS_REGION: ${{ vars.RDS_BENCH_REGION }} - BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }} - BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} - run: | - set -Eeuo pipefail - curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ - -o "${RUNNER_TEMP}/rds-global-bundle.pem" - DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem" - uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \ - scripts/post-ingest.py results.v3.jsonl \ - --postgres "${DSN}" \ - --commit-sha "${{ github.sha }}" \ - --region "${AWS_REGION}" - - - name: Alert incident.io - if: failure() - uses: ./.github/actions/alert-incident-io - with: - api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} - alert-title: "${{ matrix.benchmark.name }} benchmark failed on develop" - deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure + uses: ./.github/workflows/bench-core.yml + secrets: inherit + with: + mode: "develop" + profile: "micro" sql: uses: ./.github/workflows/sql-benchmarks.yml diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index a954b9c3268..e83db450096 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -156,8 +156,9 @@ vx-bench clean --older-than "30 days" [options] ## Declarative benchmark matrix To change what CI benchmarks, edit **`bench_orchestrator/benchmarks.py`**. `BENCHMARKS` is the list -of benchmark suites and their supported targets; `PROFILES` says how much coverage each workflow -runs. Matrix rendering is kept separately in `bench_orchestrator/matrix.py`. +of SQL suites and their supported targets, `PROFILES` says how much coverage each SQL workflow +runs, and `MICRO_BENCHMARKS` lists the Rust benchmark binaries. Matrix rendering is kept +separately in `bench_orchestrator/matrix.py`. This replaces the parallel `pr_targets`/`develop_targets` arrays and duplicated `full`/`base` matrices that previously lived in workflow YAML. @@ -184,7 +185,8 @@ support rules that validate a run. ### Built-in profiles `develop` runs full coverage on merge, `pr` runs the default targets for every SQL benchmark, and -`nightly` runs SF=100 TPC-H on NVMe and S3. List them with `vx-bench matrix`. +`nightly` runs SF=100 TPC-H on NVMe and S3. The `micro` profile describes the two Rust benchmark +binaries used by the shared develop/PR workflow. List them with `vx-bench matrix`. ### Adding a benchmark diff --git a/bench-orchestrator/bench_orchestrator/benchmarks.py b/bench-orchestrator/bench_orchestrator/benchmarks.py index 6857a3b4a15..cf4b29f3f59 100644 --- a/bench-orchestrator/bench_orchestrator/benchmarks.py +++ b/bench-orchestrator/bench_orchestrator/benchmarks.py @@ -1,19 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -"""The SQL benchmarks and profiles run by CI. +"""The benchmarks and profiles run by CI. This is the file to edit when changing benchmark coverage: * ``BENCHMARKS`` declares which benchmark suites exist and every target they support. * ``PROFILES`` declares how much of that coverage each CI workflow runs. +* ``MICRO_BENCHMARKS`` declares the Rust benchmark binaries run by the shared workflow. Matrix rendering lives in :mod:`bench_orchestrator.matrix` and should not need to change when a benchmark is added or removed. """ from .config import Benchmark, Engine, Format -from .matrix import STANDARD, BenchmarkDef, Profile, Storage, all_targets, defaults, df, duck +from .matrix import STANDARD, BenchmarkDef, MicroDef, Profile, Storage, all_targets, defaults, df, duck def _tpch(scale_factor: float | int, storage: Storage, *, iterations: int | None = 10) -> BenchmarkDef: @@ -162,3 +163,26 @@ def _fineweb(storage: Storage) -> BenchmarkDef: description="Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", ), } + + +# ================================================================================================== +# RUST MICRO-BENCHMARK DEFINITIONS — ADD OR REMOVE MICRO-BENCHMARKS HERE +# ================================================================================================== + +MICRO_BENCHMARKS: list[MicroDef] = [ + MicroDef( + id="random-access-bench", + name="Random Access", + build_args="--features lance", + formats=(Format.PARQUET, Format.LANCE, Format.VORTEX), + split=True, + ), + MicroDef( + id="compress-bench", + name="Compression", + build_args="--features lance", + formats=(Format.PARQUET, Format.LANCE, Format.VORTEX), + ), +] + +MICRO_PROFILE_DESCRIPTION = "Rust micro-benchmarks: random access and compression." diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index ef8308389d3..afef5bc0320 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -16,7 +16,7 @@ from rich.console import Console from rich.table import Table -from .benchmarks import BENCHMARKS, PROFILES +from .benchmarks import BENCHMARKS, MICRO_BENCHMARKS, MICRO_PROFILE_DESCRIPTION, PROFILES from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( @@ -31,7 +31,7 @@ parse_targets_json, resolve_axis_targets, ) -from .matrix import resolve_matrix +from .matrix import resolve_matrix, resolve_micro_matrix from .runner.builder import BenchmarkBuilder from .runner.executor import BenchmarkExecutor from .storage.store import ResultStore @@ -234,11 +234,17 @@ def matrix( if profile is None or list_profiles: for name, prof in PROFILES.items(): console.print(f"[bold cyan]{name}[/bold cyan]: {prof.description}") + console.print(f"[bold cyan]micro[/bold cyan]: {MICRO_PROFILE_DESCRIPTION}") + return + + if profile == "micro": + entries = resolve_micro_matrix(MICRO_BENCHMARKS) + typer.echo(json.dumps(entries, indent=2 if pretty else None)) return prof = PROFILES.get(profile) if prof is None: - known = ", ".join(PROFILES) + known = ", ".join((*PROFILES, "micro")) console.print(f"[red]Unknown profile '{profile}'. Available: {known}[/red]") raise typer.Exit(1) diff --git a/bench-orchestrator/bench_orchestrator/matrix.py b/bench-orchestrator/bench_orchestrator/matrix.py index 9f1a4bbd989..c716fd78614 100644 --- a/bench-orchestrator/bench_orchestrator/matrix.py +++ b/bench-orchestrator/bench_orchestrator/matrix.py @@ -156,6 +156,17 @@ def subcommand(self) -> str: return self.benchmark.value +@dataclass(frozen=True) +class MicroDef: + """A Rust micro-benchmark binary and the arguments needed to build and run it.""" + + id: str + name: str + build_args: str + formats: tuple[Format, ...] + split: bool = False + + # A target policy maps a benchmark's declared targets to the targets a profile actually runs. TargetPolicy = Callable[[BenchmarkDef], TargetSet] @@ -234,3 +245,17 @@ def resolve_matrix(profile: Profile, benchmarks: Iterable[BenchmarkDef]) -> list continue entries.append(_matrix_entry(benchmark, run_targets)) return entries + + +def resolve_micro_matrix(benchmarks: Iterable[MicroDef]) -> list[dict[str, object]]: + """Resolve Rust micro-benchmark declarations into GitHub Actions matrix entries.""" + return [ + { + "id": benchmark.id, + "name": benchmark.name, + "build_args": benchmark.build_args, + "formats": ",".join(fmt.value for fmt in benchmark.formats), + "split": benchmark.split, + } + for benchmark in benchmarks + ] diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index 01691ceea22..1cb57415cbc 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -7,7 +7,7 @@ from typing import cast from bench_orchestrator import cli as cli_module -from bench_orchestrator.benchmarks import BENCHMARKS, PROFILES +from bench_orchestrator.benchmarks import BENCHMARKS, MICRO_BENCHMARKS, PROFILES from bench_orchestrator.config import Benchmark, Engine, Format from bench_orchestrator.matrix import ( DEFAULTS, @@ -19,6 +19,7 @@ df, duck, resolve_matrix, + resolve_micro_matrix, ) from typer.testing import CliRunner @@ -127,3 +128,27 @@ def test_matrix_command_emits_json_and_rejects_unknown_profiles() -> None: result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) assert result.exit_code == 1 + + +def test_micro_matrix_matches_the_two_shared_workflow_jobs() -> None: + entries = resolve_micro_matrix(MICRO_BENCHMARKS) + assert entries == [ + { + "id": "random-access-bench", + "name": "Random Access", + "build_args": "--features lance", + "formats": "parquet,lance,vortex", + "split": True, + }, + { + "id": "compress-bench", + "name": "Compression", + "build_args": "--features lance", + "formats": "parquet,lance,vortex", + "split": False, + }, + ] + + result = runner.invoke(cli_module.app, ["matrix", "micro"]) + assert result.exit_code == 0 + assert json.loads(result.stdout) == entries