From 5002d1fc720a3e092f7b3adf4e19b3ad4c9512a1 Mon Sep 17 00:00:00 2001 From: jkaczman Date: Wed, 28 Oct 2026 14:33:08 -0700 Subject: [PATCH] ci: add x86 spot metal CI runner to bench, remove Bencher --- .github/workflows/bencher.yml | 154 ----------------------- .github/workflows/bencher_followup.yml | 27 ---- .github/workflows/bencher_fork_run.yml | 48 ------- .github/workflows/bencher_fork_track.yml | 117 ----------------- .github/workflows/metal.yml | 20 ++- benches/perf/Dockerfile | 18 --- integration/ci/metal-report.sh | 81 +++++++----- 7 files changed, 70 insertions(+), 395 deletions(-) delete mode 100644 .github/workflows/bencher.yml delete mode 100644 .github/workflows/bencher_followup.yml delete mode 100644 .github/workflows/bencher_fork_run.yml delete mode 100644 .github/workflows/bencher_fork_track.yml delete mode 100644 benches/perf/Dockerfile diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml deleted file mode 100644 index b3b8ef1a5..000000000 --- a/.github/workflows/bencher.yml +++ /dev/null @@ -1,154 +0,0 @@ -# Handles running Bencher perf tests to detect regressions. -# Runs only when PRs are made from an internal branch (NOT A FORK) to main. -# Forks are processed via 2 steps [bencher_fork_run.yml, bencher_fork_track.yml] -# TODO: I have some big plans for this file. Please contact @jkaczman before planning to modify with new tests. - -name: bencher -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened] - workflow_dispatch: - -env: - IMAGE: registry.bencher.dev/pgdog:${{ github.sha }} - -jobs: - build: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ${{ matrix.runner || 'blacksmith-4vcpu-ubuntu-2404' }} - strategy: - matrix: - include: - - bin: pgdog - run: | - cargo build --release --bin pgdog --config 'target."cfg(all())".rustflags=["-C","llvm-args=-align-all-functions=6"]' - cp target/release/pgdog out/pgdog - - bin: pgdog-microbench - runner: blacksmith-16vcpu-ubuntu-2404 - run: | - bin=$(cargo test --profile microbench -p pgdog --bin pgdog --features bench --no-run --message-format=json --config 'target."cfg(all())".rustflags=["-C","llvm-args=-align-all-functions=6"]' \ - | jq -r 'select(.profile.test == true and .target.name == "pgdog") | .executable') - cp "$bin" out/pgdog-microbench - steps: - - uses: actions/checkout@v6 - - name: Install CI deps - run: bash integration/ci/install-deps.sh - - uses: Swatinem/rust-cache@v2 - with: - prefix-key: build-v1 - key: ${{ matrix.bin }} - - name: Build ${{ matrix.bin }} - env: - GIT_HASH: bench - run: | - mkdir out - ${{ matrix.run }} - - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.bin }} - path: out/${{ matrix.bin }} - retention-days: 1 - - image: - needs: build - # This initial runner is to assemble the docker image for the Bencher runner. It doesn't run any tests itself. - # Bencher's runners do not have outside network access; requires us to have everything ready-to-go. - runs-on: blacksmith-4vcpu-ubuntu-2404 - steps: - - uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 - with: - path: bin - merge-multiple: true - - - name: Build image - run: | - chmod +x bin/* - docker build --build-context bin=bin -f benches/perf/Dockerfile -t $IMAGE . - - - name: Push image - run: | - echo '${{ secrets.BENCHER_API_KEY }}' | docker login registry.bencher.dev -u pgdog --password-stdin - docker push $IMAGE - - bench: - needs: image - runs-on: blacksmith-2vcpu-ubuntu-2404 - strategy: - fail-fast: false - matrix: - # sharding-w2 runs sharding with 2 PgDog workers, so that we can catch **contention** between - # workers (e.g., a new lock on the hot path) through ctx-switches-per-txn. - # - # TODO: we need another test on a runner with more CPUs to catch contention with a larger - # amount of workers (e.g., 4-8). 4 vCPUs can only fit 2 workers while leaving enough - # cores for Postgres and pgbench. - include: - - suite: pooler - - suite: lb - cpu_bound: 0.015 - - suite: sharding-w2 - - suite: microbench - # - suite: sharding-w8 - steps: - - uses: actions/checkout@v6 - - uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a - - name: Run ${{ matrix.suite }} on Bencher bare metal - env: - START_POINT: ${{ github.event_name == 'pull_request' && format('--start-point {0} --start-point-hash {1} --start-point-clone-thresholds --start-point-reset', github.base_ref, github.event.pull_request.base.sha) || '' }} - # This will run one test per job (all in parallel). - # - # Why --github-actions? This allows it to comment on the PR with the result. - # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts - # Why --detach & --ci-callback-token? This allows us to exit early, and defer posting a comment until repository_dispatch is fired on - # a separate workflow (bencher_followup.yml) - run: | - bencher run \ - --project pgdog \ - --key '${{ secrets.BENCHER_API_KEY }}' \ - --branch '${{ github.head_ref || github.ref_name }}' \ - $START_POINT \ - --spec intel-v1 \ - --threshold-measure throughput \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary 0.01 \ - --threshold-upper-boundary _ \ - --threshold-measure cpu-per-txn \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary ${{ matrix.cpu_bound || '0.01' }} \ - --threshold-measure ctx-switches-per-txn \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary 0.03 \ - --threshold-measure alloc_count \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary 0 \ - --threshold-measure alloc_bytes \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary 0.01 \ - --threshold-measure latency \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary 0.10 \ - --thresholds-reset \ - --error-on-alert \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - --ci-id ${{ matrix.suite }} \ - --ci-only-on-alert \ - --adapter json \ - --image $IMAGE \ - --detach \ - --ci-callback-token '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ - "bash benches/perf/bencher.sh ${{ matrix.suite }}" diff --git a/.github/workflows/bencher_followup.yml b/.github/workflows/bencher_followup.yml deleted file mode 100644 index bbba73327..000000000 --- a/.github/workflows/bencher_followup.yml +++ /dev/null @@ -1,27 +0,0 @@ -# This is just the workflow Bencher has in its docs; haven't customized it yet -# -on: - repository_dispatch: - types: [bencher_run] - -jobs: - attach_pr_benchmarks: - name: Post PR Benchmark Results from Bencher Bare Metal - permissions: - checks: write - pull-requests: write - runs-on: ubuntu-latest - steps: - - uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a - - name: Post PR Benchmark Results with Bencher - env: - JOB_UUID: ${{ github.event.client_payload.bencher.job }} - run: | - bencher run \ - --project pgdog \ - --key '${{ secrets.BENCHER_API_KEY }}' \ - --job "$JOB_UUID" \ - --error-on-alert \ - --ci-only-on-alert \ - --ci-only-thresholds \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' diff --git a/.github/workflows/bencher_fork_run.yml b/.github/workflows/bencher_fork_run.yml deleted file mode 100644 index 119b19190..000000000 --- a/.github/workflows/bencher_fork_run.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Fork version of Bencher (Step 1 of 2) -# -# This file is more-or-less a repeat of what we do in the main workflow for building the image -# except, now, the image gets saved as an artifact, and ran in a separate workflow. -# -# Why the separation? This allows for us to build directly on the fork, WITHOUT USING GITHUB SECRETS, -# and then have a separate workflow, that runs after this one, with privileged secrets, -# that can use the image built, and run on the main branch (mitigating any security concerns) -# -# This is the documented 2-step workflow that Bencher mentions in their docs. -name: bencher-fork-run -on: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - image: - if: github.event.pull_request.head.repo.full_name != github.repository - # This initial runner is to assemble the docker image for this PR for the Bencher runner. It doesn't run any tests itself. - # Bencher's runners do not have outside network access; requires us to have everything ready-to-go. - runs-on: blacksmith-4vcpu-ubuntu-2404 - steps: - - uses: actions/checkout@v6 - - name: Install CI deps - run: bash integration/ci/install-deps.sh - - uses: Swatinem/rust-cache@v2 - with: - prefix-key: build-v1 - - name: Build - env: - GIT_HASH: bench - run: cargo build --release --bin pgdog --config 'target."cfg(all())".rustflags=["-C","llvm-args=-align-all-functions=6"]' - - name: Build image - run: | - docker build --build-context bin=target/release -f benches/perf/Dockerfile -t pgdog-bench . - docker save pgdog-bench | gzip > image.tar.gz - - # Deviation from main workflow (as seen in header of this file) - - uses: actions/upload-artifact@v4 - with: - name: image - path: image.tar.gz - - # Save which PR requested the run, so the workflow running on main knows what PR to comment on - - uses: actions/upload-artifact@v4 - with: - name: event.json - path: ${{ github.event_path }} diff --git a/.github/workflows/bencher_fork_track.yml b/.github/workflows/bencher_fork_track.yml deleted file mode 100644 index b64b5b6f8..000000000 --- a/.github/workflows/bencher_fork_track.yml +++ /dev/null @@ -1,117 +0,0 @@ -# Fork version of Bencher (Step 2 of 2) -# -# We built the image in bencher_fork_run.yml on the fork PR. -# Now, we can use it (ON THE MAIN BRANCH, where it's okay to use secrets), -# and basically do what we do in the main Bencher workflow, with a pre-built image this time. - -name: bencher-fork-track -on: - workflow_run: - workflows: [bencher-fork-run] - types: [completed] - -env: - IMAGE: registry.bencher.dev/pgdog:${{ github.event.workflow_run.head_sha }} - -jobs: - image: - # Download image from previous workflow (2-step). Push image to bencher. - if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name != github.repository - permissions: - actions: read - runs-on: blacksmith-4vcpu-ubuntu-2404 - steps: - - uses: actions/download-artifact@v4 - with: - name: image - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Push image - run: | - docker load < image.tar.gz - docker tag pgdog-bench $IMAGE - echo '${{ secrets.BENCHER_API_KEY }}' | docker login registry.bencher.dev -u pgdog --password-stdin - docker push $IMAGE - - bench: - needs: image - permissions: - actions: read - runs-on: blacksmith-2vcpu-ubuntu-2404 - strategy: - fail-fast: false - matrix: - # sharding-w2 runs sharding with 2 PgDog workers, so that we can catch **contention** between - # workers (e.g., a new lock on the hot path) through ctx-switches-per-txn. - # - # TODO: we need another test on a runner with more CPUs to catch contention with a larger - # amount of workers (e.g., 4-8). 4 vCPUs can only fit 2 workers while leaving enough - # cores for Postgres and pgbench. - include: - - suite: pooler - - suite: lb - cpu_bound: 0.015 - - suite: sharding-w2 - # - suite: sharding-w8 - steps: - - uses: actions/download-artifact@v4 - with: - name: event.json - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - # Since we're running this on main branch, we have to download the artifact - # from the PR that requested this run to figure out which one it was, - # so that we can safely run this on main and have all the required info. - - name: Export PR event data - run: | - echo "PR_HEAD=$(jq -r .pull_request.head.ref event.json)" >> "$GITHUB_ENV" - echo "PR_HEAD_SHA=$(jq -r .pull_request.head.sha event.json)" >> "$GITHUB_ENV" - echo "PR_BASE=$(jq -r .pull_request.base.ref event.json)" >> "$GITHUB_ENV" - echo "PR_BASE_SHA=$(jq -r .pull_request.base.sha event.json)" >> "$GITHUB_ENV" - echo "PR_NUMBER=$(jq -r .number event.json)" >> "$GITHUB_ENV" - - uses: bencherdev/bencher@f80847436d7157936b40525db4c2a1064124f04a - - name: Run ${{ matrix.suite }} on Bencher bare metal - # This will run one test per job (all in parallel). - # - # Why --github-actions? This allows it to comment on the PR with the result. - # Why --adapter json? This allows us to use a 'custom harness'; basically nothing like rust's cargo test / criterion, we're running custom scripts - # Why --detach & --ci-callback-token? This allows us to exit early, and defer posting a comment until repository_dispatch is fired on - # a separate workflow (bencher_followup.yml) - run: | - bencher run \ - --project pgdog \ - --key '${{ secrets.BENCHER_API_KEY }}' \ - --branch "$PR_HEAD" \ - --hash "$PR_HEAD_SHA" \ - --start-point "$PR_BASE" \ - --start-point-hash "$PR_BASE_SHA" \ - --start-point-clone-thresholds \ - --start-point-reset \ - --spec intel-v1 \ - --threshold-measure throughput \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary 0.01 \ - --threshold-upper-boundary _ \ - --threshold-measure cpu-per-txn \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary ${{ matrix.cpu_bound || '0.01' }} \ - --threshold-measure ctx-switches-per-txn \ - --threshold-test percentage \ - --threshold-max-sample-size 1 \ - --threshold-lower-boundary _ \ - --threshold-upper-boundary 0.03 \ - --thresholds-reset \ - --error-on-alert \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - --ci-number "$PR_NUMBER" \ - --ci-id ${{ matrix.suite }} \ - --ci-only-on-alert \ - --adapter json \ - --image $IMAGE \ - --detach \ - --ci-callback-token '${{ secrets.BENCHER_GITHUB_TOKEN }}' \ - "bash benches/perf/bencher.sh ${{ matrix.suite }}" diff --git a/.github/workflows/metal.yml b/.github/workflows/metal.yml index 7889aa1d6..d7e5b3825 100644 --- a/.github/workflows/metal.yml +++ b/.github/workflows/metal.yml @@ -27,6 +27,8 @@ env: BUCKET: pgdog-metal-bench QUEUE_URL: https://sqs.us-east-1.amazonaws.com/588738614642/pgdog-metal-bench.fifo AUTOSCALING_GROUP: pgdog-metal-bench + QUEUE_URL_X86: https://sqs.us-east-1.amazonaws.com/588738614642/pgdog-metal-bench-x86.fifo + AUTOSCALING_GROUP_X86: pgdog-metal-bench-x86 SHA: ${{ github.event.pull_request.head.sha || github.sha }} REF: ${{ github.head_ref || github.ref_name }} @@ -58,8 +60,16 @@ jobs: --message-deduplication-id "$SHA-${{ github.run_id }}" \ --message-body "$MESSAGE" - - name: Start the runner if it's down - run: aws autoscaling set-desired-capacity --auto-scaling-group-name "$AUTOSCALING_GROUP" --desired-capacity 1 + aws sqs send-message \ + --queue-url "$QUEUE_URL_X86" \ + --message-group-id metal \ + --message-deduplication-id "$SHA-${{ github.run_id }}" \ + --message-body "$MESSAGE" + + - name: Start the runners if they're down + run: | + aws autoscaling set-desired-capacity --auto-scaling-group-name "$AUTOSCALING_GROUP" --desired-capacity 1 + aws autoscaling set-desired-capacity --auto-scaling-group-name "$AUTOSCALING_GROUP_X86" --desired-capacity 1 - name: Mark commit as queued env: @@ -76,6 +86,11 @@ jobs: -f state=pending \ -f description="Queued. $queued in line" + gh api "repos/${{ github.repository }}/statuses/$SHA" \ + -f context=metal/bench-x86 \ + -f state=pending \ + -f description="Queued" + # TODO(jkaczman): need to get a PAT with commit statuses + pull requests write, so we can # drop this unnecessary polling job (current PAT does not have necessary perms) report: @@ -95,6 +110,7 @@ jobs: with: role-to-assume: arn:aws:iam::588738614642:role/pgdog-metal-bench-ci aws-region: us-east-1 + role-duration-seconds: 21600 - uses: actions/checkout@v6 diff --git a/benches/perf/Dockerfile b/benches/perf/Dockerfile deleted file mode 100644 index 11433ca7c..000000000 --- a/benches/perf/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# Re-uses the base runtime image; injects this branch's built pgdog binary and -# the perf scripts we're gonna run in Bencher. - -FROM ghcr.io/pgdogdev/pgdog-base-runtime:latest - -RUN apt update && apt install -y --no-install-recommends postgresql-18 sudo iproute2 - -WORKDIR /pgdog -COPY --from=bin pgdog pgdog-microbench /usr/local/bin/ -COPY integration/common.sh integration/setup.sh integration/ -COPY integration/ci/setup.sh integration/ci/ -COPY pgdog/src/backend/schema/setup.sql pgdog/src/backend/schema/ -COPY benches/perf benches/perf - -# This happens here because we don't have network access in Bencher. -RUN bash integration/ci/setup.sh && pg_ctlcluster 18 main stop - -CMD ["bash", "benches/perf/bencher.sh", "pooler", "lb", "sharding-w2"] diff --git a/integration/ci/metal-report.sh b/integration/ci/metal-report.sh index 88cb012ec..ddc14f5b4 100755 --- a/integration/ci/metal-report.sh +++ b/integration/ci/metal-report.sh @@ -1,37 +1,46 @@ #!/bin/bash -# Waits for the metal runner's result in S3 and posts it as a commit status and a PR comment. +# Waits for the metal runners' results in S3 and posts them as commit statuses and PR comments. # TODO(jkaczman): get rid of this with PAT with necessary perms set -euo pipefail -readonly RESULTS="s3://$BUCKET/results/$REF/$SHA" -readonly COMMENT_MARKER="" +readonly X86_WAIT_SECONDS=5400 + +lane_context() { [ "$1" = x86 ] && echo metal/bench-x86 || echo metal/bench; } +lane_prefix() { [ "$1" = x86 ] && echo results-x86 || echo results; } +lane_marker() { [ "$1" = x86 ] && echo "" || echo ""; } set_status() { - local state=$1 description=$2 + local lane=$1 state=$2 description=$3 gh api "repos/$GITHUB_REPOSITORY/statuses/$SHA" \ - -f context=metal/bench \ + -f context="$(lane_context "$lane")" \ -f state="$state" \ -f description="$description" \ - -f target_url="https://s3.console.aws.amazon.com/s3/buckets/$BUCKET?prefix=results/$REF/$SHA/" > /dev/null + -f target_url="https://s3.console.aws.amazon.com/s3/buckets/$BUCKET?prefix=$(lane_prefix "$lane")/$REF/$SHA/" > /dev/null } -current_state() { - local modified - modified=$(aws s3api head-object --bucket "$BUCKET" --key "results/$REF/$SHA/state" --query LastModified --output text 2> /dev/null) || return 0 +lane_state() { + local key modified + key="$(lane_prefix "$1")/$REF/$SHA/state" + modified=$(aws s3api head-object --bucket "$BUCKET" --key "$key" --query LastModified --output text 2> /dev/null) || return 0 [ "$(date -d "$modified" +%s)" -ge "$ENQUEUED_AT" ] || return 0 - aws s3 cp "$RESULTS/state" - + aws s3 cp "s3://$BUCKET/$key" - } -wait_for_runner() { - local state="" last="" +wait_for_lane() { + local lane=$1 deadline=${2:-} state="" last="" until [ "$state" = done ] || [ "$state" = skipped ]; do + if [ -n "$deadline" ] && [ "$(date +%s)" -ge "$deadline" ]; then + echo timeout + return + fi + sleep 30 - state=$(current_state) + state=$(lane_state "$lane") if [ -n "$state" ] && [ "$state" != "$last" ]; then - set_status pending "$state" + set_status "$lane" pending "$state" last=$state fi done @@ -40,35 +49,49 @@ wait_for_runner() { } post_comment() { - local existing + local marker=$1 file=$2 existing existing=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments?per_page=100" \ - --jq "map(select(.body | startswith(\"$COMMENT_MARKER\")))[0].id") + --jq "map(select(.body | startswith(\"$marker\")))[0].id") if [ -n "$existing" ]; then - gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@comment.md > /dev/null + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@"$file" > /dev/null else - gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@comment.md > /dev/null + gh api "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$file" > /dev/null fi } -main() { - if [ "$(wait_for_runner)" = skipped ]; then - set_status pending "Skipped, superseded by a newer push" - return - fi +report_lane() { + local lane=$1 state=$2 results - aws s3 cp "$RESULTS/verdict.json" verdict.json - aws s3 cp "$RESULTS/comment.md" comment.md + case "$state" in + skipped) + set_status "$lane" pending "Skipped, superseded by a newer push" + return + ;; + timeout) + set_status "$lane" error "No result within $((X86_WAIT_SECONDS / 60)) minutes of arm64" + return + ;; + esac - if [ "$(jq -r .passed verdict.json)" = true ]; then - set_status success "$(jq -r .summary verdict.json)" + results="s3://$BUCKET/$(lane_prefix "$lane")/$REF/$SHA" + aws s3 cp "$results/verdict.json" "$lane.verdict.json" + aws s3 cp "$results/comment.md" "$lane.md" + + if [ "$(jq -r .passed "$lane.verdict.json")" = true ]; then + set_status "$lane" success "$(jq -r .summary "$lane.verdict.json")" else - set_status failure "$(jq -r .summary verdict.json)" + set_status "$lane" failure "$(jq -r .summary "$lane.verdict.json")" fi if [ -n "$PR" ]; then - post_comment + post_comment "$(lane_marker "$lane")" "$lane.md" fi } +main() { + report_lane arm64 "$(wait_for_lane arm64)" + report_lane x86 "$(wait_for_lane x86 $(($(date +%s) + X86_WAIT_SECONDS)))" +} + main