diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index b422ad9697d..7776f16f4f3 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -12,25 +12,6 @@ permissions: contents: write jobs: - commit-metadata: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Setup AWS CLI - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole - aws-region: us-east-1 - - name: Upload Commit Metadata - shell: bash - run: | - set -Eeu -o pipefail -x - - sudo apt-get update && sudo apt-get install -y jq - bash scripts/commit-json.sh > new-commit.json - bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json - build: timeout-minutes: 60 runs-on: >- @@ -217,15 +198,10 @@ jobs: run: | python3 scripts/string-split.py - - name: Upload Benchmark Results - shell: bash - run: | - bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json - # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live # benchmarks website for suites that emit v4 records; a failure here fails the job. - # All suites also upload S3 results above. string-bench does not emit - # results.ingest.jsonl, so these steps are disabled for it. + # string-bench does not emit results.ingest.jsonl, so these steps are disabled for it + # and its results are not stored. # Also gated on the ingest-role ARN var (the assume-role input that MUST exist for # OIDC to succeed). post-ingest.py mints the RDS IAM token internally (boto3) from # the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert. diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index f1661284475..487b067892b 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -282,12 +282,6 @@ jobs: Benchmark `${{ matrix.name }}` (${{ inputs.matrix_preset }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. comment-tag: bench-pr-comment-${{ matrix.id }}-${{ inputs.matrix_preset }} - - name: Upload Benchmark Results - if: inputs.mode == 'develop' - shell: bash - run: | - bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json - # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live # benchmarks website (see develop-bench.yml for the full rationale); a failure here fails the # job. Gated on inputs.mode == 'develop' + the ingest-role ARN var (the assume-role diff --git a/scripts/cat-s3.sh b/scripts/cat-s3.sh deleted file mode 100644 index 5864149759b..00000000000 --- a/scripts/cat-s3.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors - -set -Eeu -o pipefail -x - -bucket="$1" -key="$2" -local_file_to_concatenate="$3" -local_copy=$(mktemp) -local_concatenated=$(mktemp) -n_failures=0 - -while (( n_failures < 100 )); do - current_etag=$(aws s3api head-object --bucket "$bucket" --key "$key" --query ETag --output text 2>/dev/null) || { - # File doesn't exist yet, try to create it fresh - echo "File does not exist in S3, creating new file." - if [[ "$key" =~ \.gz$ ]]; then - gzip -c "$local_file_to_concatenate" > "$local_concatenated" - else - cp "$local_file_to_concatenate" "$local_concatenated" - fi - # Use --if-none-match to avoid race with concurrent writers - aws s3api put-object --bucket "$bucket" --key "$key" --body "$local_concatenated" --if-none-match "*" && { - echo "File created and uploaded successfully." - exit 0 - } - # Another writer created the file first, retry to concatenate with it - echo "File was created by another writer, retrying to concatenate." - n_failures=$(( n_failures + 1 )) - sleep 0.1 - continue - } - if [[ "$current_etag" == "null" ]]; then - echo "Failed to retrieve ETag. Exiting." - exit 1 - fi - - aws s3api get-object --bucket "$bucket" --key "$key" --if-match "$current_etag" "$local_copy" || { - echo "ETag does not match. Trying again." - n_failures=$(( n_failures + 1 )) - continue - } - - if [[ "$key" =~ \.gz$ ]]; then - local_decompressed=$(mktemp) - local_decompressed_concat=$(mktemp) - gzip -d -c "$local_copy" > "$local_decompressed" - cat "$local_decompressed" "$local_file_to_concatenate" > "$local_decompressed_concat" - gzip -c "$local_decompressed_concat" > "$local_concatenated" - rm "$local_decompressed" "$local_decompressed_concat" - else - cat "$local_copy" "$local_file_to_concatenate" > "$local_concatenated" - fi - - - aws s3api put-object --bucket "$bucket" --key "$key" --body "$local_concatenated" --if-match "$current_etag" || { - echo "ETag does not match during upload. Trying again." - n_failures=$(( n_failures + 1 )) - # wait for before retrying - sleep 0.1 - continue - } - - echo "File updated and uploaded successfully." - exit 0 -done - -echo "Too many failures: $n_failures." -exit 1 diff --git a/scripts/commit-json.sh b/scripts/commit-json.sh deleted file mode 100644 index e4a21f87d55..00000000000 --- a/scripts/commit-json.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors - -set -Eeu -o pipefail -x - -function string_escape() { - sed -e 's,\\,\\\\,g' -e 's,",\\",g' -} - -commit_id=$GITHUB_SHA -commit_title=$(git log -1 --pretty=%B $GITHUB_SHA | head -n 1 | string_escape) -commit_timestamp=$(git log -1 --format=%cd --date=iso-strict $GITHUB_SHA | string_escape) - -author_email=$(git log -1 --pretty=format:'%ae' $GITHUB_SHA | string_escape) -author_name=$(git log -1 --pretty=format:'%an' $GITHUB_SHA | string_escape) - -committer_email=$(git log -1 --pretty=format:'%ce' $GITHUB_SHA | string_escape) -committer_name=$(git log -1 --pretty=format:'%cn' $GITHUB_SHA | string_escape) - -tree_id=$(git rev-parse --verify ${GITHUB_SHA}^{tree}) -repo_url=$(git config --get remote.origin.url | sed 's/git@github.com:/https:\/\/github.com\//' | sed 's/.git$//' | string_escape) - -jq --compact-output '.' <