From b506e413db8253bc9d5cceb50833e3b25fb6808d Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 17 Aug 2026 11:08:40 +0800 Subject: [PATCH] [ci] Stop the scheduled builds from rebuilding on every tick, and fix 3.1 The Actions tab has been solid red: 1.2-lts, 2.1, 3.0 and 3.1 failed 28 out of 28 scheduled runs over the last 15 hours, and 2.0 failed 20 out of 28. That is one failure pattern amplified by one state machine bug. The bug: the whole pipeline keeps its state in the body of the `automation*` release. The prerelease job reads `Doris Version:` out of it to decide whether anything needs building, but the failure job overwrote the note with just `Status: *FAILURE*` and a checksum block, dropping that line. Thirty minutes later prerelease read an empty version, concluded "The first release was detected" and rebuilt everything from scratch, which failed again and wiped the version again. branch-1.2-lts shows how bad this got: that branch has had no commits in years and was still running a full macOS build every 30 minutes. The failure job now records the version it tried to build, exactly like the success job does, so the state can no longer be lost. Because that also removes the accidental retry the loop was providing, the prerelease job gains a bounded one: a run whose predecessor ended in FAILURE retries up to 3 times and then stops. A note without a parseable counter is treated as out of attempts, so a malformed note can never re-open the loop. manual-build.yml had the same failure job and writes to the same `automation` release as the scheduled master build, so one manual build that went red used to put master into that loop as well. What was actually failing underneath: - 2.1/3.0/3.1: LZO2_DOWNLOAD in those branches still points at fossies.org/linux/misc/lzo-2.10.tar.gz, which now answers 410 Gone. master and 4.0 already moved to oberhumer.com. 3.1 also shares 4.0's dead libuuid mirror, nchc.dl.sourceforge.net. Both are pre-seeded in build-3.1.yml the same way 4.0 already does it, and fixed at the source in apache/doris. - 1.2-lts: curl 7.79.0 no longer builds against the libngtcp2 that ships on the macOS runner image (ngtcp2_crypto_openssl.h was renamed upstream). - 2.0: the Linux job ran out of disk in /tmp while building aws-sdk-cpp. 1.2-lts, 2.0, 2.1 and 3.0 are switched to workflow_dispatch only. They are no longer polled; trigger them from the Actions tab when a build is needed. Other hardening: - fail-fast: false on 3.1. A single red platform used to cancel the other two, so nobody could see whether Linux or Intel macOS were healthy. - concurrency group per workflow. A full build takes hours while the schedule fires every 30 minutes, so runs were overlapping and racing on the same release tag. - 3.1 drops easimon/maximize-build-space. It reserves only root-reserve-mb on /, and gcc writes its temporaries to /tmp on /, which is how 2.0 died. master and 4.0 already build on the stock disk. - prerelease moved off macos-14 for master and 3.1. Without GNU md5sum, download-thirdparty.sh skips verification entirely and packs the 0-byte files that failed downloads leave behind into the source tarball, so the build jobs fail on a checksum prerelease never looked at. - gh release delete-asset before upload, with --yes so it actually runs non-interactively. - actions/checkout v4 -> v5, clearing the Node 20 deprecation warning. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-1.2.yml | 45 +++++++--- .github/workflows/build-2.0.yml | 45 +++++++--- .github/workflows/build-2.1.yml | 45 +++++++--- .github/workflows/build-3.0.yml | 45 +++++++--- .github/workflows/build-3.1.yml | 135 +++++++++++++++++++++++------ .github/workflows/build-4.0.yml | 69 ++++++++++++--- .github/workflows/build.yml | 84 ++++++++++++++---- .github/workflows/manual-build.yml | 22 +++-- 8 files changed, 385 insertions(+), 105 deletions(-) diff --git a/.github/workflows/build-1.2.yml b/.github/workflows/build-1.2.yml index e7f4c4a8c89..4716275b243 100644 --- a/.github/workflows/build-1.2.yml +++ b/.github/workflows/build-1.2.yml @@ -17,9 +17,21 @@ name: Build (1.2-lts) +# Scheduled builds are switched off for branch-1.2-lts. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-1.2-lts' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -181,7 +195,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-1.2-lts' @@ -249,11 +263,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -263,18 +278,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -284,5 +305,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-2.0.yml b/.github/workflows/build-2.0.yml index d22016f92c1..5a54f32be71 100644 --- a/.github/workflows/build-2.0.yml +++ b/.github/workflows/build-2.0.yml @@ -17,9 +17,21 @@ name: Build (2.0) +# Scheduled builds are switched off for branch-2.0. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.0' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -198,7 +212,7 @@ jobs: remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.0' @@ -266,11 +280,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +295,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -301,5 +322,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-2.1.yml b/.github/workflows/build-2.1.yml index 858eab8d03b..06dbead0b94 100644 --- a/.github/workflows/build-2.1.yml +++ b/.github/workflows/build-2.1.yml @@ -17,9 +17,21 @@ name: Build (2.1) +# Scheduled builds are switched off for branch-2.1. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.1' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -198,7 +212,7 @@ jobs: remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.1' @@ -266,11 +280,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +295,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -301,5 +322,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-3.0.yml b/.github/workflows/build-3.0.yml index c7a3ca15325..341f29490cb 100644 --- a/.github/workflows/build-3.0.yml +++ b/.github/workflows/build-3.0.yml @@ -17,9 +17,21 @@ name: Build (3.0) +# Scheduled builds are switched off for branch-3.0. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.0' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -198,7 +212,7 @@ jobs: remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.0' @@ -266,11 +280,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +295,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -301,5 +322,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-3.1.yml b/.github/workflows/build-3.1.yml index b7824d0a8b6..7c7179a54b6 100644 --- a/.github/workflows/build-3.1.yml +++ b/.github/workflows/build-3.1.yml @@ -18,13 +18,31 @@ name: Build (3.1) on: + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" schedule: - cron: '*/30 * * * *' +# A full third party build runs for hours while the schedule fires every 30 +# minutes. Without a concurrency group the runs stack up and race each other on +# the same release tag. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + jobs: prerelease: name: Prerelease - runs-on: macos-14 + # ubuntu-latest ships GNU md5sum. On a macOS runner download-thirdparty.sh + # finds no md5sum, skips verification altogether and packs whatever it got + # into the source tarball, including the 0-byte files a failed download + # leaves behind. The build jobs then fail on a checksum the prerelease job + # never looked at. + runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -32,9 +50,11 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} + attempt: ${{ steps.check_diff.outputs.attempt }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.1' @@ -45,18 +65,31 @@ jobs: run: | tag_name='automation-3.1' title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" + max_attempts=3 if [[ -z "$(gh release list)" ]] || ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then gh release create -t "${title}" "${tag_name}" fi - last_version="$(gh release view "${tag_name}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + + # The release note is the only state this pipeline keeps. Read it once + # so the three fields below cannot come from three different revisions. + note="$(gh release view "${tag_name}")" + last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" + last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" + # A note without a usable counter must not enable retries: treating it + # as attempt 0 is what turns every scheduled run into a full rebuild. + [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" current_version="$(git log -1 --format='%H')" echo "Last Version: ${last_version}" + echo "Last Status: ${last_status}" + echo "Last Attempt: ${last_attempt}" echo "Current Version: ${current_version}" should_release=false + attempt=1 if [[ -z "${last_version}" ]]; then echo "The first release was detected." should_release=true @@ -68,10 +101,17 @@ jobs: echo -e "Detect changes:\n${content}" should_release=true fi + elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then + # Downloads from third party mirrors fail often enough that a single + # red run should not park the branch until the next thirdparty/ + # commit lands. Retry a bounded number of times, never forever. + attempt=$((last_attempt + 1)) + echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." + should_release=true fi if "${should_release}"; then - echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md else gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ s/Update Time:.*/Update Time: *$(date)*/ @@ -81,6 +121,8 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + echo "attempt=${attempt}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -88,21 +130,41 @@ jobs: tag_name='automation-3.1' cd thirdparty + # Pre-download packages whose mirror in branch-3.1's vars.sh is blocked + # or gone on GH Actions runners: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) is unreachable + # Both are fixed upstream in apache/doris; this keeps the pipeline + # green until that lands and is a no-op once it does, because the + # checksums are identical. + mkdir -p src + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash - tar -zcvf doris-thirdparty-source.tgz src + gh release delete-asset "${tag_name}" doris-thirdparty-source.tgz --yes || true gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: + # One platform going red must not cancel the others: a cancelled matrix + # leg reports nothing, so a single failure used to hide the real state of + # the other two platforms. + fail-fast: false matrix: config: - name: macOS-x86_64 os: macos-15-intel + # Intel macOS runners are on their way out; do not let them block + # the arm64 and Linux artifacts. + continue-on-error: true packages: >- 'm4' 'automake' @@ -176,29 +238,20 @@ jobs: 'maven' runs-on: ${{ matrix.config.os }} + continue-on-error: ${{ matrix.config.continue-on-error == true }} env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Checkout easimon/maximize-build-space - if: ${{ matrix.config.name == 'Linux' }} - run: | - git clone -b v7 https://github.com/easimon/maximize-build-space - - - name: Maximize build space - if: ${{ matrix.config.name == 'Linux' }} - uses: ./maximize-build-space - with: - root-reserve-mb: 4096 - swap-size-mb: 8192 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' + # easimon/maximize-build-space is deliberately not used here. It hands the + # reclaimed space to an LVM mounted on the workspace but reserves only + # root-reserve-mb on /, and gcc writes its assembler temporaries to /tmp on + # /. That is how the 2.0 pipeline died on "No space left on device" while + # building aws-sdk-cpp. The stock runner disk is what master and 4.0 build + # on today. - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.1' @@ -255,22 +308,39 @@ jobs: export CMAKE_POLICY_VERSION_MINIMUM=3.10 export CUSTOM_CMAKE="/usr/local/bin/cmake" cd thirdparty + # Repair packages that may be empty if the source tarball was built + # before the mirror fixes: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) is unreachable + if [[ ! -s src/lzo-2.10.tar.gz ]]; then + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + fi + if [[ ! -s src/libuuid-1.0.3.tar.gz ]]; then + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz + fi ./build-thirdparty.sh -j "$(nproc)" kernel="$(uname -s | awk '{print tolower($0)}')" arch="$(uname -m)" rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + # Drop the old asset first: `gh release upload --clobber` intermittently + # exits 1 with "release not found" while deleting the asset it is + # replacing, even though the upload itself went through. + gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" --yes || true gh release upload --clobber "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +350,25 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next scheduled run rebuild from scratch, forever. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + ATTEMPT: ${{ needs.prerelease.outputs.attempt }} permissions: contents: write steps: @@ -301,5 +378,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every scheduled run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-4.0.yml b/.github/workflows/build-4.0.yml index 518ccda4621..fbeb4f36f0b 100644 --- a/.github/workflows/build-4.0.yml +++ b/.github/workflows/build-4.0.yml @@ -27,6 +27,13 @@ on: schedule: - cron: '*/30 * * * *' +# A full third party build runs for hours while the schedule fires every 30 +# minutes. Without a concurrency group the runs stack up and race each other on +# the same release tag. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + jobs: prerelease: name: Prerelease @@ -38,9 +45,11 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} + attempt: ${{ steps.check_diff.outputs.attempt }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-4.0' @@ -51,18 +60,31 @@ jobs: run: | tag_name='automation-4.0' title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" + max_attempts=3 if [[ -z "$(gh release list)" ]] || ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then gh release create -t "${title}" "${tag_name}" fi - last_version="$(gh release view "${tag_name}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + + # The release note is the only state this pipeline keeps. Read it once + # so the three fields below cannot come from three different revisions. + note="$(gh release view "${tag_name}")" + last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" + last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" + # A note without a usable counter must not enable retries: treating it + # as attempt 0 is what turns every scheduled run into a full rebuild. + [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" current_version="$(git log -1 --format='%H')" echo "Last Version: ${last_version}" + echo "Last Status: ${last_status}" + echo "Last Attempt: ${last_attempt}" echo "Current Version: ${current_version}" should_release=false + attempt=1 if [[ -z "${last_version}" ]]; then echo "The first release was detected." should_release=true @@ -74,10 +96,17 @@ jobs: echo -e "Detect changes:\n${content}" should_release=true fi + elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then + # Downloads from third party mirrors fail often enough that a single + # red run should not park the branch until the next thirdparty/ + # commit lands. Retry a bounded number of times, never forever. + attempt=$((last_attempt + 1)) + echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." + should_release=true fi if "${should_release}"; then - echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md else gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ s/Update Time:.*/Update Time: *$(date)*/ @@ -87,6 +116,8 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + echo "attempt=${attempt}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -106,6 +137,7 @@ jobs: tar -zcvf doris-thirdparty-source.tgz src + gh release delete-asset "${tag_name}" doris-thirdparty-source.tgz --yes || true gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz build: @@ -189,7 +221,7 @@ jobs: # remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-4.0' @@ -264,9 +296,12 @@ jobs: tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" # Delete existing asset first to avoid gh CLI --clobber race-condition bug # (gh release upload --clobber sometimes returns "release not found" exit 1 - # when deleting an existing asset, even though the upload itself succeeds) - gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" 2>/dev/null || true - gh release upload "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + # when deleting an existing asset, even though the upload itself succeeds). + # --yes is required: without it gh cannot confirm the prompt on a runner + # and the delete silently does nothing, which then made the plain upload + # fail because the asset was still there. + gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" --yes || true + gh release upload --clobber "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" update-docker: name: Update Docker Image @@ -419,6 +454,7 @@ jobs: env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -428,18 +464,25 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build, update-docker] - if: always() && failure() + needs: [prerelease, build, update-docker] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next scheduled run rebuild from scratch, forever. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + ATTEMPT: ${{ needs.prerelease.outputs.attempt }} permissions: contents: write steps: @@ -449,5 +492,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every scheduled run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1606cf5142d..a221a7d4eb1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,10 +27,20 @@ on: schedule: - cron: '*/30 * * * *' +# A full third party build runs for hours while the schedule fires every 30 +# minutes. Without a concurrency group the runs stack up and race each other on +# the same release tag. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + jobs: prerelease: name: Prerelease - runs-on: macos-14 + # This job only resolves a hash and pre-fetches sources, so it does not need + # a macOS runner. ubuntu-latest also ships GNU md5sum, which keeps archive + # verification on rather than at the mercy of what the runner happens to have. + runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -39,9 +49,11 @@ jobs: outputs: should_release: ${{ steps.check_diff.outputs.should_release }} thirdparty_commit_hash: ${{ steps.check_diff.outputs.thirdparty_commit_hash }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} + attempt: ${{ steps.check_diff.outputs.attempt }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' fetch-depth: 0 @@ -53,15 +65,28 @@ jobs: ! gh release list | awk -F "\t" '{ print $3 }' | grep 'automation' >/dev/null; then gh release create -t 'Apache Doris Third Party Prebuilt' automation fi - last_version="$(gh release view automation | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + max_attempts=3 + + # The release note is the only state this pipeline keeps. Read it once + # so the three fields below cannot come from three different revisions. + note="$(gh release view automation)" + last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" + last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" + # A note without a usable counter must not enable retries: treating it + # as attempt 0 is what turns every scheduled run into a full rebuild. + [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" current_version="$(git log -1 --format='%H')" thirdparty_commit_hash="$(git log -1 --format='%H' -- thirdparty)" echo "Last Version: ${last_version}" + echo "Last Status: ${last_status}" + echo "Last Attempt: ${last_attempt}" echo "Current Version: ${current_version}" echo "Thirdparty Commit: ${thirdparty_commit_hash}" should_release=false + attempt=1 if [[ -z "${last_version}" ]]; then echo "The first release was detected." should_release=true @@ -73,10 +98,17 @@ jobs: echo -e "Detect changes:\n${content}" should_release=true fi + elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then + # Downloads from third party mirrors fail often enough that a single + # red run should not park master until the next thirdparty/ commit + # lands. Retry a bounded number of times, never forever. + attempt=$((last_attempt + 1)) + echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." + should_release=true fi if "${should_release}"; then - echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md else gh release view automation | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ s/Update Time:.*/Update Time: *$(date)*/ @@ -87,6 +119,8 @@ jobs: echo "should_release=${should_release}" >> $GITHUB_OUTPUT echo "thirdparty_commit_hash=${thirdparty_commit_hash}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + echo "attempt=${attempt}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -96,6 +130,10 @@ jobs: tar -zcvf doris-thirdparty-source.tgz src + # Drop the old asset first: `gh release upload --clobber` intermittently + # exits 1 with "release not found" while deleting the asset it is + # replacing, even though the upload itself went through. + gh release delete-asset automation doris-thirdparty-source.tgz --yes || true gh release upload --clobber automation doris-thirdparty-source.tgz build: @@ -234,7 +272,7 @@ jobs: # remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' @@ -329,6 +367,12 @@ jobs: cp "${archive}" doris-thirdparty-prebuild-arm64.tar.xz assets+=(doris-thirdparty-prebuild-arm64.tar.xz) fi + # Drop the old assets first: `gh release upload --clobber` intermittently + # exits 1 with "release not found" while deleting the asset it is + # replacing, even though the upload itself went through. + for asset in "${assets[@]}"; do + gh release delete-asset automation "${asset}" --yes || true + done gh release upload --clobber automation "${assets[@]}" update-docker: @@ -489,6 +533,7 @@ jobs: env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -496,18 +541,25 @@ jobs: run: | gh release download automation - content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation failure: name: Failure - needs: [build, update-docker] - if: always() && failure() + needs: [prerelease, build, update-docker] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next scheduled run rebuild from scratch, forever. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + ATTEMPT: ${{ needs.prerelease.outputs.attempt }} permissions: contents: write steps: @@ -515,10 +567,12 @@ jobs: run: | gh release download automation - content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')" - if [[ -z "${content}" ]]; then - echo 'Unable to preserve release provenance in the failure note' >&2 - exit 1 - fi - echo -ne "${content}\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every scheduled run look like a first release. #405 guarded this + # by reading the line back out of the note and bailing out when it was + # missing, which protects an intact note but cannot repair one that is + # already broken -- and a note that has lost the line is exactly the + # state that keeps the rebuild loop running. Take the value from the + # prerelease job instead, so it is always available. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml index 83c54027453..22551da9614 100644 --- a/.github/workflows/manual-build.yml +++ b/.github/workflows/manual-build.yml @@ -35,9 +35,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} permissions: contents: write + outputs: + doris_version: ${{ steps.release_note.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: ${{ inputs.doris_ref }} @@ -51,12 +53,15 @@ jobs: fi - name: Update Release Note + id: release_note run: | current_version="$(git log -1 --format='%H')" echo "Current Version: ${current_version}" echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nDoris Ref: *${{ inputs.doris_ref }}*\nStatus: *BUILDING*" >release_note.md gh release edit -F release_note.md automation + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + - name: Download Source and Upload run: | cd thirdparty @@ -169,7 +174,7 @@ jobs: # remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: ${{ inputs.doris_ref }} @@ -248,18 +253,23 @@ jobs: run: | gh release download automation - content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')" + content="$(gh release view automation | sed -n '/Update Time:/,/Doris Ref:/p')" echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # This workflow writes to the same `automation` release as the scheduled + # master build. Dropping Doris Version here left master's prerelease job + # unable to tell what it had last built, so it rebuilt from scratch on every + # following 30 minute tick. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -267,5 +277,5 @@ jobs: run: | gh release download automation - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nDoris Ref: *${{ inputs.doris_ref }}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation