diff --git a/.github/workflows/build-1.2.yml b/.github/workflows/build-1.2.yml index e7f4c4a8c89..4716275b243 100644 --- a/.github/workflows/build-1.2.yml +++ b/.github/workflows/build-1.2.yml @@ -17,9 +17,21 @@ name: Build (1.2-lts) +# Scheduled builds are switched off for branch-1.2-lts. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-1.2-lts' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -181,7 +195,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-1.2-lts' @@ -249,11 +263,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -263,18 +278,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -284,5 +305,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-2.0.yml b/.github/workflows/build-2.0.yml index d22016f92c1..5a54f32be71 100644 --- a/.github/workflows/build-2.0.yml +++ b/.github/workflows/build-2.0.yml @@ -17,9 +17,21 @@ name: Build (2.0) +# Scheduled builds are switched off for branch-2.0. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.0' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -198,7 +212,7 @@ jobs: remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.0' @@ -266,11 +280,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +295,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -301,5 +322,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-2.1.yml b/.github/workflows/build-2.1.yml index 858eab8d03b..06dbead0b94 100644 --- a/.github/workflows/build-2.1.yml +++ b/.github/workflows/build-2.1.yml @@ -17,9 +17,21 @@ name: Build (2.1) +# Scheduled builds are switched off for branch-2.1. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.1' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -198,7 +212,7 @@ jobs: remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-2.1' @@ -266,11 +280,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +295,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -301,5 +322,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-3.0.yml b/.github/workflows/build-3.0.yml index c7a3ca15325..341f29490cb 100644 --- a/.github/workflows/build-3.0.yml +++ b/.github/workflows/build-3.0.yml @@ -17,9 +17,21 @@ name: Build (3.0) +# Scheduled builds are switched off for branch-3.0. Every polled run rebuilt the +# whole third party tree and went red, 48 times a day, and the branch no longer +# receives thirdparty/ changes often enough to justify polling for them. Trigger +# it by hand from the Actions tab when a build is actually needed. on: - schedule: - - cron: '*/30 * * * *' + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: prerelease: @@ -32,9 +44,10 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.0' @@ -81,6 +94,7 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -97,7 +111,7 @@ jobs: build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: matrix: config: @@ -198,7 +212,7 @@ jobs: remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.0' @@ -266,11 +280,12 @@ jobs: success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +295,24 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next run rebuild from scratch. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -301,5 +322,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every following run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-3.1.yml b/.github/workflows/build-3.1.yml index b7824d0a8b6..7c7179a54b6 100644 --- a/.github/workflows/build-3.1.yml +++ b/.github/workflows/build-3.1.yml @@ -18,13 +18,31 @@ name: Build (3.1) on: + workflow_dispatch: + inputs: + force_build: + description: "Force run build job when manually triggered" + required: false + default: "true" schedule: - cron: '*/30 * * * *' +# A full third party build runs for hours while the schedule fires every 30 +# minutes. Without a concurrency group the runs stack up and race each other on +# the same release tag. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + jobs: prerelease: name: Prerelease - runs-on: macos-14 + # ubuntu-latest ships GNU md5sum. On a macOS runner download-thirdparty.sh + # finds no md5sum, skips verification altogether and packs whatever it got + # into the source tarball, including the 0-byte files a failed download + # leaves behind. The build jobs then fail on a checksum the prerelease job + # never looked at. + runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -32,9 +50,11 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} + attempt: ${{ steps.check_diff.outputs.attempt }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.1' @@ -45,18 +65,31 @@ jobs: run: | tag_name='automation-3.1' title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" + max_attempts=3 if [[ -z "$(gh release list)" ]] || ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then gh release create -t "${title}" "${tag_name}" fi - last_version="$(gh release view "${tag_name}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + + # The release note is the only state this pipeline keeps. Read it once + # so the three fields below cannot come from three different revisions. + note="$(gh release view "${tag_name}")" + last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" + last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" + # A note without a usable counter must not enable retries: treating it + # as attempt 0 is what turns every scheduled run into a full rebuild. + [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" current_version="$(git log -1 --format='%H')" echo "Last Version: ${last_version}" + echo "Last Status: ${last_status}" + echo "Last Attempt: ${last_attempt}" echo "Current Version: ${current_version}" should_release=false + attempt=1 if [[ -z "${last_version}" ]]; then echo "The first release was detected." should_release=true @@ -68,10 +101,17 @@ jobs: echo -e "Detect changes:\n${content}" should_release=true fi + elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then + # Downloads from third party mirrors fail often enough that a single + # red run should not park the branch until the next thirdparty/ + # commit lands. Retry a bounded number of times, never forever. + attempt=$((last_attempt + 1)) + echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." + should_release=true fi if "${should_release}"; then - echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md else gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ s/Update Time:.*/Update Time: *$(date)*/ @@ -81,6 +121,8 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + echo "attempt=${attempt}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -88,21 +130,41 @@ jobs: tag_name='automation-3.1' cd thirdparty + # Pre-download packages whose mirror in branch-3.1's vars.sh is blocked + # or gone on GH Actions runners: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) is unreachable + # Both are fixed upstream in apache/doris; this keeps the pipeline + # green until that lands and is a no-op once it does, because the + # checksums are identical. + mkdir -p src + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz sed '/# unpacking thirdpart archives/,$d' download-thirdparty.sh | bash - tar -zcvf doris-thirdparty-source.tgz src + gh release delete-asset "${tag_name}" doris-thirdparty-source.tgz --yes || true gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz build: name: Build needs: prerelease - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') strategy: + # One platform going red must not cancel the others: a cancelled matrix + # leg reports nothing, so a single failure used to hide the real state of + # the other two platforms. + fail-fast: false matrix: config: - name: macOS-x86_64 os: macos-15-intel + # Intel macOS runners are on their way out; do not let them block + # the arm64 and Linux artifacts. + continue-on-error: true packages: >- 'm4' 'automake' @@ -176,29 +238,20 @@ jobs: 'maven' runs-on: ${{ matrix.config.os }} + continue-on-error: ${{ matrix.config.continue-on-error == true }} env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Checkout easimon/maximize-build-space - if: ${{ matrix.config.name == 'Linux' }} - run: | - git clone -b v7 https://github.com/easimon/maximize-build-space - - - name: Maximize build space - if: ${{ matrix.config.name == 'Linux' }} - uses: ./maximize-build-space - with: - root-reserve-mb: 4096 - swap-size-mb: 8192 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' + # easimon/maximize-build-space is deliberately not used here. It hands the + # reclaimed space to an LVM mounted on the workspace but reserves only + # root-reserve-mb on /, and gcc writes its assembler temporaries to /tmp on + # /. That is how the 2.0 pipeline died on "No space left on device" while + # building aws-sdk-cpp. The stock runner disk is what master and 4.0 build + # on today. - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-3.1' @@ -255,22 +308,39 @@ jobs: export CMAKE_POLICY_VERSION_MINIMUM=3.10 export CUSTOM_CMAKE="/usr/local/bin/cmake" cd thirdparty + # Repair packages that may be empty if the source tarball was built + # before the mirror fixes: + # lzo: fossies.org returns 410 Gone + # libuuid: nchc.dl.sourceforge.net (TW mirror) is unreachable + if [[ ! -s src/lzo-2.10.tar.gz ]]; then + curl -fL "https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz" \ + -o src/lzo-2.10.tar.gz + fi + if [[ ! -s src/libuuid-1.0.3.tar.gz ]]; then + curl -fL "https://downloads.sourceforge.net/project/libuuid/libuuid-1.0.3.tar.gz" \ + -o src/libuuid-1.0.3.tar.gz + fi ./build-thirdparty.sh -j "$(nproc)" kernel="$(uname -s | awk '{print tolower($0)}')" arch="$(uname -m)" rm -rf "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + # Drop the old asset first: `gh release upload --clobber` intermittently + # exits 1 with "release not found" while deleting the asset it is + # replacing, even though the upload itself went through. + gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" --yes || true gh release upload --clobber "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" success: name: Success needs: [prerelease, build] - if: needs.prerelease.outputs.should_release == 'true' + if: needs.prerelease.outputs.should_release == 'true' || (github.event_name == 'workflow_dispatch' && github.event.inputs.force_build == 'true') runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -280,18 +350,25 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next scheduled run rebuild from scratch, forever. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + ATTEMPT: ${{ needs.prerelease.outputs.attempt }} permissions: contents: write steps: @@ -301,5 +378,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every scheduled run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build-4.0.yml b/.github/workflows/build-4.0.yml index 518ccda4621..fbeb4f36f0b 100644 --- a/.github/workflows/build-4.0.yml +++ b/.github/workflows/build-4.0.yml @@ -27,6 +27,13 @@ on: schedule: - cron: '*/30 * * * *' +# A full third party build runs for hours while the schedule fires every 30 +# minutes. Without a concurrency group the runs stack up and race each other on +# the same release tag. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + jobs: prerelease: name: Prerelease @@ -38,9 +45,11 @@ jobs: contents: write outputs: should_release: ${{ steps.check_diff.outputs.should_release }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} + attempt: ${{ steps.check_diff.outputs.attempt }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-4.0' @@ -51,18 +60,31 @@ jobs: run: | tag_name='automation-4.0' title="Apache Doris Third Party Prebuilt (${tag_name/automation-/})" + max_attempts=3 if [[ -z "$(gh release list)" ]] || ! gh release list | awk -F "\t" '{ print $3 }' | grep "${tag_name}" >/dev/null; then gh release create -t "${title}" "${tag_name}" fi - last_version="$(gh release view "${tag_name}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + + # The release note is the only state this pipeline keeps. Read it once + # so the three fields below cannot come from three different revisions. + note="$(gh release view "${tag_name}")" + last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" + last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" + # A note without a usable counter must not enable retries: treating it + # as attempt 0 is what turns every scheduled run into a full rebuild. + [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" current_version="$(git log -1 --format='%H')" echo "Last Version: ${last_version}" + echo "Last Status: ${last_status}" + echo "Last Attempt: ${last_attempt}" echo "Current Version: ${current_version}" should_release=false + attempt=1 if [[ -z "${last_version}" ]]; then echo "The first release was detected." should_release=true @@ -74,10 +96,17 @@ jobs: echo -e "Detect changes:\n${content}" should_release=true fi + elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then + # Downloads from third party mirrors fail often enough that a single + # red run should not park the branch until the next thirdparty/ + # commit lands. Retry a bounded number of times, never forever. + attempt=$((last_attempt + 1)) + echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." + should_release=true fi if "${should_release}"; then - echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md else gh release view "${tag_name}" | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ s/Update Time:.*/Update Time: *$(date)*/ @@ -87,6 +116,8 @@ jobs: gh release edit -F release_note.md "${tag_name}" echo "should_release=${should_release}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + echo "attempt=${attempt}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -106,6 +137,7 @@ jobs: tar -zcvf doris-thirdparty-source.tgz src + gh release delete-asset "${tag_name}" doris-thirdparty-source.tgz --yes || true gh release upload --clobber "${tag_name}" doris-thirdparty-source.tgz build: @@ -189,7 +221,7 @@ jobs: # remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: 'branch-4.0' @@ -264,9 +296,12 @@ jobs: tar -cf - installed | xz -z -T0 - >"doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" # Delete existing asset first to avoid gh CLI --clobber race-condition bug # (gh release upload --clobber sometimes returns "release not found" exit 1 - # when deleting an existing asset, even though the upload itself succeeds) - gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" 2>/dev/null || true - gh release upload "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" + # when deleting an existing asset, even though the upload itself succeeds). + # --yes is required: without it gh cannot confirm the prompt on a runner + # and the delete silently does nothing, which then made the plain upload + # fail because the asset was still there. + gh release delete-asset "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" --yes || true + gh release upload --clobber "${tag_name}" "doris-thirdparty-prebuilt-${kernel}-${arch}.tar.xz" update-docker: name: Update Docker Image @@ -419,6 +454,7 @@ jobs: env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -428,18 +464,25 @@ jobs: gh release download "${tag_name}" - content="$(gh release view "${tag_name}" | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" failure: name: Failure - needs: [build, update-docker] - if: always() && failure() + needs: [prerelease, build, update-docker] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next scheduled run rebuild from scratch, forever. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + ATTEMPT: ${{ needs.prerelease.outputs.attempt }} permissions: contents: write steps: @@ -449,5 +492,7 @@ jobs: gh release download "${tag_name}" - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every scheduled run look like a first release. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit -F release_note.md "${tag_name}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1606cf5142d..a221a7d4eb1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,10 +27,20 @@ on: schedule: - cron: '*/30 * * * *' +# A full third party build runs for hours while the schedule fires every 30 +# minutes. Without a concurrency group the runs stack up and race each other on +# the same release tag. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + jobs: prerelease: name: Prerelease - runs-on: macos-14 + # This job only resolves a hash and pre-fetches sources, so it does not need + # a macOS runner. ubuntu-latest also ships GNU md5sum, which keeps archive + # verification on rather than at the mercy of what the runner happens to have. + runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -39,9 +49,11 @@ jobs: outputs: should_release: ${{ steps.check_diff.outputs.should_release }} thirdparty_commit_hash: ${{ steps.check_diff.outputs.thirdparty_commit_hash }} + doris_version: ${{ steps.check_diff.outputs.doris_version }} + attempt: ${{ steps.check_diff.outputs.attempt }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' fetch-depth: 0 @@ -53,15 +65,28 @@ jobs: ! gh release list | awk -F "\t" '{ print $3 }' | grep 'automation' >/dev/null; then gh release create -t 'Apache Doris Third Party Prebuilt' automation fi - last_version="$(gh release view automation | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + max_attempts=3 + + # The release note is the only state this pipeline keeps. Read it once + # so the three fields below cannot come from three different revisions. + note="$(gh release view automation)" + last_version="$(echo "${note}" | sed -n -E 's/Doris Version: \*(.*)\*.*/\1/p')" + last_status="$(echo "${note}" | sed -n -E 's/Status: \*(.*)\*.*/\1/p')" + last_attempt="$(echo "${note}" | sed -n -E 's/Attempts: \*([0-9]+)\*.*/\1/p')" + # A note without a usable counter must not enable retries: treating it + # as attempt 0 is what turns every scheduled run into a full rebuild. + [[ "${last_attempt}" =~ ^[0-9]+$ ]] || last_attempt="${max_attempts}" current_version="$(git log -1 --format='%H')" thirdparty_commit_hash="$(git log -1 --format='%H' -- thirdparty)" echo "Last Version: ${last_version}" + echo "Last Status: ${last_status}" + echo "Last Attempt: ${last_attempt}" echo "Current Version: ${current_version}" echo "Thirdparty Commit: ${thirdparty_commit_hash}" should_release=false + attempt=1 if [[ -z "${last_version}" ]]; then echo "The first release was detected." should_release=true @@ -73,10 +98,17 @@ jobs: echo -e "Detect changes:\n${content}" should_release=true fi + elif [[ "${last_status}" == 'FAILURE' ]] && [[ "${last_attempt}" -lt "${max_attempts}" ]]; then + # Downloads from third party mirrors fail often enough that a single + # red run should not park master until the next thirdparty/ commit + # lands. Retry a bounded number of times, never forever. + attempt=$((last_attempt + 1)) + echo "Previous build failed, retrying (attempt ${attempt}/${max_attempts})." + should_release=true fi if "${should_release}"; then - echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nStatus: *BUILDING*\nAttempts: *${attempt}*" >release_note.md else gh release view automation | sed -n '/--/,$p' | awk '{ if (NR > 1) print $0 }' | sed "{ s/Update Time:.*/Update Time: *$(date)*/ @@ -87,6 +119,8 @@ jobs: echo "should_release=${should_release}" >> $GITHUB_OUTPUT echo "thirdparty_commit_hash=${thirdparty_commit_hash}" >> $GITHUB_OUTPUT + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + echo "attempt=${attempt}" >> $GITHUB_OUTPUT - name: Download Source and Upload if: steps.check_diff.outputs.should_release == 'true' @@ -96,6 +130,10 @@ jobs: tar -zcvf doris-thirdparty-source.tgz src + # Drop the old asset first: `gh release upload --clobber` intermittently + # exits 1 with "release not found" while deleting the asset it is + # replacing, even though the upload itself went through. + gh release delete-asset automation doris-thirdparty-source.tgz --yes || true gh release upload --clobber automation doris-thirdparty-source.tgz build: @@ -234,7 +272,7 @@ jobs: # remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' @@ -329,6 +367,12 @@ jobs: cp "${archive}" doris-thirdparty-prebuild-arm64.tar.xz assets+=(doris-thirdparty-prebuild-arm64.tar.xz) fi + # Drop the old assets first: `gh release upload --clobber` intermittently + # exits 1 with "release not found" while deleting the asset it is + # replacing, even though the upload itself went through. + for asset in "${assets[@]}"; do + gh release delete-asset automation "${asset}" --yes || true + done gh release upload --clobber automation "${assets[@]}" update-docker: @@ -489,6 +533,7 @@ jobs: env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -496,18 +541,25 @@ jobs: run: | gh release download automation - content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')" - echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Write the version this run actually built rather than parsing it back + # out of the note, so the terminal state never depends on the note + # having survived intact. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation failure: name: Failure - needs: [build, update-docker] - if: always() && failure() + needs: [prerelease, build, update-docker] + # Only report on a build that actually ran. If prerelease itself failed there + # is no version to record, and writing an empty one is exactly what used to + # make the next scheduled run rebuild from scratch, forever. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} + ATTEMPT: ${{ needs.prerelease.outputs.attempt }} permissions: contents: write steps: @@ -515,10 +567,12 @@ jobs: run: | gh release download automation - content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')" - if [[ -z "${content}" ]]; then - echo 'Unable to preserve release provenance in the failure note' >&2 - exit 1 - fi - echo -ne "${content}\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + # Keep Doris Version: the prerelease job keys off it, and dropping it + # made every scheduled run look like a first release. #405 guarded this + # by reading the line back out of the note and bailing out when it was + # missing, which protects an intact note but cannot repair one that is + # already broken -- and a note that has lost the line is exactly the + # state that keeps the rebuild loop running. Take the value from the + # prerelease job instead, so it is always available. + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nStatus: *FAILURE*\nAttempts: *${ATTEMPT}*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation diff --git a/.github/workflows/manual-build.yml b/.github/workflows/manual-build.yml index 83c54027453..22551da9614 100644 --- a/.github/workflows/manual-build.yml +++ b/.github/workflows/manual-build.yml @@ -35,9 +35,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} permissions: contents: write + outputs: + doris_version: ${{ steps.release_note.outputs.doris_version }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: ${{ inputs.doris_ref }} @@ -51,12 +53,15 @@ jobs: fi - name: Update Release Note + id: release_note run: | current_version="$(git log -1 --format='%H')" echo "Current Version: ${current_version}" echo -ne "Update Time: *$(date)*\nDoris Version: *${current_version}*\nDoris Ref: *${{ inputs.doris_ref }}*\nStatus: *BUILDING*" >release_note.md gh release edit -F release_note.md automation + echo "doris_version=${current_version}" >> $GITHUB_OUTPUT + - name: Download Source and Upload run: | cd thirdparty @@ -169,7 +174,7 @@ jobs: # remove-docker-images: 'true' - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: repository: 'apache/doris' ref: ${{ inputs.doris_ref }} @@ -248,18 +253,23 @@ jobs: run: | gh release download automation - content="$(gh release view automation | sed -n '/Update Time:/,/Doris Version:/p')" + content="$(gh release view automation | sed -n '/Update Time:/,/Doris Ref:/p')" echo -ne "${content}\nStatus: *SUCCESS*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation failure: name: Failure - needs: [build] - if: failure() + needs: [prerelease, build] + # This workflow writes to the same `automation` release as the scheduled + # master build. Dropping Doris Version here left master's prerelease job + # unable to tell what it had last built, so it rebuilt from scratch on every + # following 30 minute tick. + if: always() && failure() && needs.prerelease.result == 'success' runs-on: ubuntu-latest env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DORIS_VERSION: ${{ needs.prerelease.outputs.doris_version }} permissions: contents: write steps: @@ -267,5 +277,5 @@ jobs: run: | gh release download automation - echo -ne "Status: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md + echo -ne "Update Time: *$(date)*\nDoris Version: *${DORIS_VERSION}*\nDoris Ref: *${{ inputs.doris_ref }}*\nStatus: *FAILURE*\n\n## SHA256 Checksums\n\`\`\`\n$(sha256sum *)\n\`\`\`" >release_note.md gh release edit --latest -F release_note.md automation