From b32cb1f8f149ebd1427d89c62c46b72129013f62 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 17 Aug 2026 17:01:36 +0800 Subject: [PATCH 1/4] [chore](thirdparty) drop the hadoop-libs 3.3.6.6 build and use 3.4.2.4 everywhere The tree carried two libhdfs builds. `build_hadoop_libs` built the 3.3.6.6 fork into `installed/{include,lib}/hadoop_hdfs/`, `build_hadoop_libs_3_4` built the 3.4.2.4 fork into `.../hadoop_hdfs_3_4/`, and on Linux both ran - roughly 10 extra minutes of every third-party build for a fork only the cloud module still consumed. The two prefixes were also mixed up on the consumer side. BE linked `hadoop_hdfs_3_4/native/libhdfs.a`, but `be/src/io/hdfs_builder.cpp` included `hadoop_hdfs/hdfs.h` - the 3.3.6.6 header - while `be/src/io/fs/hdfs.h` included the 3.4.2.4 one. Both headers share the LIBHDFS_HDFS_H include guard, so that translation unit compiled against 3.3.6.6 and linked 3.4.2.4. The cloud meta-service used 3.3.6.6 for both. Build 3.4.2.4 only, and move the remaining 3.3.6.6 consumers onto it: - thirdparty: drop `build_hadoop_libs` and the `HADOOP_LIBS_*` variables; `hadoop_libs_3_4` now runs on Linux and macOS alike. The install prefix stays `hadoop_hdfs_3_4/` so a build-env image from this change can still compile older Doris branches. - cloud: link `hadoop_hdfs_3_4/native/libhdfs.a` and include the matching header. - `be/src/io/hdfs_builder.cpp`: drop the 3.3.6.6 include - `io/fs/hdfs.h`, two lines below, already re-exports the 3.4.2.4 one. - `build.sh`, `run-be-ut.sh`, `run-cloud-ut.sh` and the thirdparty lifecycle test follow the same prefix. --- be/src/io/hdfs_builder.cpp | 3 -- build.sh | 2 +- cloud/CMakeLists.txt | 2 +- cloud/src/recycler/hdfs_accessor.cpp | 2 +- cloud/src/recycler/hdfs_accessor.h | 2 +- run-be-ut.sh | 4 +-- run-cloud-ut.sh | 4 +-- thirdparty/CHANGELOG.md | 6 ++++ thirdparty/build-thirdparty.sh | 29 ++----------------- .../test/arrow-paimon-lifecycle-test.sh | 8 ++--- thirdparty/vars.sh | 7 ----- 11 files changed, 21 insertions(+), 48 deletions(-) diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index cb9c75fb29debc..85248d178d0878 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -28,9 +28,6 @@ #include "common/config.h" #include "common/kerberos/kerberos_ticket_mgr.h" #include "common/logging.h" -#ifdef USE_HADOOP_HDFS -#include "hadoop_hdfs/hdfs.h" -#endif #include "io/fs/hdfs.h" #include "runtime/exec_env.h" #include "util/string_util.h" diff --git a/build.sh b/build.sh index f2d476c04ce1bf..7168a355d6839f 100755 --- a/build.sh +++ b/build.sh @@ -482,7 +482,7 @@ fi if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then LAST_THIRDPARTY_LIB='libbrotlienc.a' else - LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a' + LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a' fi # The final-library sentinel only proves that some third-party build completed. It cannot diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index d4e0b3466598e2..e0b44d9cdff1d2 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -302,7 +302,7 @@ set(WL_END_GROUP "-Wl,--end-group") # Set Doris libraries if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX) add_library(hadoop_hdfs STATIC IMPORTED) - set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs/native/libhdfs.a) + set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 8de638f81d8e14..4fddeab693f518 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -24,7 +24,7 @@ #include "recycler/util.h" #ifdef USE_HADOOP_HDFS -#include // IWYU pragma: export +#include // IWYU pragma: export #else #include // IWYU pragma: export #endif diff --git a/cloud/src/recycler/hdfs_accessor.h b/cloud/src/recycler/hdfs_accessor.h index b8d6b16b53fbc7..7eb182f9122bcf 100644 --- a/cloud/src/recycler/hdfs_accessor.h +++ b/cloud/src/recycler/hdfs_accessor.h @@ -18,7 +18,7 @@ #pragma once #ifdef USE_HADOOP_HDFS -#include // IWYU pragma: export +#include // IWYU pragma: export #else #include // IWYU pragma: export #endif diff --git a/run-be-ut.sh b/run-be-ut.sh index 7eeece8329effc..d04af7036ed1a6 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -435,8 +435,8 @@ touch "${UT_TMP_DIR}/tmp_file" LIB_DIR="${DORIS_TEST_BINARY_DIR}/lib/" rm -rf "${LIB_DIR}" mkdir "${LIB_DIR}" -if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then - cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}" +if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then + cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs" fi if [[ -f "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" ]]; then cp "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" "${LIB_DIR}/" diff --git a/run-cloud-ut.sh b/run-cloud-ut.sh index 0b467382be0b26..d74dcb122b2659 100755 --- a/run-cloud-ut.sh +++ b/run-cloud-ut.sh @@ -289,8 +289,8 @@ mkdir -p "${CMAKE_BUILD_DIR}/test/log" LIB_DIR="${CMAKE_BUILD_DIR}/test/lib" rm -rf "${LIB_DIR}" mkdir "${LIB_DIR}" -if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then - cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}" +if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then + cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs" fi if [[ "${RUN}" -ne 1 ]]; then diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8c049ea51bb3a7..8d1455c3e7b557 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,12 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20260817 + +- Removed: hadoop-libs 3.3.6.6, along with the `installed/{include,lib}/hadoop_hdfs/` + prefix it produced. hadoop-libs 3.4.2.4 under `hadoop_hdfs_3_4/` is the only libhdfs + built now. + ## 20260816 - Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 6f0676e565a33f..0af17ab2bbbe4d 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1904,24 +1904,6 @@ build_fast_float() { cp -r ./include/fast_float "${TP_INSTALL_DIR}/include/" } -# hadoop_libs -build_hadoop_libs() { - check_if_source_exist "${HADOOP_LIBS_SOURCE}" - cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_SOURCE}" - echo "THIRDPARTY_INSTALLED=${TP_INSTALL_DIR}" >env.sh - ./build.sh - - rm -rf "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/* "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/*.a" - find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type f ! -name '*.a' -exec cp {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \; - find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type l -exec cp -P {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \; -} - # hadoop_libs_3_4 build_hadoop_libs_3_4() { check_if_source_exist "${HADOOP_LIBS_3_4_SOURCE}" @@ -2372,14 +2354,10 @@ if [[ "${#packages[@]}" -eq 0 ]]; then ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" - # hadoop_libs, the 3.3.6 fork, stays Linux-only: it carries none of the macOS fixes - # apache/doris-thirdparty#407 made to the 3.4 fork, and nothing built here reads its - # hadoop_hdfs/ prefix - the cloud module, its only other consumer, is Linux-only too. - read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" - elif [[ "$(uname -s)" == 'Linux' ]]; then - read -r -a packages <<<"${packages[*]} hadoop_libs" - read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" fi + # hadoop_libs_3_4 runs last on every platform: its native build links against + # what the packages above install into ${TP_INSTALL_DIR}. + read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" fi # Map a package name to its source directory variable(s) and remove them to free disk space. @@ -2457,7 +2435,6 @@ cleanup_package_source() { xxhash) src_var="XXHASH_SOURCE" ;; concurrentqueue) src_var="CONCURRENTQUEUE_SOURCE" ;; fast_float) src_var="FAST_FLOAT_SOURCE" ;; - hadoop_libs) src_var="HADOOP_LIBS_SOURCE" ;; hadoop_libs_3_4) src_var="HADOOP_LIBS_3_4_SOURCE" ;; avx2neon) src_var="AVX2NEON_SOURCE" ;; libdeflate) src_var="LIBDEFLATE_SOURCE" ;; diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh index ab48d3bcd2278b..097849fc516728 100755 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh @@ -161,14 +161,14 @@ exercise_generic_recovery_dispatch() { local package2 local extra - mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ - "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ + mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ + "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ "${generic}/gensrc" "${generic}/fe" "${generic}/be/build_Release" \ "${generic}/be/output" cp "${ROOT}/../build.sh" "${generic}/build.sh" cp "${ROOT}/arrow-paimon-vars.sh" "${thirdparty_dir}/arrow-paimon-vars.sh" - touch "${thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" - touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" + touch "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" + touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" printf '%s\n' 'clean: ; @:' >"${generic}/gensrc/Makefile" # shellcheck disable=SC2016 printf '%s\n' '#!/usr/bin/env bash' '[[ "$1" == "clean" ]]' >"${fake_mvn}" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 6a42bd00cf7963..951d1b9d78604b 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -498,12 +498,6 @@ FAST_FLOAT_SOURCE=fast_float-3.9.0 FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea" # libhdfs -HADOOP_LIBS_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.3.6.6-for-doris.tar.gz" -HADOOP_LIBS_NAME="hadoop-3.3.6.6-for-doris.tar.gz" -HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.6-for-doris" -HADOOP_LIBS_MD5SUM="13b66d5f2abffd1740e692b65df5962e" - -# libhdfs 3.4 HADOOP_LIBS_3_4_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.4.2.4-for-doris.tar.gz" HADOOP_LIBS_3_4_NAME="hadoop-3.4.2.4-for-doris.tar.gz" HADOOP_LIBS_3_4_SOURCE="doris-thirdparty-hadoop-3.4.2.4-for-doris" @@ -664,7 +658,6 @@ export TP_ARCHIVES=( 'XXHASH' 'CONCURRENTQUEUE' 'FAST_FLOAT' - 'HADOOP_LIBS' 'HADOOP_LIBS_3_4' 'AVX2NEON' 'LIBDEFLATE' From 17d56270f78f582481bfc6864a3d06e610c8e73c Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 17 Aug 2026 17:06:26 +0800 Subject: [PATCH 2/4] [chore](thirdparty) build azure on every platform and stop building what Doris does not link Two things about azure-sdk-for-cpp. It only ever got built on x86_64 Linux. env.sh forced DISABLE_BUILD_AZURE=ON on aarch64 and macOS, and build_azure skipped Darwin a second time, so BE and the cloud meta-service lost +AZURE_BLOB and +AZURE_STORAGE_VAULT on those platforms. Nothing in the recipe was actually x86_64-Linux-specific except two details: `-ldl`, which fails on Apple because there is no libdl there, and vcpkg needing VCPKG_FORCE_SYSTEM_BINARIES on aarch64 Linux, where it ships no prebuilt tools. Both are handled now, and the triplet is derived from uname instead of being left to vcpkg's host detection. And it was by far the most expensive package in the tree - 26m06s of the 2h43m x86_64 Linux third-party build in apache/doris-thirdparty run 31988123966 - almost none of it for Doris's benefit: - 22m of the 26m was `vcpkg install`, and 19m of that was opentelemetry-cpp (4m35s) plus the protobuf 5.29.3 (12m29s), abseil (1m58s) and utf8-range it drags in. Doris does not use azure's OpenTelemetry tracing at all; the dependency was in vcpkg.json unconditionally. - vcpkg builds every port twice, debug and release, and Doris links only the release halves. - The SDK builds appconfiguration, attestation, eventhubs, keyvault, tables, template, uAMQP, datalake, file shares and queues. Doris links azure-core, azure-identity, azure-storage-common and azure-storage-blobs. So the patch drops opentelemetry-cpp and the uAMQP C libraries from vcpkg.json, flips DISABLE_AMQP and DISABLE_AZURE_CORE_OPENTELEMETRY to ON, and trims the sub-projects to the four Doris links; build_azure adds an overlay triplet that sets VCPKG_BUILD_TYPE=release. What is left to build is curl, libxml2, openssl 1.1.1n and zlib, release only. Also add an opt-in ENABLE_THIRDPARTY_CCACHE. A third-party build is cold every time, so a warm ccache turns a rebuild triggered by one changed package into minutes rather than hours. It exports CMAKE__COMPILER_LAUNCHER, which CMake picks up from the environment, rather than prefixing CC/CXX - that would put the compiler name into CMAKE__FLAGS. Autotools packages are left alone, and it is off unless asked for. --- env.sh | 8 +- thirdparty/CHANGELOG.md | 5 + thirdparty/build-thirdparty.sh | 132 ++++++++++++--- .../azure-sdk-for-cpp-azure-core_1.16.0.patch | 155 +++++++++++++++--- 4 files changed, 256 insertions(+), 44 deletions(-) diff --git a/env.sh b/env.sh index d5370195b05365..510b52a224b772 100755 --- a/env.sh +++ b/env.sh @@ -106,12 +106,10 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then fi # set DISABLE_BUILD_AZURE +# Azure is built on every platform now. It used to be skipped on aarch64 and macOS +# because the third-party recipe was x86_64-Linux-only; that is fixed in build_azure. if [[ -z "${DISABLE_BUILD_AZURE}" ]]; then - if [[ "${TARGET_ARCH}" == *arm* || "${TARGET_ARCH}" == "aarch64" || "${TARGET_SYSTEM}" == 'Darwin' ]]; then - export DISABLE_BUILD_AZURE='ON' - else - export DISABLE_BUILD_AZURE='OFF' - fi + export DISABLE_BUILD_AZURE='OFF' fi # check python diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8d1455c3e7b557..0704f691451630 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -7,6 +7,11 @@ This file contains version of the third-party dependency libraries in the build- - Removed: hadoop-libs 3.3.6.6, along with the `installed/{include,lib}/hadoop_hdfs/` prefix it produced. hadoop-libs 3.4.2.4 under `hadoop_hdfs_3_4/` is the only libhdfs built now. +- Modified: azure-core 1.16.0 is now built on aarch64 and macOS as well, and only the + parts Doris links (azure-core, azure-identity, azure-storage-common, + azure-storage-blobs) are built. Its vcpkg dependency closure no longer contains + opentelemetry-cpp, protobuf, abseil, utf8-range, uAMQP, and vcpkg builds the + remaining ports release-only. ## 20260816 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 0af17ab2bbbe4d..e6e5f9d9142b8f 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -56,6 +56,26 @@ if [[ -f "${DORIS_HOME}/env.sh" ]]; then export DO_NOT_CHECK_JAVA_ENV= fi +# Optional ccache for the cmake-based packages. A full third-party build is cold every +# time, so a warm ccache turns a rebuild triggered by one changed package into a few +# minutes instead of hours - that is what this is for, and CI is where it pays off. +# +# CMake initialises CMAKE__COMPILER_LAUNCHER from the environment variables of the +# same name, so this needs no change to the cmake invocations below. It is also why this +# does not go through CC/CXX: "ccache " would land the compiler name in +# CMAKE__FLAGS and leak into whatever the package exports. Autotools packages are +# deliberately left alone. Off by default, since prefixing the compiler changes how every +# package configures itself. +if [[ "${ENABLE_THIRDPARTY_CCACHE:-OFF}" == "ON" ]]; then + if ! command -v ccache &>/dev/null; then + echo "ENABLE_THIRDPARTY_CCACHE=ON, but ccache is not in PATH" >&2 + exit 1 + fi + export CMAKE_C_COMPILER_LAUNCHER='ccache' + export CMAKE_CXX_COMPILER_LAUNCHER='ccache' + echo "ccache is enabled for the cmake-based third-party packages" +fi + # Check args usage() { echo " @@ -64,6 +84,10 @@ Usage: $0 [options...] [packages...] -j build thirdparty parallel --clean clean the extracted data --continue continue to build the remaining packages (starts from the specified package) + + Environment variables: + ENABLE_THIRDPARTY_CCACHE=ON compile the cmake-based packages through ccache + DISABLE_BUILD_AZURE=ON skip the azure-sdk-for-cpp package " exit 1 } @@ -2025,30 +2049,100 @@ build_base64() { # azure blob storage build_azure() { - if [[ "${BUILD_AZURE}" == "OFF" || "$(uname -s)" == 'Darwin' ]]; then + if [[ "${BUILD_AZURE}" == "OFF" ]]; then echo "Skip build azure" - else - check_if_source_exist "${AZURE_SOURCE}" - cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" - azure_dir=$(pwd) + return + fi - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" + check_if_source_exist "${AZURE_SOURCE}" + cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" + azure_dir="$(pwd)" + + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" - # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports - AZURE_PORTS="vcpkg-custom-ports" - AZURE_MANIFEST_DIR="." + # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports + AZURE_PORTS="vcpkg-custom-ports" + AZURE_MANIFEST_DIR="." - # Add -ldl for clang compatibility (libcrypto.a requires dlopen/dlsym/dlclose/dlerror) - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ - -DCMAKE_EXE_LINKER_FLAGS="-ldl" \ - -DCMAKE_SHARED_LINKER_FLAGS="-ldl" \ - -DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. - "${BUILD_SYSTEM}" -j "${PARALLEL}" - "${BUILD_SYSTEM}" install + local azure_machine_type + local vcpkg_arch + azure_machine_type="$(uname -m)" + case "${azure_machine_type}" in + aarch64 | arm64) + vcpkg_arch='arm64' + ;; + x86_64 | amd64) + vcpkg_arch='x64' + ;; + *) + echo "azure: unsupported machine type ${azure_machine_type}" >&2 + exit 1 + ;; + esac + + # vcpkg builds every port twice, debug and release, and installs both. Doris only + # ever links the release halves, and VCPKG_BUILD_TYPE - the only supported way to + # ask for release only - can be set from a triplet file, so shadow the built-in + # triplet with our own. Naming the file after the built-in triplet is what makes + # the overlay take precedence. + local vcpkg_triplet + local vcpkg_triplet_dir="${PWD}/doris-vcpkg-triplets" + mkdir -p "${vcpkg_triplet_dir}" + if [[ "${KERNEL}" == 'Darwin' ]]; then + local vcpkg_osx_arch='x86_64' + if [[ "${vcpkg_arch}" == 'arm64' ]]; then vcpkg_osx_arch='arm64'; fi + vcpkg_triplet="${vcpkg_arch}-osx" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" <>"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" + fi + else + vcpkg_triplet="${vcpkg_arch}-linux" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" < Date: Tue, 15 Jul 2025 12:57:25 +0800 -Subject: [PATCH 1/2] resolve missing uint8_t define +Subject: [PATCH 1/3] resolve missing uint8_t define --- .../azure-security-attestation/src/private/crypto/inc/crypto.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp -index 9a98f204..8f338436 100644 +index 9a98f20..8f33843 100644 --- a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp +++ b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. - + #pragma once +#include #include #include #include --- -2.43.5 +-- +2.50.1 (Apple Git-155) + -From 0c18bd91955f6ad48582c01901ffadd4ad5a149d Mon Sep 17 00:00:00 2001 -From: Claude -Date: Wed, 29 Jan 2026 16:00:00 +0800 -Subject: [PATCH 2/2] fix clang link error: always link libdl for openssl +From 5f30d39de70fadcd1963a2c37fb64cc18068ea0a Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Thu, 29 Jan 2026 16:00:00 +0800 +Subject: [PATCH 2/3] link libdl for openssl on platforms that have it -libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. -With clang, find_library may not find libdl, but -ldl is still needed. +libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. With clang, +find_library may not find libdl, but -ldl is still needed. Apple has no libdl +at all - those symbols live in libSystem - so -ldl must not be added there. --- - vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) + vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -index 1234567..abcdefg 100644 +index 4a5ee89..4211066 100644 --- a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in +++ b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -@@ -53,7 +53,8 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") +@@ -53,7 +53,13 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "crypt32;ws2_32") endif() else() - find_library(OPENSSL_DL_LIBRARY NAMES dl) -+ # Always link dl for clang compatibility -+ set(OPENSSL_DL_LIBRARY "dl") ++ # Always link dl for clang compatibility. Apple has no libdl at all - dlopen ++ # and friends live in libSystem - so -ldl fails the link there. ++ if(APPLE) ++ set(OPENSSL_DL_LIBRARY "") ++ else() ++ set(OPENSSL_DL_LIBRARY "dl") ++ endif() if(OPENSSL_DL_LIBRARY) list(APPEND OPENSSL_LIBRARIES "dl") if(TARGET OpenSSL::Crypto) --- -2.43.5 +-- +2.50.1 (Apple Git-155) + + +From 3318809e01a27b20a253dad916d48edc8c25b467 Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Mon, 17 Aug 2026 12:00:00 +0800 +Subject: [PATCH 3/3] build only the parts of the SDK that Doris links + +Doris links azure-core, azure-identity, azure-storage-common and +azure-storage-blobs. Everything else the SDK builds by default is dead weight, +and the opentelemetry-cpp dependency in vcpkg.json drags in protobuf, abseil +and utf8-range, which dominate the build: 19 of the 26 minutes this package +took in the apache/doris-thirdparty x86_64 build were spent on that closure. +--- + CMakeLists.txt | 19 ++++++++----------- + sdk/storage/CMakeLists.txt | 5 ++--- + vcpkg.json | 18 ------------------ + 3 files changed, 10 insertions(+), 32 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 433f22d..993fb4f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -8,8 +8,12 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake-modules") + + # Disable Rust based APIs in C++ SDK. + set(DISABLE_RUST_IN_BUILD OFF CACHE BOOL "Disable Rust based APIs in package") +-set(DISABLE_AMQP OFF CACHE BOOL "Disable AMQP based functionality") +-set(DISABLE_AZURE_CORE_OPENTELEMETRY OFF CACHE BOOL "Disable OpenTelemetry based functionality") ++# Doris consumes azure-core, azure-identity and azure-storage-blobs only. AMQP and the ++# OpenTelemetry tracing adapter are off by default here so that vcpkg.json can drop ++# opentelemetry-cpp, uAMQP and their dependency closure (protobuf, abseil, utf8-range), ++# which is by far the most expensive part of this build. ++set(DISABLE_AMQP ON CACHE BOOL "Disable AMQP based functionality") ++set(DISABLE_AZURE_CORE_OPENTELEMETRY ON CACHE BOOL "Disable OpenTelemetry based functionality") + + # The Rust OpenSSL library conflicts with the vcpkg based OpenSSL library, which causes TLS operations to fail on Linux. + # For now, disabling rust functionality on Linux operating systems. +@@ -177,18 +181,11 @@ if(BUILD_SAMPLES) + endif() + + # sub-projects ++# Only the ones Doris links: appconfiguration, attestation, eventhubs, keyvault, ++# tables and template are not built. + add_subdirectory(sdk/core) +-add_subdirectory(sdk/appconfiguration) +-add_subdirectory(sdk/attestation) +-# AMQP doesn't work for UWP yet, and eventhubs depends on AMQP, so we cannot include eventhubs on UWP. +-if (NOT DISABLE_AMQP) +- add_subdirectory(sdk/eventhubs) +-endif() + add_subdirectory(sdk/identity) +-add_subdirectory(sdk/keyvault) + add_subdirectory(sdk/storage) +-add_subdirectory(sdk/template) +-add_subdirectory(sdk/tables) + + if(BUILD_SAMPLES) + add_subdirectory(samples/integration/vcpkg-all-smoke) +diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt +index 55bc5d0..f199564 100644 +--- a/sdk/storage/CMakeLists.txt ++++ b/sdk/storage/CMakeLists.txt +@@ -5,8 +5,7 @@ cmake_minimum_required (VERSION 3.13) + + project (azure-storage LANGUAGES CXX) + ++# Doris links azure-storage-common and azure-storage-blobs only; datalake, file ++# shares and queues are not built. + add_subdirectory(azure-storage-common) + add_subdirectory(azure-storage-blobs) +-add_subdirectory(azure-storage-files-datalake) +-add_subdirectory(azure-storage-files-shares) +-add_subdirectory(azure-storage-queues) +diff --git a/vcpkg.json b/vcpkg.json +index 23e6f7c..c697e03 100644 +--- a/vcpkg.json ++++ b/vcpkg.json +@@ -14,27 +14,9 @@ + { + "name": "openssl" + }, +- { +- "name": "opentelemetry-cpp", +- "features": ["otlp-http"], +- "platform": "!(windows & !static)", +- "version>=": "1.3.0" +- }, + { + "name": "wil", + "platform": "windows" +- }, +- { +- "name": "azure-c-shared-utility", +- "platform": "!uwp" +- }, +- { +- "name": "azure-macro-utils-c", +- "platform": "!uwp" +- }, +- { +- "name": "umock-c", +- "platform": "!uwp" + } + ], + "overrides": [ +-- +2.50.1 (Apple Git-155) + From c596cd50d991e63464d3dbe4b70c8ca122245efe Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 17 Aug 2026 17:38:07 +0800 Subject: [PATCH 3/4] [ci](thirdparty) stop the macOS thirdparty jobs stalling on dotnet, and cache ccache Two of the four fixes in apache/doris-thirdparty#410 apply to this workflow too. thrift's configure finds the dotnet the runner images ship, so `make` runs `dotnet build -c Release` for lib/netstd. The build itself finishes in seconds and then the compiler server it leaves behind holds the stdout it inherited for its full keep-alive while nothing happens. Between "Build succeeded" and "Making install in compiler/cpp" in apache/doris-thirdparty run 31988123966: 600.03s on macos-14, 598.79s on macos-15-intel, 0.01s on Linux. Both macOS jobs here pay it. `UseSharedCompilation=false` and `MSBUILDDISABLENODEREUSE=1` stop the server from being started. And nothing was cached between runs: ccache is installed on every runner but nothing routed compilation through it, so a change to one package recompiled all ~80, at `-j 2` on two of the three jobs. Turn on ENABLE_THIRDPARTY_CCACHE and carry the cache with the ccache-action this repository already vendors. max-size is 1G per job so that all three together stay inside what is left of the repository's 10 GB Actions cache budget next to the 5G BE-UT-macOS cache. Note that a cache saved by a pull request run is only visible to that same pull request, so this pays off across pushes to one branch; sharing it across pull requests would need this workflow to run on master. The other two fixes in that PR do not apply here: there is no Docker job to decouple, and no macOS x86_64 job to build less often. --- .github/workflows/build-thirdparty.yml | 84 ++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 1fe25b781e89b5..4b5d535ab59734 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -103,6 +103,8 @@ jobs: needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: ubuntu-22.04 + env: + ENABLE_THIRDPARTY_CCACHE: 'ON' steps: - name: Checkout easimon/maximize-build-space run: | @@ -123,6 +125,24 @@ jobs: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 + # ccache-action is vendored as a submodule and the plain checkout above does not + # fetch it. Initialise only this one; `submodules: recursive` would also clone + # contrib/clucene, which a third party build does not need. + - name: Checkout ccache-action + run: git submodule update --init .github/actions/ccache-action + + # This build is cold every time, so changing one package recompiles all ~80 of + # them. build-thirdparty.sh routes the cmake-based packages through ccache when + # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run + # is only visible to that same pull request, so this pays off across pushes to + # one branch; sharing it across pull requests needs a run on master. + - name: Ccache ${{ github.ref }} + uses: ./.github/actions/ccache-action + with: + key: THIRDPARTY-Linux + max-size: "1G" + restore-keys: THIRDPARTY-Linux- + - name: Download run: | cd thirdparty @@ -197,10 +217,42 @@ jobs: needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: macos-15 + env: + ENABLE_THIRDPARTY_CCACHE: 'ON' + # thrift's configure finds the dotnet these images ship, so `make` runs + # `dotnet build -c Release` for lib/netstd. The build itself takes seconds, and + # then the compiler server it leaves behind holds the stdout it inherited for its + # full 10 minute keep-alive while nothing happens. Measured in + # apache/doris-thirdparty run 31988123966: 600.03s on macos-14 and 598.79s on + # macos-15-intel between "Build succeeded" and "Making install in compiler/cpp", + # against 0.01s on Linux. Do not start a shared compiler server, and do not keep + # MSBuild worker nodes around either. + UseSharedCompilation: 'false' + MSBUILDDISABLENODEREUSE: '1' + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 + # ccache-action is vendored as a submodule and the plain checkout above does not + # fetch it. Initialise only this one; `submodules: recursive` would also clone + # contrib/clucene, which a third party build does not need. + - name: Checkout ccache-action + run: git submodule update --init .github/actions/ccache-action + + # This build is cold every time, so changing one package recompiles all ~80 of + # them. build-thirdparty.sh routes the cmake-based packages through ccache when + # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run + # is only visible to that same pull request, so this pays off across pushes to + # one branch; sharing it across pull requests needs a run on master. + - name: Ccache ${{ github.ref }} + uses: ./.github/actions/ccache-action + with: + key: THIRDPARTY-macOS + max-size: "1G" + restore-keys: THIRDPARTY-macOS- + - name: Download run: | cd thirdparty @@ -262,10 +314,42 @@ jobs: needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: macos-14 + env: + ENABLE_THIRDPARTY_CCACHE: 'ON' + # thrift's configure finds the dotnet these images ship, so `make` runs + # `dotnet build -c Release` for lib/netstd. The build itself takes seconds, and + # then the compiler server it leaves behind holds the stdout it inherited for its + # full 10 minute keep-alive while nothing happens. Measured in + # apache/doris-thirdparty run 31988123966: 600.03s on macos-14 and 598.79s on + # macos-15-intel between "Build succeeded" and "Making install in compiler/cpp", + # against 0.01s on Linux. Do not start a shared compiler server, and do not keep + # MSBuild worker nodes around either. + UseSharedCompilation: 'false' + MSBUILDDISABLENODEREUSE: '1' + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 + # ccache-action is vendored as a submodule and the plain checkout above does not + # fetch it. Initialise only this one; `submodules: recursive` would also clone + # contrib/clucene, which a third party build does not need. + - name: Checkout ccache-action + run: git submodule update --init .github/actions/ccache-action + + # This build is cold every time, so changing one package recompiles all ~80 of + # them. build-thirdparty.sh routes the cmake-based packages through ccache when + # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run + # is only visible to that same pull request, so this pays off across pushes to + # one branch; sharing it across pull requests needs a run on master. + - name: Ccache ${{ github.ref }} + uses: ./.github/actions/ccache-action + with: + key: THIRDPARTY-macOS-arm64 + max-size: "1G" + restore-keys: THIRDPARTY-macOS-arm64- + - name: Download run: | cd thirdparty From ea52f9d5d7c6cf034c7a11a9286ad2ae912a916a Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 18 Aug 2026 17:16:25 +0800 Subject: [PATCH 4/4] [fix](thirdparty) wait out a failed vcpkg download instead of losing the azure build All three jobs of apache/doris run 32032837067 built 73 of the 74 packages and then failed on the 74th, azure, in the same place: vcpkg downloading github.com/madler/zlib/archive/v1.3.1.tar.gz. Linux 2h19m curl: (22) The requested URL returned error: 429 macOS 1h26m curl: (56) The requested URL returned error: 429 macOS-arm64 1h40m curl: (56) The requested URL returned error: 500 The same three runs also show the Actions cache service answering "Our services aren't available right now", so this was GitHub having a bad hour rather than anything on this branch. It is worth handling anyway. azure is the only package that fetches its own sources. Everything else goes through download-thirdparty.sh, which has a mirror and a fallback; vcpkg reaches straight out to github.com while cmake configures, and its own retry is three attempts inside one second, which rides out nothing. azure is also the last package built, so those three jobs each threw away a finished tree over one file. And the exposure just tripled: before this branch azure was built on x86_64 Linux only, and it now builds on all three platforms. So retry the configure, with a backoff long enough for a rate limit window to pass - 2, 4, 6 then 8 minutes - and only when the failure was a download. A port that will not compile, or a bad option, fails identically every time and still stops on the first attempt rather than repeating itself four more times. Ports that did build come back from vcpkg's binary cache, so a retry only redoes what failed, and VCPKG_DOWNLOADS now points outside the directory build_azure wipes on entry, so nothing that already landed is fetched twice. thirdparty/test/azure-vcpkg-retry-test.sh drives build_azure against a stubbed cmake and covers both halves of that. It runs in the workflow's script test job, in seconds, next to the download-thirdparty.sh tests. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-thirdparty.yml | 11 +- thirdparty/build-thirdparty.sh | 66 +++++++-- thirdparty/test/azure-vcpkg-retry-test.sh | 167 ++++++++++++++++++++++ 3 files changed, 234 insertions(+), 10 deletions(-) create mode 100755 thirdparty/test/azure-vcpkg-retry-test.sh diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 4b5d535ab59734..e4f8bd4056c24c 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -80,8 +80,8 @@ jobs: - name: Test focused thirdparty lifecycle run: thirdparty/test/arrow-paimon-lifecycle-test.sh - download_script_test: - name: Download Script Test + script_test: + name: Thirdparty Script Test needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: ubuntu-22.04 @@ -98,6 +98,13 @@ jobs: thirdparty/test/juicefs-default-mirror-test.sh thirdparty/test/download-thirdparty-md5-test.sh + # azure is the only package that fetches its own sources, and it is the + # last one built, so a download that fails there costs the whole tree. + # This drives build_azure against a stubbed cmake, in seconds. + - name: Test build-thirdparty.sh + run: | + thirdparty/test/azure-vcpkg-retry-test.sh + build_linux: name: Build Third Party Libraries (Linux) needs: changes diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index e6e5f9d9142b8f..0bb85221697f92 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -2131,16 +2131,66 @@ EOF azure_link_flags=(-DCMAKE_EXE_LINKER_FLAGS="-ldl" -DCMAKE_SHARED_LINKER_FLAGS="-ldl") fi + # vcpkg fetches the sources of curl, libxml2, openssl and zlib from their upstream + # hosts while cmake configures, and none of that goes through download-thirdparty.sh + # and its mirror. Keep those tarballs outside "${BUILD_DIR}", which was wiped above, + # so a retry - or a later run in the same tree - only fetches what the attempt + # before it missed. "thirdparty/src*" is already gitignored. + VCPKG_DOWNLOADS="${TP_SOURCE_DIR}/vcpkg-downloads" + export VCPKG_DOWNLOADS + mkdir -p "${VCPKG_DOWNLOADS}" + + # vcpkg's own retry is three attempts inside one second, which rides out nothing: + # in apache/doris run 32032837067 all three jobs died on + # github.com/madler/zlib/archive/v1.3.1.tar.gz answering 429 (500 on macOS arm64), + # and because azure is the last package this script builds, each of them threw away + # a finished tree over one file. So wait out a rate limit window instead. Only a + # download is worth waiting on - a port that will not compile, or a bad option, + # fails identically every time - so the configure is retried on nothing else, and + # the ports that did build come back from vcpkg's binary cache, which lives outside + # this directory. cmake is piped through tee rather than redirected so that a + # vcpkg install that takes twenty minutes still shows progress; this script runs + # under `set -o pipefail`, so the status tested below is cmake's, not tee's. + # # DISABLE_AMQP and DISABLE_AZURE_CORE_OPENTELEMETRY are already the patched # defaults; passing them here keeps the reason visible from the build script. - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ - "${azure_link_flags[@]}" \ - -DVCPKG_TARGET_TRIPLET="${vcpkg_triplet}" \ - -DVCPKG_OVERLAY_TRIPLETS="${vcpkg_triplet_dir}" \ - -DDISABLE_RUST_IN_BUILD=ON -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ - -DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_PERFORMANCE_TESTS=OFF \ - -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. + local azure_attempt + local azure_attempts=5 + local azure_backoff + local azure_log="${PWD}/doris-azure-configure.log" + for ((azure_attempt = 1; azure_attempt <= azure_attempts; azure_attempt++)); do + if "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ + "${azure_link_flags[@]}" \ + -DVCPKG_TARGET_TRIPLET="${vcpkg_triplet}" \ + -DVCPKG_OVERLAY_TRIPLETS="${vcpkg_triplet_dir}" \ + -DDISABLE_RUST_IN_BUILD=ON -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_PERFORMANCE_TESTS=OFF \ + -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. 2>&1 | tee "${azure_log}"; then + break + fi + + if ! grep -qE 'Download failed, halting portfile|error: curl: \(' "${azure_log}"; then + echo "azure: cmake configure failed, and not on a download - see above" >&2 + exit 1 + fi + + if [[ "${azure_attempt}" -eq "${azure_attempts}" ]]; then + echo "azure: vcpkg could not download its sources in ${azure_attempts} attempts" >&2 + exit 1 + fi + + # A configure that died inside the vcpkg toolchain file leaves a cache with no + # compiler in it, and cmake would report that instead of running vcpkg again. + rm -rf CMakeCache.txt CMakeFiles + + azure_backoff=$((azure_attempt * 120)) + echo "azure: vcpkg could not download a source, retrying in ${azure_backoff}s" \ + "(attempt $((azure_attempt + 1)) of ${azure_attempts})" >&2 + sleep "${azure_backoff}" + done + rm -f "${azure_log}" + "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install } diff --git a/thirdparty/test/azure-vcpkg-retry-test.sh b/thirdparty/test/azure-vcpkg-retry-test.sh new file mode 100755 index 00000000000000..8719b7ae128c1c --- /dev/null +++ b/thirdparty/test/azure-vcpkg-retry-test.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# azure is the last package build-thirdparty.sh builds, and it is the only one +# that fetches sources itself: vcpkg downloads curl, libxml2, openssl and zlib +# while cmake configures, outside download-thirdparty.sh and its mirror. In +# apache/doris run 32032837067 all three jobs spent between 1h26m and 2h19m +# building the other 73 packages and then threw the tree away because +# github.com/madler/zlib/archive/v1.3.1.tar.gz answered 429. This test pins the +# two halves of the answer to that: a failed download is waited out, and any +# other failure still stops on the first attempt rather than repeating a broken +# build four more times. + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_eq() { + local expected="$1" + local actual="$2" + local what="$3" + [[ "${actual}" == "${expected}" ]] || fail "${what}: expected '${expected}', got '${actual}'" +} + +tmpdir="$(mktemp -d)" +trap 'rm -rf "${tmpdir}"' EXIT + +# build-thirdparty.sh downloads and compiles the whole tree before it defines a +# single build_* function, so the function under test is lifted out of it rather +# than sourced. Extracting it keeps the test honest: it runs the same text that +# ships, and it breaks loudly if build_azure is renamed or reindented. +azure_fn="${tmpdir}/build_azure.sh" +sed -n '/^build_azure() {$/,/^}$/p' "${ROOT}/build-thirdparty.sh" >"${azure_fn}" +[[ -s "${azure_fn}" ]] || fail "could not extract build_azure() from build-thirdparty.sh" +grep -q '^}$' "${azure_fn}" || fail "extracted build_azure() is not terminated" + +# A cmake that fails the first ${CMAKE_FAILURES} times it is called, the way +# vcpkg reports whatever CMAKE_FAILURE_KIND asks for, and counts its calls. +cmake_stub="${tmpdir}/cmake" +cat >"${cmake_stub}" <<'EOF' +#!/usr/bin/env bash +set -eo pipefail +calls="$(cat "${CMAKE_CALL_COUNT}" 2>/dev/null || echo 0)" +calls=$((calls + 1)) +echo "${calls}" >"${CMAKE_CALL_COUNT}" +if [[ "${calls}" -gt "${CMAKE_FAILURES}" ]]; then + echo '-- Configuring done' + exit 0 +fi +if [[ "${CMAKE_FAILURE_KIND}" == 'download' ]]; then + echo 'error: curl: (22) The requested URL returned error: 429' + echo 'CMake Error at scripts/cmake/vcpkg_download_distfile.cmake:136 (message):' + echo ' Download failed, halting portfile.' +else + echo 'CMake Error: CMAKE_CXX_COMPILER not set, after EnableLanguage' +fi +exit 1 +EOF +chmod +x "${cmake_stub}" + +ninja_stub="${tmpdir}/ninja" +printf '#!/usr/bin/env bash\nexit 0\n' >"${ninja_stub}" +chmod +x "${ninja_stub}" + +# Everything build_azure reads that the script normally sets up around it. +export BUILD_AZURE='ON' +export AZURE_SOURCE='azure-sdk-for-cpp-azure-core_1.16.0' +export BUILD_DIR='doris_build' +export TP_SOURCE_DIR="${tmpdir}/src" +export TP_INSTALL_DIR="${tmpdir}/installed" +export CMAKE_CMD="${cmake_stub}" +export BUILD_SYSTEM="${ninja_stub}" +export GENERATOR='Ninja' +export PARALLEL=1 +KERNEL="$(uname -s)" +export KERNEL +export CMAKE_CALL_COUNT="${tmpdir}/cmake-calls" +mkdir -p "${TP_SOURCE_DIR}/${AZURE_SOURCE}/vcpkg-custom-ports" "${TP_INSTALL_DIR}" + +# shellcheck source=/dev/null +. "${azure_fn}" + +# The script checks the unpacked source is there; the fixture above is enough. +check_if_source_exist() { :; } + +SLEEP_LOG="${tmpdir}/sleeps" +recorded_downloads="${tmpdir}/vcpkg-downloads" + +# build_azure calls exit on a failure it will not retry, so each case runs in a +# subshell and reports back through files. The backoff is minutes by design; the +# stub records what it was asked to wait instead of waiting. +run_case() { + export CMAKE_FAILURES="$1" + export CMAKE_FAILURE_KIND="$2" + rm -f "${CMAKE_CALL_COUNT}" "${SLEEP_LOG}" + : >"${SLEEP_LOG}" + status=0 + ( + # Called by build_azure, and VCPKG_DOWNLOADS is set by it. + # shellcheck disable=SC2329 + sleep() { printf '%s\n' "$1" >>"${SLEEP_LOG}"; } + build_azure + # shellcheck disable=SC2153,SC2154 + printf '%s\n' "${VCPKG_DOWNLOADS}" >"${recorded_downloads}" + ) >"${tmpdir}/out.log" 2>&1 || status=$? + calls="$(cat "${CMAKE_CALL_COUNT}" 2>/dev/null || echo 0)" + sleeps="$(awk 'END { print NR }' "${SLEEP_LOG}")" +} + +# 1. Two failed downloads then a good one: build_azure rides it out. +run_case 2 download +assert_eq 0 "${status}" 'a download that recovers should not fail the build' +assert_eq 3 "${calls}" 'cmake should be retried until the download works' +assert_eq 2 "${sleeps}" 'each retry should back off first' +first_backoff="$(sed -n '1p' "${SLEEP_LOG}")" +second_backoff="$(sed -n '2p' "${SLEEP_LOG}")" +[[ "${first_backoff}" -lt "${second_backoff}" ]] || + fail "backoff should grow, got ${first_backoff} then ${second_backoff}" + +# 2. vcpkg keeps its downloads outside the build directory, which build_azure +# wipes on entry, so a rerun does not refetch what already landed. +vcpkg_downloads="$(cat "${recorded_downloads}")" +[[ -n "${vcpkg_downloads}" ]] || fail 'VCPKG_DOWNLOADS should be set for vcpkg' +[[ -d "${vcpkg_downloads}" ]] || fail "VCPKG_DOWNLOADS ${vcpkg_downloads} should exist" +case "${vcpkg_downloads}" in +*"/${BUILD_DIR}/"* | *"/${BUILD_DIR}") + fail "VCPKG_DOWNLOADS ${vcpkg_downloads} is inside the wiped build directory" + ;; +*) ;; +esac + +# 3. A failure that is not a download stops on the first attempt. Repeating a +# configure that cannot work only buries the error under four more copies. +run_case 99 hard +[[ "${status}" -ne 0 ]] || fail 'a broken configure should fail the build' +assert_eq 1 "${calls}" 'a non-download failure should not be retried' +assert_eq 0 "${sleeps}" 'a non-download failure should not sleep' +grep -q 'not on a download' "${tmpdir}/out.log" || + fail 'the build should say why it did not retry' + +# 4. A download that never recovers gives up, but only after it has waited. +run_case 99 download +[[ "${status}" -ne 0 ]] || fail 'an unreachable source should fail the build' +[[ "${calls}" -ge 3 ]] || fail "expected several attempts, got ${calls}" +assert_eq "$((calls - 1))" "${sleeps}" 'every attempt but the first should follow a sleep' + +echo 'PASS: build_azure waits out a failed vcpkg download and fails fast on anything else'