diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 1fe25b781e89b5..e4f8bd4056c24c 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -80,8 +80,8 @@ jobs: - name: Test focused thirdparty lifecycle run: thirdparty/test/arrow-paimon-lifecycle-test.sh - download_script_test: - name: Download Script Test + script_test: + name: Thirdparty Script Test needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: ubuntu-22.04 @@ -98,11 +98,20 @@ jobs: thirdparty/test/juicefs-default-mirror-test.sh thirdparty/test/download-thirdparty-md5-test.sh + # azure is the only package that fetches its own sources, and it is the + # last one built, so a download that fails there costs the whole tree. + # This drives build_azure against a stubbed cmake, in seconds. + - name: Test build-thirdparty.sh + run: | + thirdparty/test/azure-vcpkg-retry-test.sh + build_linux: name: Build Third Party Libraries (Linux) needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: ubuntu-22.04 + env: + ENABLE_THIRDPARTY_CCACHE: 'ON' steps: - name: Checkout easimon/maximize-build-space run: | @@ -123,6 +132,24 @@ jobs: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 + # ccache-action is vendored as a submodule and the plain checkout above does not + # fetch it. Initialise only this one; `submodules: recursive` would also clone + # contrib/clucene, which a third party build does not need. + - name: Checkout ccache-action + run: git submodule update --init .github/actions/ccache-action + + # This build is cold every time, so changing one package recompiles all ~80 of + # them. build-thirdparty.sh routes the cmake-based packages through ccache when + # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run + # is only visible to that same pull request, so this pays off across pushes to + # one branch; sharing it across pull requests needs a run on master. + - name: Ccache ${{ github.ref }} + uses: ./.github/actions/ccache-action + with: + key: THIRDPARTY-Linux + max-size: "1G" + restore-keys: THIRDPARTY-Linux- + - name: Download run: | cd thirdparty @@ -197,10 +224,42 @@ jobs: needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: macos-15 + env: + ENABLE_THIRDPARTY_CCACHE: 'ON' + # thrift's configure finds the dotnet these images ship, so `make` runs + # `dotnet build -c Release` for lib/netstd. The build itself takes seconds, and + # then the compiler server it leaves behind holds the stdout it inherited for its + # full 10 minute keep-alive while nothing happens. Measured in + # apache/doris-thirdparty run 31988123966: 600.03s on macos-14 and 598.79s on + # macos-15-intel between "Build succeeded" and "Making install in compiler/cpp", + # against 0.01s on Linux. Do not start a shared compiler server, and do not keep + # MSBuild worker nodes around either. + UseSharedCompilation: 'false' + MSBUILDDISABLENODEREUSE: '1' + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 + # ccache-action is vendored as a submodule and the plain checkout above does not + # fetch it. Initialise only this one; `submodules: recursive` would also clone + # contrib/clucene, which a third party build does not need. + - name: Checkout ccache-action + run: git submodule update --init .github/actions/ccache-action + + # This build is cold every time, so changing one package recompiles all ~80 of + # them. build-thirdparty.sh routes the cmake-based packages through ccache when + # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run + # is only visible to that same pull request, so this pays off across pushes to + # one branch; sharing it across pull requests needs a run on master. + - name: Ccache ${{ github.ref }} + uses: ./.github/actions/ccache-action + with: + key: THIRDPARTY-macOS + max-size: "1G" + restore-keys: THIRDPARTY-macOS- + - name: Download run: | cd thirdparty @@ -262,10 +321,42 @@ jobs: needs: changes if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }} runs-on: macos-14 + env: + ENABLE_THIRDPARTY_CCACHE: 'ON' + # thrift's configure finds the dotnet these images ship, so `make` runs + # `dotnet build -c Release` for lib/netstd. The build itself takes seconds, and + # then the compiler server it leaves behind holds the stdout it inherited for its + # full 10 minute keep-alive while nothing happens. Measured in + # apache/doris-thirdparty run 31988123966: 600.03s on macos-14 and 598.79s on + # macos-15-intel between "Build succeeded" and "Making install in compiler/cpp", + # against 0.01s on Linux. Do not start a shared compiler server, and do not keep + # MSBuild worker nodes around either. + UseSharedCompilation: 'false' + MSBUILDDISABLENODEREUSE: '1' + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' steps: - name: Checkout ${{ github.ref }} uses: actions/checkout@v4 + # ccache-action is vendored as a submodule and the plain checkout above does not + # fetch it. Initialise only this one; `submodules: recursive` would also clone + # contrib/clucene, which a third party build does not need. + - name: Checkout ccache-action + run: git submodule update --init .github/actions/ccache-action + + # This build is cold every time, so changing one package recompiles all ~80 of + # them. build-thirdparty.sh routes the cmake-based packages through ccache when + # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run + # is only visible to that same pull request, so this pays off across pushes to + # one branch; sharing it across pull requests needs a run on master. + - name: Ccache ${{ github.ref }} + uses: ./.github/actions/ccache-action + with: + key: THIRDPARTY-macOS-arm64 + max-size: "1G" + restore-keys: THIRDPARTY-macOS-arm64- + - name: Download run: | cd thirdparty diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index cb9c75fb29debc..85248d178d0878 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -28,9 +28,6 @@ #include "common/config.h" #include "common/kerberos/kerberos_ticket_mgr.h" #include "common/logging.h" -#ifdef USE_HADOOP_HDFS -#include "hadoop_hdfs/hdfs.h" -#endif #include "io/fs/hdfs.h" #include "runtime/exec_env.h" #include "util/string_util.h" diff --git a/build.sh b/build.sh index f2d476c04ce1bf..7168a355d6839f 100755 --- a/build.sh +++ b/build.sh @@ -482,7 +482,7 @@ fi if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then LAST_THIRDPARTY_LIB='libbrotlienc.a' else - LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a' + LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a' fi # The final-library sentinel only proves that some third-party build completed. It cannot diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index d4e0b3466598e2..e0b44d9cdff1d2 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -302,7 +302,7 @@ set(WL_END_GROUP "-Wl,--end-group") # Set Doris libraries if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX) add_library(hadoop_hdfs STATIC IMPORTED) - set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs/native/libhdfs.a) + set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 8de638f81d8e14..4fddeab693f518 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -24,7 +24,7 @@ #include "recycler/util.h" #ifdef USE_HADOOP_HDFS -#include // IWYU pragma: export +#include // IWYU pragma: export #else #include // IWYU pragma: export #endif diff --git a/cloud/src/recycler/hdfs_accessor.h b/cloud/src/recycler/hdfs_accessor.h index b8d6b16b53fbc7..7eb182f9122bcf 100644 --- a/cloud/src/recycler/hdfs_accessor.h +++ b/cloud/src/recycler/hdfs_accessor.h @@ -18,7 +18,7 @@ #pragma once #ifdef USE_HADOOP_HDFS -#include // IWYU pragma: export +#include // IWYU pragma: export #else #include // IWYU pragma: export #endif diff --git a/env.sh b/env.sh index d5370195b05365..510b52a224b772 100755 --- a/env.sh +++ b/env.sh @@ -106,12 +106,10 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then fi # set DISABLE_BUILD_AZURE +# Azure is built on every platform now. It used to be skipped on aarch64 and macOS +# because the third-party recipe was x86_64-Linux-only; that is fixed in build_azure. if [[ -z "${DISABLE_BUILD_AZURE}" ]]; then - if [[ "${TARGET_ARCH}" == *arm* || "${TARGET_ARCH}" == "aarch64" || "${TARGET_SYSTEM}" == 'Darwin' ]]; then - export DISABLE_BUILD_AZURE='ON' - else - export DISABLE_BUILD_AZURE='OFF' - fi + export DISABLE_BUILD_AZURE='OFF' fi # check python diff --git a/run-be-ut.sh b/run-be-ut.sh index 7eeece8329effc..d04af7036ed1a6 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -435,8 +435,8 @@ touch "${UT_TMP_DIR}/tmp_file" LIB_DIR="${DORIS_TEST_BINARY_DIR}/lib/" rm -rf "${LIB_DIR}" mkdir "${LIB_DIR}" -if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then - cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}" +if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then + cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs" fi if [[ -f "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" ]]; then cp "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" "${LIB_DIR}/" diff --git a/run-cloud-ut.sh b/run-cloud-ut.sh index 0b467382be0b26..d74dcb122b2659 100755 --- a/run-cloud-ut.sh +++ b/run-cloud-ut.sh @@ -289,8 +289,8 @@ mkdir -p "${CMAKE_BUILD_DIR}/test/log" LIB_DIR="${CMAKE_BUILD_DIR}/test/lib" rm -rf "${LIB_DIR}" mkdir "${LIB_DIR}" -if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then - cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}" +if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then + cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs" fi if [[ "${RUN}" -ne 1 ]]; then diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8c049ea51bb3a7..0704f691451630 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,17 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20260817 + +- Removed: hadoop-libs 3.3.6.6, along with the `installed/{include,lib}/hadoop_hdfs/` + prefix it produced. hadoop-libs 3.4.2.4 under `hadoop_hdfs_3_4/` is the only libhdfs + built now. +- Modified: azure-core 1.16.0 is now built on aarch64 and macOS as well, and only the + parts Doris links (azure-core, azure-identity, azure-storage-common, + azure-storage-blobs) are built. Its vcpkg dependency closure no longer contains + opentelemetry-cpp, protobuf, abseil, utf8-range, uAMQP, and vcpkg builds the + remaining ports release-only. + ## 20260816 - Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 6f0676e565a33f..0bb85221697f92 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -56,6 +56,26 @@ if [[ -f "${DORIS_HOME}/env.sh" ]]; then export DO_NOT_CHECK_JAVA_ENV= fi +# Optional ccache for the cmake-based packages. A full third-party build is cold every +# time, so a warm ccache turns a rebuild triggered by one changed package into a few +# minutes instead of hours - that is what this is for, and CI is where it pays off. +# +# CMake initialises CMAKE__COMPILER_LAUNCHER from the environment variables of the +# same name, so this needs no change to the cmake invocations below. It is also why this +# does not go through CC/CXX: "ccache " would land the compiler name in +# CMAKE__FLAGS and leak into whatever the package exports. Autotools packages are +# deliberately left alone. Off by default, since prefixing the compiler changes how every +# package configures itself. +if [[ "${ENABLE_THIRDPARTY_CCACHE:-OFF}" == "ON" ]]; then + if ! command -v ccache &>/dev/null; then + echo "ENABLE_THIRDPARTY_CCACHE=ON, but ccache is not in PATH" >&2 + exit 1 + fi + export CMAKE_C_COMPILER_LAUNCHER='ccache' + export CMAKE_CXX_COMPILER_LAUNCHER='ccache' + echo "ccache is enabled for the cmake-based third-party packages" +fi + # Check args usage() { echo " @@ -64,6 +84,10 @@ Usage: $0 [options...] [packages...] -j build thirdparty parallel --clean clean the extracted data --continue continue to build the remaining packages (starts from the specified package) + + Environment variables: + ENABLE_THIRDPARTY_CCACHE=ON compile the cmake-based packages through ccache + DISABLE_BUILD_AZURE=ON skip the azure-sdk-for-cpp package " exit 1 } @@ -1904,24 +1928,6 @@ build_fast_float() { cp -r ./include/fast_float "${TP_INSTALL_DIR}/include/" } -# hadoop_libs -build_hadoop_libs() { - check_if_source_exist "${HADOOP_LIBS_SOURCE}" - cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_SOURCE}" - echo "THIRDPARTY_INSTALLED=${TP_INSTALL_DIR}" >env.sh - ./build.sh - - rm -rf "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/* "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/*.a" - find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type f ! -name '*.a' -exec cp {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \; - find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type l -exec cp -P {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \; -} - # hadoop_libs_3_4 build_hadoop_libs_3_4() { check_if_source_exist "${HADOOP_LIBS_3_4_SOURCE}" @@ -2043,30 +2049,150 @@ build_base64() { # azure blob storage build_azure() { - if [[ "${BUILD_AZURE}" == "OFF" || "$(uname -s)" == 'Darwin' ]]; then + if [[ "${BUILD_AZURE}" == "OFF" ]]; then echo "Skip build azure" + return + fi + + check_if_source_exist "${AZURE_SOURCE}" + cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" + azure_dir="$(pwd)" + + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" + + # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports + AZURE_PORTS="vcpkg-custom-ports" + AZURE_MANIFEST_DIR="." + + local azure_machine_type + local vcpkg_arch + azure_machine_type="$(uname -m)" + case "${azure_machine_type}" in + aarch64 | arm64) + vcpkg_arch='arm64' + ;; + x86_64 | amd64) + vcpkg_arch='x64' + ;; + *) + echo "azure: unsupported machine type ${azure_machine_type}" >&2 + exit 1 + ;; + esac + + # vcpkg builds every port twice, debug and release, and installs both. Doris only + # ever links the release halves, and VCPKG_BUILD_TYPE - the only supported way to + # ask for release only - can be set from a triplet file, so shadow the built-in + # triplet with our own. Naming the file after the built-in triplet is what makes + # the overlay take precedence. + local vcpkg_triplet + local vcpkg_triplet_dir="${PWD}/doris-vcpkg-triplets" + mkdir -p "${vcpkg_triplet_dir}" + if [[ "${KERNEL}" == 'Darwin' ]]; then + local vcpkg_osx_arch='x86_64' + if [[ "${vcpkg_arch}" == 'arm64' ]]; then vcpkg_osx_arch='arm64'; fi + vcpkg_triplet="${vcpkg_arch}-osx" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" <>"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" + fi else - check_if_source_exist "${AZURE_SOURCE}" - cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" - azure_dir=$(pwd) + vcpkg_triplet="${vcpkg_arch}-linux" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" <&1 | tee "${azure_log}"; then + break + fi - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" + if ! grep -qE 'Download failed, halting portfile|error: curl: \(' "${azure_log}"; then + echo "azure: cmake configure failed, and not on a download - see above" >&2 + exit 1 + fi - # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports - AZURE_PORTS="vcpkg-custom-ports" - AZURE_MANIFEST_DIR="." + if [[ "${azure_attempt}" -eq "${azure_attempts}" ]]; then + echo "azure: vcpkg could not download its sources in ${azure_attempts} attempts" >&2 + exit 1 + fi - # Add -ldl for clang compatibility (libcrypto.a requires dlopen/dlsym/dlclose/dlerror) - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ - -DCMAKE_EXE_LINKER_FLAGS="-ldl" \ - -DCMAKE_SHARED_LINKER_FLAGS="-ldl" \ - -DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. - "${BUILD_SYSTEM}" -j "${PARALLEL}" - "${BUILD_SYSTEM}" install - fi + # A configure that died inside the vcpkg toolchain file leaves a cache with no + # compiler in it, and cmake would report that instead of running vcpkg again. + rm -rf CMakeCache.txt CMakeFiles + + azure_backoff=$((azure_attempt * 120)) + echo "azure: vcpkg could not download a source, retrying in ${azure_backoff}s" \ + "(attempt $((azure_attempt + 1)) of ${azure_attempts})" >&2 + sleep "${azure_backoff}" + done + rm -f "${azure_log}" + + "${BUILD_SYSTEM}" -j "${PARALLEL}" + "${BUILD_SYSTEM}" install } # dragonbox @@ -2372,14 +2498,10 @@ if [[ "${#packages[@]}" -eq 0 ]]; then ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" - # hadoop_libs, the 3.3.6 fork, stays Linux-only: it carries none of the macOS fixes - # apache/doris-thirdparty#407 made to the 3.4 fork, and nothing built here reads its - # hadoop_hdfs/ prefix - the cloud module, its only other consumer, is Linux-only too. - read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" - elif [[ "$(uname -s)" == 'Linux' ]]; then - read -r -a packages <<<"${packages[*]} hadoop_libs" - read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" fi + # hadoop_libs_3_4 runs last on every platform: its native build links against + # what the packages above install into ${TP_INSTALL_DIR}. + read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" fi # Map a package name to its source directory variable(s) and remove them to free disk space. @@ -2457,7 +2579,6 @@ cleanup_package_source() { xxhash) src_var="XXHASH_SOURCE" ;; concurrentqueue) src_var="CONCURRENTQUEUE_SOURCE" ;; fast_float) src_var="FAST_FLOAT_SOURCE" ;; - hadoop_libs) src_var="HADOOP_LIBS_SOURCE" ;; hadoop_libs_3_4) src_var="HADOOP_LIBS_3_4_SOURCE" ;; avx2neon) src_var="AVX2NEON_SOURCE" ;; libdeflate) src_var="LIBDEFLATE_SOURCE" ;; diff --git a/thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch b/thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch index fe0c9340c3be45..1beb42ec68eec8 100644 --- a/thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch +++ b/thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch @@ -1,51 +1,166 @@ -From 0c18bd91955f6ad48582c01901ffadd4ad5a149c Mon Sep 17 00:00:00 2001 +From c155d4c6a000da7ae9a8e66dc7cfada8bf2bfc8a Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Tue, 15 Jul 2025 12:57:25 +0800 -Subject: [PATCH 1/2] resolve missing uint8_t define +Subject: [PATCH 1/3] resolve missing uint8_t define --- .../azure-security-attestation/src/private/crypto/inc/crypto.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp -index 9a98f204..8f338436 100644 +index 9a98f20..8f33843 100644 --- a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp +++ b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. - + #pragma once +#include #include #include #include --- -2.43.5 +-- +2.50.1 (Apple Git-155) + -From 0c18bd91955f6ad48582c01901ffadd4ad5a149d Mon Sep 17 00:00:00 2001 -From: Claude -Date: Wed, 29 Jan 2026 16:00:00 +0800 -Subject: [PATCH 2/2] fix clang link error: always link libdl for openssl +From 5f30d39de70fadcd1963a2c37fb64cc18068ea0a Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Thu, 29 Jan 2026 16:00:00 +0800 +Subject: [PATCH 2/3] link libdl for openssl on platforms that have it -libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. -With clang, find_library may not find libdl, but -ldl is still needed. +libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. With clang, +find_library may not find libdl, but -ldl is still needed. Apple has no libdl +at all - those symbols live in libSystem - so -ldl must not be added there. --- - vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) + vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -index 1234567..abcdefg 100644 +index 4a5ee89..4211066 100644 --- a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in +++ b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -@@ -53,7 +53,8 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") +@@ -53,7 +53,13 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "crypt32;ws2_32") endif() else() - find_library(OPENSSL_DL_LIBRARY NAMES dl) -+ # Always link dl for clang compatibility -+ set(OPENSSL_DL_LIBRARY "dl") ++ # Always link dl for clang compatibility. Apple has no libdl at all - dlopen ++ # and friends live in libSystem - so -ldl fails the link there. ++ if(APPLE) ++ set(OPENSSL_DL_LIBRARY "") ++ else() ++ set(OPENSSL_DL_LIBRARY "dl") ++ endif() if(OPENSSL_DL_LIBRARY) list(APPEND OPENSSL_LIBRARIES "dl") if(TARGET OpenSSL::Crypto) --- -2.43.5 +-- +2.50.1 (Apple Git-155) + + +From 3318809e01a27b20a253dad916d48edc8c25b467 Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Mon, 17 Aug 2026 12:00:00 +0800 +Subject: [PATCH 3/3] build only the parts of the SDK that Doris links + +Doris links azure-core, azure-identity, azure-storage-common and +azure-storage-blobs. Everything else the SDK builds by default is dead weight, +and the opentelemetry-cpp dependency in vcpkg.json drags in protobuf, abseil +and utf8-range, which dominate the build: 19 of the 26 minutes this package +took in the apache/doris-thirdparty x86_64 build were spent on that closure. +--- + CMakeLists.txt | 19 ++++++++----------- + sdk/storage/CMakeLists.txt | 5 ++--- + vcpkg.json | 18 ------------------ + 3 files changed, 10 insertions(+), 32 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 433f22d..993fb4f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -8,8 +8,12 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake-modules") + + # Disable Rust based APIs in C++ SDK. + set(DISABLE_RUST_IN_BUILD OFF CACHE BOOL "Disable Rust based APIs in package") +-set(DISABLE_AMQP OFF CACHE BOOL "Disable AMQP based functionality") +-set(DISABLE_AZURE_CORE_OPENTELEMETRY OFF CACHE BOOL "Disable OpenTelemetry based functionality") ++# Doris consumes azure-core, azure-identity and azure-storage-blobs only. AMQP and the ++# OpenTelemetry tracing adapter are off by default here so that vcpkg.json can drop ++# opentelemetry-cpp, uAMQP and their dependency closure (protobuf, abseil, utf8-range), ++# which is by far the most expensive part of this build. ++set(DISABLE_AMQP ON CACHE BOOL "Disable AMQP based functionality") ++set(DISABLE_AZURE_CORE_OPENTELEMETRY ON CACHE BOOL "Disable OpenTelemetry based functionality") + + # The Rust OpenSSL library conflicts with the vcpkg based OpenSSL library, which causes TLS operations to fail on Linux. + # For now, disabling rust functionality on Linux operating systems. +@@ -177,18 +181,11 @@ if(BUILD_SAMPLES) + endif() + + # sub-projects ++# Only the ones Doris links: appconfiguration, attestation, eventhubs, keyvault, ++# tables and template are not built. + add_subdirectory(sdk/core) +-add_subdirectory(sdk/appconfiguration) +-add_subdirectory(sdk/attestation) +-# AMQP doesn't work for UWP yet, and eventhubs depends on AMQP, so we cannot include eventhubs on UWP. +-if (NOT DISABLE_AMQP) +- add_subdirectory(sdk/eventhubs) +-endif() + add_subdirectory(sdk/identity) +-add_subdirectory(sdk/keyvault) + add_subdirectory(sdk/storage) +-add_subdirectory(sdk/template) +-add_subdirectory(sdk/tables) + + if(BUILD_SAMPLES) + add_subdirectory(samples/integration/vcpkg-all-smoke) +diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt +index 55bc5d0..f199564 100644 +--- a/sdk/storage/CMakeLists.txt ++++ b/sdk/storage/CMakeLists.txt +@@ -5,8 +5,7 @@ cmake_minimum_required (VERSION 3.13) + + project (azure-storage LANGUAGES CXX) + ++# Doris links azure-storage-common and azure-storage-blobs only; datalake, file ++# shares and queues are not built. + add_subdirectory(azure-storage-common) + add_subdirectory(azure-storage-blobs) +-add_subdirectory(azure-storage-files-datalake) +-add_subdirectory(azure-storage-files-shares) +-add_subdirectory(azure-storage-queues) +diff --git a/vcpkg.json b/vcpkg.json +index 23e6f7c..c697e03 100644 +--- a/vcpkg.json ++++ b/vcpkg.json +@@ -14,27 +14,9 @@ + { + "name": "openssl" + }, +- { +- "name": "opentelemetry-cpp", +- "features": ["otlp-http"], +- "platform": "!(windows & !static)", +- "version>=": "1.3.0" +- }, + { + "name": "wil", + "platform": "windows" +- }, +- { +- "name": "azure-c-shared-utility", +- "platform": "!uwp" +- }, +- { +- "name": "azure-macro-utils-c", +- "platform": "!uwp" +- }, +- { +- "name": "umock-c", +- "platform": "!uwp" + } + ], + "overrides": [ +-- +2.50.1 (Apple Git-155) + diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh index ab48d3bcd2278b..097849fc516728 100755 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh @@ -161,14 +161,14 @@ exercise_generic_recovery_dispatch() { local package2 local extra - mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ - "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ + mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ + "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ "${generic}/gensrc" "${generic}/fe" "${generic}/be/build_Release" \ "${generic}/be/output" cp "${ROOT}/../build.sh" "${generic}/build.sh" cp "${ROOT}/arrow-paimon-vars.sh" "${thirdparty_dir}/arrow-paimon-vars.sh" - touch "${thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" - touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" + touch "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" + touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" printf '%s\n' 'clean: ; @:' >"${generic}/gensrc/Makefile" # shellcheck disable=SC2016 printf '%s\n' '#!/usr/bin/env bash' '[[ "$1" == "clean" ]]' >"${fake_mvn}" diff --git a/thirdparty/test/azure-vcpkg-retry-test.sh b/thirdparty/test/azure-vcpkg-retry-test.sh new file mode 100755 index 00000000000000..8719b7ae128c1c --- /dev/null +++ b/thirdparty/test/azure-vcpkg-retry-test.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# azure is the last package build-thirdparty.sh builds, and it is the only one +# that fetches sources itself: vcpkg downloads curl, libxml2, openssl and zlib +# while cmake configures, outside download-thirdparty.sh and its mirror. In +# apache/doris run 32032837067 all three jobs spent between 1h26m and 2h19m +# building the other 73 packages and then threw the tree away because +# github.com/madler/zlib/archive/v1.3.1.tar.gz answered 429. This test pins the +# two halves of the answer to that: a failed download is waited out, and any +# other failure still stops on the first attempt rather than repeating a broken +# build four more times. + +set -eo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_eq() { + local expected="$1" + local actual="$2" + local what="$3" + [[ "${actual}" == "${expected}" ]] || fail "${what}: expected '${expected}', got '${actual}'" +} + +tmpdir="$(mktemp -d)" +trap 'rm -rf "${tmpdir}"' EXIT + +# build-thirdparty.sh downloads and compiles the whole tree before it defines a +# single build_* function, so the function under test is lifted out of it rather +# than sourced. Extracting it keeps the test honest: it runs the same text that +# ships, and it breaks loudly if build_azure is renamed or reindented. +azure_fn="${tmpdir}/build_azure.sh" +sed -n '/^build_azure() {$/,/^}$/p' "${ROOT}/build-thirdparty.sh" >"${azure_fn}" +[[ -s "${azure_fn}" ]] || fail "could not extract build_azure() from build-thirdparty.sh" +grep -q '^}$' "${azure_fn}" || fail "extracted build_azure() is not terminated" + +# A cmake that fails the first ${CMAKE_FAILURES} times it is called, the way +# vcpkg reports whatever CMAKE_FAILURE_KIND asks for, and counts its calls. +cmake_stub="${tmpdir}/cmake" +cat >"${cmake_stub}" <<'EOF' +#!/usr/bin/env bash +set -eo pipefail +calls="$(cat "${CMAKE_CALL_COUNT}" 2>/dev/null || echo 0)" +calls=$((calls + 1)) +echo "${calls}" >"${CMAKE_CALL_COUNT}" +if [[ "${calls}" -gt "${CMAKE_FAILURES}" ]]; then + echo '-- Configuring done' + exit 0 +fi +if [[ "${CMAKE_FAILURE_KIND}" == 'download' ]]; then + echo 'error: curl: (22) The requested URL returned error: 429' + echo 'CMake Error at scripts/cmake/vcpkg_download_distfile.cmake:136 (message):' + echo ' Download failed, halting portfile.' +else + echo 'CMake Error: CMAKE_CXX_COMPILER not set, after EnableLanguage' +fi +exit 1 +EOF +chmod +x "${cmake_stub}" + +ninja_stub="${tmpdir}/ninja" +printf '#!/usr/bin/env bash\nexit 0\n' >"${ninja_stub}" +chmod +x "${ninja_stub}" + +# Everything build_azure reads that the script normally sets up around it. +export BUILD_AZURE='ON' +export AZURE_SOURCE='azure-sdk-for-cpp-azure-core_1.16.0' +export BUILD_DIR='doris_build' +export TP_SOURCE_DIR="${tmpdir}/src" +export TP_INSTALL_DIR="${tmpdir}/installed" +export CMAKE_CMD="${cmake_stub}" +export BUILD_SYSTEM="${ninja_stub}" +export GENERATOR='Ninja' +export PARALLEL=1 +KERNEL="$(uname -s)" +export KERNEL +export CMAKE_CALL_COUNT="${tmpdir}/cmake-calls" +mkdir -p "${TP_SOURCE_DIR}/${AZURE_SOURCE}/vcpkg-custom-ports" "${TP_INSTALL_DIR}" + +# shellcheck source=/dev/null +. "${azure_fn}" + +# The script checks the unpacked source is there; the fixture above is enough. +check_if_source_exist() { :; } + +SLEEP_LOG="${tmpdir}/sleeps" +recorded_downloads="${tmpdir}/vcpkg-downloads" + +# build_azure calls exit on a failure it will not retry, so each case runs in a +# subshell and reports back through files. The backoff is minutes by design; the +# stub records what it was asked to wait instead of waiting. +run_case() { + export CMAKE_FAILURES="$1" + export CMAKE_FAILURE_KIND="$2" + rm -f "${CMAKE_CALL_COUNT}" "${SLEEP_LOG}" + : >"${SLEEP_LOG}" + status=0 + ( + # Called by build_azure, and VCPKG_DOWNLOADS is set by it. + # shellcheck disable=SC2329 + sleep() { printf '%s\n' "$1" >>"${SLEEP_LOG}"; } + build_azure + # shellcheck disable=SC2153,SC2154 + printf '%s\n' "${VCPKG_DOWNLOADS}" >"${recorded_downloads}" + ) >"${tmpdir}/out.log" 2>&1 || status=$? + calls="$(cat "${CMAKE_CALL_COUNT}" 2>/dev/null || echo 0)" + sleeps="$(awk 'END { print NR }' "${SLEEP_LOG}")" +} + +# 1. Two failed downloads then a good one: build_azure rides it out. +run_case 2 download +assert_eq 0 "${status}" 'a download that recovers should not fail the build' +assert_eq 3 "${calls}" 'cmake should be retried until the download works' +assert_eq 2 "${sleeps}" 'each retry should back off first' +first_backoff="$(sed -n '1p' "${SLEEP_LOG}")" +second_backoff="$(sed -n '2p' "${SLEEP_LOG}")" +[[ "${first_backoff}" -lt "${second_backoff}" ]] || + fail "backoff should grow, got ${first_backoff} then ${second_backoff}" + +# 2. vcpkg keeps its downloads outside the build directory, which build_azure +# wipes on entry, so a rerun does not refetch what already landed. +vcpkg_downloads="$(cat "${recorded_downloads}")" +[[ -n "${vcpkg_downloads}" ]] || fail 'VCPKG_DOWNLOADS should be set for vcpkg' +[[ -d "${vcpkg_downloads}" ]] || fail "VCPKG_DOWNLOADS ${vcpkg_downloads} should exist" +case "${vcpkg_downloads}" in +*"/${BUILD_DIR}/"* | *"/${BUILD_DIR}") + fail "VCPKG_DOWNLOADS ${vcpkg_downloads} is inside the wiped build directory" + ;; +*) ;; +esac + +# 3. A failure that is not a download stops on the first attempt. Repeating a +# configure that cannot work only buries the error under four more copies. +run_case 99 hard +[[ "${status}" -ne 0 ]] || fail 'a broken configure should fail the build' +assert_eq 1 "${calls}" 'a non-download failure should not be retried' +assert_eq 0 "${sleeps}" 'a non-download failure should not sleep' +grep -q 'not on a download' "${tmpdir}/out.log" || + fail 'the build should say why it did not retry' + +# 4. A download that never recovers gives up, but only after it has waited. +run_case 99 download +[[ "${status}" -ne 0 ]] || fail 'an unreachable source should fail the build' +[[ "${calls}" -ge 3 ]] || fail "expected several attempts, got ${calls}" +assert_eq "$((calls - 1))" "${sleeps}" 'every attempt but the first should follow a sleep' + +echo 'PASS: build_azure waits out a failed vcpkg download and fails fast on anything else' diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 6a42bd00cf7963..951d1b9d78604b 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -498,12 +498,6 @@ FAST_FLOAT_SOURCE=fast_float-3.9.0 FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea" # libhdfs -HADOOP_LIBS_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.3.6.6-for-doris.tar.gz" -HADOOP_LIBS_NAME="hadoop-3.3.6.6-for-doris.tar.gz" -HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.6-for-doris" -HADOOP_LIBS_MD5SUM="13b66d5f2abffd1740e692b65df5962e" - -# libhdfs 3.4 HADOOP_LIBS_3_4_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.4.2.4-for-doris.tar.gz" HADOOP_LIBS_3_4_NAME="hadoop-3.4.2.4-for-doris.tar.gz" HADOOP_LIBS_3_4_SOURCE="doris-thirdparty-hadoop-3.4.2.4-for-doris" @@ -664,7 +658,6 @@ export TP_ARCHIVES=( 'XXHASH' 'CONCURRENTQUEUE' 'FAST_FLOAT' - 'HADOOP_LIBS' 'HADOOP_LIBS_3_4' 'AVX2NEON' 'LIBDEFLATE'