From a95831ca2b5e208f06856780c23409412a82a54e Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 17 Aug 2026 17:06:26 +0800 Subject: [PATCH] [chore](thirdparty) build azure on every platform and stop building what Doris does not link Two things about azure-sdk-for-cpp. It only ever got built on x86_64 Linux. env.sh forced DISABLE_BUILD_AZURE=ON on aarch64 and macOS, and build_azure skipped Darwin a second time, so BE and the cloud meta-service lost +AZURE_BLOB and +AZURE_STORAGE_VAULT on those platforms. Nothing in the recipe was actually x86_64-Linux-specific except two details: `-ldl`, which fails on Apple because there is no libdl there, and vcpkg needing VCPKG_FORCE_SYSTEM_BINARIES on aarch64 Linux, where it ships no prebuilt tools. Both are handled now, and the triplet is derived from uname instead of being left to vcpkg's host detection. And it was by far the most expensive package in the tree - 26m06s of the 2h43m x86_64 Linux third-party build in apache/doris-thirdparty run 31988123966 - almost none of it for Doris's benefit: - 22m of the 26m was `vcpkg install`, and 19m of that was opentelemetry-cpp (4m35s) plus the protobuf 5.29.3 (12m29s), abseil (1m58s) and utf8-range it drags in. Doris does not use azure's OpenTelemetry tracing at all; the dependency was in vcpkg.json unconditionally. - vcpkg builds every port twice, debug and release, and Doris links only the release halves. - The SDK builds appconfiguration, attestation, eventhubs, keyvault, tables, template, uAMQP, datalake, file shares and queues. Doris links azure-core, azure-identity, azure-storage-common and azure-storage-blobs. So the patch drops opentelemetry-cpp and the uAMQP C libraries from vcpkg.json, flips DISABLE_AMQP and DISABLE_AZURE_CORE_OPENTELEMETRY to ON, and trims the sub-projects to the four Doris links; build_azure adds an overlay triplet that sets VCPKG_BUILD_TYPE=release. What is left to build is curl, libxml2, openssl 1.1.1n and zlib, release only. Also add an opt-in ENABLE_THIRDPARTY_CCACHE. A third-party build is cold every time, so a warm ccache turns a rebuild triggered by one changed package into minutes rather than hours. It exports CMAKE__COMPILER_LAUNCHER, which CMake picks up from the environment, rather than prefixing CC/CXX - that would put the compiler name into CMAKE__FLAGS. Autotools packages are left alone, and it is off unless asked for. --- env.sh | 8 +- thirdparty/CHANGELOG.md | 8 + thirdparty/build-thirdparty.sh | 132 ++++++++++++--- .../azure-sdk-for-cpp-azure-core_1.16.0.patch | 155 +++++++++++++++--- 4 files changed, 259 insertions(+), 44 deletions(-) diff --git a/env.sh b/env.sh index d5370195b05365..510b52a224b772 100755 --- a/env.sh +++ b/env.sh @@ -106,12 +106,10 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then fi # set DISABLE_BUILD_AZURE +# Azure is built on every platform now. It used to be skipped on aarch64 and macOS +# because the third-party recipe was x86_64-Linux-only; that is fixed in build_azure. if [[ -z "${DISABLE_BUILD_AZURE}" ]]; then - if [[ "${TARGET_ARCH}" == *arm* || "${TARGET_ARCH}" == "aarch64" || "${TARGET_SYSTEM}" == 'Darwin' ]]; then - export DISABLE_BUILD_AZURE='ON' - else - export DISABLE_BUILD_AZURE='OFF' - fi + export DISABLE_BUILD_AZURE='OFF' fi # check python diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8c049ea51bb3a7..c988d5644f0c8f 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,14 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20260817 + +- Modified: azure-core 1.16.0 is now built on aarch64 and macOS as well, and only the + parts Doris links (azure-core, azure-identity, azure-storage-common, + azure-storage-blobs) are built. Its vcpkg dependency closure no longer contains + opentelemetry-cpp, protobuf, abseil, utf8-range, uAMQP, and vcpkg builds the + remaining ports release-only. + ## 20260816 - Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 6f0676e565a33f..47b3ac76b47333 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -56,6 +56,26 @@ if [[ -f "${DORIS_HOME}/env.sh" ]]; then export DO_NOT_CHECK_JAVA_ENV= fi +# Optional ccache for the cmake-based packages. A full third-party build is cold every +# time, so a warm ccache turns a rebuild triggered by one changed package into a few +# minutes instead of hours - that is what this is for, and CI is where it pays off. +# +# CMake initialises CMAKE__COMPILER_LAUNCHER from the environment variables of the +# same name, so this needs no change to the cmake invocations below. It is also why this +# does not go through CC/CXX: "ccache " would land the compiler name in +# CMAKE__FLAGS and leak into whatever the package exports. Autotools packages are +# deliberately left alone. Off by default, since prefixing the compiler changes how every +# package configures itself. +if [[ "${ENABLE_THIRDPARTY_CCACHE:-OFF}" == "ON" ]]; then + if ! command -v ccache &>/dev/null; then + echo "ENABLE_THIRDPARTY_CCACHE=ON, but ccache is not in PATH" >&2 + exit 1 + fi + export CMAKE_C_COMPILER_LAUNCHER='ccache' + export CMAKE_CXX_COMPILER_LAUNCHER='ccache' + echo "ccache is enabled for the cmake-based third-party packages" +fi + # Check args usage() { echo " @@ -64,6 +84,10 @@ Usage: $0 [options...] [packages...] -j build thirdparty parallel --clean clean the extracted data --continue continue to build the remaining packages (starts from the specified package) + + Environment variables: + ENABLE_THIRDPARTY_CCACHE=ON compile the cmake-based packages through ccache + DISABLE_BUILD_AZURE=ON skip the azure-sdk-for-cpp package " exit 1 } @@ -2043,30 +2067,100 @@ build_base64() { # azure blob storage build_azure() { - if [[ "${BUILD_AZURE}" == "OFF" || "$(uname -s)" == 'Darwin' ]]; then + if [[ "${BUILD_AZURE}" == "OFF" ]]; then echo "Skip build azure" - else - check_if_source_exist "${AZURE_SOURCE}" - cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" - azure_dir=$(pwd) + return + fi - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" + check_if_source_exist "${AZURE_SOURCE}" + cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" + azure_dir="$(pwd)" + + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" - # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports - AZURE_PORTS="vcpkg-custom-ports" - AZURE_MANIFEST_DIR="." + # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports + AZURE_PORTS="vcpkg-custom-ports" + AZURE_MANIFEST_DIR="." - # Add -ldl for clang compatibility (libcrypto.a requires dlopen/dlsym/dlclose/dlerror) - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ - -DCMAKE_EXE_LINKER_FLAGS="-ldl" \ - -DCMAKE_SHARED_LINKER_FLAGS="-ldl" \ - -DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. - "${BUILD_SYSTEM}" -j "${PARALLEL}" - "${BUILD_SYSTEM}" install + local azure_machine_type + local vcpkg_arch + azure_machine_type="$(uname -m)" + case "${azure_machine_type}" in + aarch64 | arm64) + vcpkg_arch='arm64' + ;; + x86_64 | amd64) + vcpkg_arch='x64' + ;; + *) + echo "azure: unsupported machine type ${azure_machine_type}" >&2 + exit 1 + ;; + esac + + # vcpkg builds every port twice, debug and release, and installs both. Doris only + # ever links the release halves, and VCPKG_BUILD_TYPE - the only supported way to + # ask for release only - can be set from a triplet file, so shadow the built-in + # triplet with our own. Naming the file after the built-in triplet is what makes + # the overlay take precedence. + local vcpkg_triplet + local vcpkg_triplet_dir="${PWD}/doris-vcpkg-triplets" + mkdir -p "${vcpkg_triplet_dir}" + if [[ "${KERNEL}" == 'Darwin' ]]; then + local vcpkg_osx_arch='x86_64' + if [[ "${vcpkg_arch}" == 'arm64' ]]; then vcpkg_osx_arch='arm64'; fi + vcpkg_triplet="${vcpkg_arch}-osx" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" <>"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" + fi + else + vcpkg_triplet="${vcpkg_arch}-linux" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" < Date: Tue, 15 Jul 2025 12:57:25 +0800 -Subject: [PATCH 1/2] resolve missing uint8_t define +Subject: [PATCH 1/3] resolve missing uint8_t define --- .../azure-security-attestation/src/private/crypto/inc/crypto.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp -index 9a98f204..8f338436 100644 +index 9a98f20..8f33843 100644 --- a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp +++ b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. - + #pragma once +#include #include #include #include --- -2.43.5 +-- +2.50.1 (Apple Git-155) + -From 0c18bd91955f6ad48582c01901ffadd4ad5a149d Mon Sep 17 00:00:00 2001 -From: Claude -Date: Wed, 29 Jan 2026 16:00:00 +0800 -Subject: [PATCH 2/2] fix clang link error: always link libdl for openssl +From 5f30d39de70fadcd1963a2c37fb64cc18068ea0a Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Thu, 29 Jan 2026 16:00:00 +0800 +Subject: [PATCH 2/3] link libdl for openssl on platforms that have it -libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. -With clang, find_library may not find libdl, but -ldl is still needed. +libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. With clang, +find_library may not find libdl, but -ldl is still needed. Apple has no libdl +at all - those symbols live in libSystem - so -ldl must not be added there. --- - vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) + vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -index 1234567..abcdefg 100644 +index 4a5ee89..4211066 100644 --- a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in +++ b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -@@ -53,7 +53,8 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") +@@ -53,7 +53,13 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "crypt32;ws2_32") endif() else() - find_library(OPENSSL_DL_LIBRARY NAMES dl) -+ # Always link dl for clang compatibility -+ set(OPENSSL_DL_LIBRARY "dl") ++ # Always link dl for clang compatibility. Apple has no libdl at all - dlopen ++ # and friends live in libSystem - so -ldl fails the link there. ++ if(APPLE) ++ set(OPENSSL_DL_LIBRARY "") ++ else() ++ set(OPENSSL_DL_LIBRARY "dl") ++ endif() if(OPENSSL_DL_LIBRARY) list(APPEND OPENSSL_LIBRARIES "dl") if(TARGET OpenSSL::Crypto) --- -2.43.5 +-- +2.50.1 (Apple Git-155) + + +From 3318809e01a27b20a253dad916d48edc8c25b467 Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Mon, 17 Aug 2026 12:00:00 +0800 +Subject: [PATCH 3/3] build only the parts of the SDK that Doris links + +Doris links azure-core, azure-identity, azure-storage-common and +azure-storage-blobs. Everything else the SDK builds by default is dead weight, +and the opentelemetry-cpp dependency in vcpkg.json drags in protobuf, abseil +and utf8-range, which dominate the build: 19 of the 26 minutes this package +took in the apache/doris-thirdparty x86_64 build were spent on that closure. +--- + CMakeLists.txt | 19 ++++++++----------- + sdk/storage/CMakeLists.txt | 5 ++--- + vcpkg.json | 18 ------------------ + 3 files changed, 10 insertions(+), 32 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 433f22d..993fb4f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -8,8 +8,12 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake-modules") + + # Disable Rust based APIs in C++ SDK. + set(DISABLE_RUST_IN_BUILD OFF CACHE BOOL "Disable Rust based APIs in package") +-set(DISABLE_AMQP OFF CACHE BOOL "Disable AMQP based functionality") +-set(DISABLE_AZURE_CORE_OPENTELEMETRY OFF CACHE BOOL "Disable OpenTelemetry based functionality") ++# Doris consumes azure-core, azure-identity and azure-storage-blobs only. AMQP and the ++# OpenTelemetry tracing adapter are off by default here so that vcpkg.json can drop ++# opentelemetry-cpp, uAMQP and their dependency closure (protobuf, abseil, utf8-range), ++# which is by far the most expensive part of this build. ++set(DISABLE_AMQP ON CACHE BOOL "Disable AMQP based functionality") ++set(DISABLE_AZURE_CORE_OPENTELEMETRY ON CACHE BOOL "Disable OpenTelemetry based functionality") + + # The Rust OpenSSL library conflicts with the vcpkg based OpenSSL library, which causes TLS operations to fail on Linux. + # For now, disabling rust functionality on Linux operating systems. +@@ -177,18 +181,11 @@ if(BUILD_SAMPLES) + endif() + + # sub-projects ++# Only the ones Doris links: appconfiguration, attestation, eventhubs, keyvault, ++# tables and template are not built. + add_subdirectory(sdk/core) +-add_subdirectory(sdk/appconfiguration) +-add_subdirectory(sdk/attestation) +-# AMQP doesn't work for UWP yet, and eventhubs depends on AMQP, so we cannot include eventhubs on UWP. +-if (NOT DISABLE_AMQP) +- add_subdirectory(sdk/eventhubs) +-endif() + add_subdirectory(sdk/identity) +-add_subdirectory(sdk/keyvault) + add_subdirectory(sdk/storage) +-add_subdirectory(sdk/template) +-add_subdirectory(sdk/tables) + + if(BUILD_SAMPLES) + add_subdirectory(samples/integration/vcpkg-all-smoke) +diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt +index 55bc5d0..f199564 100644 +--- a/sdk/storage/CMakeLists.txt ++++ b/sdk/storage/CMakeLists.txt +@@ -5,8 +5,7 @@ cmake_minimum_required (VERSION 3.13) + + project (azure-storage LANGUAGES CXX) + ++# Doris links azure-storage-common and azure-storage-blobs only; datalake, file ++# shares and queues are not built. + add_subdirectory(azure-storage-common) + add_subdirectory(azure-storage-blobs) +-add_subdirectory(azure-storage-files-datalake) +-add_subdirectory(azure-storage-files-shares) +-add_subdirectory(azure-storage-queues) +diff --git a/vcpkg.json b/vcpkg.json +index 23e6f7c..c697e03 100644 +--- a/vcpkg.json ++++ b/vcpkg.json +@@ -14,27 +14,9 @@ + { + "name": "openssl" + }, +- { +- "name": "opentelemetry-cpp", +- "features": ["otlp-http"], +- "platform": "!(windows & !static)", +- "version>=": "1.3.0" +- }, + { + "name": "wil", + "platform": "windows" +- }, +- { +- "name": "azure-c-shared-utility", +- "platform": "!uwp" +- }, +- { +- "name": "azure-macro-utils-c", +- "platform": "!uwp" +- }, +- { +- "name": "umock-c", +- "platform": "!uwp" + } + ], + "overrides": [ +-- +2.50.1 (Apple Git-155) +