diff --git a/env.sh b/env.sh index d5370195b05365..510b52a224b772 100755 --- a/env.sh +++ b/env.sh @@ -106,12 +106,10 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then fi # set DISABLE_BUILD_AZURE +# Azure is built on every platform now. It used to be skipped on aarch64 and macOS +# because the third-party recipe was x86_64-Linux-only; that is fixed in build_azure. if [[ -z "${DISABLE_BUILD_AZURE}" ]]; then - if [[ "${TARGET_ARCH}" == *arm* || "${TARGET_ARCH}" == "aarch64" || "${TARGET_SYSTEM}" == 'Darwin' ]]; then - export DISABLE_BUILD_AZURE='ON' - else - export DISABLE_BUILD_AZURE='OFF' - fi + export DISABLE_BUILD_AZURE='OFF' fi # check python diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8c049ea51bb3a7..c988d5644f0c8f 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,14 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20260817 + +- Modified: azure-core 1.16.0 is now built on aarch64 and macOS as well, and only the + parts Doris links (azure-core, azure-identity, azure-storage-common, + azure-storage-blobs) are built. Its vcpkg dependency closure no longer contains + opentelemetry-cpp, protobuf, abseil, utf8-range, uAMQP, and vcpkg builds the + remaining ports release-only. + ## 20260816 - Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 6f0676e565a33f..47b3ac76b47333 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -56,6 +56,26 @@ if [[ -f "${DORIS_HOME}/env.sh" ]]; then export DO_NOT_CHECK_JAVA_ENV= fi +# Optional ccache for the cmake-based packages. A full third-party build is cold every +# time, so a warm ccache turns a rebuild triggered by one changed package into a few +# minutes instead of hours - that is what this is for, and CI is where it pays off. +# +# CMake initialises CMAKE__COMPILER_LAUNCHER from the environment variables of the +# same name, so this needs no change to the cmake invocations below. It is also why this +# does not go through CC/CXX: "ccache " would land the compiler name in +# CMAKE__FLAGS and leak into whatever the package exports. Autotools packages are +# deliberately left alone. Off by default, since prefixing the compiler changes how every +# package configures itself. +if [[ "${ENABLE_THIRDPARTY_CCACHE:-OFF}" == "ON" ]]; then + if ! command -v ccache &>/dev/null; then + echo "ENABLE_THIRDPARTY_CCACHE=ON, but ccache is not in PATH" >&2 + exit 1 + fi + export CMAKE_C_COMPILER_LAUNCHER='ccache' + export CMAKE_CXX_COMPILER_LAUNCHER='ccache' + echo "ccache is enabled for the cmake-based third-party packages" +fi + # Check args usage() { echo " @@ -64,6 +84,10 @@ Usage: $0 [options...] [packages...] -j build thirdparty parallel --clean clean the extracted data --continue continue to build the remaining packages (starts from the specified package) + + Environment variables: + ENABLE_THIRDPARTY_CCACHE=ON compile the cmake-based packages through ccache + DISABLE_BUILD_AZURE=ON skip the azure-sdk-for-cpp package " exit 1 } @@ -2043,30 +2067,100 @@ build_base64() { # azure blob storage build_azure() { - if [[ "${BUILD_AZURE}" == "OFF" || "$(uname -s)" == 'Darwin' ]]; then + if [[ "${BUILD_AZURE}" == "OFF" ]]; then echo "Skip build azure" - else - check_if_source_exist "${AZURE_SOURCE}" - cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" - azure_dir=$(pwd) + return + fi - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - cd "${BUILD_DIR}" + check_if_source_exist "${AZURE_SOURCE}" + cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" + azure_dir="$(pwd)" + + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" - # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports - AZURE_PORTS="vcpkg-custom-ports" - AZURE_MANIFEST_DIR="." + # We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports + AZURE_PORTS="vcpkg-custom-ports" + AZURE_MANIFEST_DIR="." - # Add -ldl for clang compatibility (libcrypto.a requires dlopen/dlsym/dlclose/dlerror) - "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \ - -DCMAKE_EXE_LINKER_FLAGS="-ldl" \ - -DCMAKE_SHARED_LINKER_FLAGS="-ldl" \ - -DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. - "${BUILD_SYSTEM}" -j "${PARALLEL}" - "${BUILD_SYSTEM}" install + local azure_machine_type + local vcpkg_arch + azure_machine_type="$(uname -m)" + case "${azure_machine_type}" in + aarch64 | arm64) + vcpkg_arch='arm64' + ;; + x86_64 | amd64) + vcpkg_arch='x64' + ;; + *) + echo "azure: unsupported machine type ${azure_machine_type}" >&2 + exit 1 + ;; + esac + + # vcpkg builds every port twice, debug and release, and installs both. Doris only + # ever links the release halves, and VCPKG_BUILD_TYPE - the only supported way to + # ask for release only - can be set from a triplet file, so shadow the built-in + # triplet with our own. Naming the file after the built-in triplet is what makes + # the overlay take precedence. + local vcpkg_triplet + local vcpkg_triplet_dir="${PWD}/doris-vcpkg-triplets" + mkdir -p "${vcpkg_triplet_dir}" + if [[ "${KERNEL}" == 'Darwin' ]]; then + local vcpkg_osx_arch='x86_64' + if [[ "${vcpkg_arch}" == 'arm64' ]]; then vcpkg_osx_arch='arm64'; fi + vcpkg_triplet="${vcpkg_arch}-osx" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" <>"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" + fi + else + vcpkg_triplet="${vcpkg_arch}-linux" + cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" < Date: Tue, 15 Jul 2025 12:57:25 +0800 -Subject: [PATCH 1/2] resolve missing uint8_t define +Subject: [PATCH 1/3] resolve missing uint8_t define --- .../azure-security-attestation/src/private/crypto/inc/crypto.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp -index 9a98f204..8f338436 100644 +index 9a98f20..8f33843 100644 --- a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp +++ b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. - + #pragma once +#include #include #include #include --- -2.43.5 +-- +2.50.1 (Apple Git-155) + -From 0c18bd91955f6ad48582c01901ffadd4ad5a149d Mon Sep 17 00:00:00 2001 -From: Claude -Date: Wed, 29 Jan 2026 16:00:00 +0800 -Subject: [PATCH 2/2] fix clang link error: always link libdl for openssl +From 5f30d39de70fadcd1963a2c37fb64cc18068ea0a Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Thu, 29 Jan 2026 16:00:00 +0800 +Subject: [PATCH 2/3] link libdl for openssl on platforms that have it -libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. -With clang, find_library may not find libdl, but -ldl is still needed. +libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. With clang, +find_library may not find libdl, but -ldl is still needed. Apple has no libdl +at all - those symbols live in libSystem - so -ldl must not be added there. --- - vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) + vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -index 1234567..abcdefg 100644 +index 4a5ee89..4211066 100644 --- a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in +++ b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in -@@ -53,7 +53,8 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") +@@ -53,7 +53,13 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static") set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "crypt32;ws2_32") endif() else() - find_library(OPENSSL_DL_LIBRARY NAMES dl) -+ # Always link dl for clang compatibility -+ set(OPENSSL_DL_LIBRARY "dl") ++ # Always link dl for clang compatibility. Apple has no libdl at all - dlopen ++ # and friends live in libSystem - so -ldl fails the link there. ++ if(APPLE) ++ set(OPENSSL_DL_LIBRARY "") ++ else() ++ set(OPENSSL_DL_LIBRARY "dl") ++ endif() if(OPENSSL_DL_LIBRARY) list(APPEND OPENSSL_LIBRARIES "dl") if(TARGET OpenSSL::Crypto) --- -2.43.5 +-- +2.50.1 (Apple Git-155) + + +From 3318809e01a27b20a253dad916d48edc8c25b467 Mon Sep 17 00:00:00 2001 +From: Doris Thirdparty +Date: Mon, 17 Aug 2026 12:00:00 +0800 +Subject: [PATCH 3/3] build only the parts of the SDK that Doris links + +Doris links azure-core, azure-identity, azure-storage-common and +azure-storage-blobs. Everything else the SDK builds by default is dead weight, +and the opentelemetry-cpp dependency in vcpkg.json drags in protobuf, abseil +and utf8-range, which dominate the build: 19 of the 26 minutes this package +took in the apache/doris-thirdparty x86_64 build were spent on that closure. +--- + CMakeLists.txt | 19 ++++++++----------- + sdk/storage/CMakeLists.txt | 5 ++--- + vcpkg.json | 18 ------------------ + 3 files changed, 10 insertions(+), 32 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 433f22d..993fb4f 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -8,8 +8,12 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake-modules") + + # Disable Rust based APIs in C++ SDK. + set(DISABLE_RUST_IN_BUILD OFF CACHE BOOL "Disable Rust based APIs in package") +-set(DISABLE_AMQP OFF CACHE BOOL "Disable AMQP based functionality") +-set(DISABLE_AZURE_CORE_OPENTELEMETRY OFF CACHE BOOL "Disable OpenTelemetry based functionality") ++# Doris consumes azure-core, azure-identity and azure-storage-blobs only. AMQP and the ++# OpenTelemetry tracing adapter are off by default here so that vcpkg.json can drop ++# opentelemetry-cpp, uAMQP and their dependency closure (protobuf, abseil, utf8-range), ++# which is by far the most expensive part of this build. ++set(DISABLE_AMQP ON CACHE BOOL "Disable AMQP based functionality") ++set(DISABLE_AZURE_CORE_OPENTELEMETRY ON CACHE BOOL "Disable OpenTelemetry based functionality") + + # The Rust OpenSSL library conflicts with the vcpkg based OpenSSL library, which causes TLS operations to fail on Linux. + # For now, disabling rust functionality on Linux operating systems. +@@ -177,18 +181,11 @@ if(BUILD_SAMPLES) + endif() + + # sub-projects ++# Only the ones Doris links: appconfiguration, attestation, eventhubs, keyvault, ++# tables and template are not built. + add_subdirectory(sdk/core) +-add_subdirectory(sdk/appconfiguration) +-add_subdirectory(sdk/attestation) +-# AMQP doesn't work for UWP yet, and eventhubs depends on AMQP, so we cannot include eventhubs on UWP. +-if (NOT DISABLE_AMQP) +- add_subdirectory(sdk/eventhubs) +-endif() + add_subdirectory(sdk/identity) +-add_subdirectory(sdk/keyvault) + add_subdirectory(sdk/storage) +-add_subdirectory(sdk/template) +-add_subdirectory(sdk/tables) + + if(BUILD_SAMPLES) + add_subdirectory(samples/integration/vcpkg-all-smoke) +diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt +index 55bc5d0..f199564 100644 +--- a/sdk/storage/CMakeLists.txt ++++ b/sdk/storage/CMakeLists.txt +@@ -5,8 +5,7 @@ cmake_minimum_required (VERSION 3.13) + + project (azure-storage LANGUAGES CXX) + ++# Doris links azure-storage-common and azure-storage-blobs only; datalake, file ++# shares and queues are not built. + add_subdirectory(azure-storage-common) + add_subdirectory(azure-storage-blobs) +-add_subdirectory(azure-storage-files-datalake) +-add_subdirectory(azure-storage-files-shares) +-add_subdirectory(azure-storage-queues) +diff --git a/vcpkg.json b/vcpkg.json +index 23e6f7c..c697e03 100644 +--- a/vcpkg.json ++++ b/vcpkg.json +@@ -14,27 +14,9 @@ + { + "name": "openssl" + }, +- { +- "name": "opentelemetry-cpp", +- "features": ["otlp-http"], +- "platform": "!(windows & !static)", +- "version>=": "1.3.0" +- }, + { + "name": "wil", + "platform": "windows" +- }, +- { +- "name": "azure-c-shared-utility", +- "platform": "!uwp" +- }, +- { +- "name": "azure-macro-utils-c", +- "platform": "!uwp" +- }, +- { +- "name": "umock-c", +- "platform": "!uwp" + } + ], + "overrides": [ +-- +2.50.1 (Apple Git-155) +