Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,10 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then
fi

# set DISABLE_BUILD_AZURE
# Azure is built on every platform now. It used to be skipped on aarch64 and macOS
# because the third-party recipe was x86_64-Linux-only; that is fixed in build_azure.
if [[ -z "${DISABLE_BUILD_AZURE}" ]]; then
if [[ "${TARGET_ARCH}" == *arm* || "${TARGET_ARCH}" == "aarch64" || "${TARGET_SYSTEM}" == 'Darwin' ]]; then
export DISABLE_BUILD_AZURE='ON'
else
export DISABLE_BUILD_AZURE='OFF'
fi
export DISABLE_BUILD_AZURE='OFF'
fi

# check python
Expand Down
8 changes: 8 additions & 0 deletions thirdparty/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@

This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`

## 20260817

- Modified: azure-core 1.16.0 is now built on aarch64 and macOS as well, and only the
parts Doris links (azure-core, azure-identity, azure-storage-common,
azure-storage-blobs) are built. Its vcpkg dependency closure no longer contains
opentelemetry-cpp, protobuf, abseil, utf8-range, uAMQP, and vcpkg builds the
remaining ports release-only.

## 20260816

- Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4
Expand Down
132 changes: 113 additions & 19 deletions thirdparty/build-thirdparty.sh
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,26 @@ if [[ -f "${DORIS_HOME}/env.sh" ]]; then
export DO_NOT_CHECK_JAVA_ENV=
fi

# Optional ccache for the cmake-based packages. A full third-party build is cold every
# time, so a warm ccache turns a rebuild triggered by one changed package into a few
# minutes instead of hours - that is what this is for, and CI is where it pays off.
#
# CMake initialises CMAKE_<LANG>_COMPILER_LAUNCHER from the environment variables of the
# same name, so this needs no change to the cmake invocations below. It is also why this
# does not go through CC/CXX: "ccache <compiler>" would land the compiler name in
# CMAKE_<LANG>_FLAGS and leak into whatever the package exports. Autotools packages are
# deliberately left alone. Off by default, since prefixing the compiler changes how every
# package configures itself.
if [[ "${ENABLE_THIRDPARTY_CCACHE:-OFF}" == "ON" ]]; then
if ! command -v ccache &>/dev/null; then
echo "ENABLE_THIRDPARTY_CCACHE=ON, but ccache is not in PATH" >&2
exit 1
fi
export CMAKE_C_COMPILER_LAUNCHER='ccache'
export CMAKE_CXX_COMPILER_LAUNCHER='ccache'
echo "ccache is enabled for the cmake-based third-party packages"
fi

# Check args
usage() {
echo "
Expand All @@ -64,6 +84,10 @@ Usage: $0 [options...] [packages...]
-j <num> build thirdparty parallel
--clean clean the extracted data
--continue <package> continue to build the remaining packages (starts from the specified package)

Environment variables:
ENABLE_THIRDPARTY_CCACHE=ON compile the cmake-based packages through ccache
DISABLE_BUILD_AZURE=ON skip the azure-sdk-for-cpp package
"
exit 1
}
Expand Down Expand Up @@ -2043,30 +2067,100 @@ build_base64() {

# azure blob storage
build_azure() {
if [[ "${BUILD_AZURE}" == "OFF" || "$(uname -s)" == 'Darwin' ]]; then
if [[ "${BUILD_AZURE}" == "OFF" ]]; then
echo "Skip build azure"
else
check_if_source_exist "${AZURE_SOURCE}"
cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}"
azure_dir=$(pwd)
return
fi

rm -rf "${BUILD_DIR}"
mkdir -p "${BUILD_DIR}"
cd "${BUILD_DIR}"
check_if_source_exist "${AZURE_SOURCE}"
cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}"
azure_dir="$(pwd)"

rm -rf "${BUILD_DIR}"
mkdir -p "${BUILD_DIR}"
cd "${BUILD_DIR}"

# We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports
AZURE_PORTS="vcpkg-custom-ports"
AZURE_MANIFEST_DIR="."
# We need use openssl 1.1.1n, which is already carried in vcpkg-custom-ports
AZURE_PORTS="vcpkg-custom-ports"
AZURE_MANIFEST_DIR="."

# Add -ldl for clang compatibility (libcrypto.a requires dlopen/dlsym/dlclose/dlerror)
"${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \
-DCMAKE_EXE_LINKER_FLAGS="-ldl" \
-DCMAKE_SHARED_LINKER_FLAGS="-ldl" \
-DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
"${BUILD_SYSTEM}" -j "${PARALLEL}"
"${BUILD_SYSTEM}" install
local azure_machine_type
local vcpkg_arch
azure_machine_type="$(uname -m)"
case "${azure_machine_type}" in
aarch64 | arm64)
vcpkg_arch='arm64'
;;
x86_64 | amd64)
vcpkg_arch='x64'
;;
*)
echo "azure: unsupported machine type ${azure_machine_type}" >&2
exit 1
;;
esac

# vcpkg builds every port twice, debug and release, and installs both. Doris only
# ever links the release halves, and VCPKG_BUILD_TYPE - the only supported way to
# ask for release only - can be set from a triplet file, so shadow the built-in
# triplet with our own. Naming the file after the built-in triplet is what makes
# the overlay take precedence.
local vcpkg_triplet
local vcpkg_triplet_dir="${PWD}/doris-vcpkg-triplets"
mkdir -p "${vcpkg_triplet_dir}"
if [[ "${KERNEL}" == 'Darwin' ]]; then
local vcpkg_osx_arch='x86_64'
if [[ "${vcpkg_arch}" == 'arm64' ]]; then vcpkg_osx_arch='arm64'; fi
vcpkg_triplet="${vcpkg_arch}-osx"
cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" <<EOF
set(VCPKG_TARGET_ARCHITECTURE ${vcpkg_arch})
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE static)
set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
set(VCPKG_OSX_ARCHITECTURES ${vcpkg_osx_arch})
set(VCPKG_BUILD_TYPE release)
EOF
if [[ -n "${MACOSX_DEPLOYMENT_TARGET}" ]]; then
echo "set(VCPKG_OSX_DEPLOYMENT_TARGET \"${MACOSX_DEPLOYMENT_TARGET}\")" \
>>"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake"
fi
else
vcpkg_triplet="${vcpkg_arch}-linux"
cat >"${vcpkg_triplet_dir}/${vcpkg_triplet}.cmake" <<EOF
set(VCPKG_TARGET_ARCHITECTURE ${vcpkg_arch})
set(VCPKG_CRT_LINKAGE dynamic)
set(VCPKG_LIBRARY_LINKAGE static)
set(VCPKG_CMAKE_SYSTEM_NAME Linux)
set(VCPKG_BUILD_TYPE release)
EOF
fi

# vcpkg ships no prebuilt cmake/ninja/curl for aarch64 Linux, so it has to reuse
# the ones already on PATH.
if [[ "${vcpkg_arch}" == 'arm64' && "${KERNEL}" != 'Darwin' ]]; then
export VCPKG_FORCE_SYSTEM_BINARIES=1
fi

# libcrypto.a needs dlopen/dlsym/dlclose/dlerror, and with clang find_library may
# not turn up libdl, so ask for it explicitly. Apple has no libdl - those symbols
# live in libSystem - and -ldl would fail the link there.
local azure_link_flags=()
if [[ "${KERNEL}" != 'Darwin' ]]; then
azure_link_flags=(-DCMAKE_EXE_LINKER_FLAGS="-ldl" -DCMAKE_SHARED_LINKER_FLAGS="-ldl")
fi

# DISABLE_AMQP and DISABLE_AZURE_CORE_OPENTELEMETRY are already the patched
# defaults; passing them here keeps the reason visible from the build script.
"${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" \
"${azure_link_flags[@]}" \
-DVCPKG_TARGET_TRIPLET="${vcpkg_triplet}" \
-DVCPKG_OVERLAY_TRIPLETS="${vcpkg_triplet_dir}" \
-DDISABLE_RUST_IN_BUILD=ON -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \
-DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_PERFORMANCE_TESTS=OFF \
-DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
"${BUILD_SYSTEM}" -j "${PARALLEL}"
"${BUILD_SYSTEM}" install
}

# dragonbox
Expand Down
155 changes: 135 additions & 20 deletions thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch
Original file line number Diff line number Diff line change
@@ -1,51 +1,166 @@
From 0c18bd91955f6ad48582c01901ffadd4ad5a149c Mon Sep 17 00:00:00 2001
From c155d4c6a000da7ae9a8e66dc7cfada8bf2bfc8a Mon Sep 17 00:00:00 2001
From: BiteTheDDDDt <xl@selectdb.com>
Date: Tue, 15 Jul 2025 12:57:25 +0800
Subject: [PATCH 1/2] resolve missing uint8_t define
Subject: [PATCH 1/3] resolve missing uint8_t define

---
.../azure-security-attestation/src/private/crypto/inc/crypto.hpp | 1 +
1 file changed, 1 insertion(+)

diff --git a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp
index 9a98f204..8f338436 100644
index 9a98f20..8f33843 100644
--- a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp
+++ b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp
@@ -2,6 +2,7 @@
// Licensed under the MIT License.

#pragma once
+#include <cstdint>
#include <ctime>
#include <memory>
#include <string>
--
2.43.5
--
2.50.1 (Apple Git-155)


From 0c18bd91955f6ad48582c01901ffadd4ad5a149d Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Wed, 29 Jan 2026 16:00:00 +0800
Subject: [PATCH 2/2] fix clang link error: always link libdl for openssl
From 5f30d39de70fadcd1963a2c37fb64cc18068ea0a Mon Sep 17 00:00:00 2001
From: Doris Thirdparty <dev@doris.apache.org>
Date: Thu, 29 Jan 2026 16:00:00 +0800
Subject: [PATCH 2/3] link libdl for openssl on platforms that have it

libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl.
With clang, find_library may not find libdl, but -ldl is still needed.
libcrypto.a requires dlopen/dlsym/dlclose/dlerror from libdl. With clang,
find_library may not find libdl, but -ldl is still needed. Apple has no libdl
at all - those symbols live in libSystem - so -ldl must not be added there.
---
vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in
index 1234567..abcdefg 100644
index 4a5ee89..4211066 100644
--- a/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in
+++ b/vcpkg-custom-ports/openssl/vcpkg-cmake-wrapper.cmake.in
@@ -53,7 +53,8 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static")
@@ -53,7 +53,13 @@ if(OPENSSL_FOUND AND "@VCPKG_LIBRARY_LINKAGE@" STREQUAL "static")
set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES "crypt32;ws2_32")
endif()
else()
- find_library(OPENSSL_DL_LIBRARY NAMES dl)
+ # Always link dl for clang compatibility
+ set(OPENSSL_DL_LIBRARY "dl")
+ # Always link dl for clang compatibility. Apple has no libdl at all - dlopen
+ # and friends live in libSystem - so -ldl fails the link there.
+ if(APPLE)
+ set(OPENSSL_DL_LIBRARY "")
+ else()
+ set(OPENSSL_DL_LIBRARY "dl")
+ endif()
if(OPENSSL_DL_LIBRARY)
list(APPEND OPENSSL_LIBRARIES "dl")
if(TARGET OpenSSL::Crypto)
--
2.43.5
--
2.50.1 (Apple Git-155)


From 3318809e01a27b20a253dad916d48edc8c25b467 Mon Sep 17 00:00:00 2001
From: Doris Thirdparty <dev@doris.apache.org>
Date: Mon, 17 Aug 2026 12:00:00 +0800
Subject: [PATCH 3/3] build only the parts of the SDK that Doris links

Doris links azure-core, azure-identity, azure-storage-common and
azure-storage-blobs. Everything else the SDK builds by default is dead weight,
and the opentelemetry-cpp dependency in vcpkg.json drags in protobuf, abseil
and utf8-range, which dominate the build: 19 of the 26 minutes this package
took in the apache/doris-thirdparty x86_64 build were spent on that closure.
---
CMakeLists.txt | 19 ++++++++-----------
sdk/storage/CMakeLists.txt | 5 ++---
vcpkg.json | 18 ------------------
3 files changed, 10 insertions(+), 32 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 433f22d..993fb4f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,8 +8,12 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake-modules")

# Disable Rust based APIs in C++ SDK.
set(DISABLE_RUST_IN_BUILD OFF CACHE BOOL "Disable Rust based APIs in package")
-set(DISABLE_AMQP OFF CACHE BOOL "Disable AMQP based functionality")
-set(DISABLE_AZURE_CORE_OPENTELEMETRY OFF CACHE BOOL "Disable OpenTelemetry based functionality")
+# Doris consumes azure-core, azure-identity and azure-storage-blobs only. AMQP and the
+# OpenTelemetry tracing adapter are off by default here so that vcpkg.json can drop
+# opentelemetry-cpp, uAMQP and their dependency closure (protobuf, abseil, utf8-range),
+# which is by far the most expensive part of this build.
+set(DISABLE_AMQP ON CACHE BOOL "Disable AMQP based functionality")
+set(DISABLE_AZURE_CORE_OPENTELEMETRY ON CACHE BOOL "Disable OpenTelemetry based functionality")

# The Rust OpenSSL library conflicts with the vcpkg based OpenSSL library, which causes TLS operations to fail on Linux.
# For now, disabling rust functionality on Linux operating systems.
@@ -177,18 +181,11 @@ if(BUILD_SAMPLES)
endif()

# sub-projects
+# Only the ones Doris links: appconfiguration, attestation, eventhubs, keyvault,
+# tables and template are not built.
add_subdirectory(sdk/core)
-add_subdirectory(sdk/appconfiguration)
-add_subdirectory(sdk/attestation)
-# AMQP doesn't work for UWP yet, and eventhubs depends on AMQP, so we cannot include eventhubs on UWP.
-if (NOT DISABLE_AMQP)
- add_subdirectory(sdk/eventhubs)
-endif()
add_subdirectory(sdk/identity)
-add_subdirectory(sdk/keyvault)
add_subdirectory(sdk/storage)
-add_subdirectory(sdk/template)
-add_subdirectory(sdk/tables)

if(BUILD_SAMPLES)
add_subdirectory(samples/integration/vcpkg-all-smoke)
diff --git a/sdk/storage/CMakeLists.txt b/sdk/storage/CMakeLists.txt
index 55bc5d0..f199564 100644
--- a/sdk/storage/CMakeLists.txt
+++ b/sdk/storage/CMakeLists.txt
@@ -5,8 +5,7 @@ cmake_minimum_required (VERSION 3.13)

project (azure-storage LANGUAGES CXX)

+# Doris links azure-storage-common and azure-storage-blobs only; datalake, file
+# shares and queues are not built.
add_subdirectory(azure-storage-common)
add_subdirectory(azure-storage-blobs)
-add_subdirectory(azure-storage-files-datalake)
-add_subdirectory(azure-storage-files-shares)
-add_subdirectory(azure-storage-queues)
diff --git a/vcpkg.json b/vcpkg.json
index 23e6f7c..c697e03 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -14,27 +14,9 @@
{
"name": "openssl"
},
- {
- "name": "opentelemetry-cpp",
- "features": ["otlp-http"],
- "platform": "!(windows & !static)",
- "version>=": "1.3.0"
- },
{
"name": "wil",
"platform": "windows"
- },
- {
- "name": "azure-c-shared-utility",
- "platform": "!uwp"
- },
- {
- "name": "azure-macro-utils-c",
- "platform": "!uwp"
- },
- {
- "name": "umock-c",
- "platform": "!uwp"
}
],
"overrides": [
--
2.50.1 (Apple Git-155)

Loading