[chore](thirdparty) build only hadoop 3.4.2.4, build azure everywhere and slim it down, and add ccache

.github/workflows/build-thirdparty.yml

@@ -103,6 +103,8 @@ jobs:
     needs: changes
     if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }}
     runs-on: ubuntu-22.04
+    env:
+      ENABLE_THIRDPARTY_CCACHE: 'ON'
     steps:
       - name: Checkout easimon/maximize-build-space
         run: |
@@ -123,6 +125,24 @@ jobs:
       - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v4
 
+      # ccache-action is vendored as a submodule and the plain checkout above does not
+      # fetch it. Initialise only this one; `submodules: recursive` would also clone
+      # contrib/clucene, which a third party build does not need.
+      - name: Checkout ccache-action
+        run: git submodule update --init .github/actions/ccache-action
+
+      # This build is cold every time, so changing one package recompiles all ~80 of
+      # them. build-thirdparty.sh routes the cmake-based packages through ccache when
+      # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run
+      # is only visible to that same pull request, so this pays off across pushes to
+      # one branch; sharing it across pull requests needs a run on master.
+      - name: Ccache ${{ github.ref }}
+        uses: ./.github/actions/ccache-action
+        with:
+          key: THIRDPARTY-Linux
+          max-size: "1G"
+          restore-keys: THIRDPARTY-Linux-
+
       - name: Download
         run: |
           cd thirdparty
@@ -197,10 +217,42 @@ jobs:
     needs: changes
     if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }}
     runs-on: macos-15
+    env:
+      ENABLE_THIRDPARTY_CCACHE: 'ON'
+      # thrift's configure finds the dotnet these images ship, so `make` runs
+      # `dotnet build -c Release` for lib/netstd. The build itself takes seconds, and
+      # then the compiler server it leaves behind holds the stdout it inherited for its
+      # full 10 minute keep-alive while nothing happens. Measured in
+      # apache/doris-thirdparty run 31988123966: 600.03s on macos-14 and 598.79s on
+      # macos-15-intel between "Build succeeded" and "Making install in compiler/cpp",
+      # against 0.01s on Linux. Do not start a shared compiler server, and do not keep
+      # MSBuild worker nodes around either.
+      UseSharedCompilation: 'false'
+      MSBUILDDISABLENODEREUSE: '1'
+      DOTNET_CLI_TELEMETRY_OPTOUT: '1'
+      DOTNET_NOLOGO: '1'
     steps:
       - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v4
 
+      # ccache-action is vendored as a submodule and the plain checkout above does not
+      # fetch it. Initialise only this one; `submodules: recursive` would also clone
+      # contrib/clucene, which a third party build does not need.
+      - name: Checkout ccache-action
+        run: git submodule update --init .github/actions/ccache-action
+
+      # This build is cold every time, so changing one package recompiles all ~80 of
+      # them. build-thirdparty.sh routes the cmake-based packages through ccache when
+      # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run
+      # is only visible to that same pull request, so this pays off across pushes to
+      # one branch; sharing it across pull requests needs a run on master.
+      - name: Ccache ${{ github.ref }}
+        uses: ./.github/actions/ccache-action
+        with:
+          key: THIRDPARTY-macOS
+          max-size: "1G"
+          restore-keys: THIRDPARTY-macOS-
+
       - name: Download
         run: |
           cd thirdparty
@@ -262,10 +314,42 @@ jobs:
     needs: changes
     if: ${{ needs.changes.outputs.thirdparty_changes == 'true' }}
     runs-on: macos-14
+    env:
+      ENABLE_THIRDPARTY_CCACHE: 'ON'
+      # thrift's configure finds the dotnet these images ship, so `make` runs
+      # `dotnet build -c Release` for lib/netstd. The build itself takes seconds, and
+      # then the compiler server it leaves behind holds the stdout it inherited for its
+      # full 10 minute keep-alive while nothing happens. Measured in
+      # apache/doris-thirdparty run 31988123966: 600.03s on macos-14 and 598.79s on
+      # macos-15-intel between "Build succeeded" and "Making install in compiler/cpp",
+      # against 0.01s on Linux. Do not start a shared compiler server, and do not keep
+      # MSBuild worker nodes around either.
+      UseSharedCompilation: 'false'
+      MSBUILDDISABLENODEREUSE: '1'
+      DOTNET_CLI_TELEMETRY_OPTOUT: '1'
+      DOTNET_NOLOGO: '1'
     steps:
       - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v4
 
+      # ccache-action is vendored as a submodule and the plain checkout above does not
+      # fetch it. Initialise only this one; `submodules: recursive` would also clone
+      # contrib/clucene, which a third party build does not need.
+      - name: Checkout ccache-action
+        run: git submodule update --init .github/actions/ccache-action
+
+      # This build is cold every time, so changing one package recompiles all ~80 of
+      # them. build-thirdparty.sh routes the cmake-based packages through ccache when
+      # ENABLE_THIRDPARTY_CCACHE is on. Note that a cache saved by a pull request run
+      # is only visible to that same pull request, so this pays off across pushes to
+      # one branch; sharing it across pull requests needs a run on master.
+      - name: Ccache ${{ github.ref }}
+        uses: ./.github/actions/ccache-action
+        with:
+          key: THIRDPARTY-macOS-arm64
+          max-size: "1G"
+          restore-keys: THIRDPARTY-macOS-arm64-
+
       - name: Download
         run: |
           cd thirdparty

be/src/io/hdfs_builder.cpp

@@ -28,9 +28,6 @@
 #include "common/config.h"
 #include "common/kerberos/kerberos_ticket_mgr.h"
 #include "common/logging.h"
-#ifdef USE_HADOOP_HDFS
-#include "hadoop_hdfs/hdfs.h"
-#endif
 #include "io/fs/hdfs.h"
 #include "runtime/exec_env.h"
 #include "util/string_util.h"

build.sh

@@ -482,7 +482,7 @@ fi
 if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then
     LAST_THIRDPARTY_LIB='libbrotlienc.a'
 else
-    LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a'
+    LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a'
 fi
 
 # The final-library sentinel only proves that some third-party build completed. It cannot

cloud/CMakeLists.txt

@@ -302,7 +302,7 @@ set(WL_END_GROUP "-Wl,--end-group")
 # Set Doris libraries
 if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX)
     add_library(hadoop_hdfs STATIC IMPORTED)
-    set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs/native/libhdfs.a)
+    set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a)
 
     set(COMMON_THIRDPARTY
         ${COMMON_THIRDPARTY}

cloud/src/recycler/hdfs_accessor.cpp

@@ -24,7 +24,7 @@
 #include "recycler/util.h"
 
 #ifdef USE_HADOOP_HDFS
-#include <hadoop_hdfs/hdfs.h> // IWYU pragma: export
+#include <hadoop_hdfs_3_4/hdfs.h> // IWYU pragma: export
 #else
 #include <hdfs/hdfs.h> // IWYU pragma: export
 #endif

cloud/src/recycler/hdfs_accessor.h

@@ -18,7 +18,7 @@
 #pragma once
 
 #ifdef USE_HADOOP_HDFS
-#include <hadoop_hdfs/hdfs.h> // IWYU pragma: export
+#include <hadoop_hdfs_3_4/hdfs.h> // IWYU pragma: export
 #else
 #include <hdfs/hdfs.h> // IWYU pragma: export
 #endif

env.sh

@@ -106,12 +106,10 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then
 fi
 
 # set DISABLE_BUILD_AZURE
+# Azure is built on every platform now. It used to be skipped on aarch64 and macOS
+# because the third-party recipe was x86_64-Linux-only; that is fixed in build_azure.
 if [[ -z "${DISABLE_BUILD_AZURE}" ]]; then
-    if [[ "${TARGET_ARCH}" == *arm* || "${TARGET_ARCH}" == "aarch64" || "${TARGET_SYSTEM}" == 'Darwin' ]]; then
-        export DISABLE_BUILD_AZURE='ON'
-    else
-        export DISABLE_BUILD_AZURE='OFF'
-    fi
+    export DISABLE_BUILD_AZURE='OFF'
 fi
 
 # check python

run-be-ut.sh

@@ -435,8 +435,8 @@ touch "${UT_TMP_DIR}/tmp_file"
 LIB_DIR="${DORIS_TEST_BINARY_DIR}/lib/"
 rm -rf "${LIB_DIR}"
 mkdir "${LIB_DIR}"
-if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then
-    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}"
+if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then
+    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs"
 fi
 if [[ -f "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" ]]; then
     cp "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" "${LIB_DIR}/"

run-cloud-ut.sh

@@ -289,8 +289,8 @@ mkdir -p "${CMAKE_BUILD_DIR}/test/log"
 LIB_DIR="${CMAKE_BUILD_DIR}/test/lib"
 rm -rf "${LIB_DIR}"
 mkdir "${LIB_DIR}"
-if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then
-    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}"
+if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then
+    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs"
 fi
 
 if [[ "${RUN}" -ne 1 ]]; then

thirdparty/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
 
+## 20260817
+
+- Removed: hadoop-libs 3.3.6.6, along with the `installed/{include,lib}/hadoop_hdfs/`
+  prefix it produced. hadoop-libs 3.4.2.4 under `hadoop_hdfs_3_4/` is the only libhdfs
+  built now.
+- Modified: azure-core 1.16.0 is now built on aarch64 and macOS as well, and only the
+  parts Doris links (azure-core, azure-identity, azure-storage-common,
+  azure-storage-blobs) are built. Its vcpkg dependency closure no longer contains
+  opentelemetry-cpp, protobuf, abseil, utf8-range, uAMQP, and vcpkg builds the
+  remaining ports release-only.
+
 ## 20260816
 
 - Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4