[chore](thirdparty) drop the hadoop-libs 3.3.6.6 build and use 3.4.2.4 everywhere

be/src/io/hdfs_builder.cpp

@@ -28,9 +28,6 @@
 #include "common/config.h"
 #include "common/kerberos/kerberos_ticket_mgr.h"
 #include "common/logging.h"
-#ifdef USE_HADOOP_HDFS
-#include "hadoop_hdfs/hdfs.h"
-#endif
 #include "io/fs/hdfs.h"
 #include "runtime/exec_env.h"
 #include "util/string_util.h"

build.sh

@@ -482,7 +482,7 @@ fi
 if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then
     LAST_THIRDPARTY_LIB='libbrotlienc.a'
 else
-    LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a'
+    LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a'
 fi
 
 # The final-library sentinel only proves that some third-party build completed. It cannot

cloud/CMakeLists.txt

@@ -302,7 +302,7 @@ set(WL_END_GROUP "-Wl,--end-group")
 # Set Doris libraries
 if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX)
     add_library(hadoop_hdfs STATIC IMPORTED)
-    set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs/native/libhdfs.a)
+    set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a)
 
     set(COMMON_THIRDPARTY
         ${COMMON_THIRDPARTY}

cloud/src/recycler/hdfs_accessor.cpp

@@ -24,7 +24,7 @@
 #include "recycler/util.h"
 
 #ifdef USE_HADOOP_HDFS
-#include <hadoop_hdfs/hdfs.h> // IWYU pragma: export
+#include <hadoop_hdfs_3_4/hdfs.h> // IWYU pragma: export
 #else
 #include <hdfs/hdfs.h> // IWYU pragma: export
 #endif

cloud/src/recycler/hdfs_accessor.h

@@ -18,7 +18,7 @@
 #pragma once
 
 #ifdef USE_HADOOP_HDFS
-#include <hadoop_hdfs/hdfs.h> // IWYU pragma: export
+#include <hadoop_hdfs_3_4/hdfs.h> // IWYU pragma: export
 #else
 #include <hdfs/hdfs.h> // IWYU pragma: export
 #endif

run-be-ut.sh

@@ -435,8 +435,8 @@ touch "${UT_TMP_DIR}/tmp_file"
 LIB_DIR="${DORIS_TEST_BINARY_DIR}/lib/"
 rm -rf "${LIB_DIR}"
 mkdir "${LIB_DIR}"
-if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then
-    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}"
+if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then
+    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs"
 fi
 if [[ -f "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" ]]; then
     cp "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" "${LIB_DIR}/"

run-cloud-ut.sh

@@ -289,8 +289,8 @@ mkdir -p "${CMAKE_BUILD_DIR}/test/log"
 LIB_DIR="${CMAKE_BUILD_DIR}/test/lib"
 rm -rf "${LIB_DIR}"
 mkdir "${LIB_DIR}"
-if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then
-    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}"
+if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then
+    cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs"
 fi
 
 if [[ "${RUN}" -ne 1 ]]; then

thirdparty/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
 
+## 20260817
+
+- Removed: hadoop-libs 3.3.6.6, along with the `installed/{include,lib}/hadoop_hdfs/`
+  prefix it produced. hadoop-libs 3.4.2.4 under `hadoop_hdfs_3_4/` is the only libhdfs
+  built now.
+
 ## 20260816
 
 - Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4

thirdparty/build-thirdparty.sh

@@ -1904,24 +1904,6 @@ build_fast_float() {
     cp -r ./include/fast_float "${TP_INSTALL_DIR}/include/"
 }
 
-# hadoop_libs
-build_hadoop_libs() {
-    check_if_source_exist "${HADOOP_LIBS_SOURCE}"
-    cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_SOURCE}"
-    echo "THIRDPARTY_INSTALLED=${TP_INSTALL_DIR}" >env.sh
-    ./build.sh
-
-    rm -rf "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
-    rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
-    mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
-    mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
-    cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/* "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
-    cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
-    rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/*.a"
-    find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type f ! -name '*.a' -exec cp {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \;
-    find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type l -exec cp -P {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \;
-}
-
 # hadoop_libs_3_4
 build_hadoop_libs_3_4() {
     check_if_source_exist "${HADOOP_LIBS_3_4_SOURCE}"
@@ -2372,14 +2354,10 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
     )
     if [[ "$(uname -s)" == 'Darwin' ]]; then
         read -r -a packages <<<"binutils gettext ${packages[*]}"
-        # hadoop_libs, the 3.3.6 fork, stays Linux-only: it carries none of the macOS fixes
-        # apache/doris-thirdparty#407 made to the 3.4 fork, and nothing built here reads its
-        # hadoop_hdfs/ prefix - the cloud module, its only other consumer, is Linux-only too.
-        read -r -a packages <<<"${packages[*]} hadoop_libs_3_4"
-    elif [[ "$(uname -s)" == 'Linux' ]]; then
-        read -r -a packages <<<"${packages[*]} hadoop_libs"
-        read -r -a packages <<<"${packages[*]} hadoop_libs_3_4"
     fi
+    # hadoop_libs_3_4 runs last on every platform: its native build links against
+    # what the packages above install into ${TP_INSTALL_DIR}.
+    read -r -a packages <<<"${packages[*]} hadoop_libs_3_4"
 fi
 
 # Map a package name to its source directory variable(s) and remove them to free disk space.
@@ -2457,7 +2435,6 @@ cleanup_package_source() {
         xxhash)          src_var="XXHASH_SOURCE" ;;
         concurrentqueue) src_var="CONCURRENTQUEUE_SOURCE" ;;
         fast_float)      src_var="FAST_FLOAT_SOURCE" ;;
-        hadoop_libs)     src_var="HADOOP_LIBS_SOURCE" ;;
         hadoop_libs_3_4) src_var="HADOOP_LIBS_3_4_SOURCE" ;;
         avx2neon)        src_var="AVX2NEON_SOURCE" ;;
         libdeflate)      src_var="LIBDEFLATE_SOURCE" ;;

thirdparty/test/arrow-paimon-lifecycle-test.sh

@@ -161,14 +161,14 @@ exercise_generic_recovery_dispatch() {
     local package2
     local extra
 
-    mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs/native" \
-        "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native" \
+    mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \
+        "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \
         "${generic}/gensrc" "${generic}/fe" "${generic}/be/build_Release" \
         "${generic}/be/output"
     cp "${ROOT}/../build.sh" "${generic}/build.sh"
     cp "${ROOT}/arrow-paimon-vars.sh" "${thirdparty_dir}/arrow-paimon-vars.sh"
-    touch "${thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a"
-    touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a"
+    touch "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a"
+    touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a"
     printf '%s\n' 'clean: ; @:' >"${generic}/gensrc/Makefile"
     # shellcheck disable=SC2016
     printf '%s\n' '#!/usr/bin/env bash' '[[ "$1" == "clean" ]]' >"${fake_mvn}"

thirdparty/vars.sh

@@ -498,12 +498,6 @@ FAST_FLOAT_SOURCE=fast_float-3.9.0
 FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea"
 
 # libhdfs
-HADOOP_LIBS_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.3.6.6-for-doris.tar.gz"
-HADOOP_LIBS_NAME="hadoop-3.3.6.6-for-doris.tar.gz"
-HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.6-for-doris"
-HADOOP_LIBS_MD5SUM="13b66d5f2abffd1740e692b65df5962e"
-
-# libhdfs 3.4
 HADOOP_LIBS_3_4_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.4.2.4-for-doris.tar.gz"
 HADOOP_LIBS_3_4_NAME="hadoop-3.4.2.4-for-doris.tar.gz"
 HADOOP_LIBS_3_4_SOURCE="doris-thirdparty-hadoop-3.4.2.4-for-doris"
@@ -664,7 +658,6 @@ export TP_ARCHIVES=(
     'XXHASH'
     'CONCURRENTQUEUE'
     'FAST_FLOAT'
-    'HADOOP_LIBS'
     'HADOOP_LIBS_3_4'
     'AVX2NEON'
     'LIBDEFLATE'