diff --git a/be/src/io/hdfs_builder.cpp b/be/src/io/hdfs_builder.cpp index cb9c75fb29debc..85248d178d0878 100644 --- a/be/src/io/hdfs_builder.cpp +++ b/be/src/io/hdfs_builder.cpp @@ -28,9 +28,6 @@ #include "common/config.h" #include "common/kerberos/kerberos_ticket_mgr.h" #include "common/logging.h" -#ifdef USE_HADOOP_HDFS -#include "hadoop_hdfs/hdfs.h" -#endif #include "io/fs/hdfs.h" #include "runtime/exec_env.h" #include "util/string_util.h" diff --git a/build.sh b/build.sh index f2d476c04ce1bf..7168a355d6839f 100755 --- a/build.sh +++ b/build.sh @@ -482,7 +482,7 @@ fi if [[ "${TARGET_SYSTEM}" == 'Darwin' ]]; then LAST_THIRDPARTY_LIB='libbrotlienc.a' else - LAST_THIRDPARTY_LIB='hadoop_hdfs/native/libhdfs.a' + LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a' fi # The final-library sentinel only proves that some third-party build completed. It cannot diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index d4e0b3466598e2..e0b44d9cdff1d2 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -302,7 +302,7 @@ set(WL_END_GROUP "-Wl,--end-group") # Set Doris libraries if ((ARCH_AMD64 OR ARCH_AARCH64) AND OS_LINUX) add_library(hadoop_hdfs STATIC IMPORTED) - set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs/native/libhdfs.a) + set_target_properties(hadoop_hdfs PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/hadoop_hdfs_3_4/native/libhdfs.a) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 8de638f81d8e14..4fddeab693f518 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -24,7 +24,7 @@ #include "recycler/util.h" #ifdef USE_HADOOP_HDFS -#include // IWYU pragma: export +#include // IWYU pragma: export #else #include // IWYU pragma: export #endif diff --git a/cloud/src/recycler/hdfs_accessor.h b/cloud/src/recycler/hdfs_accessor.h index b8d6b16b53fbc7..7eb182f9122bcf 100644 --- a/cloud/src/recycler/hdfs_accessor.h +++ b/cloud/src/recycler/hdfs_accessor.h @@ -18,7 +18,7 @@ #pragma once #ifdef USE_HADOOP_HDFS -#include // IWYU pragma: export +#include // IWYU pragma: export #else #include // IWYU pragma: export #endif diff --git a/run-be-ut.sh b/run-be-ut.sh index 7eeece8329effc..d04af7036ed1a6 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -435,8 +435,8 @@ touch "${UT_TMP_DIR}/tmp_file" LIB_DIR="${DORIS_TEST_BINARY_DIR}/lib/" rm -rf "${LIB_DIR}" mkdir "${LIB_DIR}" -if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then - cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}" +if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then + cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs" fi if [[ -f "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" ]]; then cp "${DORIS_HOME}/output/be/lib/java-udf-jar-with-dependencies.jar" "${LIB_DIR}/" diff --git a/run-cloud-ut.sh b/run-cloud-ut.sh index 0b467382be0b26..d74dcb122b2659 100755 --- a/run-cloud-ut.sh +++ b/run-cloud-ut.sh @@ -289,8 +289,8 @@ mkdir -p "${CMAKE_BUILD_DIR}/test/log" LIB_DIR="${CMAKE_BUILD_DIR}/test/lib" rm -rf "${LIB_DIR}" mkdir "${LIB_DIR}" -if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then - cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${LIB_DIR}" +if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" ]]; then + cp -r "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs_3_4/" "${LIB_DIR}/hadoop_hdfs" fi if [[ "${RUN}" -ne 1 ]]; then diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 8c049ea51bb3a7..8d1455c3e7b557 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,12 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20260817 + +- Removed: hadoop-libs 3.3.6.6, along with the `installed/{include,lib}/hadoop_hdfs/` + prefix it produced. hadoop-libs 3.4.2.4 under `hadoop_hdfs_3_4/` is the only libhdfs + built now. + ## 20260816 - Modified: hadoop-libs 3.4.2.3 -> 3.4.2.4 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 6f0676e565a33f..0af17ab2bbbe4d 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1904,24 +1904,6 @@ build_fast_float() { cp -r ./include/fast_float "${TP_INSTALL_DIR}/include/" } -# hadoop_libs -build_hadoop_libs() { - check_if_source_exist "${HADOOP_LIBS_SOURCE}" - cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_SOURCE}" - echo "THIRDPARTY_INSTALLED=${TP_INSTALL_DIR}" >env.sh - ./build.sh - - rm -rf "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/* "${TP_INSTALL_DIR}/lib/hadoop_hdfs/" - cp -r ./hadoop-dist/target/hadoop-libhdfs-3.3.6/include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/" - rm -rf "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/*.a" - find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type f ! -name '*.a' -exec cp {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \; - find ./hadoop-dist/target/hadoop-3.3.6/lib/native/ -type l -exec cp -P {} "${TP_INSTALL_DIR}/lib/hadoop_hdfs/native/" \; -} - # hadoop_libs_3_4 build_hadoop_libs_3_4() { check_if_source_exist "${HADOOP_LIBS_3_4_SOURCE}" @@ -2372,14 +2354,10 @@ if [[ "${#packages[@]}" -eq 0 ]]; then ) if [[ "$(uname -s)" == 'Darwin' ]]; then read -r -a packages <<<"binutils gettext ${packages[*]}" - # hadoop_libs, the 3.3.6 fork, stays Linux-only: it carries none of the macOS fixes - # apache/doris-thirdparty#407 made to the 3.4 fork, and nothing built here reads its - # hadoop_hdfs/ prefix - the cloud module, its only other consumer, is Linux-only too. - read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" - elif [[ "$(uname -s)" == 'Linux' ]]; then - read -r -a packages <<<"${packages[*]} hadoop_libs" - read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" fi + # hadoop_libs_3_4 runs last on every platform: its native build links against + # what the packages above install into ${TP_INSTALL_DIR}. + read -r -a packages <<<"${packages[*]} hadoop_libs_3_4" fi # Map a package name to its source directory variable(s) and remove them to free disk space. @@ -2457,7 +2435,6 @@ cleanup_package_source() { xxhash) src_var="XXHASH_SOURCE" ;; concurrentqueue) src_var="CONCURRENTQUEUE_SOURCE" ;; fast_float) src_var="FAST_FLOAT_SOURCE" ;; - hadoop_libs) src_var="HADOOP_LIBS_SOURCE" ;; hadoop_libs_3_4) src_var="HADOOP_LIBS_3_4_SOURCE" ;; avx2neon) src_var="AVX2NEON_SOURCE" ;; libdeflate) src_var="LIBDEFLATE_SOURCE" ;; diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh index ab48d3bcd2278b..097849fc516728 100755 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh @@ -161,14 +161,14 @@ exercise_generic_recovery_dispatch() { local package2 local extra - mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ - "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native" \ + mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ + "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ "${generic}/gensrc" "${generic}/fe" "${generic}/be/build_Release" \ "${generic}/be/output" cp "${ROOT}/../build.sh" "${generic}/build.sh" cp "${ROOT}/arrow-paimon-vars.sh" "${thirdparty_dir}/arrow-paimon-vars.sh" - touch "${thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" - touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs/native/libhdfs.a" + touch "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" + touch "${external_thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native/libhdfs.a" printf '%s\n' 'clean: ; @:' >"${generic}/gensrc/Makefile" # shellcheck disable=SC2016 printf '%s\n' '#!/usr/bin/env bash' '[[ "$1" == "clean" ]]' >"${fake_mvn}" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 6a42bd00cf7963..951d1b9d78604b 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -498,12 +498,6 @@ FAST_FLOAT_SOURCE=fast_float-3.9.0 FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea" # libhdfs -HADOOP_LIBS_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.3.6.6-for-doris.tar.gz" -HADOOP_LIBS_NAME="hadoop-3.3.6.6-for-doris.tar.gz" -HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.6-for-doris" -HADOOP_LIBS_MD5SUM="13b66d5f2abffd1740e692b65df5962e" - -# libhdfs 3.4 HADOOP_LIBS_3_4_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.4.2.4-for-doris.tar.gz" HADOOP_LIBS_3_4_NAME="hadoop-3.4.2.4-for-doris.tar.gz" HADOOP_LIBS_3_4_SOURCE="doris-thirdparty-hadoop-3.4.2.4-for-doris" @@ -664,7 +658,6 @@ export TP_ARCHIVES=( 'XXHASH' 'CONCURRENTQUEUE' 'FAST_FLOAT' - 'HADOOP_LIBS' 'HADOOP_LIBS_3_4' 'AVX2NEON' 'LIBDEFLATE'