From 1559a21a53cadc9b90b3bf62aec3734d4717acaa Mon Sep 17 00:00:00 2001 From: Yuanzhan Gao Date: Thu, 26 Mar 2026 15:51:08 +0800 Subject: [PATCH 1/6] [2.6] fix: wheel build fails silently when setup.py crashes (#1541) * fix: wheel build fails silently when setup.py crashes Root cause: setup_requires=["numpy", "setuptools_scm"] triggers the deprecated fetch_build_eggs() on every build, making network calls to PyPI. When the fetch fails in CI (flaky K8s pod networking), setup.py crashes before SWIG compilation starts. The failure is then masked by bash's `local var=$(cmd)` pattern (SC2155), which always returns exit code 0 regardless of the command's exit code. This causes the script to continue with an empty wheel path, passing "" to auditwheel which converts Path("") to PosixPath(".") and errors with "cannot access .. No such file". Changes: - Remove unused setuptools_scm from setup_requires (use_scm_version is commented out) and drop setup_requires entirely since numpy is pre-installed by install_deps.sh - Split `local` declarations from command substitution assignments so set -e can catch failures (SC2155) - Redirect auditwheel output to stderr only, preventing error text from leaking into captured variables via tee - Add numpy import check in build script for early failure Signed-off-by: jamesgao-jpg * fix: update PVC claimName to db-data-pvc-ci-cluster Signed-off-by: jamesgao-jpg --------- Signed-off-by: jamesgao-jpg --- ci/pod/e2e-cpu.yaml | 2 +- ci/pod/e2e-gpu.yaml | 2 +- ci/pod/e2e-sse.yaml | 2 +- python/build_portable_wheel.sh | 24 ++++++++++++++++-------- python/setup.py | 4 +++- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/ci/pod/e2e-cpu.yaml b/ci/pod/e2e-cpu.yaml index 23a6611d6..b4a5c3b78 100644 --- a/ci/pod/e2e-cpu.yaml +++ b/ci/pod/e2e-cpu.yaml @@ -40,4 +40,4 @@ spec: volumes: - name: db-data persistentVolumeClaim: - claimName: db-data + claimName: db-data-pvc-ci-cluster diff --git a/ci/pod/e2e-gpu.yaml b/ci/pod/e2e-gpu.yaml index 015a19844..e248ee370 100644 --- a/ci/pod/e2e-gpu.yaml +++ b/ci/pod/e2e-gpu.yaml @@ -32,4 +32,4 @@ spec: volumes: - name: db-data persistentVolumeClaim: - claimName: db-data + claimName: db-data-pvc-ci-cluster diff --git a/ci/pod/e2e-sse.yaml b/ci/pod/e2e-sse.yaml index db4754a32..74c08c86a 100644 --- a/ci/pod/e2e-sse.yaml +++ b/ci/pod/e2e-sse.yaml @@ -41,4 +41,4 @@ spec: volumes: - name: db-data persistentVolumeClaim: - claimName: db-data + claimName: db-data-pvc-ci-cluster diff --git a/python/build_portable_wheel.sh b/python/build_portable_wheel.sh index ee6393ec4..a22633b52 100755 --- a/python/build_portable_wheel.sh +++ b/python/build_portable_wheel.sh @@ -73,6 +73,11 @@ check_deps() { command -v "$PYTHON" >/dev/null || { log_error "Python not found: $PYTHON"; exit 1; } + $PYTHON -c "import numpy" 2>/dev/null || { + log_error "numpy not found. Install with: pip3 install 'numpy<2'" + exit 1 + } + [ -f "$BUILD_DIR/libknowhere.so" ] || { log_error "libknowhere.so not found at $BUILD_DIR" log_error "Please build Knowhere C++ library first:" @@ -145,15 +150,16 @@ repair_wheel() { log_info "Repairing wheel for $platform..." >&2 # Export library paths from libknowhere RUNPATH - local lib_paths=$(readelf -d "$BUILD_DIR/libknowhere.so" | \ - grep -E 'RUNPATH|RPATH' | \ - sed 's/.*\[\(.*\)\]/\1/' | \ - tr ':' '\n' | grep -v '^$' | tr '\n' ':') + local lib_paths + lib_paths=$(readelf -d "$BUILD_DIR/libknowhere.so" | \ + grep -E 'RUNPATH|RPATH' | \ + sed 's/.*\[\(.*\)\]/\1/' | \ + tr ':' '\n' | grep -v '^$' | tr '\n' ':') export LD_LIBRARY_PATH="${lib_paths}:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu" - # Run auditwheel repair with full output - if ! auditwheel repair "$wheel" -w dist/ --plat "$platform" 2>&1 | tee /dev/stderr; then + # Run auditwheel repair (all output to stderr so it doesn't leak into command substitution) + if ! auditwheel repair "$wheel" -w dist/ --plat "$platform" 2>&1 >&2; then log_error "auditwheel repair failed" >&2 exit 1 fi @@ -237,11 +243,13 @@ main() { log_info "Target platform: $platform" # Build wheel - local wheel=$(build_wheel) + local wheel + wheel=$(build_wheel) log_info "Built: $(basename "$wheel")" # Repair wheel - local final_wheel=$(repair_wheel "$wheel" "$platform") + local final_wheel + final_wheel=$(repair_wheel "$wheel" "$platform") log_info "Final: $(basename "$final_wheel")" # Verify diff --git a/python/setup.py b/python/setup.py index 04cbe5ccb..3e7983346 100644 --- a/python/setup.py +++ b/python/setup.py @@ -106,7 +106,9 @@ def get_readme(): author_email="milvus-team@zilliz.com", license='Apache License 2.0', keywords="search nearest neighbors", - setup_requires=["numpy", "setuptools_scm"], + # numpy must be pre-installed (install_deps.sh handles this). + # Do NOT use setup_requires — it triggers the deprecated fetch_build_eggs + # which makes network calls to PyPI and fails intermittently in CI. #use_scm_version={'root': '..', 'local_scheme': 'no-local-version', 'version_scheme': 'release-branch-semver'}, long_description=get_readme(), long_description_content_type="text/markdown", From 961d50fd62951c173d108c222382457779841317 Mon Sep 17 00:00:00 2001 From: Gao Date: Thu, 26 Mar 2026 17:43:25 +0800 Subject: [PATCH 2/6] update cardinal version (#1539) Signed-off-by: chasingegg Revert "upgrade index version to 10 (#1494)" This reverts commit 25ce6b5e85ffd48caaa5e0d82f75d3fe5d0f617c. --- cmake/libs/cardinal/v1/CMakeLists.txt | 2 +- cmake/libs/cardinal/v2/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/libs/cardinal/v1/CMakeLists.txt b/cmake/libs/cardinal/v1/CMakeLists.txt index fbc26aad0..4f2a6384a 100644 --- a/cmake/libs/cardinal/v1/CMakeLists.txt +++ b/cmake/libs/cardinal/v1/CMakeLists.txt @@ -3,7 +3,7 @@ project(knowhere CXX C) # Use short SHA1 as version # v2.5 tag is used for cardinal v1 -set(CARDINAL_VERSION v2.5.103) +set(CARDINAL_VERSION v2.5.104) set(CARDINAL_REPO_URL "https://github.com/zilliztech/cardinal.git") set(CARDINAL_ROOT "${KNOWHERE_THRID_ROOT}/cardinalv1") diff --git a/cmake/libs/cardinal/v2/CMakeLists.txt b/cmake/libs/cardinal/v2/CMakeLists.txt index f383c0189..e39f3eecb 100644 --- a/cmake/libs/cardinal/v2/CMakeLists.txt +++ b/cmake/libs/cardinal/v2/CMakeLists.txt @@ -3,7 +3,7 @@ project(knowhere CXX C) # Use short SHA1 as version # v2.6 tag is used for cardinal v2 -set(CARDINAL_VERSION v2.6.8) +set(CARDINAL_VERSION v2.6.9) set(CARDINAL_REPO_URL "https://github.com/zilliztech/cardinal.git") set(CARDINAL_ROOT "${KNOWHERE_THRID_ROOT}/cardinalv2") From b5c2f88e405f0b3f7fbf3e4ceaf4a76c074cea5b Mon Sep 17 00:00:00 2001 From: foxspy Date: Tue, 31 Mar 2026 02:51:29 +0800 Subject: [PATCH 3/6] fix: check brute-force threshold before iterator path in HNSW RangeSearch (#1536) Previously, RangeSearch always delegated to the iterator-based path when is_ann_iterator_supported() returned true (all common data types). This bypassed WhetherPerformBruteForceRangeSearch(), making the brute-force fallback dead code. At high filter ratios (>=97%), the iterator visits too many filtered-out graph nodes, causing severe performance degradation (18 QPS vs 3166 QPS with the fix on cohere 1M dataset at 99% filtering). Signed-off-by: xianliang.li --- src/index/hnsw/faiss_hnsw.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/index/hnsw/faiss_hnsw.cc b/src/index/hnsw/faiss_hnsw.cc index 7e11dc854..5a22033c8 100644 --- a/src/index/hnsw/faiss_hnsw.cc +++ b/src/index/hnsw/faiss_hnsw.cc @@ -1565,8 +1565,19 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode { expected RangeSearch(const DataSetPtr dataset, std::unique_ptr cfg, const BitsetView& bitset_, milvus::OpContext* op_context) const override { - // if support ann_iterator, use iterator-based range_search (IndexNode::RangeSearch) - if (is_ann_iterator_supported()) { + // Check brute-force threshold BEFORE iterator path. + // At high filter ratios (>=97%), brute force is much faster than graph traversal + // because the iterator visits too many filtered-out nodes. + if (is_ann_iterator_supported() && !this->indexes.empty() && indexes[0] != nullptr) { + const auto& hnsw_cfg_check = static_cast(*cfg); + BitsetView bitset_check(bitset_); + auto whether_bf = WhetherPerformBruteForceRangeSearch(indexes[0].get(), hnsw_cfg_check, bitset_check); + if (!whether_bf.has_value() || !whether_bf.value()) { + // Not brute-force worthy: use iterator path + return IndexNode::RangeSearch(dataset, std::move(cfg), bitset_, op_context); + } + // Fall through to brute-force range search path below + } else if (is_ann_iterator_supported()) { return IndexNode::RangeSearch(dataset, std::move(cfg), bitset_, op_context); } if (this->indexes.empty()) { From 150979f2f3bae26c222c1fdd04a22666aadd62f6 Mon Sep 17 00:00:00 2001 From: Gao Date: Tue, 31 Mar 2026 13:19:27 +0800 Subject: [PATCH 4/6] [2.6] make 2.6 branch compile with cardinal (#1549) * [2.6] make 2.6 branch compile with cardinal Signed-off-by: chasingegg * update Signed-off-by: chasingegg --------- Signed-off-by: chasingegg Co-authored-by: foxspy --- CMakeLists.txt | 4 ++++ cmake/libs/libmilvus-common.cmake | 2 +- src/simd/hook.h | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a3da2ac9e..ce93c0bc2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,6 +105,10 @@ find_package(nlohmann_json REQUIRED) find_package(glog REQUIRED) find_package(prometheus-cpp REQUIRED) find_package(fmt REQUIRED) +if(NOT TARGET fmt::fmt) + add_library(fmt::fmt INTERFACE IMPORTED) + target_link_libraries(fmt::fmt INTERFACE fmt::fmt-header-only) +endif() find_package(xxHash REQUIRED) include_directories(${xxHash_INCLUDE_DIRS}) find_package(simde REQUIRED) diff --git a/cmake/libs/libmilvus-common.cmake b/cmake/libs/libmilvus-common.cmake index c6e71a7ff..1aea0773f 100644 --- a/cmake/libs/libmilvus-common.cmake +++ b/cmake/libs/libmilvus-common.cmake @@ -1,6 +1,6 @@ set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "") -set( MILVUS-COMMON-VERSION c37f138 ) +set( MILVUS-COMMON-VERSION 9dc0923 ) set( GIT_REPOSITORY "https://github.com/zilliztech/milvus-common.git" ) message(STATUS "milvus-common repo: ${GIT_REPOSITORY}") diff --git a/src/simd/hook.h b/src/simd/hook.h index 243bb9175..456834336 100644 --- a/src/simd/hook.h +++ b/src/simd/hook.h @@ -159,6 +159,20 @@ disable_patch_for_fp32_bf16(); void fvec_hook(std::string&); +namespace cppcontrib { +namespace knowhere { +#if defined(__x86_64__) +using faiss::cpu_support_avx2; +using faiss::cpu_support_avx512; +using faiss::cpu_support_f16c; +using faiss::cpu_support_sse4_2; +#endif +#if defined(__aarch64__) +using faiss::supports_sve; +#endif +} // namespace knowhere +} // namespace cppcontrib + } // namespace faiss #endif /* HOOK_H */ From 4bc3908f6c47defbeeb88978f13c26bdaad1904c Mon Sep 17 00:00:00 2001 From: ChenLiqing <2208529306@qq.com> Date: Fri, 3 Apr 2026 16:28:57 +0800 Subject: [PATCH 5/6] fix: initialize nearest_idx to -1 in k=1 L2 fast path (#1556) When all L2 distances overflow to infinity (e.g. max-float vectors), the loop in exhaustive_L2sqr_nearest_imp never updates nearest_idx, returning 0 instead of -1 (no valid result). Backport of zilliztech/knowhere#1533 to 2.6. Signed-off-by: CLiQing <2208529306@qq.com> Co-authored-by: Claude Opus 4.6 (1M context) --- thirdparty/faiss/faiss/utils/distances.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/faiss/faiss/utils/distances.cpp b/thirdparty/faiss/faiss/utils/distances.cpp index 205da713b..bbfad63cf 100644 --- a/thirdparty/faiss/faiss/utils/distances.cpp +++ b/thirdparty/faiss/faiss/utils/distances.cpp @@ -1015,7 +1015,7 @@ void exhaustive_L2sqr_nearest_imp( float dis_buffer[ny_batch_size]; for (size_t i = i0; i < i1; i++) { const float* x_i = x + i * d; - size_t nearest_idx = 0; + int64_t nearest_idx = -1; float min_dis = HUGE_VALF; // compute distances for (auto j = 0; j < ny; j += ny_batch_size) { From 9a03088de739bf2f961fab9a62d39e9d79193e4a Mon Sep 17 00:00:00 2001 From: Nate Wilkinson Date: Tue, 7 Jul 2026 11:17:09 -0600 Subject: [PATCH 6/6] Add KNOWHERE_DISABLE_HNSW_BRUTE_FORCE env to disable sealed HNSW BF paths Query nodes can set KNOWHERE_DISABLE_HNSW_BRUTE_FORCE to skip upfront and fallback IndexBruteForceWrapper usage on sealed HNSW segments without per-query config changes. Co-authored-by: Cursor --- src/index/hnsw/base_hnsw_config.h | 3 +- src/index/hnsw/faiss_hnsw.cc | 6 ++- .../hnsw/impl/IndexConditionalWrapper.cc | 9 ++++ src/index/hnsw/impl/hnsw_brute_force_env.cc | 41 +++++++++++++++++++ src/index/hnsw/impl/hnsw_brute_force_env.h | 27 ++++++++++++ 5 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 src/index/hnsw/impl/hnsw_brute_force_env.cc create mode 100644 src/index/hnsw/impl/hnsw_brute_force_env.h diff --git a/src/index/hnsw/base_hnsw_config.h b/src/index/hnsw/base_hnsw_config.h index 1f3c2cbb2..9addd1189 100644 --- a/src/index/hnsw/base_hnsw_config.h +++ b/src/index/hnsw/base_hnsw_config.h @@ -31,8 +31,7 @@ class BaseHnswConfig : public BaseConfig { CFG_INT efConstruction; CFG_INT ef; CFG_INT overview_levels; - CFG_BOOL disable_fallback_brute_force; // default is false, means we will use fallback brute force when hnsw search - // does not get enough topk results + CFG_BOOL disable_fallback_brute_force; // per-query override (see also KNOWHERE_DISABLE_HNSW_BRUTE_FORCE env) KNOHWERE_DECLARE_CONFIG(BaseHnswConfig) { KNOWHERE_CONFIG_DECLARE_FIELD(M).description("hnsw M").set_default(30).set_range(2, 2048).for_train(); KNOWHERE_CONFIG_DECLARE_FIELD(efConstruction) diff --git a/src/index/hnsw/faiss_hnsw.cc b/src/index/hnsw/faiss_hnsw.cc index 5a22033c8..264379ec2 100644 --- a/src/index/hnsw/faiss_hnsw.cc +++ b/src/index/hnsw/faiss_hnsw.cc @@ -41,6 +41,7 @@ #include "index/hnsw/impl/IndexConditionalWrapper.h" #include "index/hnsw/impl/IndexHNSWWrapper.h" #include "index/hnsw/impl/IndexWrapperCosine.h" +#include "index/hnsw/impl/hnsw_brute_force_env.h" #include "index/refine/refine_utils.h" #include "io/memory_io.h" #include "knowhere/bitsetview_idselector.h" @@ -1335,7 +1336,7 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode { // set up a bf wrapper as fallback std::unique_ptr bf_index_wrapper = nullptr; faiss::Index* bf_index_wrapper_ptr = nullptr; - if (!whether_bf_search.value_or(false)) { + if (!whether_bf_search.value_or(false) && !IsHnswBruteForceDisabledByEnv()) { std::tie(bf_index_wrapper, is_refined) = create_conditional_hnsw_wrapper(indexes[index_id].get(), hnsw_cfg, true, whether_to_enable_refine); if (bf_index_wrapper == nullptr) { @@ -1405,7 +1406,8 @@ class BaseFaissRegularIndexHNSWNode : public BaseFaissRegularIndexNode { real_topk++; } if (real_topk < k && real_topk < bitset.size() - bitset.count() && - bf_index_wrapper_ptr != nullptr && !hnsw_cfg.disable_fallback_brute_force.value()) { + bf_index_wrapper_ptr != nullptr && !IsHnswBruteForceDisabledByEnv() && + !hnsw_cfg.disable_fallback_brute_force.value()) { LOG_KNOWHERE_WARNING_ << "required topk: " << k << ", but the actual num of results got from hnsw: " << real_topk << ", trigger brute force search as fallback for hnsw search"; diff --git a/src/index/hnsw/impl/IndexConditionalWrapper.cc b/src/index/hnsw/impl/IndexConditionalWrapper.cc index 69410cbdf..7f3d5f31e 100644 --- a/src/index/hnsw/impl/IndexConditionalWrapper.cc +++ b/src/index/hnsw/impl/IndexConditionalWrapper.cc @@ -20,6 +20,7 @@ #include "index/hnsw/impl/IndexBruteForceWrapper.h" #include "index/hnsw/impl/IndexHNSWWrapper.h" #include "index/hnsw/impl/IndexWrapperCosine.h" +#include "index/hnsw/impl/hnsw_brute_force_env.h" #include "knowhere/utils.h" #if defined(NOT_COMPILE_FOR_SWIG) && !defined(KNOWHERE_WITH_LIGHT) @@ -38,6 +39,10 @@ WhetherPerformBruteForceSearch(const faiss::Index* index, const BaseConfig& cfg, return std::nullopt; } + if (IsHnswBruteForceDisabledByEnv()) { + return false; + } + // decide const auto k = cfg.k.value(); @@ -71,6 +76,10 @@ WhetherPerformBruteForceRangeSearch(const faiss::Index* index, const FaissHnswCo return std::nullopt; } + if (IsHnswBruteForceDisabledByEnv()) { + return false; + } + // decide const auto ef = cfg.ef.value(); diff --git a/src/index/hnsw/impl/hnsw_brute_force_env.cc b/src/index/hnsw/impl/hnsw_brute_force_env.cc new file mode 100644 index 000000000..76c7426a2 --- /dev/null +++ b/src/index/hnsw/impl/hnsw_brute_force_env.cc @@ -0,0 +1,41 @@ +// Copyright (C) 2019-2024 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#include "index/hnsw/impl/hnsw_brute_force_env.h" + +#include + +#include + +namespace knowhere { + +namespace { + +constexpr const char* kDisableHnswBruteForceEnv = "KNOWHERE_DISABLE_HNSW_BRUTE_FORCE"; + +bool +ParseTruthyEnv(const char* value) { + if (value == nullptr || value[0] == '\0') { + return false; + } + return strcasecmp(value, "1") == 0 || strcasecmp(value, "true") == 0 || strcasecmp(value, "yes") == 0 || + strcasecmp(value, "on") == 0; +} + +} // namespace + +bool +IsHnswBruteForceDisabledByEnv() { + static const bool disabled = ParseTruthyEnv(std::getenv(kDisableHnswBruteForceEnv)); + return disabled; +} + +} // namespace knowhere diff --git a/src/index/hnsw/impl/hnsw_brute_force_env.h b/src/index/hnsw/impl/hnsw_brute_force_env.h new file mode 100644 index 000000000..8617e0632 --- /dev/null +++ b/src/index/hnsw/impl/hnsw_brute_force_env.h @@ -0,0 +1,27 @@ +// Copyright (C) 2019-2024 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +#pragma once + +namespace knowhere { + +// Env: KNOWHERE_DISABLE_HNSW_BRUTE_FORCE +// Truthy values: 1, true, yes, on (case-insensitive). +// +// When enabled, sealed HNSW segments never use IndexBruteForceWrapper: +// - upfront BF for selective filters / large topk +// - post-HNSW fallback when graph search returns too few results +// +// Evaluated once per process on first call (intended for query-node pod env). +bool +IsHnswBruteForceDisabledByEnv(); + +} // namespace knowhere