From 837208a53879a2ca43383b46eece74deb876ae12 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 10 Nov 2026 22:57:23 +0000 Subject: [PATCH 01/10] Use tiled GEMM brute force for half with k <= 64 and grow tile rows for small indexes fusedL2Knn has no tensor-core path, so half inputs with k <= 64 now take tiled_brute_force_knn. Float stays on the fused kernel. When the whole index fits in one column tile, chooseTileSize now grows tileRows up to the existing memory target (capped at numQueries) instead of stopping at 512, so small indexes need fewer GEMM + select_k rounds. Large indexes are unchanged. The per-column-tile merge buffers are only allocated when tiling over columns. Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/faiss_distance_utils.h | 5 ++++- cpp/src/neighbors/detail/knn_brute_force.cuh | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cpp/src/neighbors/detail/faiss_distance_utils.h b/cpp/src/neighbors/detail/faiss_distance_utils.h index 894b5f4bc6..63ff80c5c9 100644 --- a/cpp/src/neighbors/detail/faiss_distance_utils.h +++ b/cpp/src/neighbors/detail/faiss_distance_utils.h @@ -1,6 +1,6 @@ /* * SPDX-FileCopyrightText: Copyright (c) Facebook, Inc. and its affiliates. - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 AND MIT */ @@ -42,6 +42,9 @@ inline void chooseTileSize(size_t numQueries, size_t targetUsage = 512 * 1024 * 1024; if (tileRows * numCentroids * elementSize * 2 <= targetUsage) { tileCols = numCentroids; + // The whole index fits in one column tile, so spend the rest of the target on query rows. + tileRows = + std::min(targetUsage / (2 * elementSize * std::max(numCentroids, 1)), numQueries); } else { // only query total memory in case it potentially impacts tilesize size_t totalMem = rmm::available_device_memory().second; diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index cac660201f..7d60279ac7 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -172,8 +172,10 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - rmm::device_uvector temp_out_distances(tile_rows * temp_out_cols, stream); - rmm::device_uvector temp_out_indices(tile_rows * temp_out_cols, stream); + // only needed to merge the per-column-tile results + size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; + rmm::device_uvector temp_out_distances(temp_out_size, stream); + rmm::device_uvector temp_out_indices(temp_out_size, stream); bool select_min = cuvs::distance::is_min_close(metric); @@ -450,7 +452,9 @@ void brute_force_knn_impl( auto stream = raft::resource::get_next_usable_stream(handle, i); - if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true && + // fusedL2Knn has no tensor-core path, so half is faster through the tiled GEMM below. + if (!std::is_same_v && k <= 64 && rowMajorQuery == rowMajorIndex && + rowMajorQuery == true && (metric == cuvs::distance::DistanceType::L2Unexpanded || metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || metric == cuvs::distance::DistanceType::L2Expanded || From 182cf17d96c245b1d91a53d8c39e884933b6bc83 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 10 Nov 2026 23:05:51 +0000 Subject: [PATCH 02/10] Clarify when the brute-force merge buffers are used Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 7d60279ac7..9471f0dcb8 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -172,7 +172,7 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - // only needed to merge the per-column-tile results + // Only used when the index is split into column tiles, to merge each tile's top-k. size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; rmm::device_uvector temp_out_distances(temp_out_size, stream); rmm::device_uvector temp_out_indices(temp_out_size, stream); From a3694728e690b9d20b9a272a6b3ad3f2ddd85c59 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 10 Nov 2026 23:49:07 +0000 Subject: [PATCH 03/10] Size brute-force tiles against k when the index has fewer than k rows tiled_brute_force_knn raises tile_cols to k after chooseTileSize, so with n < k the row tile sized against n made the distance buffer far larger than the memory target (n=10, k=1000, m=1e6: 4 GB). Pass max(n, k) as the column count so rows are sized against the actual tile width. Unchanged for k <= n. Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 9471f0dcb8..6b435ecfe2 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -91,7 +91,7 @@ void tiled_brute_force_knn(const raft::resources& handle, auto stream = raft::resource::get_cuda_stream(handle); cuvs::neighbors::detail::faiss_select::chooseTileSize( - m, n, d, sizeof(DistanceT), tile_rows, tile_cols); + m, std::max(n, k), d, sizeof(DistanceT), tile_rows, tile_cols); // for unittesting, its convenient to be able to put a max size on the tiles // so we can test the tiling logic without having to use huge inputs. From 27cf1e3b32e368ec9986acdf53d89379e9af2f8c Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 12 Nov 2026 05:02:36 +0000 Subject: [PATCH 04/10] Score expanded-L2 brute-force tiles with a cuBLASLt bias epilogue For L2Expanded and L2SqrtExpanded over float or half rows (FP32 distances, no custom epilogue), each tile is now one cuBLASLt matmul that writes |x|^2 - 2 q.x directly: alpha = -2 scales the dot products and the bias epilogue adds the FP32 index norms. This replaces the inner-product GEMM plus the pass that turned every dot product into a distance. select_k ranks the scores as they are, since |q|^2 is constant along a row, and only the k kept per row get |q|^2 added, clamped and rooted as l2_exp_cutlass_op does. The matmul is planned per tile shape and per operand alignment (tiles start at arbitrary rows), cached through a custom resource, and falls back to the previous path if cuBLASLt has no algorithm for it. Float stays plain FP32 (no TF32); half runs on the tensor cores. H100, brute_force::search vs the parent commit: half 1.3-1.66x faster at 256-4096 queries over 8k-262k rows, float with k = 100 up to 1.28x, results identical up to ties. Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 73 +++++-- .../neighbors/detail/l2_expanded_scores.cuh | 189 ++++++++++++++++++ 2 files changed, 248 insertions(+), 14 deletions(-) create mode 100644 cpp/src/neighbors/detail/l2_expanded_scores.cuh diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 6b435ecfe2..79851977d0 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -15,6 +15,7 @@ #include "./fused_l2_knn.cuh" #include "./haversine_distance.cuh" #include "./knn_utils.cuh" +#include "./l2_expanded_scores.cuh" #include #include @@ -179,6 +180,14 @@ void tiled_brute_force_knn(const raft::resources& handle, bool select_min = cuvs::distance::is_min_close(metric); + // Expanded L2 ranks a query's rows by |x|^2 - 2 q.x, which one cuBLASLt matmul writes with the + // index norms added in its epilogue (l2_expanded_scores), so no pass over the tile turns dot + // products into distances. |q|^2 is added after the selection, to the k kept per row. + constexpr bool l2_scores_supported = + std::is_same_v && + (std::is_same_v || std::is_same_v) && + std::is_same_v; + for (size_t i = 0; i < m; i += tile_rows) { size_t current_query_size = std::min(tile_rows, m - i); @@ -186,21 +195,38 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t current_centroid_size = std::min(tile_cols, n - j); size_t current_k = std::min(current_centroid_size, k); - // calculate the top-k elements for the current tile, by calculating the - // full pairwise distance for the tile - and then selecting the top-k from that - cuvs::distance::pairwise_distance( - handle, - raft::make_device_matrix_view( - search + i * d, current_query_size, d), - raft::make_device_matrix_view( - index + j * d, current_centroid_size, d), - raft::make_device_matrix_view( - temp_distances.data(), current_query_size, current_centroid_size), - pairwise_metric, - metric_arg); + bool l2_scored = false; + if constexpr (l2_scores_supported) { + if (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded) { + auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); + l2_scored = l2_expanded_scores(handle, + search + i * d, + index + j * d, + current_query_size, + current_centroid_size, + d, + col_norms + j, + temp_distances.data()); + } + } + if (!l2_scored) { + // calculate the top-k elements for the current tile, by calculating the + // full pairwise distance for the tile - and then selecting the top-k from that + cuvs::distance::pairwise_distance( + handle, + raft::make_device_matrix_view( + search + i * d, current_query_size, d), + raft::make_device_matrix_view( + index + j * d, current_centroid_size, d), + raft::make_device_matrix_view( + temp_distances.data(), current_query_size, current_centroid_size), + pairwise_metric, + metric_arg); + } - if (metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded) { + if (!l2_scored && (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { auto row_norms = precomputed_search_norms ? precomputed_search_norms : search_norms.data(); auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); auto dist = temp_distances.data(); @@ -284,6 +310,25 @@ void tiled_brute_force_knn(const raft::resources& handle, select_min, true); + if constexpr (l2_scores_supported) { + if (l2_scored && current_k > 0) { + // |q|^2 + (|x|^2 - 2 q.x) for the k kept per row, clamped and rooted as + // l2_exp_cutlass_op. + auto row_norms = + precomputed_search_norms ? precomputed_search_norms : search_norms.data(); + auto kept = distances + i * k; + bool sqrt = metric == cuvs::distance::DistanceType::L2SqrtExpanded; + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(kept, current_query_size * current_k), + [=] __device__(IndexType idx) { + DistanceT val = row_norms[i + idx / current_k] + kept[idx]; + val = val < DistanceT{0} ? DistanceT{0} : val; + return sqrt ? raft::sqrt(val) : val; + }); + } + } + // if we're tiling over columns, we need to do a couple things to fix up // the output of select_k // 1. The column id's in the output are relative to the tile, so we need diff --git a/cpp/src/neighbors/detail/l2_expanded_scores.cuh b/cpp/src/neighbors/detail/l2_expanded_scores.cuh new file mode 100644 index 0000000000..f2cffa3136 --- /dev/null +++ b/cpp/src/neighbors/detail/l2_expanded_scores.cuh @@ -0,0 +1,189 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace cuvs::neighbors::detail { + +/** The largest power of two (at most 256) dividing the address: what cuBLASLt may assume of it. */ +inline auto l2_scores_alignment(const void* ptr) -> uint32_t +{ + auto const address = reinterpret_cast(ptr); + uint32_t alignment = 256; + while (alignment > 1 && address % alignment != 0) { + alignment /= 2; + } + return alignment; +} + +struct l2_scores_key { + uint64_t m; + uint64_t n; + uint64_t d; + uint32_t search_alignment; + uint32_t index_alignment; + uint32_t scores_alignment; + + auto operator==(const l2_scores_key&) const -> bool = default; +}; + +struct l2_scores_key_hash { + auto operator()(const l2_scores_key& x) const noexcept -> std::size_t + { + return x.m * x.n * x.d + x.search_alignment + x.index_alignment * 512 + + x.scores_alignment * 512 * 512; + } +}; + +/** A matmul with the bias epilogue, and the algorithm cuBLASLt chose for it (if any). */ +struct l2_scores_plan { + raft::linalg::detail::cublastlt_matmul_desc desc; + raft::linalg::detail::cublastlt_matrix_layout index_layout; + raft::linalg::detail::cublastlt_matrix_layout search_layout; + raft::linalg::detail::cublastlt_matrix_layout scores_layout; + cublasLtMatmulHeuristicResult_t heuristics{}; + bool supported = false; +}; + +template +struct l2_scores_cache { + raft::cache:: + lru, std::shared_ptr> + value{}; +}; + +template +auto make_l2_scores_plan(raft::resources const& handle, const l2_scores_key& key) -> l2_scores_plan +{ + namespace lt = raft::linalg::detail; + // Column-major, the [m x n] row-major scores are [n x m] = index [n x d] * search^T: the index + // rows are the rows of the result, so the bias epilogue adds one norm per index row. + l2_scores_plan plan{ + lt::cublastlt_matmul_desc::for_matmul(/*transpose_a=*/true, + /*transpose_b=*/false), + lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), key.d, key.n, key.d}, + lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), key.d, key.m, key.d}, + lt::cublastlt_matrix_layout{CUDA_R_32F, key.n, key.m, key.n}}; + const cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_BIAS; + RAFT_CUBLAS_TRY(cublasLtMatmulDescSetAttribute( + plan.desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); + const cudaDataType_t bias_type = CUDA_R_32F; + RAFT_CUBLAS_TRY(cublasLtMatmulDescSetAttribute( + plan.desc, CUBLASLT_MATMUL_DESC_BIAS_DATA_TYPE, &bias_type, sizeof(bias_type))); + + // Tiles start at arbitrary rows, so the operands are only as aligned as their addresses say. + lt::cublastlt_matmul_preference preference; + auto const set_alignment = [&](cublasLtMatmulPreferenceAttributes_t attribute, uint32_t bytes) { + RAFT_CUBLAS_TRY( + cublasLtMatmulPreferenceSetAttribute(preference, attribute, &bytes, sizeof(bytes))); + }; + set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_A_BYTES, key.index_alignment); + set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_B_BYTES, key.search_alignment); + set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_C_BYTES, key.scores_alignment); + set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_D_BYTES, key.scores_alignment); + + // The FP32 matmul in RAFT steers cuBLASLt 13.6.0 off a faulty algorithm; this one does the same. + const lt::matmul_key_t args{key.n, key.m, key.d, key.d, key.d, key.n, true, false}; + bool const workaround = + std::is_same_v && lt::needs_cublaslt_13_6_workaround(args, cublasLtGetVersion()); + auto const heuristic_index = + workaround ? lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), + key.d, + key.n, + lt::get_cublaslt_13_6_heuristic_args(args).lda} + : lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), key.d, key.n, key.d}; + int algo_count = 0; + RAFT_CUBLAS_TRY(cublasLtMatmulAlgoGetHeuristic(raft::resource::get_cublaslt_handle(handle), + plan.desc, + heuristic_index, + plan.search_layout, + plan.scores_layout, + plan.scores_layout, + preference, + 1, + &plan.heuristics, + &algo_count)); + constexpr int faulty_algorithm = 68; + plan.supported = + algo_count > 0 && + !(workaround && lt::get_cublaslt_algorithm_id(plan.heuristics) == faulty_algorithm); + return plan; +} + +/** + * @brief Scores a tile for expanded L2 ranking in one cuBLASLt matmul: + * scores(i, j) = |index_j|^2 - 2 search_i . index_j, row-major [m x n]. + * + * alpha = -2 scales the dot products and the bias epilogue adds the index norms, so no pass over + * the tile turns dot products into distances. |search_i|^2 is the same along a row and does not + * change its ranking; the caller adds it to the rows it selects. Inputs are multiplied as given + * (FP16 on the tensor cores, FP32 on CUDA cores; no TF32) and accumulated in FP32. + * + * @return false, leaving @p scores untouched, if cuBLASLt has no algorithm for this matmul. + */ +template +auto l2_expanded_scores(raft::resources const& handle, + const T* search, + const T* index, + uint64_t m, + uint64_t n, + uint64_t d, + const float* index_norms, + float* scores) -> bool +{ + static_assert(std::is_same_v || std::is_same_v); + if (raft::resource::get_dry_run_flag(handle)) { return true; } + const l2_scores_key key{ + m, n, d, l2_scores_alignment(search), l2_scores_alignment(index), l2_scores_alignment(scores)}; + auto& cache = raft::resource::get_custom_resource>(handle)->value; + std::shared_ptr plan; + if (!cache.get(key, &plan)) { + plan = std::make_shared(make_l2_scores_plan(handle, key)); + cache.set(key, plan); + } + if (!plan->supported) { return false; } + + RAFT_CUBLAS_TRY(cublasLtMatmulDescSetAttribute( + plan->desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &index_norms, sizeof(index_norms))); + const float alpha = -2.0f; + const float beta = 0.0f; + RAFT_CUBLAS_TRY(cublasLtMatmul(raft::resource::get_cublaslt_handle(handle), + plan->desc, + &alpha, + index, + plan->index_layout, + search, + plan->search_layout, + &beta, + scores, + plan->scores_layout, + scores, + plan->scores_layout, + &plan->heuristics.algo, + nullptr, + 0, + raft::resource::get_cuda_stream(handle).get())); + return true; +} + +} // namespace cuvs::neighbors::detail From df06c958ecd17c76f502b261ea9841d4a8297a47 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 12 Nov 2026 05:52:25 +0000 Subject: [PATCH 05/10] Use raft::linalg::gemm's bias epilogue instead of calling cuBLASLt The tile's |x|^2 - 2 q.x now comes from raft::linalg::gemm (column-major: index * search^T, alpha = -2) with the index norms as its bias, which NVIDIA/raft#3205 adds. This drops l2_expanded_scores.cuh and its direct cuBLASLt calls, descriptor cache and alignment handling. Same arithmetic, tests and timings as before. Requires NVIDIA/raft#3205. Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 37 ++-- .../neighbors/detail/l2_expanded_scores.cuh | 189 ------------------ 2 files changed, 25 insertions(+), 201 deletions(-) delete mode 100644 cpp/src/neighbors/detail/l2_expanded_scores.cuh diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 79851977d0..2e58c6b8a9 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -15,7 +15,6 @@ #include "./fused_l2_knn.cuh" #include "./haversine_distance.cuh" #include "./knn_utils.cuh" -#include "./l2_expanded_scores.cuh" #include #include @@ -26,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -180,9 +180,9 @@ void tiled_brute_force_knn(const raft::resources& handle, bool select_min = cuvs::distance::is_min_close(metric); - // Expanded L2 ranks a query's rows by |x|^2 - 2 q.x, which one cuBLASLt matmul writes with the - // index norms added in its epilogue (l2_expanded_scores), so no pass over the tile turns dot - // products into distances. |q|^2 is added after the selection, to the k kept per row. + // Expanded L2 ranks a query's rows by |x|^2 - 2 q.x, which one GEMM writes with the index norms + // added in its epilogue (raft::linalg::gemm's bias), so no pass over the tile turns dot products + // into distances. |q|^2 is added after the selection, to the k kept per row. constexpr bool l2_scores_supported = std::is_same_v && (std::is_same_v || std::is_same_v) && @@ -199,15 +199,28 @@ void tiled_brute_force_knn(const raft::resources& handle, if constexpr (l2_scores_supported) { if (metric == cuvs::distance::DistanceType::L2Expanded || metric == cuvs::distance::DistanceType::L2SqrtExpanded) { + // Column-major, the row-major [queries x index rows] tile is index * search^T, and the + // bias adds one norm per index row. auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); - l2_scored = l2_expanded_scores(handle, - search + i * d, - index + j * d, - current_query_size, - current_centroid_size, - d, - col_norms + j, - temp_distances.data()); + DistanceT const alpha = -2; + DistanceT const beta = 0; + raft::linalg::gemm(handle, + /*trans_a=*/true, + /*trans_b=*/false, + static_cast(current_centroid_size), + static_cast(current_query_size), + static_cast(d), + &alpha, + index + j * d, + static_cast(d), + search + i * d, + static_cast(d), + &beta, + temp_distances.data(), + static_cast(current_centroid_size), + stream.get(), + col_norms + j); + l2_scored = true; } } if (!l2_scored) { diff --git a/cpp/src/neighbors/detail/l2_expanded_scores.cuh b/cpp/src/neighbors/detail/l2_expanded_scores.cuh deleted file mode 100644 index f2cffa3136..0000000000 --- a/cpp/src/neighbors/detail/l2_expanded_scores.cuh +++ /dev/null @@ -1,189 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include - -namespace cuvs::neighbors::detail { - -/** The largest power of two (at most 256) dividing the address: what cuBLASLt may assume of it. */ -inline auto l2_scores_alignment(const void* ptr) -> uint32_t -{ - auto const address = reinterpret_cast(ptr); - uint32_t alignment = 256; - while (alignment > 1 && address % alignment != 0) { - alignment /= 2; - } - return alignment; -} - -struct l2_scores_key { - uint64_t m; - uint64_t n; - uint64_t d; - uint32_t search_alignment; - uint32_t index_alignment; - uint32_t scores_alignment; - - auto operator==(const l2_scores_key&) const -> bool = default; -}; - -struct l2_scores_key_hash { - auto operator()(const l2_scores_key& x) const noexcept -> std::size_t - { - return x.m * x.n * x.d + x.search_alignment + x.index_alignment * 512 + - x.scores_alignment * 512 * 512; - } -}; - -/** A matmul with the bias epilogue, and the algorithm cuBLASLt chose for it (if any). */ -struct l2_scores_plan { - raft::linalg::detail::cublastlt_matmul_desc desc; - raft::linalg::detail::cublastlt_matrix_layout index_layout; - raft::linalg::detail::cublastlt_matrix_layout search_layout; - raft::linalg::detail::cublastlt_matrix_layout scores_layout; - cublasLtMatmulHeuristicResult_t heuristics{}; - bool supported = false; -}; - -template -struct l2_scores_cache { - raft::cache:: - lru, std::shared_ptr> - value{}; -}; - -template -auto make_l2_scores_plan(raft::resources const& handle, const l2_scores_key& key) -> l2_scores_plan -{ - namespace lt = raft::linalg::detail; - // Column-major, the [m x n] row-major scores are [n x m] = index [n x d] * search^T: the index - // rows are the rows of the result, so the bias epilogue adds one norm per index row. - l2_scores_plan plan{ - lt::cublastlt_matmul_desc::for_matmul(/*transpose_a=*/true, - /*transpose_b=*/false), - lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), key.d, key.n, key.d}, - lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), key.d, key.m, key.d}, - lt::cublastlt_matrix_layout{CUDA_R_32F, key.n, key.m, key.n}}; - const cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_BIAS; - RAFT_CUBLAS_TRY(cublasLtMatmulDescSetAttribute( - plan.desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue))); - const cudaDataType_t bias_type = CUDA_R_32F; - RAFT_CUBLAS_TRY(cublasLtMatmulDescSetAttribute( - plan.desc, CUBLASLT_MATMUL_DESC_BIAS_DATA_TYPE, &bias_type, sizeof(bias_type))); - - // Tiles start at arbitrary rows, so the operands are only as aligned as their addresses say. - lt::cublastlt_matmul_preference preference; - auto const set_alignment = [&](cublasLtMatmulPreferenceAttributes_t attribute, uint32_t bytes) { - RAFT_CUBLAS_TRY( - cublasLtMatmulPreferenceSetAttribute(preference, attribute, &bytes, sizeof(bytes))); - }; - set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_A_BYTES, key.index_alignment); - set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_B_BYTES, key.search_alignment); - set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_C_BYTES, key.scores_alignment); - set_alignment(CUBLASLT_MATMUL_PREF_MIN_ALIGNMENT_D_BYTES, key.scores_alignment); - - // The FP32 matmul in RAFT steers cuBLASLt 13.6.0 off a faulty algorithm; this one does the same. - const lt::matmul_key_t args{key.n, key.m, key.d, key.d, key.d, key.n, true, false}; - bool const workaround = - std::is_same_v && lt::needs_cublaslt_13_6_workaround(args, cublasLtGetVersion()); - auto const heuristic_index = - workaround ? lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), - key.d, - key.n, - lt::get_cublaslt_13_6_heuristic_args(args).lda} - : lt::cublastlt_matrix_layout{raft::get_cuda_data_type(), key.d, key.n, key.d}; - int algo_count = 0; - RAFT_CUBLAS_TRY(cublasLtMatmulAlgoGetHeuristic(raft::resource::get_cublaslt_handle(handle), - plan.desc, - heuristic_index, - plan.search_layout, - plan.scores_layout, - plan.scores_layout, - preference, - 1, - &plan.heuristics, - &algo_count)); - constexpr int faulty_algorithm = 68; - plan.supported = - algo_count > 0 && - !(workaround && lt::get_cublaslt_algorithm_id(plan.heuristics) == faulty_algorithm); - return plan; -} - -/** - * @brief Scores a tile for expanded L2 ranking in one cuBLASLt matmul: - * scores(i, j) = |index_j|^2 - 2 search_i . index_j, row-major [m x n]. - * - * alpha = -2 scales the dot products and the bias epilogue adds the index norms, so no pass over - * the tile turns dot products into distances. |search_i|^2 is the same along a row and does not - * change its ranking; the caller adds it to the rows it selects. Inputs are multiplied as given - * (FP16 on the tensor cores, FP32 on CUDA cores; no TF32) and accumulated in FP32. - * - * @return false, leaving @p scores untouched, if cuBLASLt has no algorithm for this matmul. - */ -template -auto l2_expanded_scores(raft::resources const& handle, - const T* search, - const T* index, - uint64_t m, - uint64_t n, - uint64_t d, - const float* index_norms, - float* scores) -> bool -{ - static_assert(std::is_same_v || std::is_same_v); - if (raft::resource::get_dry_run_flag(handle)) { return true; } - const l2_scores_key key{ - m, n, d, l2_scores_alignment(search), l2_scores_alignment(index), l2_scores_alignment(scores)}; - auto& cache = raft::resource::get_custom_resource>(handle)->value; - std::shared_ptr plan; - if (!cache.get(key, &plan)) { - plan = std::make_shared(make_l2_scores_plan(handle, key)); - cache.set(key, plan); - } - if (!plan->supported) { return false; } - - RAFT_CUBLAS_TRY(cublasLtMatmulDescSetAttribute( - plan->desc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &index_norms, sizeof(index_norms))); - const float alpha = -2.0f; - const float beta = 0.0f; - RAFT_CUBLAS_TRY(cublasLtMatmul(raft::resource::get_cublaslt_handle(handle), - plan->desc, - &alpha, - index, - plan->index_layout, - search, - plan->search_layout, - &beta, - scores, - plan->scores_layout, - scores, - plan->scores_layout, - &plan->heuristics.algo, - nullptr, - 0, - raft::resource::get_cuda_stream(handle).get())); - return true; -} - -} // namespace cuvs::neighbors::detail From 5634bffa15973a5a049428232d8d60714dc5490d Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Fri, 13 Nov 2026 00:02:12 +0000 Subject: [PATCH 06/10] Keep the L2 GEMM's bias and tile rows 16-byte aligned cuBLASLt runs slower kernels when the bias (the index norms at norms + j) or the rows of the distance tile aren't 16-byte aligned. When the index is split into column tiles, each tile now spans a multiple of 4 index rows, and precomputed index norms that aren't 16-byte aligned, such as a slice of a larger norm vector, are copied once per search into an aligned buffer. Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 35 +++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 2e58c6b8a9..4f3299b52d 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -102,6 +102,20 @@ void tiled_brute_force_knn(const raft::resources& handle, // tile_cols must be at least k items tile_cols = std::max(tile_cols, k); + // Expanded L2 ranks a query's rows by |x|^2 - 2 q.x, which one GEMM writes with the index norms + // added in its epilogue (raft::linalg::gemm's bias), so no pass over the tile turns dot products + // into distances. |q|^2 is added after the selection, to the k kept per row. + constexpr bool l2_scores_supported = + std::is_same_v && + (std::is_same_v || std::is_same_v) && + std::is_same_v; + bool const l2_scored = + l2_scores_supported && (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded); + // cuBLASLt's fast kernels need the tile's rows and the bias (the norms at norms + j) 16-byte + // aligned, so column tiles span multiples of 4 index rows. + if (l2_scored && tile_cols < n) { tile_cols = std::max(tile_cols - tile_cols % 4, k); } + // stores pairwise distances for the current tile rmm::device_uvector temp_distances(tile_rows * tile_cols, stream); @@ -149,6 +163,14 @@ void tiled_brute_force_knn(const raft::resources& handle, } pairwise_metric = cuvs::distance::DistanceType::InnerProduct; } + // Precomputed norms that aren't 16-byte aligned, such as a slice of a larger vector, would make + // every tile's bias misaligned, so they are copied to an aligned buffer once per search. + if (l2_scored && precomputed_index_norms != nullptr && + reinterpret_cast(precomputed_index_norms) % 16 != 0) { + index_norms.resize(n, stream); + raft::copy(index_norms.data(), precomputed_index_norms, n, stream); + precomputed_index_norms = index_norms.data(); + } // if we're tiling over columns, we need additional buffers for temporary output // distances/indices @@ -180,14 +202,6 @@ void tiled_brute_force_knn(const raft::resources& handle, bool select_min = cuvs::distance::is_min_close(metric); - // Expanded L2 ranks a query's rows by |x|^2 - 2 q.x, which one GEMM writes with the index norms - // added in its epilogue (raft::linalg::gemm's bias), so no pass over the tile turns dot products - // into distances. |q|^2 is added after the selection, to the k kept per row. - constexpr bool l2_scores_supported = - std::is_same_v && - (std::is_same_v || std::is_same_v) && - std::is_same_v; - for (size_t i = 0; i < m; i += tile_rows) { size_t current_query_size = std::min(tile_rows, m - i); @@ -195,10 +209,8 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t current_centroid_size = std::min(tile_cols, n - j); size_t current_k = std::min(current_centroid_size, k); - bool l2_scored = false; if constexpr (l2_scores_supported) { - if (metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded) { + if (l2_scored) { // Column-major, the row-major [queries x index rows] tile is index * search^T, and the // bias adds one norm per index row. auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); @@ -220,7 +232,6 @@ void tiled_brute_force_knn(const raft::resources& handle, static_cast(current_centroid_size), stream.get(), col_norms + j); - l2_scored = true; } } if (!l2_scored) { From 9f09e8cf83c8592f67c1f03c27bcc090bb2e8c2a Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 19 Nov 2026 02:04:51 +0000 Subject: [PATCH 07/10] Pad brute-force results when the index has fewer than k rows With n < k the tiled path wrote the n-wide preliminary selection packed into the k-wide output and then asked select_k for k results from n inputs. Copy the already sorted neighbors to the front of each row and fill the remaining slots instead, and add a multi-query half test for this case. Co-Authored-By: Claude Fable 5.2 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 37 +++++++---- cpp/tests/neighbors/brute_force.cu | 64 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 6b435ecfe2..456a5e51ce 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -160,18 +160,6 @@ void tiled_brute_force_knn(const raft::resources& handle, auto last_col_tile_size = n % tile_cols; if (last_col_tile_size && (last_col_tile_size < k)) { temp_out_cols -= k - last_col_tile_size; } - // if we have less than k items in the index, we should fill out the result - // to indicate that we are missing items (and match behaviour in faiss) - if (n < k) { - raft::matrix::fill(handle, - raft::make_device_matrix_view(distances, m, k), - std::numeric_limits::lowest()); - - if constexpr (std::is_signed_v) { - raft::matrix::fill(handle, raft::make_device_matrix_view(indices, m, k), IndexType{-1}); - } - } - // Only used when the index is split into column tiles, to merge each tile's top-k. size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; rmm::device_uvector temp_out_distances(temp_out_size, stream); @@ -312,7 +300,30 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - if (tile_cols != n) { + if (n < k) { + // if we have less than k items in the index, the single column tile already holds all n + // neighbors in sorted order. Copy them to the front of each k-wide output row and fill out + // the rest to indicate that we are missing items (and match behaviour in faiss) + const DistanceT* in_distances = temp_out_distances.data(); + const IndexType* in_indices = temp_out_indices.data(); + DistanceT* out_distances = distances + i * k; + IndexType* out_indices = indices + i * k; + + auto count = cuda::make_counting_iterator(0); + thrust::for_each(raft::resource::get_thrust_policy(handle), + count, + count + current_query_size * k, + [=] __device__(IndexType idx) { + size_t row = idx / k, col = idx % k; + if (col < n) { + out_distances[idx] = in_distances[row * n + col]; + out_indices[idx] = in_indices[row * n + col]; + } else { + out_distances[idx] = std::numeric_limits::lowest(); + if constexpr (std::is_signed_v) { out_indices[idx] = -1; } + } + }); + } else if (tile_cols != n) { // select the actual top-k items here from the temporary output cuvs::selection::select_k( handle, diff --git a/cpp/tests/neighbors/brute_force.cu b/cpp/tests/neighbors/brute_force.cu index 1c09b9a739..f6c8dd4fac 100644 --- a/cpp/tests/neighbors/brute_force.cu +++ b/cpp/tests/neighbors/brute_force.cu @@ -19,6 +19,8 @@ #include #include +#include +#include #include namespace cuvs::neighbors::brute_force { @@ -273,6 +275,68 @@ TEST(BruteForceEqualNorms, PreservesNeighborOrdering) } } +TEST(BruteForceSmallIndex, PadsMissingNeighbors) +{ + raft::resources handle; + auto stream = raft::resource::get_cuda_stream(handle); + constexpr int64_t n_database = 4; + constexpr int64_t n_queries = 5; + constexpr int64_t k = 6; + + // 1-d points, so the expected neighbors of each query are unambiguous. + std::vector database_x{0.0f, 1.0f, 2.0f, 3.0f}; + std::vector queries_x{0.125f, 1.125f, 2.125f, 3.125f, -1.0f}; + std::vector h_database(n_database); + std::vector h_queries(n_queries); + std::ranges::transform(database_x, h_database.begin(), [](float v) { return __float2half(v); }); + std::ranges::transform(queries_x, h_queries.begin(), [](float v) { return __float2half(v); }); + + auto database = raft::make_device_matrix(handle, n_database, 1); + auto queries = raft::make_device_matrix(handle, n_queries, 1); + raft::update_device(database.data_handle(), h_database.data(), h_database.size(), stream); + raft::update_device(queries.data_handle(), h_queries.data(), h_queries.size(), stream); + + index_params index_params; + index_params.metric = cuvs::distance::DistanceType::L2Expanded; + auto index = build(handle, index_params, raft::make_const_mdspan(database.view())); + + auto neighbors = raft::make_device_matrix(handle, n_queries, k); + auto distances = raft::make_device_matrix(handle, n_queries, k); + search(handle, + search_params{}, + index, + raft::make_const_mdspan(queries.view()), + neighbors.view(), + distances.view(), + filtering::none_sample_filter{}); + + std::vector h_neighbors(n_queries * k); + std::vector h_distances(n_queries * k); + raft::update_host(h_neighbors.data(), neighbors.data_handle(), h_neighbors.size(), stream); + raft::update_host(h_distances.data(), distances.data_handle(), h_distances.size(), stream); + raft::resource::sync_stream(handle, stream); + + for (int64_t q = 0; q < n_queries; ++q) { + std::vector expected(n_database); + std::iota(expected.begin(), expected.end(), int64_t{0}); + std::ranges::sort(expected, [&](int64_t lhs, int64_t rhs) -> bool { + return std::abs(queries_x[q] - database_x[lhs]) < std::abs(queries_x[q] - database_x[rhs]); + }); + + for (int64_t i = 0; i < k; ++i) { + if (i < n_database) { + float diff = queries_x[q] - database_x[expected[i]]; + EXPECT_EQ(h_neighbors[q * k + i], expected[i]) << "query=" << q << " rank=" << i; + EXPECT_NEAR(h_distances[q * k + i], diff * diff, 1e-4) << "query=" << q << " rank=" << i; + } else { + EXPECT_EQ(h_neighbors[q * k + i], -1) << "query=" << q << " rank=" << i; + EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::lowest()) + << "query=" << q << " rank=" << i; + } + } + } +} + // Also test with larger random inputs, including col-major inputs struct RandomKNNInputs { int num_queries; From a4e839d43933203d726c5eb6c33af4a687e27969 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 19 Nov 2026 02:20:07 +0000 Subject: [PATCH 08/10] Use raft::linalg::map_offset to pad brute-force results for small indexes Co-Authored-By: Claude Fable 5.2 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 32 +++++++++----------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 456a5e51ce..f3e0715bcd 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -306,23 +306,21 @@ void tiled_brute_force_knn(const raft::resources& handle, // the rest to indicate that we are missing items (and match behaviour in faiss) const DistanceT* in_distances = temp_out_distances.data(); const IndexType* in_indices = temp_out_indices.data(); - DistanceT* out_distances = distances + i * k; - IndexType* out_indices = indices + i * k; - - auto count = cuda::make_counting_iterator(0); - thrust::for_each(raft::resource::get_thrust_policy(handle), - count, - count + current_query_size * k, - [=] __device__(IndexType idx) { - size_t row = idx / k, col = idx % k; - if (col < n) { - out_distances[idx] = in_distances[row * n + col]; - out_indices[idx] = in_indices[row * n + col]; - } else { - out_distances[idx] = std::numeric_limits::lowest(); - if constexpr (std::is_signed_v) { out_indices[idx] = -1; } - } - }); + + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(distances + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_distances[row * n + col] : std::numeric_limits::lowest(); + }); + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(indices + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_indices[row * n + col] : static_cast(-1); + }); } else if (tile_cols != n) { // select the actual top-k items here from the temporary output cuvs::selection::select_k( From 5ddf3b6c9a7c27302bd2b05c6183f201a2ea92c9 Mon Sep 17 00:00:00 2001 From: Tarang Jain <40517122+tarang-jain@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:25:39 -0800 Subject: [PATCH 09/10] Clarify comment on handling fewer neighbors than k --- cpp/src/neighbors/detail/knn_brute_force.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index f3e0715bcd..b9829d21cb 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -303,7 +303,7 @@ void tiled_brute_force_knn(const raft::resources& handle, if (n < k) { // if we have less than k items in the index, the single column tile already holds all n // neighbors in sorted order. Copy them to the front of each k-wide output row and fill out - // the rest to indicate that we are missing items (and match behaviour in faiss) + // the rest to indicate that we are missing items const DistanceT* in_distances = temp_out_distances.data(); const IndexType* in_indices = temp_out_indices.data(); From 88547e1aedc0595bad281f7b3a44183670f3f94c Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 19 Nov 2026 03:42:18 +0000 Subject: [PATCH 10/10] Keep half unexpanded L2 on fusedL2Knn and budget select_k's workspace - Only half L2Expanded/L2SqrtExpanded searches go through the tiled GEMM; the unexpanded metrics have no GEMM and stay on fusedL2Knn. - chooseTileSize counts select_k's per-row workspace when it grows the row tile, so the distance tile and the selection share the target. - Missing neighbors are padded with the worst distance for the metric (max() when selecting the smallest), as fusedL2Knn does. Co-Authored-By: Claude Fable 5.2 --- .../neighbors/detail/faiss_distance_utils.h | 6 +++-- cpp/src/neighbors/detail/knn_brute_force.cuh | 25 +++++++++++++------ cpp/tests/neighbors/brute_force.cu | 2 +- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cpp/src/neighbors/detail/faiss_distance_utils.h b/cpp/src/neighbors/detail/faiss_distance_utils.h index 63ff80c5c9..71771b2148 100644 --- a/cpp/src/neighbors/detail/faiss_distance_utils.h +++ b/cpp/src/neighbors/detail/faiss_distance_utils.h @@ -20,6 +20,7 @@ inline void chooseTileSize(size_t numQueries, size_t numCentroids, size_t dim, size_t elementSize, + size_t rowWorkspaceSize, size_t& tileRows, size_t& tileCols) { @@ -43,8 +44,9 @@ inline void chooseTileSize(size_t numQueries, if (tileRows * numCentroids * elementSize * 2 <= targetUsage) { tileCols = numCentroids; // The whole index fits in one column tile, so spend the rest of the target on query rows. - tileRows = - std::min(targetUsage / (2 * elementSize * std::max(numCentroids, 1)), numQueries); + // Each row also takes rowWorkspaceSize bytes of temporary memory after the distance tile. + size_t rowSize = elementSize * std::max(numCentroids, 1) + rowWorkspaceSize; + tileRows = std::min(std::max(tileRows, targetUsage / (2 * rowSize)), numQueries); } else { // only query total memory in case it potentially impacts tilesize size_t totalMem = rmm::available_device_memory().second; diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index b9829d21cb..c87cfd4f38 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -90,8 +90,14 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t tile_cols = 0; auto stream = raft::resource::get_cuda_stream(handle); - cuvs::neighbors::detail::faiss_select::chooseTileSize( - m, std::max(n, k), d, sizeof(DistanceT), tile_rows, tile_cols); + // select_k keeps k distances and indices per query row in its workspace + cuvs::neighbors::detail::faiss_select::chooseTileSize(m, + std::max(n, k), + d, + sizeof(DistanceT), + k * (sizeof(DistanceT) + sizeof(IndexType)), + tile_rows, + tile_cols); // for unittesting, its convenient to be able to put a max size on the tiles // so we can test the tiling logic without having to use huge inputs. @@ -306,13 +312,15 @@ void tiled_brute_force_knn(const raft::resources& handle, // the rest to indicate that we are missing items const DistanceT* in_distances = temp_out_distances.data(); const IndexType* in_indices = temp_out_indices.data(); + DistanceT missing_distance = select_min ? std::numeric_limits::max() + : std::numeric_limits::lowest(); raft::linalg::map_offset( handle, raft::make_device_vector_view(distances + i * k, current_query_size * k), [=] __device__(size_t idx) { size_t row = idx / k, col = idx % k; - return col < n ? in_distances[row * n + col] : std::numeric_limits::lowest(); + return col < n ? in_distances[row * n + col] : missing_distance; }); raft::linalg::map_offset( handle, @@ -461,13 +469,14 @@ void brute_force_knn_impl( auto stream = raft::resource::get_next_usable_stream(handle, i); - // fusedL2Knn has no tensor-core path, so half is faster through the tiled GEMM below. - if (!std::is_same_v && k <= 64 && rowMajorQuery == rowMajorIndex && - rowMajorQuery == true && + // fusedL2Knn has no tensor-core path, so half expanded L2 is faster through the tiled GEMM + // below. + if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true && (metric == cuvs::distance::DistanceType::L2Unexpanded || metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || - metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { + (!std::is_same_v && + (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded)))) { fusedL2Knn(D, out_i_ptr, out_d_ptr, diff --git a/cpp/tests/neighbors/brute_force.cu b/cpp/tests/neighbors/brute_force.cu index f6c8dd4fac..7d5e9d648d 100644 --- a/cpp/tests/neighbors/brute_force.cu +++ b/cpp/tests/neighbors/brute_force.cu @@ -330,7 +330,7 @@ TEST(BruteForceSmallIndex, PadsMissingNeighbors) EXPECT_NEAR(h_distances[q * k + i], diff * diff, 1e-4) << "query=" << q << " rank=" << i; } else { EXPECT_EQ(h_neighbors[q * k + i], -1) << "query=" << q << " rank=" << i; - EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::lowest()) + EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::max()) << "query=" << q << " rank=" << i; } }