From 837208a53879a2ca43383b46eece74deb876ae12 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 10 Nov 2026 22:57:23 +0000 Subject: [PATCH 1/7] Use tiled GEMM brute force for half with k <= 64 and grow tile rows for small indexes fusedL2Knn has no tensor-core path, so half inputs with k <= 64 now take tiled_brute_force_knn. Float stays on the fused kernel. When the whole index fits in one column tile, chooseTileSize now grows tileRows up to the existing memory target (capped at numQueries) instead of stopping at 512, so small indexes need fewer GEMM + select_k rounds. Large indexes are unchanged. The per-column-tile merge buffers are only allocated when tiling over columns. Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/faiss_distance_utils.h | 5 ++++- cpp/src/neighbors/detail/knn_brute_force.cuh | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cpp/src/neighbors/detail/faiss_distance_utils.h b/cpp/src/neighbors/detail/faiss_distance_utils.h index 894b5f4bc6..63ff80c5c9 100644 --- a/cpp/src/neighbors/detail/faiss_distance_utils.h +++ b/cpp/src/neighbors/detail/faiss_distance_utils.h @@ -1,6 +1,6 @@ /* * SPDX-FileCopyrightText: Copyright (c) Facebook, Inc. and its affiliates. - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 AND MIT */ @@ -42,6 +42,9 @@ inline void chooseTileSize(size_t numQueries, size_t targetUsage = 512 * 1024 * 1024; if (tileRows * numCentroids * elementSize * 2 <= targetUsage) { tileCols = numCentroids; + // The whole index fits in one column tile, so spend the rest of the target on query rows. + tileRows = + std::min(targetUsage / (2 * elementSize * std::max(numCentroids, 1)), numQueries); } else { // only query total memory in case it potentially impacts tilesize size_t totalMem = rmm::available_device_memory().second; diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index cac660201f..7d60279ac7 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -172,8 +172,10 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - rmm::device_uvector temp_out_distances(tile_rows * temp_out_cols, stream); - rmm::device_uvector temp_out_indices(tile_rows * temp_out_cols, stream); + // only needed to merge the per-column-tile results + size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; + rmm::device_uvector temp_out_distances(temp_out_size, stream); + rmm::device_uvector temp_out_indices(temp_out_size, stream); bool select_min = cuvs::distance::is_min_close(metric); @@ -450,7 +452,9 @@ void brute_force_knn_impl( auto stream = raft::resource::get_next_usable_stream(handle, i); - if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true && + // fusedL2Knn has no tensor-core path, so half is faster through the tiled GEMM below. + if (!std::is_same_v && k <= 64 && rowMajorQuery == rowMajorIndex && + rowMajorQuery == true && (metric == cuvs::distance::DistanceType::L2Unexpanded || metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || metric == cuvs::distance::DistanceType::L2Expanded || From 182cf17d96c245b1d91a53d8c39e884933b6bc83 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 10 Nov 2026 23:05:51 +0000 Subject: [PATCH 2/7] Clarify when the brute-force merge buffers are used Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 7d60279ac7..9471f0dcb8 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -172,7 +172,7 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - // only needed to merge the per-column-tile results + // Only used when the index is split into column tiles, to merge each tile's top-k. size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; rmm::device_uvector temp_out_distances(temp_out_size, stream); rmm::device_uvector temp_out_indices(temp_out_size, stream); From a3694728e690b9d20b9a272a6b3ad3f2ddd85c59 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Tue, 10 Nov 2026 23:49:07 +0000 Subject: [PATCH 3/7] Size brute-force tiles against k when the index has fewer than k rows tiled_brute_force_knn raises tile_cols to k after chooseTileSize, so with n < k the row tile sized against n made the distance buffer far larger than the memory target (n=10, k=1000, m=1e6: 4 GB). Pass max(n, k) as the column count so rows are sized against the actual tile width. Unchanged for k <= n. Co-Authored-By: Claude Opus 5.5 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 9471f0dcb8..6b435ecfe2 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -91,7 +91,7 @@ void tiled_brute_force_knn(const raft::resources& handle, auto stream = raft::resource::get_cuda_stream(handle); cuvs::neighbors::detail::faiss_select::chooseTileSize( - m, n, d, sizeof(DistanceT), tile_rows, tile_cols); + m, std::max(n, k), d, sizeof(DistanceT), tile_rows, tile_cols); // for unittesting, its convenient to be able to put a max size on the tiles // so we can test the tiling logic without having to use huge inputs. From 9f09e8cf83c8592f67c1f03c27bcc090bb2e8c2a Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 19 Nov 2026 02:04:51 +0000 Subject: [PATCH 4/7] Pad brute-force results when the index has fewer than k rows With n < k the tiled path wrote the n-wide preliminary selection packed into the k-wide output and then asked select_k for k results from n inputs. Copy the already sorted neighbors to the front of each row and fill the remaining slots instead, and add a multi-query half test for this case. Co-Authored-By: Claude Fable 5.2 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 37 +++++++---- cpp/tests/neighbors/brute_force.cu | 64 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 6b435ecfe2..456a5e51ce 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -160,18 +160,6 @@ void tiled_brute_force_knn(const raft::resources& handle, auto last_col_tile_size = n % tile_cols; if (last_col_tile_size && (last_col_tile_size < k)) { temp_out_cols -= k - last_col_tile_size; } - // if we have less than k items in the index, we should fill out the result - // to indicate that we are missing items (and match behaviour in faiss) - if (n < k) { - raft::matrix::fill(handle, - raft::make_device_matrix_view(distances, m, k), - std::numeric_limits::lowest()); - - if constexpr (std::is_signed_v) { - raft::matrix::fill(handle, raft::make_device_matrix_view(indices, m, k), IndexType{-1}); - } - } - // Only used when the index is split into column tiles, to merge each tile's top-k. size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; rmm::device_uvector temp_out_distances(temp_out_size, stream); @@ -312,7 +300,30 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - if (tile_cols != n) { + if (n < k) { + // if we have less than k items in the index, the single column tile already holds all n + // neighbors in sorted order. Copy them to the front of each k-wide output row and fill out + // the rest to indicate that we are missing items (and match behaviour in faiss) + const DistanceT* in_distances = temp_out_distances.data(); + const IndexType* in_indices = temp_out_indices.data(); + DistanceT* out_distances = distances + i * k; + IndexType* out_indices = indices + i * k; + + auto count = cuda::make_counting_iterator(0); + thrust::for_each(raft::resource::get_thrust_policy(handle), + count, + count + current_query_size * k, + [=] __device__(IndexType idx) { + size_t row = idx / k, col = idx % k; + if (col < n) { + out_distances[idx] = in_distances[row * n + col]; + out_indices[idx] = in_indices[row * n + col]; + } else { + out_distances[idx] = std::numeric_limits::lowest(); + if constexpr (std::is_signed_v) { out_indices[idx] = -1; } + } + }); + } else if (tile_cols != n) { // select the actual top-k items here from the temporary output cuvs::selection::select_k( handle, diff --git a/cpp/tests/neighbors/brute_force.cu b/cpp/tests/neighbors/brute_force.cu index 1c09b9a739..f6c8dd4fac 100644 --- a/cpp/tests/neighbors/brute_force.cu +++ b/cpp/tests/neighbors/brute_force.cu @@ -19,6 +19,8 @@ #include #include +#include +#include #include namespace cuvs::neighbors::brute_force { @@ -273,6 +275,68 @@ TEST(BruteForceEqualNorms, PreservesNeighborOrdering) } } +TEST(BruteForceSmallIndex, PadsMissingNeighbors) +{ + raft::resources handle; + auto stream = raft::resource::get_cuda_stream(handle); + constexpr int64_t n_database = 4; + constexpr int64_t n_queries = 5; + constexpr int64_t k = 6; + + // 1-d points, so the expected neighbors of each query are unambiguous. + std::vector database_x{0.0f, 1.0f, 2.0f, 3.0f}; + std::vector queries_x{0.125f, 1.125f, 2.125f, 3.125f, -1.0f}; + std::vector h_database(n_database); + std::vector h_queries(n_queries); + std::ranges::transform(database_x, h_database.begin(), [](float v) { return __float2half(v); }); + std::ranges::transform(queries_x, h_queries.begin(), [](float v) { return __float2half(v); }); + + auto database = raft::make_device_matrix(handle, n_database, 1); + auto queries = raft::make_device_matrix(handle, n_queries, 1); + raft::update_device(database.data_handle(), h_database.data(), h_database.size(), stream); + raft::update_device(queries.data_handle(), h_queries.data(), h_queries.size(), stream); + + index_params index_params; + index_params.metric = cuvs::distance::DistanceType::L2Expanded; + auto index = build(handle, index_params, raft::make_const_mdspan(database.view())); + + auto neighbors = raft::make_device_matrix(handle, n_queries, k); + auto distances = raft::make_device_matrix(handle, n_queries, k); + search(handle, + search_params{}, + index, + raft::make_const_mdspan(queries.view()), + neighbors.view(), + distances.view(), + filtering::none_sample_filter{}); + + std::vector h_neighbors(n_queries * k); + std::vector h_distances(n_queries * k); + raft::update_host(h_neighbors.data(), neighbors.data_handle(), h_neighbors.size(), stream); + raft::update_host(h_distances.data(), distances.data_handle(), h_distances.size(), stream); + raft::resource::sync_stream(handle, stream); + + for (int64_t q = 0; q < n_queries; ++q) { + std::vector expected(n_database); + std::iota(expected.begin(), expected.end(), int64_t{0}); + std::ranges::sort(expected, [&](int64_t lhs, int64_t rhs) -> bool { + return std::abs(queries_x[q] - database_x[lhs]) < std::abs(queries_x[q] - database_x[rhs]); + }); + + for (int64_t i = 0; i < k; ++i) { + if (i < n_database) { + float diff = queries_x[q] - database_x[expected[i]]; + EXPECT_EQ(h_neighbors[q * k + i], expected[i]) << "query=" << q << " rank=" << i; + EXPECT_NEAR(h_distances[q * k + i], diff * diff, 1e-4) << "query=" << q << " rank=" << i; + } else { + EXPECT_EQ(h_neighbors[q * k + i], -1) << "query=" << q << " rank=" << i; + EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::lowest()) + << "query=" << q << " rank=" << i; + } + } + } +} + // Also test with larger random inputs, including col-major inputs struct RandomKNNInputs { int num_queries; From a4e839d43933203d726c5eb6c33af4a687e27969 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 19 Nov 2026 02:20:07 +0000 Subject: [PATCH 5/7] Use raft::linalg::map_offset to pad brute-force results for small indexes Co-Authored-By: Claude Fable 5.2 --- cpp/src/neighbors/detail/knn_brute_force.cuh | 32 +++++++++----------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index 456a5e51ce..f3e0715bcd 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -306,23 +306,21 @@ void tiled_brute_force_knn(const raft::resources& handle, // the rest to indicate that we are missing items (and match behaviour in faiss) const DistanceT* in_distances = temp_out_distances.data(); const IndexType* in_indices = temp_out_indices.data(); - DistanceT* out_distances = distances + i * k; - IndexType* out_indices = indices + i * k; - - auto count = cuda::make_counting_iterator(0); - thrust::for_each(raft::resource::get_thrust_policy(handle), - count, - count + current_query_size * k, - [=] __device__(IndexType idx) { - size_t row = idx / k, col = idx % k; - if (col < n) { - out_distances[idx] = in_distances[row * n + col]; - out_indices[idx] = in_indices[row * n + col]; - } else { - out_distances[idx] = std::numeric_limits::lowest(); - if constexpr (std::is_signed_v) { out_indices[idx] = -1; } - } - }); + + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(distances + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_distances[row * n + col] : std::numeric_limits::lowest(); + }); + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(indices + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_indices[row * n + col] : static_cast(-1); + }); } else if (tile_cols != n) { // select the actual top-k items here from the temporary output cuvs::selection::select_k( From 5ddf3b6c9a7c27302bd2b05c6183f201a2ea92c9 Mon Sep 17 00:00:00 2001 From: Tarang Jain <40517122+tarang-jain@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:25:39 -0800 Subject: [PATCH 6/7] Clarify comment on handling fewer neighbors than k --- cpp/src/neighbors/detail/knn_brute_force.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index f3e0715bcd..b9829d21cb 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -303,7 +303,7 @@ void tiled_brute_force_knn(const raft::resources& handle, if (n < k) { // if we have less than k items in the index, the single column tile already holds all n // neighbors in sorted order. Copy them to the front of each k-wide output row and fill out - // the rest to indicate that we are missing items (and match behaviour in faiss) + // the rest to indicate that we are missing items const DistanceT* in_distances = temp_out_distances.data(); const IndexType* in_indices = temp_out_indices.data(); From 88547e1aedc0595bad281f7b3a44183670f3f94c Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 19 Nov 2026 03:42:18 +0000 Subject: [PATCH 7/7] Keep half unexpanded L2 on fusedL2Knn and budget select_k's workspace - Only half L2Expanded/L2SqrtExpanded searches go through the tiled GEMM; the unexpanded metrics have no GEMM and stay on fusedL2Knn. - chooseTileSize counts select_k's per-row workspace when it grows the row tile, so the distance tile and the selection share the target. - Missing neighbors are padded with the worst distance for the metric (max() when selecting the smallest), as fusedL2Knn does. Co-Authored-By: Claude Fable 5.2 --- .../neighbors/detail/faiss_distance_utils.h | 6 +++-- cpp/src/neighbors/detail/knn_brute_force.cuh | 25 +++++++++++++------ cpp/tests/neighbors/brute_force.cu | 2 +- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cpp/src/neighbors/detail/faiss_distance_utils.h b/cpp/src/neighbors/detail/faiss_distance_utils.h index 63ff80c5c9..71771b2148 100644 --- a/cpp/src/neighbors/detail/faiss_distance_utils.h +++ b/cpp/src/neighbors/detail/faiss_distance_utils.h @@ -20,6 +20,7 @@ inline void chooseTileSize(size_t numQueries, size_t numCentroids, size_t dim, size_t elementSize, + size_t rowWorkspaceSize, size_t& tileRows, size_t& tileCols) { @@ -43,8 +44,9 @@ inline void chooseTileSize(size_t numQueries, if (tileRows * numCentroids * elementSize * 2 <= targetUsage) { tileCols = numCentroids; // The whole index fits in one column tile, so spend the rest of the target on query rows. - tileRows = - std::min(targetUsage / (2 * elementSize * std::max(numCentroids, 1)), numQueries); + // Each row also takes rowWorkspaceSize bytes of temporary memory after the distance tile. + size_t rowSize = elementSize * std::max(numCentroids, 1) + rowWorkspaceSize; + tileRows = std::min(std::max(tileRows, targetUsage / (2 * rowSize)), numQueries); } else { // only query total memory in case it potentially impacts tilesize size_t totalMem = rmm::available_device_memory().second; diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index b9829d21cb..c87cfd4f38 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -90,8 +90,14 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t tile_cols = 0; auto stream = raft::resource::get_cuda_stream(handle); - cuvs::neighbors::detail::faiss_select::chooseTileSize( - m, std::max(n, k), d, sizeof(DistanceT), tile_rows, tile_cols); + // select_k keeps k distances and indices per query row in its workspace + cuvs::neighbors::detail::faiss_select::chooseTileSize(m, + std::max(n, k), + d, + sizeof(DistanceT), + k * (sizeof(DistanceT) + sizeof(IndexType)), + tile_rows, + tile_cols); // for unittesting, its convenient to be able to put a max size on the tiles // so we can test the tiling logic without having to use huge inputs. @@ -306,13 +312,15 @@ void tiled_brute_force_knn(const raft::resources& handle, // the rest to indicate that we are missing items const DistanceT* in_distances = temp_out_distances.data(); const IndexType* in_indices = temp_out_indices.data(); + DistanceT missing_distance = select_min ? std::numeric_limits::max() + : std::numeric_limits::lowest(); raft::linalg::map_offset( handle, raft::make_device_vector_view(distances + i * k, current_query_size * k), [=] __device__(size_t idx) { size_t row = idx / k, col = idx % k; - return col < n ? in_distances[row * n + col] : std::numeric_limits::lowest(); + return col < n ? in_distances[row * n + col] : missing_distance; }); raft::linalg::map_offset( handle, @@ -461,13 +469,14 @@ void brute_force_knn_impl( auto stream = raft::resource::get_next_usable_stream(handle, i); - // fusedL2Knn has no tensor-core path, so half is faster through the tiled GEMM below. - if (!std::is_same_v && k <= 64 && rowMajorQuery == rowMajorIndex && - rowMajorQuery == true && + // fusedL2Knn has no tensor-core path, so half expanded L2 is faster through the tiled GEMM + // below. + if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true && (metric == cuvs::distance::DistanceType::L2Unexpanded || metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || - metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { + (!std::is_same_v && + (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded)))) { fusedL2Knn(D, out_i_ptr, out_d_ptr, diff --git a/cpp/tests/neighbors/brute_force.cu b/cpp/tests/neighbors/brute_force.cu index f6c8dd4fac..7d5e9d648d 100644 --- a/cpp/tests/neighbors/brute_force.cu +++ b/cpp/tests/neighbors/brute_force.cu @@ -330,7 +330,7 @@ TEST(BruteForceSmallIndex, PadsMissingNeighbors) EXPECT_NEAR(h_distances[q * k + i], diff * diff, 1e-4) << "query=" << q << " rank=" << i; } else { EXPECT_EQ(h_neighbors[q * k + i], -1) << "query=" << q << " rank=" << i; - EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::lowest()) + EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::max()) << "query=" << q << " rank=" << i; } }