diff --git a/cpp/src/neighbors/detail/faiss_distance_utils.h b/cpp/src/neighbors/detail/faiss_distance_utils.h index 894b5f4bc6..71771b2148 100644 --- a/cpp/src/neighbors/detail/faiss_distance_utils.h +++ b/cpp/src/neighbors/detail/faiss_distance_utils.h @@ -1,6 +1,6 @@ /* * SPDX-FileCopyrightText: Copyright (c) Facebook, Inc. and its affiliates. - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 AND MIT */ @@ -20,6 +20,7 @@ inline void chooseTileSize(size_t numQueries, size_t numCentroids, size_t dim, size_t elementSize, + size_t rowWorkspaceSize, size_t& tileRows, size_t& tileCols) { @@ -42,6 +43,10 @@ inline void chooseTileSize(size_t numQueries, size_t targetUsage = 512 * 1024 * 1024; if (tileRows * numCentroids * elementSize * 2 <= targetUsage) { tileCols = numCentroids; + // The whole index fits in one column tile, so spend the rest of the target on query rows. + // Each row also takes rowWorkspaceSize bytes of temporary memory after the distance tile. + size_t rowSize = elementSize * std::max(numCentroids, 1) + rowWorkspaceSize; + tileRows = std::min(std::max(tileRows, targetUsage / (2 * rowSize)), numQueries); } else { // only query total memory in case it potentially impacts tilesize size_t totalMem = rmm::available_device_memory().second; diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index cac660201f..c87cfd4f38 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -90,8 +90,14 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t tile_cols = 0; auto stream = raft::resource::get_cuda_stream(handle); - cuvs::neighbors::detail::faiss_select::chooseTileSize( - m, n, d, sizeof(DistanceT), tile_rows, tile_cols); + // select_k keeps k distances and indices per query row in its workspace + cuvs::neighbors::detail::faiss_select::chooseTileSize(m, + std::max(n, k), + d, + sizeof(DistanceT), + k * (sizeof(DistanceT) + sizeof(IndexType)), + tile_rows, + tile_cols); // for unittesting, its convenient to be able to put a max size on the tiles // so we can test the tiling logic without having to use huge inputs. @@ -160,20 +166,10 @@ void tiled_brute_force_knn(const raft::resources& handle, auto last_col_tile_size = n % tile_cols; if (last_col_tile_size && (last_col_tile_size < k)) { temp_out_cols -= k - last_col_tile_size; } - // if we have less than k items in the index, we should fill out the result - // to indicate that we are missing items (and match behaviour in faiss) - if (n < k) { - raft::matrix::fill(handle, - raft::make_device_matrix_view(distances, m, k), - std::numeric_limits::lowest()); - - if constexpr (std::is_signed_v) { - raft::matrix::fill(handle, raft::make_device_matrix_view(indices, m, k), IndexType{-1}); - } - } - - rmm::device_uvector temp_out_distances(tile_rows * temp_out_cols, stream); - rmm::device_uvector temp_out_indices(tile_rows * temp_out_cols, stream); + // Only used when the index is split into column tiles, to merge each tile's top-k. + size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; + rmm::device_uvector temp_out_distances(temp_out_size, stream); + rmm::device_uvector temp_out_indices(temp_out_size, stream); bool select_min = cuvs::distance::is_min_close(metric); @@ -310,7 +306,30 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - if (tile_cols != n) { + if (n < k) { + // if we have less than k items in the index, the single column tile already holds all n + // neighbors in sorted order. Copy them to the front of each k-wide output row and fill out + // the rest to indicate that we are missing items + const DistanceT* in_distances = temp_out_distances.data(); + const IndexType* in_indices = temp_out_indices.data(); + DistanceT missing_distance = select_min ? std::numeric_limits::max() + : std::numeric_limits::lowest(); + + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(distances + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_distances[row * n + col] : missing_distance; + }); + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(indices + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_indices[row * n + col] : static_cast(-1); + }); + } else if (tile_cols != n) { // select the actual top-k items here from the temporary output cuvs::selection::select_k( handle, @@ -450,11 +469,14 @@ void brute_force_knn_impl( auto stream = raft::resource::get_next_usable_stream(handle, i); + // fusedL2Knn has no tensor-core path, so half expanded L2 is faster through the tiled GEMM + // below. if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true && (metric == cuvs::distance::DistanceType::L2Unexpanded || metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || - metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { + (!std::is_same_v && + (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded)))) { fusedL2Knn(D, out_i_ptr, out_d_ptr, diff --git a/cpp/tests/neighbors/brute_force.cu b/cpp/tests/neighbors/brute_force.cu index 1c09b9a739..7d5e9d648d 100644 --- a/cpp/tests/neighbors/brute_force.cu +++ b/cpp/tests/neighbors/brute_force.cu @@ -19,6 +19,8 @@ #include #include +#include +#include #include namespace cuvs::neighbors::brute_force { @@ -273,6 +275,68 @@ TEST(BruteForceEqualNorms, PreservesNeighborOrdering) } } +TEST(BruteForceSmallIndex, PadsMissingNeighbors) +{ + raft::resources handle; + auto stream = raft::resource::get_cuda_stream(handle); + constexpr int64_t n_database = 4; + constexpr int64_t n_queries = 5; + constexpr int64_t k = 6; + + // 1-d points, so the expected neighbors of each query are unambiguous. + std::vector database_x{0.0f, 1.0f, 2.0f, 3.0f}; + std::vector queries_x{0.125f, 1.125f, 2.125f, 3.125f, -1.0f}; + std::vector h_database(n_database); + std::vector h_queries(n_queries); + std::ranges::transform(database_x, h_database.begin(), [](float v) { return __float2half(v); }); + std::ranges::transform(queries_x, h_queries.begin(), [](float v) { return __float2half(v); }); + + auto database = raft::make_device_matrix(handle, n_database, 1); + auto queries = raft::make_device_matrix(handle, n_queries, 1); + raft::update_device(database.data_handle(), h_database.data(), h_database.size(), stream); + raft::update_device(queries.data_handle(), h_queries.data(), h_queries.size(), stream); + + index_params index_params; + index_params.metric = cuvs::distance::DistanceType::L2Expanded; + auto index = build(handle, index_params, raft::make_const_mdspan(database.view())); + + auto neighbors = raft::make_device_matrix(handle, n_queries, k); + auto distances = raft::make_device_matrix(handle, n_queries, k); + search(handle, + search_params{}, + index, + raft::make_const_mdspan(queries.view()), + neighbors.view(), + distances.view(), + filtering::none_sample_filter{}); + + std::vector h_neighbors(n_queries * k); + std::vector h_distances(n_queries * k); + raft::update_host(h_neighbors.data(), neighbors.data_handle(), h_neighbors.size(), stream); + raft::update_host(h_distances.data(), distances.data_handle(), h_distances.size(), stream); + raft::resource::sync_stream(handle, stream); + + for (int64_t q = 0; q < n_queries; ++q) { + std::vector expected(n_database); + std::iota(expected.begin(), expected.end(), int64_t{0}); + std::ranges::sort(expected, [&](int64_t lhs, int64_t rhs) -> bool { + return std::abs(queries_x[q] - database_x[lhs]) < std::abs(queries_x[q] - database_x[rhs]); + }); + + for (int64_t i = 0; i < k; ++i) { + if (i < n_database) { + float diff = queries_x[q] - database_x[expected[i]]; + EXPECT_EQ(h_neighbors[q * k + i], expected[i]) << "query=" << q << " rank=" << i; + EXPECT_NEAR(h_distances[q * k + i], diff * diff, 1e-4) << "query=" << q << " rank=" << i; + } else { + EXPECT_EQ(h_neighbors[q * k + i], -1) << "query=" << q << " rank=" << i; + EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::max()) + << "query=" << q << " rank=" << i; + } + } + } +} + // Also test with larger random inputs, including col-major inputs struct RandomKNNInputs { int num_queries;