diff --git a/cpp/src/neighbors/detail/faiss_distance_utils.h b/cpp/src/neighbors/detail/faiss_distance_utils.h index 894b5f4bc6..71771b2148 100644 --- a/cpp/src/neighbors/detail/faiss_distance_utils.h +++ b/cpp/src/neighbors/detail/faiss_distance_utils.h @@ -1,6 +1,6 @@ /* * SPDX-FileCopyrightText: Copyright (c) Facebook, Inc. and its affiliates. - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 AND MIT */ @@ -20,6 +20,7 @@ inline void chooseTileSize(size_t numQueries, size_t numCentroids, size_t dim, size_t elementSize, + size_t rowWorkspaceSize, size_t& tileRows, size_t& tileCols) { @@ -42,6 +43,10 @@ inline void chooseTileSize(size_t numQueries, size_t targetUsage = 512 * 1024 * 1024; if (tileRows * numCentroids * elementSize * 2 <= targetUsage) { tileCols = numCentroids; + // The whole index fits in one column tile, so spend the rest of the target on query rows. + // Each row also takes rowWorkspaceSize bytes of temporary memory after the distance tile. + size_t rowSize = elementSize * std::max(numCentroids, 1) + rowWorkspaceSize; + tileRows = std::min(std::max(tileRows, targetUsage / (2 * rowSize)), numQueries); } else { // only query total memory in case it potentially impacts tilesize size_t totalMem = rmm::available_device_memory().second; diff --git a/cpp/src/neighbors/detail/knn_brute_force.cuh b/cpp/src/neighbors/detail/knn_brute_force.cuh index cac660201f..daf48f2481 100644 --- a/cpp/src/neighbors/detail/knn_brute_force.cuh +++ b/cpp/src/neighbors/detail/knn_brute_force.cuh @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -90,8 +91,14 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t tile_cols = 0; auto stream = raft::resource::get_cuda_stream(handle); - cuvs::neighbors::detail::faiss_select::chooseTileSize( - m, n, d, sizeof(DistanceT), tile_rows, tile_cols); + // select_k keeps k distances and indices per query row in its workspace + cuvs::neighbors::detail::faiss_select::chooseTileSize(m, + std::max(n, k), + d, + sizeof(DistanceT), + k * (sizeof(DistanceT) + sizeof(IndexType)), + tile_rows, + tile_cols); // for unittesting, its convenient to be able to put a max size on the tiles // so we can test the tiling logic without having to use huge inputs. @@ -101,6 +108,20 @@ void tiled_brute_force_knn(const raft::resources& handle, // tile_cols must be at least k items tile_cols = std::max(tile_cols, k); + // Expanded L2 ranks a query's rows by |x|^2 - 2 q.x, which one GEMM writes with the index norms + // added in its epilogue (raft::linalg::gemm's bias), so no pass over the tile turns dot products + // into distances. |q|^2 is added after the selection, to the k kept per row. + constexpr bool l2_scores_supported = + std::is_same_v && + (std::is_same_v || std::is_same_v) && + std::is_same_v; + bool const l2_scored = + l2_scores_supported && (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded); + // cuBLASLt's fast kernels need the tile's rows and the bias (the norms at norms + j) 16-byte + // aligned, so column tiles span multiples of 4 index rows. + if (l2_scored && tile_cols < n) { tile_cols = std::max(tile_cols - tile_cols % 4, k); } + // stores pairwise distances for the current tile rmm::device_uvector temp_distances(tile_rows * tile_cols, stream); @@ -148,6 +169,14 @@ void tiled_brute_force_knn(const raft::resources& handle, } pairwise_metric = cuvs::distance::DistanceType::InnerProduct; } + // Precomputed norms that aren't 16-byte aligned, such as a slice of a larger vector, would make + // every tile's bias misaligned, so they are copied to an aligned buffer once per search. + if (l2_scored && precomputed_index_norms != nullptr && + reinterpret_cast(precomputed_index_norms) % 16 != 0) { + index_norms.resize(n, stream); + raft::copy(index_norms.data(), precomputed_index_norms, n, stream); + precomputed_index_norms = index_norms.data(); + } // if we're tiling over columns, we need additional buffers for temporary output // distances/indices @@ -160,20 +189,10 @@ void tiled_brute_force_knn(const raft::resources& handle, auto last_col_tile_size = n % tile_cols; if (last_col_tile_size && (last_col_tile_size < k)) { temp_out_cols -= k - last_col_tile_size; } - // if we have less than k items in the index, we should fill out the result - // to indicate that we are missing items (and match behaviour in faiss) - if (n < k) { - raft::matrix::fill(handle, - raft::make_device_matrix_view(distances, m, k), - std::numeric_limits::lowest()); - - if constexpr (std::is_signed_v) { - raft::matrix::fill(handle, raft::make_device_matrix_view(indices, m, k), IndexType{-1}); - } - } - - rmm::device_uvector temp_out_distances(tile_rows * temp_out_cols, stream); - rmm::device_uvector temp_out_indices(tile_rows * temp_out_cols, stream); + // Only used when the index is split into column tiles, to merge each tile's top-k. + size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0; + rmm::device_uvector temp_out_distances(temp_out_size, stream); + rmm::device_uvector temp_out_indices(temp_out_size, stream); bool select_min = cuvs::distance::is_min_close(metric); @@ -184,21 +203,48 @@ void tiled_brute_force_knn(const raft::resources& handle, size_t current_centroid_size = std::min(tile_cols, n - j); size_t current_k = std::min(current_centroid_size, k); - // calculate the top-k elements for the current tile, by calculating the - // full pairwise distance for the tile - and then selecting the top-k from that - cuvs::distance::pairwise_distance( - handle, - raft::make_device_matrix_view( - search + i * d, current_query_size, d), - raft::make_device_matrix_view( - index + j * d, current_centroid_size, d), - raft::make_device_matrix_view( - temp_distances.data(), current_query_size, current_centroid_size), - pairwise_metric, - metric_arg); + if constexpr (l2_scores_supported) { + if (l2_scored) { + // Column-major, the row-major [queries x index rows] tile is index * search^T, and the + // bias adds one norm per index row. + auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); + DistanceT const alpha = -2; + DistanceT const beta = 0; + raft::linalg::gemm(handle, + /*trans_a=*/true, + /*trans_b=*/false, + static_cast(current_centroid_size), + static_cast(current_query_size), + static_cast(d), + &alpha, + index + j * d, + static_cast(d), + search + i * d, + static_cast(d), + &beta, + temp_distances.data(), + static_cast(current_centroid_size), + stream.get(), + col_norms + j); + } + } + if (!l2_scored) { + // calculate the top-k elements for the current tile, by calculating the + // full pairwise distance for the tile - and then selecting the top-k from that + cuvs::distance::pairwise_distance( + handle, + raft::make_device_matrix_view( + search + i * d, current_query_size, d), + raft::make_device_matrix_view( + index + j * d, current_centroid_size, d), + raft::make_device_matrix_view( + temp_distances.data(), current_query_size, current_centroid_size), + pairwise_metric, + metric_arg); + } - if (metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded) { + if (!l2_scored && (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { auto row_norms = precomputed_search_norms ? precomputed_search_norms : search_norms.data(); auto col_norms = precomputed_index_norms ? precomputed_index_norms : index_norms.data(); auto dist = temp_distances.data(); @@ -282,6 +328,25 @@ void tiled_brute_force_knn(const raft::resources& handle, select_min, true); + if constexpr (l2_scores_supported) { + if (l2_scored && current_k > 0) { + // |q|^2 + (|x|^2 - 2 q.x) for the k kept per row, clamped and rooted as + // l2_exp_cutlass_op. + auto row_norms = + precomputed_search_norms ? precomputed_search_norms : search_norms.data(); + auto kept = distances + i * k; + bool sqrt = metric == cuvs::distance::DistanceType::L2SqrtExpanded; + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(kept, current_query_size * current_k), + [=] __device__(IndexType idx) { + DistanceT val = row_norms[i + idx / current_k] + kept[idx]; + val = val < DistanceT{0} ? DistanceT{0} : val; + return sqrt ? raft::sqrt(val) : val; + }); + } + } + // if we're tiling over columns, we need to do a couple things to fix up // the output of select_k // 1. The column id's in the output are relative to the tile, so we need @@ -310,7 +375,30 @@ void tiled_brute_force_knn(const raft::resources& handle, } } - if (tile_cols != n) { + if (n < k) { + // if we have less than k items in the index, the single column tile already holds all n + // neighbors in sorted order. Copy them to the front of each k-wide output row and fill out + // the rest to indicate that we are missing items + const DistanceT* in_distances = temp_out_distances.data(); + const IndexType* in_indices = temp_out_indices.data(); + DistanceT missing_distance = select_min ? std::numeric_limits::max() + : std::numeric_limits::lowest(); + + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(distances + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_distances[row * n + col] : missing_distance; + }); + raft::linalg::map_offset( + handle, + raft::make_device_vector_view(indices + i * k, current_query_size * k), + [=] __device__(size_t idx) { + size_t row = idx / k, col = idx % k; + return col < n ? in_indices[row * n + col] : static_cast(-1); + }); + } else if (tile_cols != n) { // select the actual top-k items here from the temporary output cuvs::selection::select_k( handle, @@ -450,11 +538,14 @@ void brute_force_knn_impl( auto stream = raft::resource::get_next_usable_stream(handle, i); + // fusedL2Knn has no tensor-core path, so half expanded L2 is faster through the tiled GEMM + // below. if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true && (metric == cuvs::distance::DistanceType::L2Unexpanded || metric == cuvs::distance::DistanceType::L2SqrtUnexpanded || - metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded)) { + (!std::is_same_v && + (metric == cuvs::distance::DistanceType::L2Expanded || + metric == cuvs::distance::DistanceType::L2SqrtExpanded)))) { fusedL2Knn(D, out_i_ptr, out_d_ptr, diff --git a/cpp/tests/neighbors/brute_force.cu b/cpp/tests/neighbors/brute_force.cu index 1c09b9a739..7d5e9d648d 100644 --- a/cpp/tests/neighbors/brute_force.cu +++ b/cpp/tests/neighbors/brute_force.cu @@ -19,6 +19,8 @@ #include #include +#include +#include #include namespace cuvs::neighbors::brute_force { @@ -273,6 +275,68 @@ TEST(BruteForceEqualNorms, PreservesNeighborOrdering) } } +TEST(BruteForceSmallIndex, PadsMissingNeighbors) +{ + raft::resources handle; + auto stream = raft::resource::get_cuda_stream(handle); + constexpr int64_t n_database = 4; + constexpr int64_t n_queries = 5; + constexpr int64_t k = 6; + + // 1-d points, so the expected neighbors of each query are unambiguous. + std::vector database_x{0.0f, 1.0f, 2.0f, 3.0f}; + std::vector queries_x{0.125f, 1.125f, 2.125f, 3.125f, -1.0f}; + std::vector h_database(n_database); + std::vector h_queries(n_queries); + std::ranges::transform(database_x, h_database.begin(), [](float v) { return __float2half(v); }); + std::ranges::transform(queries_x, h_queries.begin(), [](float v) { return __float2half(v); }); + + auto database = raft::make_device_matrix(handle, n_database, 1); + auto queries = raft::make_device_matrix(handle, n_queries, 1); + raft::update_device(database.data_handle(), h_database.data(), h_database.size(), stream); + raft::update_device(queries.data_handle(), h_queries.data(), h_queries.size(), stream); + + index_params index_params; + index_params.metric = cuvs::distance::DistanceType::L2Expanded; + auto index = build(handle, index_params, raft::make_const_mdspan(database.view())); + + auto neighbors = raft::make_device_matrix(handle, n_queries, k); + auto distances = raft::make_device_matrix(handle, n_queries, k); + search(handle, + search_params{}, + index, + raft::make_const_mdspan(queries.view()), + neighbors.view(), + distances.view(), + filtering::none_sample_filter{}); + + std::vector h_neighbors(n_queries * k); + std::vector h_distances(n_queries * k); + raft::update_host(h_neighbors.data(), neighbors.data_handle(), h_neighbors.size(), stream); + raft::update_host(h_distances.data(), distances.data_handle(), h_distances.size(), stream); + raft::resource::sync_stream(handle, stream); + + for (int64_t q = 0; q < n_queries; ++q) { + std::vector expected(n_database); + std::iota(expected.begin(), expected.end(), int64_t{0}); + std::ranges::sort(expected, [&](int64_t lhs, int64_t rhs) -> bool { + return std::abs(queries_x[q] - database_x[lhs]) < std::abs(queries_x[q] - database_x[rhs]); + }); + + for (int64_t i = 0; i < k; ++i) { + if (i < n_database) { + float diff = queries_x[q] - database_x[expected[i]]; + EXPECT_EQ(h_neighbors[q * k + i], expected[i]) << "query=" << q << " rank=" << i; + EXPECT_NEAR(h_distances[q * k + i], diff * diff, 1e-4) << "query=" << q << " rank=" << i; + } else { + EXPECT_EQ(h_neighbors[q * k + i], -1) << "query=" << q << " rank=" << i; + EXPECT_EQ(h_distances[q * k + i], std::numeric_limits::max()) + << "query=" << q << " rank=" << i; + } + } + } +} + // Also test with larger random inputs, including col-major inputs struct RandomKNNInputs { int num_queries;