Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cpp/src/neighbors/detail/faiss_distance_utils.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* SPDX-FileCopyrightText: Copyright (c) Facebook, Inc. and its affiliates.
* SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0 AND MIT
*/

Expand All @@ -20,6 +20,7 @@ inline void chooseTileSize(size_t numQueries,
size_t numCentroids,
size_t dim,
size_t elementSize,
size_t rowWorkspaceSize,
size_t& tileRows,
size_t& tileCols)
{
Expand All @@ -42,6 +43,10 @@ inline void chooseTileSize(size_t numQueries,
size_t targetUsage = 512 * 1024 * 1024;
if (tileRows * numCentroids * elementSize * 2 <= targetUsage) {
tileCols = numCentroids;
// The whole index fits in one column tile, so spend the rest of the target on query rows.
// Each row also takes rowWorkspaceSize bytes of temporary memory after the distance tile.
size_t rowSize = elementSize * std::max<size_t>(numCentroids, 1) + rowWorkspaceSize;
tileRows = std::min(std::max(tileRows, targetUsage / (2 * rowSize)), numQueries);
} else {
// only query total memory in case it potentially impacts tilesize
size_t totalMem = rmm::available_device_memory().second;
Expand Down
60 changes: 41 additions & 19 deletions cpp/src/neighbors/detail/knn_brute_force.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,14 @@ void tiled_brute_force_knn(const raft::resources& handle,
size_t tile_cols = 0;
auto stream = raft::resource::get_cuda_stream(handle);

cuvs::neighbors::detail::faiss_select::chooseTileSize(
m, n, d, sizeof(DistanceT), tile_rows, tile_cols);
// select_k keeps k distances and indices per query row in its workspace
cuvs::neighbors::detail::faiss_select::chooseTileSize(m,
std::max(n, k),
d,
sizeof(DistanceT),
k * (sizeof(DistanceT) + sizeof(IndexType)),
tile_rows,
tile_cols);

// for unittesting, its convenient to be able to put a max size on the tiles
// so we can test the tiling logic without having to use huge inputs.
Expand Down Expand Up @@ -160,20 +166,10 @@ void tiled_brute_force_knn(const raft::resources& handle,
auto last_col_tile_size = n % tile_cols;
if (last_col_tile_size && (last_col_tile_size < k)) { temp_out_cols -= k - last_col_tile_size; }

// if we have less than k items in the index, we should fill out the result
// to indicate that we are missing items (and match behaviour in faiss)
if (n < k) {
raft::matrix::fill(handle,
raft::make_device_matrix_view(distances, m, k),
std::numeric_limits<DistanceT>::lowest());

if constexpr (std::is_signed_v<IndexType>) {
raft::matrix::fill(handle, raft::make_device_matrix_view(indices, m, k), IndexType{-1});
}
}

rmm::device_uvector<DistanceT> temp_out_distances(tile_rows * temp_out_cols, stream);
rmm::device_uvector<IndexType> temp_out_indices(tile_rows * temp_out_cols, stream);
// Only used when the index is split into column tiles, to merge each tile's top-k.
size_t temp_out_size = tile_cols != n ? tile_rows * temp_out_cols : 0;
rmm::device_uvector<DistanceT> temp_out_distances(temp_out_size, stream);
rmm::device_uvector<IndexType> temp_out_indices(temp_out_size, stream);

bool select_min = cuvs::distance::is_min_close(metric);

Expand Down Expand Up @@ -310,7 +306,30 @@ void tiled_brute_force_knn(const raft::resources& handle,
}
}

if (tile_cols != n) {
if (n < k) {
// if we have less than k items in the index, the single column tile already holds all n
// neighbors in sorted order. Copy them to the front of each k-wide output row and fill out
// the rest to indicate that we are missing items
const DistanceT* in_distances = temp_out_distances.data();
const IndexType* in_indices = temp_out_indices.data();
DistanceT missing_distance = select_min ? std::numeric_limits<DistanceT>::max()
: std::numeric_limits<DistanceT>::lowest();

raft::linalg::map_offset(
handle,
raft::make_device_vector_view(distances + i * k, current_query_size * k),
[=] __device__(size_t idx) {
size_t row = idx / k, col = idx % k;
return col < n ? in_distances[row * n + col] : missing_distance;
});
raft::linalg::map_offset(
handle,
raft::make_device_vector_view(indices + i * k, current_query_size * k),
[=] __device__(size_t idx) {
size_t row = idx / k, col = idx % k;
return col < n ? in_indices[row * n + col] : static_cast<IndexType>(-1);
});
} else if (tile_cols != n) {
// select the actual top-k items here from the temporary output
cuvs::selection::select_k(
handle,
Expand Down Expand Up @@ -450,11 +469,14 @@ void brute_force_knn_impl(

auto stream = raft::resource::get_next_usable_stream(handle, i);

// fusedL2Knn has no tensor-core path, so half expanded L2 is faster through the tiled GEMM
// below.
if (k <= 64 && rowMajorQuery == rowMajorIndex && rowMajorQuery == true &&
(metric == cuvs::distance::DistanceType::L2Unexpanded ||
metric == cuvs::distance::DistanceType::L2SqrtUnexpanded ||
metric == cuvs::distance::DistanceType::L2Expanded ||
metric == cuvs::distance::DistanceType::L2SqrtExpanded)) {
(!std::is_same_v<value_t, half> &&
(metric == cuvs::distance::DistanceType::L2Expanded ||
metric == cuvs::distance::DistanceType::L2SqrtExpanded)))) {
fusedL2Knn(D,
out_i_ptr,
out_d_ptr,
Expand Down
64 changes: 64 additions & 0 deletions cpp/tests/neighbors/brute_force.cu
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

#include <algorithm>
#include <cmath>
#include <limits>
#include <numeric>
#include <vector>

namespace cuvs::neighbors::brute_force {
Expand Down Expand Up @@ -273,6 +275,68 @@ TEST(BruteForceEqualNorms, PreservesNeighborOrdering)
}
}

TEST(BruteForceSmallIndex, PadsMissingNeighbors)
{
raft::resources handle;
auto stream = raft::resource::get_cuda_stream(handle);
constexpr int64_t n_database = 4;
constexpr int64_t n_queries = 5;
constexpr int64_t k = 6;

// 1-d points, so the expected neighbors of each query are unambiguous.
std::vector<float> database_x{0.0f, 1.0f, 2.0f, 3.0f};
std::vector<float> queries_x{0.125f, 1.125f, 2.125f, 3.125f, -1.0f};
std::vector<half> h_database(n_database);
std::vector<half> h_queries(n_queries);
std::ranges::transform(database_x, h_database.begin(), [](float v) { return __float2half(v); });
std::ranges::transform(queries_x, h_queries.begin(), [](float v) { return __float2half(v); });

auto database = raft::make_device_matrix<half, int64_t>(handle, n_database, 1);
auto queries = raft::make_device_matrix<half, int64_t>(handle, n_queries, 1);
raft::update_device(database.data_handle(), h_database.data(), h_database.size(), stream);
raft::update_device(queries.data_handle(), h_queries.data(), h_queries.size(), stream);

index_params index_params;
index_params.metric = cuvs::distance::DistanceType::L2Expanded;
auto index = build(handle, index_params, raft::make_const_mdspan(database.view()));

auto neighbors = raft::make_device_matrix<int64_t, int64_t>(handle, n_queries, k);
auto distances = raft::make_device_matrix<float, int64_t>(handle, n_queries, k);
search(handle,
search_params{},
index,
raft::make_const_mdspan(queries.view()),
neighbors.view(),
distances.view(),
filtering::none_sample_filter{});

std::vector<int64_t> h_neighbors(n_queries * k);
std::vector<float> h_distances(n_queries * k);
raft::update_host(h_neighbors.data(), neighbors.data_handle(), h_neighbors.size(), stream);
raft::update_host(h_distances.data(), distances.data_handle(), h_distances.size(), stream);
raft::resource::sync_stream(handle, stream);

for (int64_t q = 0; q < n_queries; ++q) {
std::vector<int64_t> expected(n_database);
std::iota(expected.begin(), expected.end(), int64_t{0});
std::ranges::sort(expected, [&](int64_t lhs, int64_t rhs) -> bool {
return std::abs(queries_x[q] - database_x[lhs]) < std::abs(queries_x[q] - database_x[rhs]);
});

for (int64_t i = 0; i < k; ++i) {
if (i < n_database) {
float diff = queries_x[q] - database_x[expected[i]];
EXPECT_EQ(h_neighbors[q * k + i], expected[i]) << "query=" << q << " rank=" << i;
EXPECT_NEAR(h_distances[q * k + i], diff * diff, 1e-4) << "query=" << q << " rank=" << i;
} else {
EXPECT_EQ(h_neighbors[q * k + i], -1) << "query=" << q << " rank=" << i;
EXPECT_EQ(h_distances[q * k + i], std::numeric_limits<float>::max())
<< "query=" << q << " rank=" << i;
}
}
}
}

// Also test with larger random inputs, including col-major inputs
struct RandomKNNInputs {
int num_queries;
Expand Down
Loading