diff --git a/c/src/neighbors/cagra.cpp b/c/src/neighbors/cagra.cpp index 99e456e23c..23e02a376c 100644 --- a/c/src/neighbors/cagra.cpp +++ b/c/src/neighbors/cagra.cpp @@ -524,7 +524,7 @@ static void make_host_standard_dataset_view(raft::resources*, } template -static void attach_dataset(raft::resources* res_ptr, +static void update_dataset(raft::resources* res_ptr, cuvsDataset_t device_padded_dataset, cuvsCagraIndex_t index) { @@ -537,7 +537,7 @@ static void attach_dataset(raft::resources* res_ptr, auto* box = reinterpret_cast(index->addr); RAFT_EXPECTS(device_padded_dataset->mem_type == CUVS_DATASET_MEM_TYPE_DEVICE && device_padded_dataset->layout == CUVS_DATASET_LAYOUT_PADDED, - "cuvsCagraAttachDataset: dataset must be device padded"); + "cuvsCagraUpdateDataset: dataset must be device padded"); using owner_t = cuvs::neighbors::device_padded_dataset; using view_t = cuvs::neighbors::device_padded_dataset_view; @@ -547,63 +547,24 @@ static void attach_dataset(raft::resources* res_ptr, "cuvsCagraUpdateDataset: null index handle", "cuvsCagraUpdateDataset: host index layout is allowed for this operation", [&](auto& idx) { - auto padded_idx = cuvs::neighbors::cagra::attach_dataset(*res_ptr, idx, padded_view); - auto* holder = - new cuvs_cagra_c_api_index_lifetime_holder{std::move(padded_idx)}; - destroy_sg_cagra_c_api_box(index->addr); - index->addr = 0; - bind_index_lifetime_holder_to_C_index(index, index->dtype, holder); + using index_t = std::remove_cvref_t; + if constexpr (std::is_same_v< + index_t, + cuvs::neighbors::cagra::device_padded_index>) { + idx.update_dataset(*res_ptr, padded_view); + } else { + auto padded_idx = + cuvs::neighbors::cagra::update_dataset(*res_ptr, std::move(idx), padded_view); + auto* holder = + new cuvs_cagra_c_api_index_lifetime_holder{std::move(padded_idx)}; + destroy_sg_cagra_c_api_box(index->addr); + index->addr = 0; + bind_index_lifetime_holder_to_C_index(index, index->dtype, holder); + } }); }); } -template -static void update_device_dataset_same_layout(raft::resources* res_ptr, - cuvsDataset_t device_dataset, - cuvsCagraIndex_t index) -{ - RAFT_EXPECTS(device_dataset != nullptr, "cuvsCagraUpdateDataset: null dataset"); - RAFT_EXPECTS(index != nullptr, "cuvsCagraUpdateDataset: null index handle"); - RAFT_EXPECTS(index->addr != 0, "cuvsCagraUpdateDataset: null index storage"); - RAFT_EXPECTS(device_dataset->addr != 0, "cuvsCagraUpdateDataset: null dataset storage"); - - auto* box = reinterpret_cast(index->addr); - if (box->layout == sg_cagra_c_api_index_box::dataset_layout::device_padded) { - RAFT_EXPECTS(device_dataset->mem_type == CUVS_DATASET_MEM_TYPE_DEVICE && - device_dataset->layout == CUVS_DATASET_LAYOUT_PADDED, - "cuvsCagraUpdateDeviceDatasetSameLayout: device-padded index " - "requires a " - "device-padded dataset"); - using owner_t = cuvs::neighbors::device_padded_dataset; - using view_t = cuvs::neighbors::device_padded_dataset_view; - with_dataset_view(device_dataset, [&](auto const& dataset_view) { - auto* idx = - reinterpret_cast*>(box->index_ptr); - RAFT_EXPECTS(idx != nullptr, "cuvsCagraUpdateDataset: null index handle"); - idx->update_device_dataset_same_layout(*res_ptr, dataset_view); - }); - } else if (box->layout == sg_cagra_c_api_index_box::dataset_layout::device_standard) { - RAFT_EXPECTS(device_dataset->mem_type == CUVS_DATASET_MEM_TYPE_DEVICE && - device_dataset->layout == CUVS_DATASET_LAYOUT_STANDARD, - "cuvsCagraUpdateDeviceDatasetSameLayout: device-standard " - "index requires a " - "device-standard dataset"); - using owner_t = cuvs::neighbors::device_standard_dataset; - using view_t = cuvs::neighbors::device_standard_dataset_view; - with_dataset_view(device_dataset, [&](auto const& dataset_view) { - auto* idx = - reinterpret_cast*>(box->index_ptr); - RAFT_EXPECTS(idx != nullptr, "cuvsCagraUpdateDataset: null index handle"); - idx->update_device_dataset_same_layout(*res_ptr, dataset_view); - }); - } else { - RAFT_FAIL( - "cuvsCagraUpdateDataset: C++ " - "update_device_dataset_same_layout " - "requires a device index and dataset"); - } -} - static void _set_graph_build_params( std::variant( *res_ptr, metric); - raw->update_device_dataset_same_layout(*res_ptr, dataset_view); + raw->update_dataset(*res_ptr, dataset_view); update_graph_from_dlpack(raw); wrap_CPP_index_in_lifetime_holder_and_bind_to_C_index< T, @@ -721,7 +682,7 @@ void _from_args(cuvsResources_t res, auto dataset_view = cuvs::neighbors::make_device_standard_dataset_view(mds); auto* raw = new cuvs::neighbors::cagra::device_standard_index( *res_ptr, metric); - raw->update_device_dataset_same_layout(*res_ptr, dataset_view); + raw->update_dataset(*res_ptr, dataset_view); update_graph_from_dlpack(raw); wrap_CPP_index_in_lifetime_holder_and_bind_to_C_index< T, @@ -1578,7 +1539,7 @@ extern "C" cuvsError_t cuvsDatasetMakeStandardView(cuvsResources_t res, }); } -static cuvsError_t dispatch_attach_dataset(cuvsResources_t res, +static cuvsError_t dispatch_update_dataset(cuvsResources_t res, cuvsDataset_t device_padded_dataset, cuvsCagraIndex_t index) { @@ -1592,40 +1553,13 @@ static cuvsError_t dispatch_attach_dataset(cuvsResources_t res, index->dtype.bits == device_padded_dataset->dtype.bits, "cuvsCagraUpdateDataset: dtype mismatch between index and dataset"); if (index->dtype.code == kDLFloat && index->dtype.bits == 32) { - attach_dataset(res_ptr, device_padded_dataset, index); + update_dataset(res_ptr, device_padded_dataset, index); } else if (index->dtype.code == kDLFloat && index->dtype.bits == 16) { - attach_dataset(res_ptr, device_padded_dataset, index); + update_dataset(res_ptr, device_padded_dataset, index); } else if (index->dtype.code == kDLInt && index->dtype.bits == 8) { - attach_dataset(res_ptr, device_padded_dataset, index); + update_dataset(res_ptr, device_padded_dataset, index); } else if (index->dtype.code == kDLUInt && index->dtype.bits == 8) { - attach_dataset(res_ptr, device_padded_dataset, index); - } else { - RAFT_FAIL("Unsupported index dtype: %d and bits: %d", index->dtype.code, index->dtype.bits); - } - }); -} - -static cuvsError_t dispatch_update_device_dataset_same_layout(cuvsResources_t res, - cuvsDataset_t device_dataset, - cuvsCagraIndex_t index) -{ - return cuvs::core::translate_exceptions([=] { - auto* res_ptr = reinterpret_cast(res); - RAFT_EXPECTS(index != nullptr, "cuvsCagraUpdateDataset: null index handle"); - RAFT_EXPECTS(device_dataset != nullptr, - "cuvsCagraUpdateDataset: null dataset view"); - RAFT_EXPECTS(index->dtype.code == device_dataset->dtype.code && - index->dtype.bits == device_dataset->dtype.bits, - "cuvsCagraUpdateDataset: dtype mismatch " - "between index and dataset"); - if (index->dtype.code == kDLFloat && index->dtype.bits == 32) { - update_device_dataset_same_layout(res_ptr, device_dataset, index); - } else if (index->dtype.code == kDLFloat && index->dtype.bits == 16) { - update_device_dataset_same_layout(res_ptr, device_dataset, index); - } else if (index->dtype.code == kDLInt && index->dtype.bits == 8) { - update_device_dataset_same_layout(res_ptr, device_dataset, index); - } else if (index->dtype.code == kDLUInt && index->dtype.bits == 8) { - update_device_dataset_same_layout(res_ptr, device_dataset, index); + update_dataset(res_ptr, device_padded_dataset, index); } else { RAFT_FAIL("Unsupported index dtype: %d and bits: %d", index->dtype.code, index->dtype.bits); } @@ -1650,12 +1584,7 @@ extern "C" cuvsError_t cuvsCagraUpdateDataset(cuvsResources_t res, "cuvsCagraUpdateDataset: dtype mismatch between index and dataset"); }); if (status != CUVS_SUCCESS) { return status; } - - auto* box = reinterpret_cast(index->addr); - if (box->layout == sg_cagra_c_api_index_box::dataset_layout::device_padded) { - return dispatch_update_device_dataset_same_layout(res, device_padded_dataset, index); - } - return dispatch_attach_dataset(res, device_padded_dataset, index); + return dispatch_update_dataset(res, device_padded_dataset, index); } /** diff --git a/c/src/neighbors/mg_cagra.cpp b/c/src/neighbors/mg_cagra.cpp index 705d53c2a6..79caf18593 100644 --- a/c/src/neighbors/mg_cagra.cpp +++ b/c/src/neighbors/mg_cagra.cpp @@ -229,7 +229,8 @@ void _mg_update_dataset(cuvsResources_t res, using padded_ann_t = cuvs::neighbors::cagra::device_padded_index; auto* standard_index = reinterpret_cast*>(box->index_ptr); auto* padded_index = new mg_cagra_index_t( - cuvs::neighbors::cagra::attach_dataset(*res_ptr, *standard_index, padded_view)); + cuvs::neighbors::cagra::update_dataset( + *res_ptr, std::move(*standard_index), padded_view)); auto* padded_box = make_mg_cagra_box(padded_index, mg_cagra_dataset_layout::device_padded); destroy_mg_cagra_c_api_box(index->addr); @@ -237,8 +238,7 @@ void _mg_update_dataset(cuvsResources_t res, } else if (box->layout == mg_cagra_dataset_layout::device_padded) { using padded_ann_t = cuvs::neighbors::cagra::device_padded_index; auto* padded_index = reinterpret_cast*>(box->index_ptr); - cuvs::neighbors::cagra::update_device_dataset_same_layout( - *res_ptr, *padded_index, padded_view); + cuvs::neighbors::cagra::update_dataset(*res_ptr, *padded_index, padded_view); } else { RAFT_FAIL("cuvsMultiGpuCagraUpdateDataset: unsupported index dataset layout"); } diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h index 6b275068b0..142756b632 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h @@ -244,8 +244,8 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) } auto ace_host_index = cuvs::neighbors::cagra::build(handle_, params, *host_pdv); auto padded = cuvs::neighbors::make_device_padded_dataset(handle_, dataset_view_host); - auto ace_index = - cuvs::neighbors::cagra::attach_dataset(handle_, ace_host_index, padded->as_dataset_view()); + auto ace_index = cuvs::neighbors::cagra::update_dataset( + handle_, std::move(ace_host_index), padded->as_dataset_view()); *dataset_ = std::move(padded->data_); index_ = std::make_shared(std::move(ace_index)); } else { @@ -283,13 +283,13 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) *input_dataset_v_ = raft::make_device_matrix_view( mds.data_handle(), static_cast(nrow), static_cast(dim_)); auto index = cuvs::neighbors::cagra::build(handle_, params, pdv); - index.update_device_dataset_same_layout(handle_, pdv); + index.update_dataset(handle_, pdv); index_ = std::make_shared(std::move(index)); } else { auto padded = cuvs::neighbors::make_device_padded_dataset(handle_, mds); auto view = padded->as_dataset_view(); auto index = cuvs::neighbors::cagra::build(handle_, params, view); - index.update_device_dataset_same_layout(handle_, view); + index.update_dataset(handle_, view); *dataset_ = std::move(padded->data_); index_ = std::make_shared(std::move(index)); } @@ -318,17 +318,17 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) raft::resource::get_cuda_stream(handle_)); cuvs::neighbors::device_padded_dataset_view dv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index.update_device_dataset_same_layout(handle_, dv); + sub_index.update_dataset(handle_, dv); } else { if (cuvs::neighbors::matrix_row_width_matches_cagra_required(sub_dev)) { auto pdv = cuvs::neighbors::make_device_padded_dataset_view(handle_, sub_dev); - sub_index.update_device_dataset_same_layout(handle_, pdv); + sub_index.update_dataset(handle_, pdv); } else { auto padded = cuvs::neighbors::make_device_padded_dataset(handle_, sub_dev); sub_dataset_buffers_->push_back(std::move(padded->data_)); cuvs::neighbors::device_padded_dataset_view pdv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index.update_device_dataset_same_layout(handle_, pdv); + sub_index.update_dataset(handle_, pdv); } } } @@ -348,8 +348,8 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) } auto ace_host_index = cuvs::neighbors::cagra::build(handle_, params, *host_pdv_sub); auto padded_sub = cuvs::neighbors::make_device_padded_dataset(handle_, sub_host); - sub_index = cuvs::neighbors::cagra::attach_dataset( - handle_, ace_host_index, padded_sub->as_dataset_view()); + sub_index = cuvs::neighbors::cagra::update_dataset( + handle_, std::move(ace_host_index), padded_sub->as_dataset_view()); sub_dataset_buffers_->push_back(std::move(padded_sub->data_)); } else if (dataset_is_on_host) { sub_dataset_buffers_->emplace_back(raft::make_device_matrix( @@ -370,12 +370,12 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) if (sub_device && src_sub == req_sub) { auto pdv_sub = cuvs::neighbors::make_device_padded_dataset_view(handle_, mds_sub); sub_index = cuvs::neighbors::cagra::build(handle_, params, pdv_sub); - sub_index.update_device_dataset_same_layout(handle_, pdv_sub); + sub_index.update_dataset(handle_, pdv_sub); } else { auto padded_sub = cuvs::neighbors::make_device_padded_dataset(handle_, mds_sub); auto view = padded_sub->as_dataset_view(); auto index = cuvs::neighbors::cagra::build(handle_, params, view); - index.update_device_dataset_same_layout(handle_, view); + index.update_dataset(handle_, view); sub_dataset_buffers_->push_back(std::move(padded_sub->data_)); sub_index = std::move(index); } @@ -391,12 +391,12 @@ void cuvs_cagra::build(const T* dataset, size_t nrow) if (sub_device && src_sub == req_sub) { auto pdv_sub = cuvs::neighbors::make_device_padded_dataset_view(handle_, mds_sub); sub_index = cuvs::neighbors::cagra::build(handle_, params, pdv_sub); - sub_index.update_device_dataset_same_layout(handle_, pdv_sub); + sub_index.update_dataset(handle_, pdv_sub); } else { auto padded_sub = cuvs::neighbors::make_device_padded_dataset(handle_, mds_sub); auto view = padded_sub->as_dataset_view(); auto index = cuvs::neighbors::cagra::build(handle_, params, view); - index.update_device_dataset_same_layout(handle_, view); + index.update_dataset(handle_, view); sub_dataset_buffers_->push_back(std::move(padded_sub->data_)); sub_index = std::move(index); } @@ -484,7 +484,7 @@ void cuvs_cagra::set_search_param(const search_param_base& param, *dataset_ = raft::make_device_matrix(handle_, 0, 0); cuvs::neighbors::device_padded_dataset_view empty_dv( raft::make_device_matrix_view(static_cast(nullptr), 0, this->dim_), this->dim_); - index_->update_device_dataset_same_layout(handle_, empty_dv); + index_->update_dataset(handle_, empty_dv); // Allocate space using the correct memory resource. RAFT_LOG_DEBUG("moving dataset to new memory space: %s", @@ -497,7 +497,7 @@ void cuvs_cagra::set_search_param(const search_param_base& param, raft::make_device_matrix_view( dataset_->data_handle(), dataset_->extent(0), dataset_->extent(1)), this->dim_); - index_->update_device_dataset_same_layout(handle_, dv); + index_->update_dataset(handle_, dv); need_dataset_update_ = false; needs_dynamic_batcher_update = true; @@ -554,17 +554,17 @@ void cuvs_cagra::set_search_dataset(const T* dataset, size_t nrow) raft::resource::get_cuda_stream(handle_)); cuvs::neighbors::device_padded_dataset_view dv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index->update_device_dataset_same_layout(handle_, dv); + sub_index->update_dataset(handle_, dv); } else { if (cuvs::neighbors::matrix_row_width_matches_cagra_required(sub_dev)) { auto pdv = cuvs::neighbors::make_device_padded_dataset_view(handle_, sub_dev); - sub_index->update_device_dataset_same_layout(handle_, pdv); + sub_index->update_dataset(handle_, pdv); } else { auto padded = cuvs::neighbors::make_device_padded_dataset(handle_, sub_dev); sub_dataset_buffers_->push_back(std::move(padded->data_)); cuvs::neighbors::device_padded_dataset_view pdv( raft::make_const_mdspan(sub_dataset_buffers_->back().view()), dim_); - sub_index->update_device_dataset_same_layout(handle_, pdv); + sub_index->update_dataset(handle_, pdv); } } } diff --git a/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff b/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff index b36a18587a..9b81a4a2b3 100644 --- a/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff +++ b/cpp/cmake/patches/faiss-1.14-cuvs-26.08.diff @@ -79,7 +79,7 @@ index b331fdc..c7b5733 100644 - cuvs_index->update_dataset(raft_handle, dataset); + auto dataset_view = + cuvs::neighbors::make_device_padded_dataset_view(raft_handle, dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, dataset_view); ++ cuvs_index->update_dataset(raft_handle, dataset_view); } else { - auto dataset = raft::make_host_matrix_view( + auto host_dataset = raft::make_host_matrix_view( @@ -87,7 +87,7 @@ index b331fdc..c7b5733 100644 - cuvs_index->update_dataset(raft_handle, dataset); + host_to_device_dataset_ = + cuvs::neighbors::make_device_padded_dataset(raft_handle, host_dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, ++ cuvs_index->update_dataset(raft_handle, + host_to_device_dataset_->as_dataset_view()); } store_dataset_ = true; @@ -210,7 +210,7 @@ index 755817f..0eb03ae 100644 - cuvs_index->update_dataset(raft_handle, dataset); + auto dataset_view = + cuvs::neighbors::make_device_padded_dataset_view(raft_handle, dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, dataset_view); ++ cuvs_index->update_dataset(raft_handle, dataset_view); } else { - auto dataset = raft::make_host_matrix_view( + auto host_dataset = raft::make_host_matrix_view( @@ -218,7 +218,7 @@ index 755817f..0eb03ae 100644 - cuvs_index->update_dataset(raft_handle, dataset); + host_to_device_dataset_ = + cuvs::neighbors::make_device_padded_dataset(raft_handle, host_dataset); -+ cuvs_index->update_device_dataset_same_layout(raft_handle, ++ cuvs_index->update_dataset(raft_handle, + host_to_device_dataset_->as_dataset_view()); } store_dataset_ = true; diff --git a/cpp/include/cuvs/neighbors/cagra.hpp b/cpp/include/cuvs/neighbors/cagra.hpp index 43ae7a6235..f05efd69a3 100644 --- a/cpp/include/cuvs/neighbors/cagra.hpp +++ b/cpp/include/cuvs/neighbors/cagra.hpp @@ -5,35 +5,28 @@ #pragma once -#include "common.hpp" #include +#include #include #include #include #include #include + #include #include -#include #include #include #include #include -#include -#include - -#include -#include -#include #include #include -#include -#include +#include #include -#include #include #include +#include #include #include @@ -203,16 +196,16 @@ struct index_params : cuvs::neighbors::index_params { * as the index is used. A device-backed index is ready to search immediately; a host-backed index * retains the dataset for operations such as serialization but is not searchable. * - `false` means `build` only builds the graph and the caller is expected to attach a dataset - * separately via `cuvs::neighbors::cagra::index::update_device_dataset_same_layout` before + * separately via `cuvs::neighbors::cagra::index::update_dataset` before * searching. * * Unlike the legacy behavior, no copy of the dataset is made: the index always stores a view. * Setting `attach_dataset_on_build = false` is useful when the caller needs to apply specific * memory placement or transformation (e.g. moving to managed memory) before attaching. * - * Host indexes are not directly searchable. Call `attach_dataset` with a user-provided - * device-padded dataset view to obtain a search-ready `device_padded_index`. Disk-based ACE - * builds manage file-backed dataset state separately and ignore this flag. + * Host indexes are not directly searchable. Call the type-changing `update_dataset` with a + * user-provided device-padded dataset view to obtain a search-ready `device_padded_index`. + * Disk-based ACE builds manage file-backed dataset state separately and ignore this flag. * * @code{.cpp} * auto dataset = cuvs::neighbors::make_device_padded_dataset(res, host_matrix.view()); @@ -222,7 +215,7 @@ struct index_params : cuvs::neighbors::index_params { * auto index = cagra::build(res, index_params, dataset->as_dataset_view()); * // ASSERT(index.size() == 0); // no dataset yet * // Attach with a view (storage owned by `dataset`). - * index.update_device_dataset_same_layout(res, dataset->as_dataset_view()); + * index.update_dataset(res, dataset->as_dataset_view()); * cagra::search(res, search_params, index, queries, neighbors, distances); * @endcode */ @@ -439,12 +432,6 @@ struct extend_params { static_assert(std::is_aggregate_v); static_assert(std::is_aggregate_v); -template > -struct index; - /** * @defgroup cagra_cpp_index CAGRA index type * @{ @@ -461,7 +448,9 @@ struct index; * @tparam DatasetViewT concrete non-owning dataset view type stored by the index * */ -template +template > struct CUVS_EXPORT index : cuvs::neighbors::index { using index_params_type = cagra::index_params; using search_params_type = cagra::search_params; @@ -558,8 +547,8 @@ struct CUVS_EXPORT index : cuvs::neighbors::index { /** \endcond */ /** Construct a graph-only index with a zero-row dataset view placeholder. */ - index(raft::resources const& res, - cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded) + explicit index(raft::resources const& res, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded) requires(cuvs::neighbors::ann_dataset_view) : cuvs::neighbors::index(), metric_(metric), @@ -623,19 +612,35 @@ struct CUVS_EXPORT index : cuvs::neighbors::index { raft::resource::sync_stream(res); } + /* Construct an index with a new dataset type by moving the old index and passing in a new + * dataset*/ + template + index(raft::resources const& res, index&& other, DatasetViewT dataset) + : metric_(other.metric_), + graph_(std::move(other.graph_)), + graph_view_(other.graph_view_), + source_indices_(std::move(other.source_indices_)), + n_rows_(other.n_rows_), + dim_(other.dim_), + graph_degree_(other.graph_degree_) + { + update_dataset(res, dataset); + } + /** * Replace the dataset with a new `dataset_view`. * * The index stores a copy of the view handle only (not the vector storage). The caller must * keep the underlying device data alive. Clears precomputed norms. */ - void update_device_dataset_same_layout(raft::resources const& res, DatasetViewT const& dataset) - requires cuvs::neighbors::is_device_dataset_view_v + void update_dataset(raft::resources const& res, DatasetViewT dataset) { dataset_ = dataset; dataset_norms_.reset(); - if (metric() == cuvs::distance::DistanceType::CosineExpanded) { - if (dataset_.n_rows() > 0) { compute_dataset_norms_(res); } + if constexpr (is_device_dataset_view_v) { + if (metric() == cuvs::distance::DistanceType::CosineExpanded) { + if (dataset_.n_rows() > 0) { compute_dataset_norms_(res); } + } } } @@ -849,6 +854,9 @@ struct CUVS_EXPORT index : cuvs::neighbors::index { } private: + template + friend struct index; + friend struct detail::fd_transfer; [[nodiscard]] inline auto steal_dataset_fd_() noexcept @@ -941,8 +949,8 @@ using cagra_index_t = index cuvs::neighbors::mg_index, uint8_t, uint32_t>; /** - * @brief Convert a standard MG CAGRA index into a padded MG CAGRA index for search. + * @brief Consume a standard MG CAGRA index and attach a padded dataset for search. * - * This returns a new padded index because standard and padded MG indexes have different C++ types. + * This moves each rank-local CAGRA graph into the returned padded MG index. */ -auto attach_dataset( +auto update_dataset( const raft::resources& clique, - const cuvs::neighbors::mg_index, float, uint32_t>& - idx, + cuvs::neighbors::mg_index, float, uint32_t>&& idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset) -> cuvs::neighbors::mg_index, float, uint32_t>; -auto attach_dataset( +auto update_dataset( const raft::resources& clique, - const cuvs::neighbors::mg_index, half, uint32_t>& - idx, + cuvs::neighbors::mg_index, half, uint32_t>&& idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset) -> cuvs::neighbors::mg_index, half, uint32_t>; -auto attach_dataset( +auto update_dataset( const raft::resources& clique, - const cuvs::neighbors::mg_index, int8_t, uint32_t>& - idx, + cuvs::neighbors::mg_index, int8_t, uint32_t>&& idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset) -> cuvs::neighbors::mg_index, int8_t, uint32_t>; -auto attach_dataset( +auto update_dataset( const raft::resources& clique, - const cuvs::neighbors:: - mg_index, uint8_t, uint32_t>& idx, + cuvs::neighbors::mg_index, uint8_t, uint32_t>&& + idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset) -> cuvs::neighbors::mg_index, uint8_t, uint32_t>; /** * @brief Update an existing padded MG CAGRA index with a padded dataset of the same layout. */ -void update_device_dataset_same_layout( +void update_dataset( const raft::resources& clique, cuvs::neighbors::mg_index, float, uint32_t>& idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset); -void update_device_dataset_same_layout( +void update_dataset( const raft::resources& clique, cuvs::neighbors::mg_index, half, uint32_t>& idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset); -void update_device_dataset_same_layout( +void update_dataset( const raft::resources& clique, cuvs::neighbors::mg_index, int8_t, uint32_t>& idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset); -void update_device_dataset_same_layout( +void update_dataset( const raft::resources& clique, cuvs::neighbors::mg_index, uint8_t, uint32_t>& idx, cuvs::neighbors::device_padded_dataset_view const& padded_dataset); @@ -4475,35 +4480,6 @@ struct fd_transfer { } }; -/** - * @brief Copy a host-resident CAGRA index graph into a new device-resident index (graph only). - * - * @internal - */ -template - requires cuvs::neighbors::is_host_dataset_view_v -auto convert_host_to_device_index(raft::resources const& res, index const& src) - -> index> -{ - using DeviceViewT = cuvs::neighbors::device_counterpart_t; - using GraphIndexType = typename index::graph_index_type; - index out(res, src.metric()); - if (src.graph().size() > 0) { - // The graph lives in device memory owned by `src`. `update_graph(device_view)` would only - // store a view (no ownership transfer), leaving `out` with a dangling pointer once `src` - // is destroyed. Copy device→host→device so that `out` owns its graph memory. - auto graph_host = - raft::make_host_matrix(src.graph().extent(0), src.graph().extent(1)); - raft::copy(graph_host.data_handle(), - src.graph().data_handle(), - src.graph().size(), - raft::resource::get_cuda_stream(res)); - raft::resource::sync_stream(res); - out.update_graph(res, raft::make_const_mdspan(graph_host.view())); // host overload: copies H→D - } - return out; -} - } // namespace detail /** @@ -4544,59 +4520,86 @@ auto convert_standard_to_padded_index( if (standard_idx.source_indices().has_value()) { out.update_source_indices(res, standard_idx.source_indices().value()); } - out.update_device_dataset_same_layout(res, padded_dataset); + out.update_dataset(res, padded_dataset); return out; } -/** - * @brief Attach a device-padded dataset and return a search-ready padded-device index. - * - * The dataset is provided by the caller and must already be device-padded. - * - * For host/standard index layouts, this function converts to and returns a new - * `device_padded_index`. - * - * If `idx` is already a `device_padded_index`, call `idx.update_device_dataset_same_layout(res, - * device_padded_dataset)` directly to avoid an unnecessary copy path. - * - * @param[in] res RAFT resources - * @param[in] idx CAGRA index in any host/device + standard/padded layout - * @param[in] device_padded_dataset caller-owned device-padded dataset view - * @return search-ready padded-device CAGRA index - */ -template - requires cuvs::neighbors::ann_dataset_view -auto attach_dataset( +auto update_dataset( raft::resources const& res, - index const& idx, - cuvs::neighbors::device_padded_dataset_view const& device_padded_dataset) - -> device_padded_index -{ - RAFT_EXPECTS(device_padded_dataset.n_rows() == idx.size(), - "Padded dataset row count must match the index size"); - - if constexpr (cuvs::neighbors::is_host_standard_dataset_view_v) { - auto dev_std = detail::convert_host_to_device_index(res, idx); - return convert_standard_to_padded_index(res, dev_std, device_padded_dataset); - } else if constexpr (cuvs::neighbors::is_host_padded_dataset_view_v) { - auto dev_pad = detail::convert_host_to_device_index(res, idx); - dev_pad.update_device_dataset_same_layout(res, device_padded_dataset); - return dev_pad; - } else if constexpr (cuvs::neighbors::is_device_standard_dataset_view_v) { - return convert_standard_to_padded_index(res, idx, device_padded_dataset); - } else if constexpr (cuvs::neighbors::is_device_padded_dataset_view_v) { - RAFT_LOG_WARN( - "cagra::attach_dataset called with an already device-padded index. " - "To avoid an unnecessary index copy, call " - "index.update_device_dataset_same_layout(res, device_padded_dataset) " - "directly on the original index."); - RAFT_FAIL( - "cagra::attach_dataset: device_padded_index input is not supported in this overload. " - "Call index.update_device_dataset_same_layout(res, device_padded_dataset) directly."); - } else { - static_assert(!sizeof(IndexViewT), "Unsupported CAGRA index dataset view type"); - } -} + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_standard_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset(raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; +auto update_dataset( + raft::resources const& res, + index>&& cagra_index, + device_padded_dataset_view dataset) + -> index>; } // namespace cagra } // namespace neighbors diff --git a/cpp/include/cuvs/preprocessing/quantize/pq.hpp b/cpp/include/cuvs/preprocessing/quantize/pq.hpp index 29d0bbb1e3..e6ffc78f53 100644 --- a/cpp/include/cuvs/preprocessing/quantize/pq.hpp +++ b/cpp/include/cuvs/preprocessing/quantize/pq.hpp @@ -268,7 +268,7 @@ template * Typical **CAGRA** usage: build the graph on dense vectors, then attach VPQ for search (metric * must remain `L2Expanded` for this path). Train VPQ from the same CAGRA-padded device layout you * used for graph build, keep the `device_vpq_dataset` alive, and call - * `index::update_device_dataset_same_layout` with a non-owning view. + * `index::update_dataset` with a non-owning view. * * @code{.cpp} * #include @@ -278,7 +278,7 @@ template * // `padded` is a `device_padded_dataset_view` view of those same rows. * cuvs::neighbors::vpq_params vpq_params{}; * auto vpq = cuvs::preprocessing::quantize::pq::make_vpq_dataset(res, vpq_params, padded.view()); - * idx.update_device_dataset_same_layout(res, vpq.as_dataset_view()); + * idx.update_dataset(res, vpq.as_dataset_view()); * @endcode */ template diff --git a/cpp/src/neighbors/cagra.cuh b/cpp/src/neighbors/cagra.cuh index 80e2f2a07e..af6e56e59d 100644 --- a/cpp/src/neighbors/cagra.cuh +++ b/cpp/src/neighbors/cagra.cuh @@ -286,8 +286,8 @@ void optimize( * stored in the returned index as a non-owning view — no copy is made. The caller must keep the * underlying storage alive for the lifetime of the index. * - * Host-backed indexes cannot be searched; call `attach_dataset` with a device-padded dataset to get - * a search-ready device index. + * Host-backed indexes cannot be searched; call the type-changing `update_dataset` with a + * device-padded dataset to get a search-ready device index. */ template requires(!cuvs::neighbors::is_empty_dataset_view_v && @@ -300,13 +300,13 @@ auto build(raft::resources const& res, const index_params& params, DatasetViewT using IdxT = uint32_t; // Dense paths build the graph and optionally attach the input dataset view. Host indexes remain - // non-searchable until attach_dataset(...) supplies a device-padded dataset. + // non-searchable until the type-changing update_dataset(...) supplies a device-padded dataset. if constexpr (cuvs::neighbors::is_device_vpq_dataset_view_v) { RAFT_FAIL("cagra::build: VPQ-compressed dataset cannot be used for dense graph construction."); } else if constexpr (cuvs::neighbors::is_dense_row_major_device_dataset_view_v) { auto idx = cuvs::neighbors::cagra::detail::build_from_device_matrix( res, params, dataset); - if (params.attach_dataset_on_build) { idx.update_device_dataset_same_layout(res, dataset); } + if (params.attach_dataset_on_build) { idx.update_dataset(res, dataset); } return idx; } else { if (std::holds_alternative(params.graph_build_params)) { @@ -587,6 +587,27 @@ void search( } } +template +auto update_dataset(raft::resources const& res, + index&& cagra_index, + DstDatasetViewT dataset) -> index +{ + static_assert(!std::is_same_v, + "For updating a dataset of the same type use the cagra_index.update_dataset() " + "function instead."); + + RAFT_EXPECTS(dataset.n_rows() == static_cast(cagra_index.size()), + "The new dataset row count must match the source dataset row count"); + RAFT_EXPECTS(cagra_index.dim() == 0 || dataset.dim() == cagra_index.dim(), + "The new dataset dimension must match the source dataset dimension"); + + index new_index(res, std::move(cagra_index), dataset); + return new_index; +} + /** @} */ // end group cagra } // namespace cuvs::neighbors::cagra diff --git a/cpp/src/neighbors/cagra_build_inst.cu.in b/cpp/src/neighbors/cagra_build_inst.cu.in index acaaa942c1..49ff8a18ce 100644 --- a/cpp/src/neighbors/cagra_build_inst.cu.in +++ b/cpp/src/neighbors/cagra_build_inst.cu.in @@ -9,6 +9,7 @@ #include #include +#include namespace { @@ -57,4 +58,29 @@ CUVS_DEFINE_CAGRA_BUILD_OVERLOAD(inst_host_standard_view_t, #undef CUVS_DEFINE_CAGRA_BUILD_OVERLOAD +#define CUVS_INST_CAGRA_UPDATE_DATASET(T, IdxT, SrcDatasetViewT, DstDatasetViewT) \ + auto update_dataset(raft::resources const& res, \ + index&& cagra_index, \ + DstDatasetViewT dataset) -> index \ + { \ + return cuvs::neighbors::cagra::update_dataset( \ + res, std::move(cagra_index), dataset); \ + } + +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, + index_t, + inst_host_standard_view_t, + inst_device_padded_view_t); +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, + index_t, + inst_host_standard_view_t, + inst_device_standard_view_t); +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, index_t, inst_host_padded_view_t, inst_device_padded_view_t); +CUVS_INST_CAGRA_UPDATE_DATASET(data_t, + index_t, + inst_device_standard_view_t, + inst_device_padded_view_t); + +#undef CUVS_INST_CAGRA_UPDATE_DATASET + } // namespace cuvs::neighbors::cagra diff --git a/cpp/src/neighbors/detail/cagra/add_nodes.cuh b/cpp/src/neighbors/detail/cagra/add_nodes.cuh index 62a77239b2..14108f8410 100644 --- a/cpp/src/neighbors/detail/cagra/add_nodes.cuh +++ b/cpp/src/neighbors/detail/cagra/add_nodes.cuh @@ -325,7 +325,7 @@ void add_graph_nodes( // add_node_core() uses CAGRA search internally, which requires a padded device dataset. // Keep this path allocation-free by requiring pre-padded chunk views. auto pdv = cuvs::neighbors::make_device_padded_dataset_view(handle, dataset_view); - internal_index.update_device_dataset_same_layout(handle, pdv); + internal_index.update_dataset(handle, pdv); // Note: The graph is copied to the device memory. internal_index.update_graph(handle, graph_view); @@ -388,7 +388,7 @@ void extend_core(raft::resources const& handle, if constexpr (cuvs::neighbors::is_empty_dataset_view_v>) { RAFT_FAIL( "cagra::extend only supports an index to which the dataset is attached. Please check if the " - "index has an empty dataset; attach one with update_device_dataset_same_layout before " + "index has an empty dataset; attach one with update_dataset before " "extend."); } else if constexpr (!cuvs::neighbors::is_padded_dataset_view_v>) { RAFT_FAIL("cagra::extend only supports an uncompressed padded dataset index"); @@ -408,7 +408,7 @@ void extend_core(raft::resources const& handle, cuvs::neighbors::cagra::add_graph_nodes( handle, extended_strided, index, updated_graph.view(), params); - index.update_device_dataset_same_layout(handle, extended_dataset); + index.update_dataset(handle, extended_dataset); index.update_graph(handle, raft::make_const_mdspan(updated_graph.view())); } } diff --git a/cpp/src/neighbors/detail/cagra/cagra_build.cuh b/cpp/src/neighbors/detail/cagra/cagra_build.cuh index c06f9b12e3..d371d213e3 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_build.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_build.cuh @@ -2390,7 +2390,7 @@ auto build_from_host_matrix(raft::resources const& res, * Build from a dense device `dataset_view` (padded or standard). VPQ views are rejected by * `cagra::build()` before this entry point is reached. Also used from ACE sub-builds and merge. * The returned index contains only the optimized graph; call - * `index::update_device_dataset_same_layout` before search. + * `index::update_dataset` before search. */ template requires cuvs::neighbors::is_dense_row_major_device_dataset_view_v diff --git a/cpp/src/neighbors/detail/cagra/cagra_merge.cuh b/cpp/src/neighbors/detail/cagra/cagra_merge.cuh index 3fd1963268..8dca813689 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_merge.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_merge.cuh @@ -80,7 +80,7 @@ cuvs::neighbors::cagra::index merge( RAFT_EXPECTS( dataset.n_rows() != 0, "cagra::merge only supports an index to which the dataset is attached. Please check if " - "the index has an empty dataset; attach one with update_device_dataset_same_layout " + "the index has an empty dataset; attach one with update_dataset " "before merge."); if (dim == 0) { dim = index->dim(); @@ -174,7 +174,7 @@ cuvs::neighbors::cagra::index merge( auto index = ::cuvs::neighbors::cagra::detail::build_from_device_matrix( handle, params, merged_dataset); - index.update_device_dataset_same_layout(handle, merged_dataset); + index.update_dataset(handle, merged_dataset); RAFT_LOG_DEBUG("cagra merge: using device memory for merged dataset"); return index; } @@ -187,7 +187,7 @@ cuvs::neighbors::cagra::index merge( merge_dataset(output_view.data_handle(), static_cast(stride)); auto index = ::cuvs::neighbors::cagra::detail::build_from_device_matrix( handle, params, merged_dataset); - index.update_device_dataset_same_layout(handle, merged_dataset); + index.update_dataset(handle, merged_dataset); RAFT_LOG_DEBUG("cagra merge: using device memory for merged dataset"); return index; } diff --git a/cpp/src/neighbors/detail/cagra/cagra_search.cuh b/cpp/src/neighbors/detail/cagra/cagra_search.cuh index 165e478337..04d3accdd6 100644 --- a/cpp/src/neighbors/detail/cagra/cagra_search.cuh +++ b/cpp/src/neighbors/detail/cagra/cagra_search.cuh @@ -233,7 +233,7 @@ void search_main(raft::resources const& res, if constexpr (cuvs::neighbors::is_empty_dataset_view_v) { RAFT_FAIL( "Attempted to search without a dataset. Please call " - "index.update_device_dataset_same_layout(...) first."); + "index.update_dataset(...) first."); } else if constexpr (cuvs::neighbors::is_device_vpq_f32_dataset_view_v) { RAFT_FAIL("FP32 VPQ dataset support is coming soon"); } else if constexpr (cuvs::neighbors::is_device_vpq_f16_dataset_view_v) { @@ -259,14 +259,13 @@ void search_main(raft::resources const& res, } else if constexpr (cuvs::neighbors::is_device_standard_dataset_view_v) { RAFT_FAIL( "CAGRA search requires a padded device dataset. Build from a standard dataset view, then " - "call " - "cagra::attach_dataset(res, index, padded_view) before search."); + "call cagra::update_dataset(res, std::move(index), padded_view) before search."); } else if constexpr (cuvs::neighbors::is_device_padded_dataset_view_v) { run_strided_like(index.dataset()); } else if constexpr (cuvs::neighbors::is_host_dataset_view_v) { static_assert(sizeof(DatasetViewT) == 0, "search requires a device-resident dataset. " - "Call cagra::attach_dataset(res, index, padded_view) " + "Call cagra::update_dataset(res, std::move(index), padded_view) " "to convert/attach into a search-ready device padded index before searching."); } else { static_assert(sizeof(DatasetViewT) == 0, "search: unsupported dataset view type"); diff --git a/cpp/src/neighbors/detail/tiered_index.cuh b/cpp/src/neighbors/detail/tiered_index.cuh index 368a41ee44..6abd57507f 100644 --- a/cpp/src/neighbors/detail/tiered_index.cuh +++ b/cpp/src/neighbors/detail/tiered_index.cuh @@ -126,8 +126,7 @@ struct index_state { { auto index = std::forward(build_fn)(res, tiered_params, dataset); if constexpr (std::is_same_v>) { - index.update_device_dataset_same_layout( - res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); + index.update_dataset(res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); } return std::make_shared(std::move(index)); } @@ -299,8 +298,7 @@ inline void update_cagra_ann_dataset_for_stride( cuvs::neighbors::cagra::device_standard_index& ann_index, raft::device_matrix_view dataset) { - ann_index.update_device_dataset_same_layout( - res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); + ann_index.update_dataset(res, cuvs::neighbors::make_device_standard_dataset_view(dataset)); } /** diff --git a/cpp/src/neighbors/iface/iface.hpp b/cpp/src/neighbors/iface/iface.hpp index 5d5ee76406..fb9905737a 100644 --- a/cpp/src/neighbors/iface/iface.hpp +++ b/cpp/src/neighbors/iface/iface.hpp @@ -57,12 +57,12 @@ void cagra_build_from_device_dataset( if constexpr (std::is_same_v>) { auto padded = cuvs::neighbors::make_device_padded_dataset_view(h, dview); auto index = cuvs::neighbors::cagra::build(h, cagra_params, padded); - index.update_device_dataset_same_layout(h, padded); + index.update_dataset(h, padded); interface.index_.emplace(std::move(index)); } else { auto standard = cuvs::neighbors::make_device_standard_dataset_view(dview); auto index = cuvs::neighbors::cagra::build(h, cagra_params, standard); - index.update_device_dataset_same_layout(h, standard); + index.update_dataset(h, standard); interface.index_.emplace(std::move(index)); } interface.cagra_owned_standard_dataset_.reset(); @@ -106,10 +106,10 @@ void build(const raft::resources& handle, if constexpr (std::is_same>::value) { cuvs::neighbors::host_padded_dataset_view host_padded( host_view, static_cast(host_view.extent(1))); - auto host_idx = cuvs::neighbors::cagra::build(handle, cagra_params, host_padded); - auto padded_r = cuvs::neighbors::make_device_padded_dataset(handle, index_dataset); - auto device_idx = - cuvs::neighbors::cagra::attach_dataset(handle, host_idx, padded_r->as_dataset_view()); + auto host_idx = cuvs::neighbors::cagra::build(handle, cagra_params, host_padded); + auto padded_r = cuvs::neighbors::make_device_padded_dataset(handle, index_dataset); + auto device_idx = cuvs::neighbors::cagra::update_dataset( + handle, std::move(host_idx), padded_r->as_dataset_view()); interface.cagra_owned_padded_dataset_ = std::move(padded_r); interface.cagra_owned_standard_dataset_.reset(); interface.index_.emplace(std::move(device_idx)); @@ -121,9 +121,8 @@ void build(const raft::resources& handle, index_dataset, static_cast(index_dataset.extent(1)), static_cast(index_dataset.stride(0))); - auto device_idx = - cuvs::neighbors::cagra::detail::convert_host_to_device_index(handle, host_idx); - device_idx.update_device_dataset_same_layout(handle, standard_r->as_dataset_view()); + auto device_idx = cuvs::neighbors::cagra::update_dataset( + handle, std::move(host_idx), standard_r->as_dataset_view()); interface.cagra_owned_standard_dataset_ = std::move(standard_r); interface.cagra_owned_padded_dataset_.reset(); interface.index_.emplace(std::move(device_idx)); diff --git a/cpp/src/neighbors/mg/mg_cagra_inst.cu.in b/cpp/src/neighbors/mg/mg_cagra_inst.cu.in index 78d6b109ef..8b2a467b03 100644 --- a/cpp/src/neighbors/mg/mg_cagra_inst.cu.in +++ b/cpp/src/neighbors/mg/mg_cagra_inst.cu.in @@ -80,9 +80,9 @@ void distribute_padded_dataset( return index; \ } \ \ - auto attach_dataset( \ + auto update_dataset( \ const raft::resources& res, \ - const cuvs::neighbors::mg_index, T, IdxT>& idx, \ + cuvs::neighbors::mg_index, T, IdxT>&& idx, \ cuvs::neighbors::device_padded_dataset_view const& padded_dataset) \ -> cuvs::neighbors::mg_index, T, IdxT> \ { \ @@ -90,10 +90,10 @@ void distribute_padded_dataset( out.ann_interfaces_.resize(idx.num_ranks_); \ detail::distribute_padded_dataset( \ res, idx, padded_dataset, [&](const raft::resources& dev_res, int rank, auto dataset) { \ - const auto& in_if = idx.ann_interfaces_[rank]; \ - auto& out_if = out.ann_interfaces_[rank]; \ - auto padded_idx = cuvs::neighbors::cagra::attach_dataset( \ - dev_res, in_if.index_.value(), dataset->as_dataset_view()); \ + auto& in_if = idx.ann_interfaces_[rank]; \ + auto& out_if = out.ann_interfaces_[rank]; \ + auto padded_idx = cuvs::neighbors::cagra::update_dataset( \ + dev_res, std::move(in_if.index_.value()), dataset->as_dataset_view()); \ out_if.cagra_owned_padded_dataset_ = std::move(dataset); \ out_if.cagra_owned_standard_dataset_.reset(); \ out_if.index_.emplace(std::move(padded_idx)); \ @@ -101,7 +101,7 @@ void distribute_padded_dataset( return out; \ } \ \ - void update_device_dataset_same_layout( \ + void update_dataset( \ const raft::resources& res, \ cuvs::neighbors::mg_index, T, IdxT>& idx, \ cuvs::neighbors::device_padded_dataset_view const& padded_dataset) \ @@ -110,7 +110,7 @@ void distribute_padded_dataset( res, idx, padded_dataset, [&](const raft::resources& dev_res, int rank, auto dataset) { \ auto& ann_if = idx.ann_interfaces_[rank]; \ auto& padded_idx = ann_if.index_.value(); \ - padded_idx.update_device_dataset_same_layout(dev_res, dataset->as_dataset_view()); \ + padded_idx.update_dataset(dev_res, dataset->as_dataset_view()); \ ann_if.cagra_owned_padded_dataset_ = std::move(dataset); \ ann_if.cagra_owned_standard_dataset_.reset(); \ }); \ diff --git a/cpp/src/neighbors/tiered_index.cu b/cpp/src/neighbors/tiered_index.cu index f5942001d7..7d1e684492 100644 --- a/cpp/src/neighbors/tiered_index.cu +++ b/cpp/src/neighbors/tiered_index.cu @@ -149,15 +149,12 @@ void extend(raft::resources const& res, if (storage->num_rows_allocated != idx->state->storage->num_rows_allocated) { // CAGRA could be holding on to a non-owning view of the previous dataset in the ann_index, // which is problematic since the underlying ownership of the dataset could be freed here - // call cagra::index::update_device_dataset_same_layout on it to update the ann_index to point - // to the - // new dataset + // call cagra::index::update_dataset on it to update the ann_index to point to the new dataset if (next_state->ann_index) { auto dataset = raft::make_device_matrix_view( storage->dataset.data(), next_state->ann_rows(), storage->dim); - // Block 'search' calls during the update_device_dataset_same_layout call to ensure that this - // doesn't cause issues in a multithreaded environment + // Block 'search' calls during update_dataset to avoid issues in a multithreaded environment std::unique_lock lock(idx->ann_mutex); detail::update_cagra_ann_dataset_for_stride(res, *next_state->ann_index, dataset); } diff --git a/cpp/tests/neighbors/ann_cagra.cuh b/cpp/tests/neighbors/ann_cagra.cuh index 529b8fe038..4b7d80e610 100644 --- a/cpp/tests/neighbors/ann_cagra.cuh +++ b/cpp/tests/neighbors/ann_cagra.cuh @@ -72,11 +72,11 @@ void cagra_build_into_index( *ace_host_dataset, static_cast(ace_host_dataset->extent(1))); auto host_idx = cagra::build(res, params, host_view); // In-memory ACE returns graph-only; attach device padded storage for search. - index = cagra::attach_dataset(res, host_idx, padded); + index = cagra::update_dataset(res, std::move(host_idx), padded); return; } index = cagra::build(res, params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); } struct test_cagra_sample_filter { @@ -489,9 +489,7 @@ class AnnCagraTest : public ::testing::TestWithParam { std::unique_ptr> loaded_dataset; cagra::deserialize(handle_, index_file.filename, &index, &loaded_dataset); - if (!ps.include_serialized_dataset) { - index.update_device_dataset_same_layout(handle_, device_padded.view); - } + if (!ps.include_serialized_dataset) { index.update_dataset(handle_, device_padded.view); } auto search_queries_view = raft::make_device_matrix_view( search_queries.data(), ps.n_queries, ps.dim); @@ -2126,7 +2124,7 @@ class AnnCagraMultiPartitionTest : public ::testing::TestWithParam padded_small( res, small_dataset_view); auto small_index = cagra::build(res, small_index_params, padded_small.view); - small_index.update_device_dataset_same_layout(res, padded_small.view); + small_index.update_dataset(res, padded_small.view); raft::resource::sync_stream(res); // Step 2: Update to FULL dataset (1000 points) but keep small graph (500 nodes) // This creates the exact bug scenario: dataset.size=1000, graph.extent(0)=500 - small_index.update_device_dataset_same_layout(res, - cuvs::neighbors::make_device_padded_dataset_view( - res, raft::make_const_mdspan(dataset.view()))); + small_index.update_dataset(res, + cuvs::neighbors::make_device_padded_dataset_view( + res, raft::make_const_mdspan(dataset.view()))); // Verify the mismatch - THIS IS THE BUG SCENARIO! ASSERT_EQ(small_index.graph().extent(0), n_graph); // Graph has 500 nodes diff --git a/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu b/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu index 9e2cde1d80..320626b211 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_issue_93_reproducer.cu @@ -75,7 +75,7 @@ TEST(Issue93Reproducer, ConcurrentSearchDifferentGraphDegrees) padded_builders.emplace_back(handle, raft::make_const_mdspan(database.view())); auto index = cagra::build(handle, ip, padded_builders.back().view); - index.update_device_dataset_same_layout(handle, padded_builders.back().view); + index.update_dataset(handle, padded_builders.back().view); indices.push_back(std::move(index)); } raft::resource::sync_stream(handle); diff --git a/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu b/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu index 3d6b5c98a2..c3774f5cfd 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_iterative_cagra_build.cu @@ -38,7 +38,7 @@ class CagraIterativeBuildBugTest : public ::testing::Test { cuvs::neighbors::test::padded_device_matrix_for_cagra padded( res, raft::make_const_mdspan(dataset->view())); auto cagra_index = cagra::build(res, index_params, padded.view); - cagra_index.update_device_dataset_same_layout(res, padded.view); + cagra_index.update_dataset(res, padded.view); raft::resource::sync_stream(res); // Verify the index was built successfully diff --git a/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu b/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu index 6e965bfcb9..85d8eb8315 100644 --- a/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu +++ b/cpp/tests/neighbors/ann_cagra/bug_multi_cta_crash.cu @@ -30,7 +30,7 @@ class AnnCagraBugMultiCTACrash : public ::testing::TestWithParamview())); auto cagra_index = cagra::build(res, cagra_index_params, build_padded_->view); - cagra_index.update_device_dataset_same_layout(res, build_padded_->view); + cagra_index.update_dataset(res, build_padded_->view); raft::resource::sync_stream(res); cagra::search_params cagra_search_params; diff --git a/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu b/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu index de06bef34a..64fb5c91af 100644 --- a/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu +++ b/cpp/tests/neighbors/ann_cagra/test_float_uint32_t.cu @@ -96,7 +96,7 @@ void expect_multi_partition_search_throws( part_padded.emplace_back(handle, view); auto const& padded = part_padded.back().view; part_indices.push_back(cagra::build(handle, index_params, padded)); - part_indices.back().update_device_dataset_same_layout(handle, padded); + part_indices.back().update_dataset(handle, padded); } std::vector*> index_ptrs; for (auto& idx : part_indices) { diff --git a/cpp/tests/neighbors/dynamic_batching/test_cagra.cu b/cpp/tests/neighbors/dynamic_batching/test_cagra.cu index 802bae4bce..336daab5f4 100644 --- a/cpp/tests/neighbors/dynamic_batching/test_cagra.cu +++ b/cpp/tests/neighbors/dynamic_batching/test_cagra.cu @@ -22,7 +22,7 @@ auto build_cagra_with_dataset(raft::resources const& res, { auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset); auto index = cagra::build(res, params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); return index; } diff --git a/cpp/tests/neighbors/hnsw.cu b/cpp/tests/neighbors/hnsw.cu index d9fedf0ed7..01eeeb511c 100644 --- a/cpp/tests/neighbors/hnsw.cu +++ b/cpp/tests/neighbors/hnsw.cu @@ -98,7 +98,7 @@ class AnnHNSWTest : public ::testing::TestWithParam { cuvs::neighbors::test::padded_device_matrix_for_cagra padded(handle_, database_view); auto index = cuvs::neighbors::cagra::build(handle_, index_params, padded.view); - index.update_device_dataset_same_layout(handle_, padded.view); + index.update_dataset(handle_, padded.view); raft::resource::sync_stream(handle_); cuvs::neighbors::hnsw::search_params search_params; diff --git a/cpp/tests/neighbors/mg.cuh b/cpp/tests/neighbors/mg.cuh index b575a7e285..336563d9c4 100644 --- a/cpp/tests/neighbors/mg.cuh +++ b/cpp/tests/neighbors/mg.cuh @@ -250,7 +250,7 @@ class AnnMGTest : public ::testing::TestWithParam { auto padded_index_dataset = cuvs::neighbors::make_device_padded_dataset_view(clique_, index_dataset_device); auto search_index = - cuvs::neighbors::cagra::attach_dataset(clique_, new_index, padded_index_dataset); + cuvs::neighbors::cagra::update_dataset(clique_, std::move(new_index), padded_index_dataset); if (ps.m_mode == m_mode_t::MERGE_ON_ROOT_RANK) search_params.merge_mode = MERGE_ON_ROOT_RANK; @@ -409,8 +409,8 @@ class AnnMGTest : public ::testing::TestWithParam { d_index_dataset.data(), ps.num_db_vecs, ps.dim); auto padded_index_dataset = cuvs::neighbors::make_device_padded_dataset_view(clique_, index_dataset_device); - auto search_index = - cuvs::neighbors::cagra::attach_dataset(clique_, distributed_index, padded_index_dataset); + auto search_index = cuvs::neighbors::cagra::update_dataset( + clique_, std::move(distributed_index), padded_index_dataset); search_params.merge_mode = TREE_MERGE; @@ -595,7 +595,7 @@ class AnnMGTest : public ::testing::TestWithParam { auto padded_index_dataset = cuvs::neighbors::make_device_padded_dataset_view(clique_, index_dataset_device); auto search_index = - cuvs::neighbors::cagra::attach_dataset(clique_, index, padded_index_dataset); + cuvs::neighbors::cagra::update_dataset(clique_, std::move(index), padded_index_dataset); int n_parallel_searches = 16; std::vector searches_correctness(n_parallel_searches); diff --git a/examples/cpp/src/cagra_bloom_filter_example.cu b/examples/cpp/src/cagra_bloom_filter_example.cu index 7d505dd4f0..9ce1ff6baa 100644 --- a/examples/cpp/src/cagra_bloom_filter_example.cu +++ b/examples/cpp/src/cagra_bloom_filter_example.cu @@ -72,7 +72,7 @@ int main() auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, raft::make_const_mdspan(dataset.view())); auto index = cuvs::neighbors::cagra::build(res, index_params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); // Build one global bloom filter over the index: bulk-insert every valid row id once. std::vector valid_ids_host; diff --git a/examples/cpp/src/cagra_example.cu b/examples/cpp/src/cagra_example.cu index bd35b4bd86..b7a96fa39a 100644 --- a/examples/cpp/src/cagra_example.cu +++ b/examples/cpp/src/cagra_example.cu @@ -34,7 +34,7 @@ void cagra_build_search_simple(raft::device_resources const& dev_resources, std::cout << "Building CAGRA index (search graph)" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(dev_resources, dataset); auto index = cagra::build(dev_resources, index_params, padded); - index.update_device_dataset_same_layout(dev_resources, padded); + index.update_dataset(dev_resources, padded); std::cout << "CAGRA index has " << index.size() << " vectors" << std::endl; std::cout << "CAGRA graph has degree " << index.graph_degree() << ", graph size [" diff --git a/examples/cpp/src/cagra_filter_udf_example.cu b/examples/cpp/src/cagra_filter_udf_example.cu index 56121ecb73..1b331719b0 100644 --- a/examples/cpp/src/cagra_filter_udf_example.cu +++ b/examples/cpp/src/cagra_filter_udf_example.cu @@ -147,7 +147,7 @@ int main() std::cout << "Building CAGRA index" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset.view()); auto index = cuvs::neighbors::cagra::build(res, index_params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); std::vector row_tenant_ids(n_rows); std::vector row_timestamps(n_rows); diff --git a/examples/cpp/src/cagra_hnsw_ace_example.cu b/examples/cpp/src/cagra_hnsw_ace_example.cu index 05d448fbc1..e27d4dfa47 100644 --- a/examples/cpp/src/cagra_hnsw_ace_example.cu +++ b/examples/cpp/src/cagra_hnsw_ace_example.cu @@ -99,8 +99,8 @@ void cagra_build_search_ace(raft::device_resources const& dev_resources, // In-memory ACE path: graph is in host memory. Upload the original dataset to device and // attach it before from_cagra builds the HNSW hierarchy in memory. padded_owner = cuvs::neighbors::make_device_padded_dataset(dev_resources, dataset_host_view); - auto device_index = - cagra::attach_dataset(dev_resources, ace_host_index, padded_owner->as_dataset_view()); + auto device_index = cagra::update_dataset( + dev_resources, std::move(ace_host_index), padded_owner->as_dataset_view()); hnsw_index = hnsw::from_cagra(dev_resources, hnsw_params, device_index, dataset_host_view); } diff --git a/examples/cpp/src/cagra_persistent_example.cu b/examples/cpp/src/cagra_persistent_example.cu index 4580526e79..494a8230a3 100644 --- a/examples/cpp/src/cagra_persistent_example.cu +++ b/examples/cpp/src/cagra_persistent_example.cu @@ -71,7 +71,7 @@ void cagra_build_search_variants(raft::device_resources const& res, std::cout << "Building CAGRA index (search graph)" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset); auto index = cagra::build(res, index_params, padded); - index.update_device_dataset_same_layout(res, padded); + index.update_dataset(res, padded); std::cout << "CAGRA index has " << index.size() << " vectors" << std::endl; std::cout << "CAGRA graph has degree " << index.graph_degree() << ", graph size [" diff --git a/examples/cpp/src/dynamic_batching_example.cu b/examples/cpp/src/dynamic_batching_example.cu index dc9a29e759..d52d59f49a 100644 --- a/examples/cpp/src/dynamic_batching_example.cu +++ b/examples/cpp/src/dynamic_batching_example.cu @@ -116,7 +116,7 @@ void dynamic_batching_example(raft::resources const& res, std::cout << "Building CAGRA index (search graph)" << std::endl; auto padded = cuvs::neighbors::make_device_padded_dataset_view(res, dataset); auto orig_index = cagra::build(res, orig_index_params, padded); - orig_index.update_device_dataset_same_layout(res, padded); + orig_index.update_dataset(res, padded); std::cout << "CAGRA index has " << orig_index.size() << " vectors" << std::endl; std::cout << "CAGRA graph has degree " << orig_index.graph_degree() << ", graph size ["