diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dc06c88b31..ffc48de094 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1241,6 +1241,28 @@ if(NOT BUILD_CPU_ONLY) "${cutile_fused_1nn_tiles}" @ONLY ) list(APPEND cutile_fused_1nn_files "${cutile_fused_1nn_tiles}") + + set(pruned_1nn_cutile_dir + "${CMAKE_CURRENT_SOURCE_DIR}/src/distance/detail/pruned_top_1_nn/cutile" + ) + set(cutile_pruned_1nn_generated_dir + "${CMAKE_CURRENT_BINARY_DIR}/generated_kernels/distance/pruned_1nn/cutile" + ) + generate_cutile_kernels( + cutile_pruned_1nn_files + KERNEL_DIR "${pruned_1nn_cutile_dir}" + KERNEL_BASENAME "pruned_1nn_fused_filter" + KERNEL_PYTHON "fused_filter_kernel.py" + EXPORT_SCRIPT "export_fused_filter.py" + OUTPUT_DIRECTORY "${cutile_pruned_1nn_generated_dir}" + MATRIX_JSON_FILE "${pruned_1nn_cutile_dir}/fused_filter_cutile_matrix.json" + PYTHON "${cutile_python}" BIN2C "${cutile_bin2c}" + FRAGMENT_TAG_FORMAT_CUBIN + "cuvs::distance::detail::pruned_nn::fragment_tag_fused_filter_cubin" + FRAGMENT_TAG_HEADER_FILES + "" + "" "" + ) endif() # Note that this matrix contains an `arch_includes` placeholder, since we don't currently have a # way to do an item-wise transform on a list after computing the matrix product and before @@ -1463,6 +1485,7 @@ if(NOT BUILD_CPU_ONLY) src/cluster/kmeans_balanced_fit_predict_int8.cu src/cluster/kmeans_balanced_predict_int8.cu src/cluster/kmeans_balanced_predict_uint8.cu + src/cluster/kmeans_superkmeans.cu src/cluster/kmeans_transform_double.cu src/cluster/kmeans_transform_float.cu src/cluster/single_linkage_float.cu @@ -1577,7 +1600,10 @@ if(NOT BUILD_CPU_ONLY) target_sources( cuvs_objs PRIVATE ${cutile_fused_1nn_files} - src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu ${cutile_smoke_files} + src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu + ${cutile_smoke_files} + ${cutile_pruned_1nn_files} + src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu ) endif() @@ -1626,8 +1652,11 @@ if(NOT BUILD_CPU_ONLY) PUBLIC "$" "$" INTERFACE "$" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" "${CMAKE_CURRENT_BINARY_DIR}/src" - "${cutile_fused_1nn_generated_dir}" "${cutile_smoke_generated_dir}" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" + "${CMAKE_CURRENT_BINARY_DIR}/src" + "${cutile_fused_1nn_generated_dir}" + "${cutile_smoke_generated_dir}" + "${cutile_pruned_1nn_generated_dir}" ) # Endian detection diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index 13882b118d..bfb21c1901 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -1987,6 +1987,179 @@ void find_k(raft::resources const& handle, */ } // namespace helpers +namespace experimental { + +/** + * @defgroup superkmeans Experimental SuperKMeans APIs + * @{ + */ + +/** + * Simple object to specify hyper-parameters to the SuperKMeans algorithm. + * + * SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every + * row with the fused 1-NN over all dimensions; later iterations compute partial distances over the + * first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the + * ADSampling bound, without its random rotation), and finish the remaining dimensions only for the + * survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: + * row strides must be divisible by 16 bytes, dataset and centroid pointers must be 16-byte + * aligned, and the build and device must provide the pruning kernel. Inputs that violate these + * requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every + * iteration uses full nearest-centroid assignment (Lloyd's algorithm). + * + * The number of clusters is taken from the `centroids` argument. The start centroids are rows + * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run + * (there is no convergence test), and an empty cluster keeps its previous centroid. + * + * Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 + * tensor cores for both float and half input; surviving candidates are refined in FP32 over + * subsequent tiles of up to 256 dimensions, with further pruning between tiles. + * + * Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional + * inertia are computed with a full assignment against the final centroids. + * + * The following metrics are currently supported: + * - L2Expanded + */ +struct superkmeans_params : base_params { + /** + * Number of training iterations (default 10). + */ + uint32_t n_iters = 10; + + /** + * Seed to the random number generator (the start rows); default seed 0. + */ + raft::random::RngState rng_state{0}; +}; + +/** + * @brief Find clusters with the SuperKMeans algorithm. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia = std::nullopt); + +/** + * @brief Find clusters with the SuperKMeans algorithm. + * + * With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the + * centroids, which are accumulated and returned in FP32. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia = std::nullopt); + +/** + * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia = std::nullopt); + +/** + * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia = std::nullopt); + +/** + * @} + */ + +} // namespace experimental + } // namespace kmeans } // namespace cluster } // namespace CUVS_EXPORT cuvs diff --git a/cpp/include/cuvs/detail/jit_lto/pruned_top_1_nn/fused_filter_fragments.hpp b/cpp/include/cuvs/detail/jit_lto/pruned_top_1_nn/fused_filter_fragments.hpp new file mode 100644 index 0000000000..c678409dd1 --- /dev/null +++ b/cpp/include/cuvs/detail/jit_lto/pruned_top_1_nn/fused_filter_fragments.hpp @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +namespace cuvs::distance::detail::pruned_nn { + +template +struct fragment_tag_fused_filter_cubin { + static constexpr int cc_major = ArchTag::cc_major; + static constexpr int cc_minor = ArchTag::cc_minor; +}; + +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh new file mode 100644 index 0000000000..a14699fb9c --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -0,0 +1,349 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* + * Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * + * This source code is licensed under the MIT license found in the + * LICENSE file thirdparty/LICENSES/LICENSE.superkmeans + */ +#pragma once + +#include "../../distance/pruned_top_1_nn.cuh" +#include "../../neighbors/detail/ann_utils.cuh" +#include "../kmeans_balanced.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace cuvs::cluster::kmeans::detail::superkmeans { + +namespace pruned_nn = cuvs::distance::detail::pruned_nn; + +// Pruning only pays off with enough dimensions and centroids to discard; below these thresholds +// every iteration assigns with the full 1-NN. +constexpr int64_t kMinPrunedK = 256; +constexpr int64_t kMinPrunedDim = 1024; + +inline bool prunes(int64_t n_clusters, int64_t dim) +{ + return dim > kMinPrunedDim && n_clusters > kMinPrunedK; +} + +/** Sum of squared distances of the rows to their assigned centroids. */ +template +auto inertia(const raft::resources& handle, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) -> float +{ + auto mr = raft::resource::get_workspace_resource_ref(handle); + auto dist = + raft::make_device_mdarray(handle, mr, raft::make_extents(X.extent(0))); + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + pruned_nn::distance_to_assigned(handle, + X.data_handle(), + X.extent(0), + centroids.data_handle(), + X.extent(1), + labels.data_handle(), + dist.data_handle()); + auto sum = raft::make_device_mdarray(handle, mr, raft::make_extents()); + raft::linalg::mapThenSumReduce( + sum.data_handle(), dist.size(), raft::identity_op{}, stream, dist.data_handle()); + float out = 0; + raft::copy(&out, sum.data_handle(), 1, stream); + raft::resource::sync_stream(handle); + return out; +} + +template +void validate(const cuvs::cluster::kmeans::experimental::superkmeans_params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids) +{ + RAFT_EXPECTS(params.metric == cuvs::distance::DistanceType::L2Expanded, + "SuperKMeans supports only the L2Expanded metric"); + RAFT_EXPECTS(centroids.extent(0) > 0, "The number of centroids must be strictly positive"); + RAFT_EXPECTS(centroids.extent(1) == X.extent(1), + "Number of features in dataset and centroids are different"); +} + +/** Each row's closest centroid over all dims. */ +template +void predict(const raft::resources& handle, + const cuvs::cluster::kmeans::experimental::superkmeans_params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) +{ + validate(params, X, centroids); + RAFT_EXPECTS(labels.extent(0) == X.extent(0), + "Number of rows in dataset and labels are different"); + cuvs::cluster::kmeans::balanced_params assign_params; + assign_params.metric = params.metric; + cuvs::cluster::kmeans_balanced::predict(handle, + assign_params, + X, + centroids, + labels, + cuvs::spatial::knn::detail::utils::mapping{}); +} + +/** + * The pruned assignment (cuvs::distance::pruned_top_1_nn) and what it keeps across iterations: the + * row norms and the workspace. Float filter rows are prepared per batch. + * The centroids the 1-NN reads are T: for half rows, a half copy of the float centroids, which is + * also the filter's. + */ +template +class pruned_assignment { + public: + pruned_assignment(const raft::resources& handle, + raft::device_matrix_view X, + int64_t n_clusters) + : pruned_assignment(handle, X, n_clusters, select_batch_size(handle, X, n_clusters)) + { + } + + pruned_assignment(const raft::resources& handle, + raft::device_matrix_view X, + int64_t n_clusters, + int64_t row_batch_size) + : handle_(handle), + X_(X), + n_rows_(X.extent(0)), + dim_(X.extent(1)), + n_clusters_(n_clusters), + filter_dim_(std::is_same_v ? dim_ : pruned_nn::kPartialDim), + row_norms_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_rows_))), + centroid_norms_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_clusters))), + dist_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_rows_))), + filter_centroids_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_clusters, filter_dim_))), + row_batch_size_(row_batch_size), + workspace_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents( + cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, row_batch_size_)))) + { + // The input rows do not change between iterations, so cache their filter norms once. + pruned_nn::squared_norms( + handle, X.data_handle(), n_rows_, dim_, pruned_nn::kPartialDim, row_norms_.data_handle()); + } + + /** + * Updates `labels`, each row's centroid from the previous assignment, for `centroids`. + */ + void assign(raft::device_matrix_view centroids, + raft::device_vector_view labels) + { + pruned_nn::squared_norms(handle_, + centroids.data_handle(), + n_clusters_, + dim_, + pruned_nn::kPartialDim, + centroid_norms_.data_handle(), + filter_centroids_.data_handle(), + filter_dim_); + const T* kernel_centroids = nullptr; + if constexpr (std::is_same_v) { + kernel_centroids = filter_centroids_.data_handle(); + } else { + kernel_centroids = centroids.data_handle(); + } + cuvs::distance::pruned_top_1_nn(handle_, + {labels.data_handle(), dist_.data_handle()}, + X_.data_handle(), + kernel_centroids, + filter_centroids_.data_handle(), + row_norms_.data_handle(), + centroid_norms_.data_handle(), + n_rows_, + n_clusters_, + dim_, + row_batch_size_, + workspace_.data_handle(), + workspace_.size()); + } + + private: + static auto select_batch_size(const raft::resources& handle, + raft::device_matrix_view X, + int64_t n_clusters) -> int64_t + { + const auto n_rows = X.extent(0); + const auto budget = std::min( + (raft::resource::get_workspace_free_bytes(handle) / 10) * 8, std::size_t{1} << 29); + const auto fits = [&](int64_t batch_size) { + try { + // Only workspace is used here; keep the probe local rather than swapping global resources. + raft::mr::dry_run_resource probe{ + raft::resource::get_workspace_resource_ref(handle)}; + raft::resources dry_handle(handle); + dry_handle.add_resource_factory( + std::make_shared(true)); + raft::resource::set_workspace_resource( + dry_handle, probe, std::numeric_limits::max()); + pruned_assignment candidate(dry_handle, X, n_clusters, batch_size); + return raft::resource::get_workspace_used_bytes(dry_handle) <= budget; + } catch (const rmm::bad_alloc&) { + return false; + } + }; + + constexpr int64_t alignment = pruned_nn::kAlignment / sizeof(float); + const auto min_batch_size = std::min(n_rows, alignment); + auto batch_size = n_rows; + while (!fits(batch_size)) { + RAFT_EXPECTS(batch_size > min_batch_size, + "Insufficient workspace for SuperKMeans pruned assignment"); + const auto half_batch = batch_size / 2; + const int64_t row_multiple = half_batch >= 1024 ? 1024 : alignment; + batch_size = std::max(min_batch_size, half_batch / row_multiple * row_multiple); + } + return batch_size; + } + + const raft::resources& handle_; + raft::device_matrix_view X_; + int64_t n_rows_, dim_, n_clusters_, filter_dim_; + raft::device_vector row_norms_, centroid_norms_, dist_; + raft::device_matrix filter_centroids_; + int64_t row_batch_size_; + raft::device_vector workspace_; +}; + +/** + * Lloyd's iterations from rows sampled at random, a fixed number of them. Each iteration assigns + * every row (the full 1-NN when pruning does not apply) and sets each centroid to the mean of its + * rows; an empty cluster keeps its centroid. `labels` and `inertia`, when given, are for the + * returned centroids. + */ +template +void fit(const raft::resources& handle, + const cuvs::cluster::kmeans::experimental::superkmeans_params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> labels, + std::optional> inertia_out) +{ + const int64_t n_rows = X.extent(0); + const int64_t dim = X.extent(1); + const int64_t n_clusters = centroids.extent(0); + validate(params, X, raft::make_const_mdspan(centroids)); + RAFT_EXPECTS(n_rows >= n_clusters, "SuperKMeans needs at least as many rows as clusters"); + RAFT_EXPECTS(!labels.has_value() || labels->extent(0) == n_rows, + "Number of rows in dataset and labels are different"); + RAFT_EXPECTS(params.n_iters > 0, "SuperKMeans needs at least one iteration"); + const bool pruned = prunes(n_clusters, dim); + if (pruned) { + // Validate before sampling centroids or allocating scratch: eligible inputs must use pruning. + RAFT_EXPECTS(dim % (pruned_nn::kAlignment / sizeof(T)) == 0, + "SuperKMeans pruning needs row strides divisible by 16 bytes"); + const auto aligned = [](const void* ptr) { + return reinterpret_cast(ptr) % pruned_nn::kAlignment == 0; + }; + RAFT_EXPECTS(aligned(X.data_handle()) && aligned(centroids.data_handle()), + "SuperKMeans pruning needs 16-byte-aligned dataset and centroid pointers"); + RAFT_EXPECTS(cuvs::distance::is_pruned_top_1_nn_available(dim), + "SuperKMeans pruning kernel is unavailable on this build or device"); + } + + auto mr = raft::resource::get_workspace_resource_ref(handle); + auto stream = raft::resource::get_cuda_stream(handle); + auto mapping_op = cuvs::spatial::knn::detail::utils::mapping{}; + + // The start centroids: rows sampled at random + raft::random::RngState random_state(params.rng_state.seed); + if constexpr (std::is_same_v) { + raft::matrix::sample_rows(handle, random_state, X, centroids); + } else { + auto start = raft::make_device_mdarray( + handle, mr, raft::make_extents(n_clusters, dim)); + raft::matrix::sample_rows(handle, random_state, X, start.view()); + raft::linalg::map(handle, centroids, mapping_op, raft::make_const_mdspan(start.view())); + } + + auto assigned = + raft::make_device_mdarray(handle, mr, raft::make_extents(n_rows)); + auto sizes = raft::make_device_mdarray( + handle, mr, raft::make_extents(n_clusters)); + auto means = raft::make_device_mdarray( + handle, mr, raft::make_extents(n_clusters, dim)); + auto assign = [&]() { + predict(handle, params, X, raft::make_const_mdspan(centroids), assigned.view()); + }; + + std::optional> pruned_assign; + if (pruned) { pruned_assign.emplace(handle, X, n_clusters); } + + for (uint32_t iter = 0; iter < params.n_iters; iter++) { + if (pruned_assign.has_value() && iter > 0) { + pruned_assign->assign(raft::make_const_mdspan(centroids), assigned.view()); + } else { + assign(); + } + cuvs::cluster::kmeans_balanced::helpers::calc_centers_and_sizes( + handle, + X, + raft::make_const_mdspan(assigned.view()), + means.view(), + sizes.view(), + true, + mapping_op); + const float* m = means.data_handle(); + const float* c = centroids.data_handle(); + const uint32_t* z = sizes.data_handle(); + raft::linalg::map_offset( + handle, centroids, [=] __device__(int64_t i) { return z[i / dim] ? m[i] : c[i]; }); + } + + if (labels.has_value() || inertia_out.has_value()) { + assign(); + if (inertia_out.has_value()) { + *inertia_out->data_handle() = inertia( + handle, X, raft::make_const_mdspan(centroids), raft::make_const_mdspan(assigned.view())); + } + if (labels.has_value()) { + raft::copy(labels->data_handle(), assigned.data_handle(), n_rows, stream); + } + } +} + +} // namespace cuvs::cluster::kmeans::detail::superkmeans diff --git a/cpp/src/cluster/kmeans_superkmeans.cu b/cpp/src/cluster/kmeans_superkmeans.cu new file mode 100644 index 0000000000..2a2353ddd5 --- /dev/null +++ b/cpp/src/cluster/kmeans_superkmeans.cu @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "detail/superkmeans.cuh" + +#include + +namespace cuvs::cluster::kmeans::experimental { + +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, std::nullopt, inertia); +} + +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, std::nullopt, inertia); +} + +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) +{ + detail::superkmeans::predict(handle, params, X, centroids, labels); +} + +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) +{ + detail::superkmeans::predict(handle, params, X, centroids, labels); +} + +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, labels, inertia); +} + +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, labels, inertia); +} + +} // namespace cuvs::cluster::kmeans::experimental diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py new file mode 100644 index 0000000000..9919b82e44 --- /dev/null +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py @@ -0,0 +1,126 @@ +# ============================================================================= +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ============================================================================= +"""Export the pruned 1-NN's fused first-filter cuTile kernel to cubin.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import cuda.tile as ct +from cuda.tile.compilation import ( + ArrayConstraint, + CallingConvention, + ConstantConstraint, + KernelSignature, + ScalarConstraint, + export_kernel, +) + +# CI enables Python safe-path mode, so the script directory is not guaranteed +# to be importable even when this file is executed directly. +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from fused_filter_kernel import kernel_symbol, make_kernel # noqa: E402 + + +def _matrix(): + """float16, row-major, unit last stride, 16-byte aligned base and row pitch.""" + return ArrayConstraint( + ct.float16, + ndim=2, + index_dtype=ct.int32, + stride_lower_bound_incl=(0, None), + alias_groups=("read_only_inputs",), + may_alias_internally=False, + stride_constant=(None, 1), + stride_divisible_by=(8, 1), + base_addr_divisible_by=16, + ) + + +def _vector(elem_dtype, alias_groups=()): + return ArrayConstraint( + elem_dtype, + ndim=1, + index_dtype=ct.int32, + stride_lower_bound_incl=(None,), + alias_groups=alias_groups, + may_alias_internally=False, + stride_constant=(1,), + base_addr_divisible_by=16, + ) + + +def _counts(): + return ArrayConstraint( + ct.int32, + ndim=2, + index_dtype=ct.int32, + stride_lower_bound_incl=(0, None), + alias_groups=(), + may_alias_internally=False, + stride_constant=(None, 1), + base_addr_divisible_by=16, + ) + + +def _signature(data_type: str, tile_m: int, tile_n: int, tile_k: int): + # Argument order of the C++ launch: each array as (pointer, shape..., stride...), then the + # scalars; the tile sizes are compiled in. + read_only = ("read_only_inputs",) + return KernelSignature( + parameters=[ + _matrix(), # rows (n_rows, d) + _matrix(), # centroids (n_centroids, d) + _vector(ct.float32, read_only), # row_norms over d' + _vector(ct.float32, read_only), # centroid_norms over d' + _vector(ct.float32, read_only), # best: current distances + _vector(ct.int64), # packed survivors + _counts(), # count (n_rows, 1) + ScalarConstraint(ct.int32), # n_rows + ScalarConstraint(ct.int32), # n_centroids + ScalarConstraint(ct.float32), # ratio + ScalarConstraint(ct.int32), # d' + ScalarConstraint(ct.int32), # row_stride + ConstantConstraint(tile_m), + ConstantConstraint(tile_n), + ConstantConstraint(tile_k), + ], + calling_convention=CallingConvention.cutile_python_v1(), + ).with_symbol(kernel_symbol(data_type)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output_file", type=Path) + parser.add_argument("--format", choices=("cubin",), default="cubin") + parser.add_argument("--data-type", choices=("half",), required=True) + parser.add_argument("--metric", choices=("l2",), required=True) + parser.add_argument("--index-type", choices=("int32",), required=True) + parser.add_argument("--tile-m", type=int, required=True) + parser.add_argument("--tile-n", type=int, required=True) + parser.add_argument("--tile-k", type=int, required=True) + parser.add_argument("--gpu-code", required=True) + parser.add_argument("--occupancy", type=int) + args = parser.parse_args() + + export_kernel( + kernel=make_kernel(gpu_code=args.gpu_code, occupancy=args.occupancy), + signatures=[ + _signature(args.data_type, args.tile_m, args.tile_n, args.tile_k) + ], + output_file=str(args.output_file), + gpu_code=args.gpu_code, + output_format=args.format, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json new file mode 100644 index 0000000000..7db659b5d4 --- /dev/null +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json @@ -0,0 +1,37 @@ +[ + { + "_data": [ + {"data_type": "half", "data_abbrev": "h"} + ], + "_metric": [{"metric": "l2", "metric_abbrev": "l2"}], + "_index": [{"index_type": "int32", "index_abbrev": "i32"}], + "output_format": "cubin", + "register": "cubin", + "artifact_basename": "@data_type@_@gpu_code@", + "artifact_ext": "cubin", + "_tile": [ + { + "tile_m": 64, "tile_n": 64, "tile_k": 64, "occupancy": 2, + "_architecture": [ + {"gpu_code": "sm_90", "arch_tag": "cutile_arch_9_0"} + ] + }, + { + "tile_m": 64, "tile_n": 128, "tile_k": 64, "occupancy": 2, + "_architecture": [ + {"gpu_code": "sm_80", "arch_tag": "cutile_arch_8_0"}, + {"gpu_code": "sm_86", "arch_tag": "cutile_arch_8_6"}, + {"gpu_code": "sm_120", "arch_tag": "cutile_arch_12_0"} + ] + }, + { + "tile_m": 128, "tile_n": 128, "tile_k": 64, "occupancy": 2, + "_architecture": [ + {"gpu_code": "sm_100", "arch_tag": "cutile_arch_10_0"}, + {"gpu_code": "sm_103", "arch_tag": "cutile_arch_10_3"}, + {"gpu_code": "sm_107", "arch_tag": "cutile_arch_10_7"} + ] + } + ] + } +] diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py new file mode 100644 index 0000000000..5bdcf17a18 --- /dev/null +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py @@ -0,0 +1,122 @@ +# ============================================================================= +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ============================================================================= +"""The pruned 1-NN's first filter: the partial GEMM over the first d' dims fused with the filter. + +Each CTA owns TM rows and loops over the centroid tiles (TN wide). For each it computes the partial +distance ||row||^2 + ||centroid||^2 - 2 row.centroid over the first d' dims (FP16 tensor cores) and +keeps the pairs under the row's threshold best[row] * ratio, where best is the row's +distance to its current centroid. + +Output, for n_rows rows against n_centroids centroids (row_stride >= n_centroids): + + packed int64, row r's survivors at r * row_stride + 0 .. count[r] - 1, in centroid order (the + rest not written); a survivor: (centroid index) << 32 | (the partial distance's float32 + bits) + count int32 (n_rows, 1): each row's survivor count + +A row's region holds every centroid, so it cannot overflow, and a survivor's slot is its rank among +the row's survivors: the CTA carries each row's count over the centroid tiles, so there is no +prefix sum across rows. + +d' is a runtime argument and must be a multiple of TK. +""" + +import cuda.tile as ct + +ConstInt = ct.Constant[int] + +# A survivor packed in one int64: the centroid index in the high 32 bits, the partial distance's +# float32 bits in the low 32. One scatter per survivor is the costly part of the kernel; separate +# 32-bit scatters per field are much slower. +ID_SHIFT = 32 + + +def kernel_symbol(data_type: str) -> str: + return {"half": "pruned_1nn_fused_filter_h"}[data_type] + + +def make_kernel(*, gpu_code: str, occupancy: int | None = None): + kernel_options = {} + if occupancy is not None: + kernel_options["occupancy"] = ct.ByTarget(**{gpu_code: occupancy}) + + @ct.kernel(**kernel_options) + def fused_filter( + rows, + centroids, + row_norms, + centroid_norms, + best, + packed, + count, + n_rows, + n_centroids, + ratio, + dp, + row_stride, + tm: ConstInt, + tn: ConstInt, + tk: ConstInt, + ): + row_tile = ct.bid(0) + zero_pad = ct.PaddingMode.ZERO + n_col_tiles = ct.num_tiles(centroids, axis=0, shape=(tn, tk)) + row_ids = row_tile * tm + ct.arange(tm, dtype=ct.int32)[:, None] + row_norm = ct.load( + row_norms, index=(row_tile,), shape=(tm,), padding_mode=zero_pad + )[:, None] + threshold = ( + ct.load( + best, index=(row_tile,), shape=(tm,), padding_mode=zero_pad + )[:, None] + * ratio + ) + # Each row's survivors so far + filled = ct.full((tm, 1), 0, dtype=ct.int32) + for centroid_tile in range(n_col_tiles): + acc = ct.full((tm, tn), 0, dtype=ct.float32) + for kk in range(dp // tk): + a = ct.load( + rows, + index=(row_tile, kk), + shape=(tm, tk), + padding_mode=zero_pad, + ) + b = ct.load( + centroids, + index=(kk, centroid_tile), + shape=(tk, tn), + padding_mode=zero_pad, + order=(1, 0), + ) + acc = ct.mma(a, b, acc) + centroid_norm = ct.load( + centroid_norms, + index=(centroid_tile,), + shape=(tn,), + padding_mode=zero_pad, + )[None, :] + dist = -2.0 * acc + row_norm + centroid_norm + centroid_ids = ( + centroid_tile * tn + ct.arange(tn, dtype=ct.int32)[None, :] + ) + keep = ( + (dist < threshold) + & (centroid_ids < n_centroids) + & (row_ids < n_rows) + ) + # Slot: the row's region, after its survivors in the earlier tiles, plus the + # survivor's rank in this tile + keep_i = keep.astype(ct.int32) + slot = row_ids * row_stride + filled + ct.cumsum(keep_i, 1) - 1 + filled = filled + ct.sum(keep_i, axis=1, keepdims=True) + # Unsigned widening keeps the distance's sign bit out of the centroid index. + index = centroid_ids.astype(ct.int64) + dist_bits = ct.bitcast(dist, ct.uint32).astype(ct.int64) + value = ct.bitwise_lshift(index, ID_SHIFT) | dist_bits + ct.scatter(packed, slot, value, mask=keep) + ct.store(count, index=(row_tile, 0), tile=filled) + + return fused_filter diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu new file mode 100644 index 0000000000..61abcaa9fd --- /dev/null +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fused_filter_tile.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +namespace cuvs::distance::detail::pruned_nn { + +namespace { + +/** Must match kernel_symbol() in fused_filter_kernel.py. */ +constexpr const char* kFusedFilterEntrypoint = "pruned_1nn_fused_filter_h"; + +struct FusedFilterTilePlanner : cuvs::detail::jit_lto::TileAlgorithmPlanner { + using DataTag = cuvs::neighbors::detail::tag_h; + + inline static cuvs::detail::jit_lto::TileLauncherCache launcher_cache{}; + + FusedFilterTilePlanner() : TileAlgorithmPlanner(kFusedFilterEntrypoint, launcher_cache) + { + using namespace cuvs::detail::jit_lto; + add_static_fragment>(); + add_static_fragment>(); + add_static_fragment>(); + add_static_fragment>(); +#if CUVS_CUTILE_SM103_ENABLED + add_static_fragment>(); +#endif +#if CUVS_CUTILE_SM107_ENABLED + add_static_fragment>(); +#endif + add_static_fragment>(); + } +}; + +} // namespace + +auto fused_filter_tile_config() -> std::optional +{ + FusedFilterTilePlanner planner; + if (planner.try_get_launcher() == nullptr) { return std::nullopt; } + const auto cfg = planner.tile_config(); + return fused_filter_tile{cfg.tile_m, cfg.tile_n, cfg.tile_k}; +} + +void launch_fused_filter(const raft::resources& handle, + const half* rows, + int n_rows, + const half* centroids, + int n_centroids, + int dim, + const float* row_norms, + const float* centroid_norms, + const float* best, + float ratio, + int partial_dim, + int64_t* packed, + int* count, + int row_stride) +{ + FusedFilterTilePlanner planner; + auto launcher = planner.try_get_launcher(); + RAFT_EXPECTS(launcher != nullptr, "Pruned 1-NN fused filter: no cuTile cubin for this device"); + const auto tile = planner.tile_config(); + + constexpr int row_pitch_divisor = 16 / sizeof(half); + RAFT_EXPECTS(n_rows > 0 && n_centroids > 0, "Pruned 1-NN fused filter: empty input"); + RAFT_EXPECTS(dim % row_pitch_divisor == 0, + "Pruned 1-NN fused filter: the row pitch must be a multiple of 16 bytes"); + RAFT_EXPECTS(partial_dim > 0 && partial_dim <= dim && partial_dim % tile.tile_k == 0, + "Pruned 1-NN fused filter: partial_dim must be a positive multiple of %d, at most " + "dim", + tile.tile_k); + for (const void* ptr : {static_cast(rows), + static_cast(centroids), + static_cast(row_norms), + static_cast(centroid_norms), + static_cast(best), + static_cast(packed), + static_cast(count)}) { + RAFT_EXPECTS(reinterpret_cast(ptr) % 16 == 0, + "Pruned 1-NN fused filter: buffers must be 16-byte aligned"); + } + RAFT_EXPECTS(row_stride >= n_centroids, + "Pruned 1-NN fused filter: a row's region must hold every centroid"); + const int64_t packed_size = int64_t{n_rows} * row_stride; + RAFT_EXPECTS(packed_size <= std::numeric_limits::max(), + "Pruned 1-NN fused filter: too many rows x centroids in one call"); + + using fused_filter_kernel_t = void(void*, + int, + int, + int, + int, + void*, + int, + int, + int, + int, + void*, + int, + int, + void*, + int, + int, + void*, + int, + int, + void*, + int, + int, + void*, + int, + int, + int, + int, + int, + int, + float, + int, + int); + void* rows_ptr = const_cast(rows); + void* centroids_ptr = const_cast(centroids); + void* row_norms_ptr = const_cast(row_norms); + void* centroid_norms_ptr = const_cast(centroid_norms); + void* best_ptr = const_cast(best); + void* packed_ptr = packed; + void* count_ptr = count; + const int packed_len = static_cast(packed_size); + const int unit = 1; + + dim3 grid((n_rows + tile.tile_m - 1) / tile.tile_m, 1, 1); + dim3 block(1, 1, 1); + // Each array as (pointer, shape..., stride...), then the scalars; the tile sizes are compiled in. + raft::launch_kernel(handle, + grid, + block, + raft::kernel_ref{launcher->get_kernel()}, + rows_ptr, + int{n_rows}, + int{dim}, + int{dim}, + int{unit}, + centroids_ptr, + int{n_centroids}, + int{dim}, + int{dim}, + int{unit}, + row_norms_ptr, + int{n_rows}, + int{unit}, + centroid_norms_ptr, + int{n_centroids}, + int{unit}, + best_ptr, + int{n_rows}, + int{unit}, + packed_ptr, + int{packed_len}, + int{unit}, + count_ptr, + int{n_rows}, + int{unit}, + int{unit}, + int{unit}, + int{n_rows}, + int{n_centroids}, + float{ratio}, + int{partial_dim}, + int{row_stride}); +} + +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp new file mode 100644 index 0000000000..3fa26e7720 --- /dev/null +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp @@ -0,0 +1,64 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include + +#include + +#include +#include + +namespace cuvs::distance::detail::pruned_nn { + +struct fused_filter_tile { + int tile_m; + int tile_n; + int tile_k; +}; + +/** The tile of the fused first-filter cubin for this device, or nullopt when there is none. */ +auto fused_filter_tile_config() -> std::optional; + +/** + * The pruned 1-NN's first filter: the partial distances of rows to centroids over the first + * `partial_dim` dims, keeping the pairs below `best[row] * ratio`. FP16 only: the caller filters + * float data on half copies of its first dims. + * + * Row r's survivors go to `packed` at r * row_stride + 0 .. count[r] - 1, each + * `centroid << 32 | float bits of the partial distance`, in centroid order. + * + * @param handle RAFT resources providing the stream and dry-run state + * @param rows [n_rows, dim] row-major, 16-byte aligned, 16-byte row pitch + * @param centroids [n_centroids, dim] same layout as rows + * @param row_norms [n_rows] squared norms over the first partial_dim dims + * @param centroid_norms [n_centroids] squared norms over the first partial_dim dims + * @param best [n_rows] each row's distance to its current centroid + * @param ratio the pruning bound's scale at partial_dim + * @param partial_dim a multiple of tile_k, at most dim + * @param packed [n_rows * row_stride] survivors + * @param count [n_rows] survivors per row + * @param row_stride each row's region in `packed`, at least n_centroids + */ +void launch_fused_filter(const raft::resources& handle, + const half* rows, + int n_rows, + const half* centroids, + int n_centroids, + int dim, + const float* row_norms, + const float* centroid_norms, + const float* best, + float ratio, + int partial_dim, + int64_t* packed, + int* count, + int row_stride); + +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh new file mode 100644 index 0000000000..38e1e9e944 --- /dev/null +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -0,0 +1,416 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* + * Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * + * This source code is licensed under the MIT license found in the + * LICENSE file thirdparty/LICENSES/LICENSE.superkmeans + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include + +namespace cuvs::distance::detail::pruned_nn { + +// Dims used by the first filter (d') +constexpr uint32_t kPartialDim = 256; +// Dims between the survivors' re-filters +constexpr uint32_t kDimTileSize = 256; + +/** ADSampling's margin, the eps0 of adsampling_ratio. */ +constexpr float kAdsamplingEps0 = 1.5f; +// Centroids per batch: one fused filter and pruning launch per row batch and centroid batch +constexpr uint32_t kCentroidBatchSize = 1024; +// Survivors whose loads are batched over a dim tile +constexpr uint32_t kSurvivorTileSize = 16; +// fused_filter_kernel.py's ID_SHIFT +constexpr int kSurvivorIdShift = 32; + +constexpr std::size_t kAlignment = 16; + +/** + * ADSampling's pruning ratio for a partial distance over the first i of dim dims: + * (i / dim) * (1 + eps0 / sqrt(i))^2, with eps0 ADSampling's margin. A partial distance over i + * dims is compared against the ratio times the row's current best distance. The ratios assume + * every dim carries the same share of a distance, which ADSampling ensures with a random rotation + * of the data; without one they are a heuristic. + */ +__host__ __device__ inline float adsampling_ratio(uint32_t i, uint32_t dim, float eps0) +{ + const float f = 1.0f + eps0 / sqrtf(static_cast(i)); + return static_cast(i) / static_cast(dim) * f * f; +} + +/** + * Squared norm of each of n rows (row stride `stride`) over their first `cols` dims; a warp per + * row. Not raft::linalg::rowNorm: raft's row reductions take whole contiguous rows (row length = + * stride), and masking the other columns would still read them all. When copying, pack the first + * `copy_cols` dims as FP16 and compute the norm from those rounded values. + */ +template +RAFT_KERNEL squared_norms_kernel(const T* __restrict__ rows, + int64_t n, + int64_t stride, + int64_t cols, + float* __restrict__ out, + Copy copy, + int64_t copy_cols) +{ + const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / raft::warp_size(); + if (row >= n) { return; } + float s = 0.f; + constexpr bool kCopy = !std::is_same_v; + for (int64_t j = threadIdx.x % raft::warp_size(); j < (kCopy ? copy_cols : cols); + j += raft::warp_size()) { + float v = static_cast(rows[row * stride + j]); + if constexpr (kCopy) { + const half h = static_cast(v); + copy[row * copy_cols + j] = h; + v = __half2float(h); + } + if (j < cols) { s += v * v; } + } + s = raft::warpReduce(s); + if (threadIdx.x % raft::warp_size() == 0) { out[row] = s; } +} + +template +void squared_norms(const raft::resources& handle, + const T* rows, + int64_t n, + int64_t stride, + int64_t cols, + float* out, + Copy copy = nullptr, + int64_t copy_cols = 0) +{ + constexpr uint32_t kThreads = 256; + const auto blocks = raft::ceildiv(n * raft::warp_size(), kThreads); + raft::launch_kernel(handle, + blocks, + kThreads, + squared_norms_kernel, + rows, + n, + stride, + cols, + out, + copy, + copy_cols); +} + +/** Each row's squared distance to its assigned centroid; optionally pack its filter dims as FP16. + */ +template +RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, + int64_t n, + const C* __restrict__ centroids, + int64_t dim, + const uint32_t* __restrict__ labels, + float* __restrict__ dist, + half* __restrict__ filter_rows) +{ + const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / raft::warp_size(); + if (row >= n) { return; } + const T* row_data = rows + row * dim; + const C* assigned_centroid = centroids + static_cast(labels[row]) * dim; + float distance = 0.f; + for (int64_t j = threadIdx.x % raft::warp_size(); j < dim; j += raft::warp_size()) { + const float value = static_cast(row_data[j]); + const float diff = value - static_cast(assigned_centroid[j]); + distance += diff * diff; + if constexpr (Copy) { + if (j < kPartialDim) { filter_rows[row * kPartialDim + j] = static_cast(value); } + } + } + distance = raft::warpReduce(distance); + if (threadIdx.x % raft::warp_size() == 0) { dist[row] = distance; } +} + +template +void distance_to_assigned(const raft::resources& handle, + const T* rows, + int64_t n, + const C* centroids, + int64_t dim, + const uint32_t* labels, + float* dist, + half* filter_rows = nullptr) +{ + constexpr uint32_t kThreads = 256; + const auto blocks = raft::ceildiv(n * raft::warp_size(), kThreads); + raft::launch_kernel(handle, + blocks, + kThreads, + distance_to_assigned_and_copy_kernel, + rows, + n, + centroids, + dim, + labels, + dist, + filter_rows); +} + +/** + * One dim tile in 16-byte loads. Non-256-aligned dimensions still have a whole-vector tail; + * zero-fill vectors past its end. Aligned instantiations omit the bounds checks. + */ +template +__device__ __forceinline__ void load_tile(const T* __restrict__ values, + uint32_t valid_dims, + T (&tile)[Dims / raft::warp_size()]) +{ + constexpr uint32_t kVec = 16 / sizeof(T); + static_assert(Dims % (raft::warp_size() * kVec) == 0, + "a dim tile is whole 16-byte loads per lane"); + const uint32_t lane = threadIdx.x % raft::warp_size(); +#pragma unroll + for (uint32_t j = 0; j < Dims / (raft::warp_size() * kVec); j++) { + const uint32_t offset = (j * raft::warp_size() + lane) * kVec; + if constexpr (!Aligned) { + if (offset >= valid_dims) { +#pragma unroll + for (uint32_t k = 0; k < kVec; k++) { + tile[j * kVec + k] = T{0}; + } + continue; + } + } + raft::TxN_t v; + v.load(values, offset); +#pragma unroll + for (uint32_t k = 0; k < kVec; k++) { + tile[j * kVec + k] = v.val.data[k]; + } + } +} + +/** + * Distances over one dim tile between the row (`row_tile`, laid out by load_tile) and the N + * survivors at window positions [first, first + N), their ids held one per lane in `ids`; the lane + * holding each survivor adds its distance to `distance`. The N centroids' loads are issued back to + * back. + */ +template +__device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / raft::warp_size()], + const T* __restrict__ centroids, + uint32_t stride, + uint32_t valid_dims, + uint32_t ids, + uint32_t first, + float& distance) +{ + const uint32_t lane = threadIdx.x % raft::warp_size(); + float partial[N]; +#pragma unroll + for (uint32_t i = 0; i < N; i++) { + const T* centroid = centroids + static_cast(raft::shfl(ids, first + i)) * stride; + T centroid_tile[Dims / raft::warp_size()]; + load_tile(centroid, valid_dims, centroid_tile); + partial[i] = 0; +#pragma unroll + for (uint32_t j = 0; j < Dims / raft::warp_size(); j++) { + const float diff = static_cast(row_tile[j]) - static_cast(centroid_tile[j]); + partial[i] = fmaf(diff, diff, partial[i]); + } + } + // Lane i ends with survivor first + i's sum; move it to that survivor's lane + using batched_reduce = cub::WarpReduceBatched; + typename batched_reduce::TempStorage temp_storage; + const float sum = raft::shfl(batched_reduce{temp_storage}.Sum(partial), lane - first); + if (lane >= first && lane < first + N) { distance += sum; } +} + +/** Calls f(integral_constant), then with N / 2, ..., 1. */ +template +__device__ __forceinline__ void for_each_halving(F&& f) +{ + f(std::integral_constant{}); + if constexpr (N > 1) { for_each_halving(f); } +} + +__device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) +{ + return (static_cast(id) << kSurvivorIdShift) | + static_cast(__float_as_int(distance)); +} + +/** + * A warp per block and row, against one centroid batch: the row's first-filter survivors (partial + * distances over the first kPartialDim dims), the tail dims in dim tiles with a re-filter after + * each but the last, then the argmin, kept if closer than the row's current centroid. Block b + * handles row row_order[b], ordered using this tile's first-filter survivor counts. + * + * Row r has count[r] survivors at packed[r * kCentroidBatchSize]. The first warp's entries are + * copied to shared memory; the warp reads one window of entries at a time, lane l holding entry l, + * and each re-filter compacts the list in place. + */ +template +RAFT_KERNEL prune_kernel(const T* __restrict__ rows, + const uint32_t* __restrict__ row_order, + const T* __restrict__ centroids, + uint32_t centroid_offset, + uint32_t dim, + int64_t* __restrict__ packed, + const int32_t* __restrict__ count, + uint32_t* __restrict__ labels, + float* __restrict__ dist) +{ + __shared__ int64_t head[raft::warp_size()]; + const uint32_t lane = threadIdx.x; + const uint32_t lanes_below = (1u << lane) - 1; + const int64_t row = row_order[blockIdx.x]; + uint32_t n_left = static_cast(count[row]); + const T* row_values = rows + row * dim; + uint32_t best_index = UINT32_MAX; // No improving candidate yet. + // The row's distance to its current centroid until the last tile's argmin + float best = dist[row]; + + int64_t* list = packed + row * kCentroidBatchSize; + if (lane < n_left) { head[lane] = list[lane]; } + __syncwarp(); + + // One pass per remaining dim tile; the last one ends with the argmin + uint32_t d = kPartialDim; + for (; n_left > 1; d += kDimTileSize) { + const bool last = d + kDimTileSize >= dim; + // The re-filter's ratio + const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); + T row_tile[kDimTileSize / raft::warp_size()]; + load_tile(row_values + d, dim - d, row_tile); + uint32_t kept = 0; + for (uint32_t base = 0; base < n_left; base += raft::warp_size()) { + const uint32_t w = min(uint32_t{raft::warp_size()}, n_left - base); + const int64_t entry = lane >= w ? 0 : base == 0 ? head[lane] : list[base + lane]; + const auto ids = static_cast(entry >> kSurvivorIdShift); + float distance = __int_as_float(static_cast(entry)); + // Survivor tiles of kSurvivorTileSize, then halving sizes down to 1 for the ones left + uint32_t i = 0; + for_each_halving([&](auto tile) { + constexpr uint32_t kTile = decltype(tile)::value; + for (; i + kTile <= w; i += kTile) { + add_tile_distances( + row_tile, centroids + d, dim, dim - d, ids, i, distance); + } + }); + if (!last) { + // The kept survivors go below this window's end, never into a later window + const bool keep = lane < w && distance < best * ratio; + const uint32_t ballot = __ballot_sync(raft::warp_full_mask(), keep); + if (keep) { + const uint32_t slot = kept + __popc(ballot & lanes_below); + (slot < raft::warp_size() ? head[slot] : list[slot]) = pack_survivor(ids, distance); + } + kept += __popc(ballot); + } else if (lane < w && distance < best) { // after the last tile the distances are full + best = distance; + best_index = centroid_offset + ids; + } + } + // The final tile leaves lane-local full distances for the warp argmin. + if (last) { break; } + __syncwarp(); // the compacted list is visible to the warp + n_left = kept; + } + if (n_left == 0) { return; } + + // A single survivor needs no windows or list compaction, but still gets re-filtered per tile. + if (n_left == 1) { + const int64_t entry = head[0]; + const auto centroid_id = static_cast(entry >> kSurvivorIdShift); + // The assigned centroid's full distance is already known. + if (centroid_offset + centroid_id == labels[row]) { return; } + float distance = __int_as_float(static_cast(entry)); + const T* centroid = centroids + static_cast(centroid_id) * dim; + for (; d < dim; d += kDimTileSize) { + T row_tile[kDimTileSize / raft::warp_size()]; + T centroid_tile[kDimTileSize / raft::warp_size()]; + load_tile(row_values + d, dim - d, row_tile); + load_tile(centroid + d, dim - d, centroid_tile); + float partial = 0.f; +#pragma unroll + for (uint32_t j = 0; j < kDimTileSize / raft::warp_size(); j++) { + const float diff = static_cast(row_tile[j]) - static_cast(centroid_tile[j]); + partial = fmaf(diff, diff, partial); + } + distance += raft::warpReduce(partial); + if (d + kDimTileSize < dim) { + const float ratio = adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); + if (!(distance < best * ratio)) { break; } + } else if (lane == 0 && distance < best) { + labels[row] = centroid_offset + centroid_id; + dist[row] = distance; + } + } + return; + } + + const auto nearest = + raft::warpReduce(raft::KeyValuePair{best_index, best}, raft::argmin_op{}); + if (lane == 0 && nearest.key != UINT32_MAX) { + labels[row] = nearest.key; + dist[row] = nearest.value; + } +} + +/** + * The pruning kernel for n_rows rows against one centroid batch (its first centroid + * centroid_offset), after the fused first filter wrote `packed` and `count`. `labels` and `dist` + * hold each row's current centroid and distance, and get the closer survivor if there is one. + * row_order: the rows' order. + */ +template +void prune(const raft::resources& handle, + const T* rows, + int64_t n_rows, + const uint32_t* row_order, + const T* centroids, + uint32_t centroid_offset, + int64_t dim, + int64_t* packed, + const int32_t* count, + uint32_t* labels, + float* dist) +{ + RAFT_EXPECTS(dim > kPartialDim, "Pruned 1-NN needs a positive tail after %u dims", kPartialDim); + RAFT_EXPECTS(dim % (kAlignment / sizeof(T)) == 0, + "Pruned 1-NN needs row strides divisible by 16 bytes"); + // Whole dimension tiles keep the vectorized kernel free of tail checks. + const auto kernel = dim % kDimTileSize == 0 ? prune_kernel : prune_kernel; + raft::launch_kernel(handle, + n_rows, + raft::warp_size(), + kernel, + rows, + row_order, + centroids, + centroid_offset, + static_cast(dim), + packed, + count, + labels, + dist); +} + +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh new file mode 100644 index 0000000000..ef6edb7d3f --- /dev/null +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -0,0 +1,334 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* + * Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * + * This source code is licensed under the MIT license found in the + * LICENSE file thirdparty/LICENSES/LICENSE.superkmeans + */ +#pragma once + +#include "detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp" +#include "detail/pruned_top_1_nn/prune.cuh" +#include "top_1_nn.cuh" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace cuvs::distance { + +namespace detail::pruned_nn { + +constexpr int64_t kMaxStreams = 2; + +/** The fused first filter's tile on this device, or nullopt when the build or device has none. */ +inline auto filter_tile() -> std::optional +{ +#if CUVS_CUTILE_ENABLED + return fused_filter_tile_config(); +#else + return std::nullopt; +#endif +} + +/** The workspace's buffers, as byte offsets, for m rows on the handle's streams. */ +struct workspace_layout { + int64_t n_row_batches; + int64_t n_streams; + int64_t row_batch_size; + int max_blocks_per_sm; + std::vector packed, count, filter_rows, row_order; + std::size_t total_bytes; + + workspace_layout(raft::resources const& handle, int64_t m, bool copy_rows, int64_t batch_size) + : row_batch_size(batch_size) + { + max_blocks_per_sm = raft::resource::get_device_properties(handle).maxBlocksPerMultiProcessor; + RAFT_EXPECTS(filter_tile().has_value(), "Pruned 1-NN needs the fused filter's cuTile kernel"); + RAFT_EXPECTS(row_batch_size > 0 && + (row_batch_size >= m || row_batch_size % (kAlignment / sizeof(float)) == 0), + "Pruned 1-NN row batches must preserve 16-byte alignment of per-row buffers"); + n_row_batches = raft::ceildiv(m, row_batch_size); + const auto pool_size = raft::resource::is_stream_pool_initialized(handle) + ? static_cast(raft::resource::get_stream_pool_size(handle)) + : 0; + n_streams = std::min({1 + pool_size, kMaxStreams, n_row_batches}); + + std::size_t end = 0; + auto take = [&](std::size_t bytes) { + const std::size_t offset = raft::round_up_safe(end, kAlignment); + end = offset + bytes; + return offset; + }; + const int64_t batch_rows = std::min(m, row_batch_size); + // Streams reuse their buffers only after the previous row batch finishes. + for (int64_t s = 0; s < n_streams; s++) { + packed.push_back(take(batch_rows * kCentroidBatchSize * sizeof(int64_t))); + count.push_back(take(batch_rows * sizeof(int32_t))); + row_order.push_back(take(batch_rows * sizeof(uint32_t))); + if (copy_rows) { filter_rows.push_back(take(batch_rows * kPartialDim * sizeof(half))); } + } + total_bytes = end; + } +}; + +/** + * Order the rows of one row batch for pruning against the current centroid batch. count[r] is + * the number of centroids that survived the fused first filter for row r. Each sorting block + * sorts (count, row id) pairs within one balanced, contiguous segment, largest counts first. + * Only row ids are written; the counts and input data stay in their original positions. + * + * Interleave the sorted segments by rank to submit heavier rows from every segment early: + * + * sorted segment 0: a(12), b(10), c(8) // row ids, with counts in parentheses + * sorted segment 1: d(7), e(4), f(1) + * row_order: a, d, b, e, c, f + * + * This approximates longest-processing-time-first (LPT) scheduling without a device-wide sort. + * LPT applies across pruning blocks; similar-cost rows should share a block when multiple + * row-processing warps are grouped together. Prune currently launches one warp (one row) per + * block, with block b processing row_order[b], so there is no within-block row grouping. + */ +template +RAFT_KERNEL order_rows_by_survivors_kernel(const int32_t* __restrict__ count, + uint32_t* __restrict__ row_order, + int64_t n_rows) +{ + using Sort = cub::BlockRadixSort; + __shared__ typename Sort::TempStorage scratch; + const int64_t n_segments = gridDim.x; + const int64_t rows_per_segment = n_rows / n_segments; + const int64_t extra_rows = n_rows % n_segments; + const int64_t base = blockIdx.x * rows_per_segment + min(int64_t{blockIdx.x}, extra_rows); + const int64_t segment_rows = rows_per_segment + (blockIdx.x < extra_rows); + uint32_t keys[ItemsPerThread]; + uint32_t ids[ItemsPerThread]; +#pragma unroll + for (int j = 0; j < ItemsPerThread; j++) { + const int64_t rank = threadIdx.x * ItemsPerThread + j; + const int64_t row = base + rank; + ids[j] = static_cast(row); + // Zero is reserved for padding, so it follows valid rows with no survivors. + keys[j] = rank < segment_rows ? static_cast(count[row]) + 1 : 0; + } + constexpr int kCountBits = std::bit_width(kCentroidBatchSize + 1); + Sort(scratch).SortDescendingBlockedToStriped(keys, ids, 0, kCountBits); +#pragma unroll + for (int j = 0; j < ItemsPerThread; j++) { + const int64_t rank = threadIdx.x + j * BlockSize; + // Longer segments come first, so the final partial rank leaves no holes. + if (rank < segment_rows) { row_order[rank * n_segments + blockIdx.x] = ids[j]; } + } +} + +inline void order_rows_by_survivors(raft::resources const& handle, + const int32_t* count, + uint32_t* row_order, + int64_t n_rows, + int max_blocks_per_sm) +{ + constexpr int64_t kMaxSegmentRows = 256 * 32; + // Use the resident-block limit as the interleaved rank width, adding segments only when + // needed to keep each segment within one sorting block's capacity. + const int64_t n_segments = + std::min(n_rows, std::max(int64_t{max_blocks_per_sm}, raft::ceildiv(n_rows, kMaxSegmentRows))); + const int64_t segment_rows = raft::ceildiv(n_rows, n_segments); + // CUB requires both sizes at compile time. Keep a compact 2048-entry variant to avoid + // excessive padding on short segments; the 8192-entry variant uses more threads to limit + // per-thread storage for larger segments. + if (segment_rows <= 128 * 16) { + raft::launch_kernel( + handle, n_segments, 128, order_rows_by_survivors_kernel<128, 16>, count, row_order, n_rows); + } else { + raft::launch_kernel( + handle, n_segments, 256, order_rows_by_survivors_kernel<256, 32>, count, row_order, n_rows); + } +} + +} // namespace detail::pruned_nn + +/** + * Whether pruned_top_1_nn runs for k features on this build and device: it needs the fused + * filter's cuTile kernel, the first detail::pruned_nn::kPartialDim dims for filtering, and a + * positive tail to prune on. + */ +inline bool is_pruned_top_1_nn_available(int64_t k) +{ + return k > detail::pruned_nn::kPartialDim && detail::pruned_nn::filter_tile().has_value(); +} + +/** The workspace pruned_top_1_nn needs for m rows on `handle` (the same handle as the call). */ +template +inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, + int64_t m, + int64_t row_batch_size) +{ + return detail::pruned_nn::workspace_layout( + handle, m, !std::is_same_v, row_batch_size) + .total_bytes; +} + +/** + * Updates each row of x to its nearest row of y in L2, starting from its current one: the pruned + * assignment of SuperKMeans (Kuffo, Hepkema, Boncz, "A Super Fast K-means for Indexing Vector + * Embeddings", https://arxiv.org/abs/2603.20009). With b a row's squared distance to its current + * neighbor and ratio(i) = (i / k) * (1 + eps0 / sqrt(i))^2 (ADSampling's bound, see + * detail::pruned_nn::adsampling_ratio): + * + * 1. a fused FP16 GEMM over the first detail::pruned_nn::kPartialDim dims keeps the rows of y + * whose partial distance is under b * ratio(kPartialDim); + * 2. the pruning kernel adds the survivors' remaining dims a tile at a time, keeping those under + * b * ratio(dims so far); + * 3. the closest survivor over all k dims replaces the current neighbor, and b, if closer. + * + * Not exact: a neighbor pruned away on a partial distance is missed. + * + * Each first-filter tile orders its rows by the newly produced survivor counts before pruning. + * Counts are sorted within segments and their ranks interleaved, approximating heavy-first order. + * The row order and survivor lists are scratch buffers private to each stream. + * + * Rows go in batches of row_batch_size against batches of kCentroidBatchSize rows of y, each + * centroid batch tightening b for the next. The row batches round-robin over the main stream and, + * with a stream pool on the handle, its first stream, which waits on and is synchronized with the + * main stream. + * + * @param output in: each row's current neighbor (nearest_idx); out: its nearest neighbor and + * squared distance + * @param x [m, k] row-major + * @param y [n, k] row-major + * @param y_fp16 [n, kPartialDim]: y's first dims in FP16, for float; ignored for half + * @param xn [m] squared norms of the original x over the first kPartialDim dims + * @param yn [n] squared norms of the FP16 filter centroids over the first kPartialDim dims + * @param row_batch_size rows per batch, matching workspace sizing + * @param workspace pruned_top_1_nn_workspace_size(handle, m, row_batch_size) bytes, + * 16-byte aligned + */ +template +void pruned_top_1_nn(raft::resources const& handle, + Top1nnOutput output, + const DataT* x, + const DataT* y, + const half* y_fp16, + const float* xn, + const float* yn, + int64_t m, + int64_t n, + int64_t k, + int64_t row_batch_size, + void* workspace, + std::size_t workspace_bytes) +{ + static_assert(std::is_same_v || std::is_same_v, + "Pruned 1-NN supports float and half"); +#if CUVS_CUTILE_ENABLED + using namespace detail::pruned_nn; + constexpr bool kHalf = std::is_same_v; + RAFT_EXPECTS(is_pruned_top_1_nn_available(k), + "Pruned 1-NN needs the fused filter's cuTile kernel and a positive tail after %u " + "dims", + kPartialDim); + RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); + RAFT_EXPECTS(kHalf || y_fp16 != nullptr, + "Pruned 1-NN on float needs an FP16 copy of y's first dims"); + RAFT_EXPECTS(xn != nullptr && yn != nullptr, "Pruned 1-NN needs row and centroid norms"); + const workspace_layout layout(handle, m, !kHalf, row_batch_size); + RAFT_EXPECTS(workspace_bytes >= layout.total_bytes, "Pruned 1-NN's workspace is too small"); + auto* ws = static_cast(workspace); + + // The filter reads FP16 rows: half data whole, float data's FP16 copies of its first dims + const int64_t filter_dim = kHalf ? k : kPartialDim; + const half* y_filter = nullptr; + if constexpr (kHalf) { + y_filter = y; + } else { + y_filter = y_fp16; + } + const bool pooled = layout.n_streams > 1; + if (pooled) { raft::resource::wait_stream_pool_on_stream(handle); } + + std::vector stream_handles(layout.n_streams, handle); + for (int64_t s = 1; s < layout.n_streams; s++) { + raft::resource::set_cuda_stream(stream_handles[s], + raft::resource::get_stream_from_stream_pool(handle, s - 1)); + } + + for (int64_t b = 0; b < layout.n_row_batches; b++) { + const int64_t i = b * layout.row_batch_size; + const int64_t n_rows = std::min(layout.row_batch_size, m - i); + const int64_t s = b % layout.n_streams; + const auto& batch_handle = stream_handles[s]; + const DataT* rows = x + i * k; + uint32_t* row_labels = output.nearest_idx + i; + float* row_dist = output.nearest_dist + i; + auto* order = reinterpret_cast(ws + layout.row_order[s]); + auto* packed = reinterpret_cast(ws + layout.packed[s]); + auto* count = reinterpret_cast(ws + layout.count[s]); + // Each row's bound: its distance to its current neighbor + const half* x_filter; + const float* row_norms = xn + i; + if constexpr (kHalf) { + x_filter = rows; + distance_to_assigned(batch_handle, rows, n_rows, y, k, row_labels, row_dist); + } else { + // Each stream reuses its filter inputs only after its previous row batch finishes. + auto* batch_rows = reinterpret_cast(ws + layout.filter_rows[s]); + distance_to_assigned( + batch_handle, rows, n_rows, y, k, row_labels, row_dist, batch_rows); + x_filter = batch_rows; + } + for (int64_t j = 0; j < n; j += kCentroidBatchSize) { + const int64_t n_centroids = std::min(kCentroidBatchSize, n - j); + launch_fused_filter(batch_handle, + x_filter, + static_cast(n_rows), + y_filter + j * filter_dim, + static_cast(n_centroids), + static_cast(filter_dim), + row_norms, + yn + j, + row_dist, + adsampling_ratio(kPartialDim, static_cast(k), kAdsamplingEps0), + static_cast(kPartialDim), + packed, + count, + static_cast(kCentroidBatchSize)); + order_rows_by_survivors(batch_handle, count, order, n_rows, layout.max_blocks_per_sm); + prune(batch_handle, + rows, + n_rows, + order, + y + j * k, + static_cast(j), + k, + packed, + count, + row_labels, + row_dist); + } + } + if (pooled) { raft::resource::sync_stream_pool(handle, std::vector{0}); } +#else + RAFT_FAIL("Pruned 1-NN needs a build with cuTile"); +#endif +} + +} // namespace cuvs::distance diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index dc97dd176f..20fb8fef88 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -135,6 +135,7 @@ ConfigureTest( PATH cluster/kmeans.cu cluster/cluster_cost.cu cluster/kmeans_balanced.cu + cluster/kmeans_superkmeans.cu cluster/kmeans_find_k.cu cluster/gmm.cu cluster/kmeans_predict_batching.cu diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu new file mode 100644 index 0000000000..f16637df86 --- /dev/null +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -0,0 +1,498 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "../test_utils.cuh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs { + +namespace { + +constexpr int64_t kRows = 4000; +constexpr int64_t kDim = 200; // below one dimension tile: no pruning +constexpr int64_t kClusters = 300; +constexpr int64_t kPrunedDim = 1536; +constexpr int64_t kRowAlignmentBytes = 16; + +double sq_dist(const float* a, const float* b, int64_t dim) +{ + double s = 0; + for (int64_t j = 0; j < dim; j++) { + const double d = double(a[j]) - double(b[j]); + s += d * d; + } + return s; +} + +double sq_norm(const float* a, int64_t dim) +{ + double s = 0; + for (int64_t j = 0; j < dim; j++) { + s += double(a[j]) * a[j]; + } + return s; +} + +/** + * Host SuperKMeans without pruning: Lloyd's from the given start centroids, an empty cluster + * keeping its centroid. + */ +void reference_superkmeans(const std::vector& X, + int64_t n_rows, + int64_t dim, + int64_t n_clusters, + uint32_t n_iters, + std::vector start, + std::vector& centroids, + std::vector& labels, + double& inertia) +{ + centroids = std::move(start); + labels.resize(n_rows); + auto assign = [&]() { + inertia = 0; + for (int64_t r = 0; r < n_rows; r++) { + double best = std::numeric_limits::max(); + for (int64_t c = 0; c < n_clusters; c++) { + const double d = sq_dist(&X[r * dim], ¢roids[c * dim], dim); + if (d < best) { + best = d; + labels[r] = c; + } + } + inertia += best; + } + }; + std::vector sums(n_clusters * dim); + std::vector sizes(n_clusters); + for (uint32_t iter = 0; iter < n_iters; iter++) { + assign(); + std::fill(sums.begin(), sums.end(), 0.0); + std::fill(sizes.begin(), sizes.end(), 0u); + for (int64_t r = 0; r < n_rows; r++) { + sizes[labels[r]]++; + for (int64_t j = 0; j < dim; j++) { + sums[labels[r] * dim + j] += X[r * dim + j]; + } + } + for (int64_t c = 0; c < n_clusters; c++) { + if (sizes[c] == 0) { continue; } + for (int64_t j = 0; j < dim; j++) { + centroids[c * dim + j] = float(sums[c * dim + j] / sizes[c]); + } + } + } + assign(); +} + +/** Well separated blobs as T, and the same values as float on the host. */ +template +auto make_data(raft::handle_t& handle, std::vector& host) +{ + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + auto blobs = raft::make_device_matrix(handle, kRows, kDim); + auto blob_labels = raft::make_device_vector(handle, kRows); + raft::random::make_blobs(blobs.data_handle(), + blob_labels.data_handle(), + kRows, + kDim, + kClusters, + stream, + true, + nullptr, + nullptr, + 0.1f, + true, + -1.0f, + 1.0f, + uint64_t{1234}); + auto X = raft::make_device_matrix(handle, kRows, kDim); + raft::linalg::unaryOp( + X.data_handle(), blobs.data_handle(), kRows * kDim, raft::cast_op(), stream); + raft::linalg::unaryOp( + blobs.data_handle(), X.data_handle(), kRows * kDim, raft::cast_op(), stream); + host.resize(kRows * kDim); + raft::copy(host.data(), blobs.data_handle(), host.size(), stream); + raft::resource::sync_stream(handle); + return X; +} + +/** The start centroids SuperKMeans draws for `seed`: rows sampled with raft::matrix::sample_rows. + */ +std::vector start_centroids(raft::handle_t& handle, + const std::vector& host, + uint64_t seed) +{ + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + auto X = raft::make_device_matrix(handle, kRows, kDim); + auto start = raft::make_device_matrix(handle, kClusters, kDim); + raft::copy(X.data_handle(), host.data(), host.size(), stream); + raft::matrix::sample_rows( + handle, raft::random::RngState(seed), raft::make_const_mdspan(X.view()), start.view()); + std::vector out(kClusters * kDim); + raft::copy(out.data(), start.data_handle(), out.size(), stream); + raft::resource::sync_stream(handle); + return out; +} + +template +void fit_predict(raft::handle_t& handle, + raft::device_matrix_view X, + uint32_t n_iters, + std::vector& centroids, + std::vector& labels, + float& inertia) +{ + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + cuvs::cluster::kmeans::experimental::superkmeans_params params; + params.n_iters = n_iters; + auto d_centroids = raft::make_device_matrix(handle, kClusters, kDim); + auto d_labels = raft::make_device_vector(handle, kRows); + cuvs::cluster::kmeans::experimental::fit_predict( + handle, params, X, d_centroids.view(), d_labels.view(), raft::make_host_scalar_view(&inertia)); + centroids.resize(kClusters * kDim); + labels.resize(kRows); + raft::copy(centroids.data(), d_centroids.data_handle(), centroids.size(), stream); + raft::copy(labels.data(), d_labels.data_handle(), labels.size(), stream); + raft::resource::sync_stream(handle); +} + +} // namespace + +/* Without pruning SuperKMeans is Lloyd's algorithm from random start rows. The GPU assignment + * rounds its GEMM operands to TF32, so a run on float data drifts from an exact reference once + * near-ties go the other way: the test compares against the reference where that cannot happen, + * and checks the invariants of Lloyd's output otherwise. + */ +TEST(KmeansSuperKMeans, Fallback) +{ + raft::handle_t handle; + + // Half values are exact in TF32 and one iteration assigns against the start rows, so the start + // rows, the assignment and the update must match the reference. + { + std::vector h_X, centroids, ref_centroids; + std::vector labels, ref_labels; + float inertia; + double ref_inertia; + auto X = make_data(handle, h_X); + fit_predict(handle, raft::make_const_mdspan(X.view()), 1, centroids, labels, inertia); + reference_superkmeans( + h_X, + kRows, + kDim, + kClusters, + 1, + start_centroids( + handle, h_X, cuvs::cluster::kmeans::experimental::superkmeans_params{}.rng_state.seed), + ref_centroids, + ref_labels, + ref_inertia); + ASSERT_TRUE( + cuvs::hostVecMatch(ref_centroids, centroids, cuvs::CompareApproxNoScaling(1e-5f))); + int64_t same_label = 0; + for (int64_t r = 0; r < kRows; r++) { + same_label += labels[r] == ref_labels[r]; + } + EXPECT_GE(double(same_label) / kRows, 0.999); + EXPECT_NEAR(inertia, ref_inertia, 1e-4 * ref_inertia); + } + + // Float data, several iterations: each row's label is its nearest centroid (up to the TF32 + // rounding of the distances), the inertia is that of the labels, and each centroid is the mean of + // its rows. + { + constexpr uint32_t n_iters = 5; + std::vector h_X, centroids; + std::vector labels; + float inertia; + auto X = make_data(handle, h_X); + fit_predict( + handle, raft::make_const_mdspan(X.view()), n_iters, centroids, labels, inertia); + + double host_inertia = 0; + for (int64_t r = 0; r < kRows; r++) { + const float* row = &h_X[r * kDim]; + const float* own = ¢roids[int64_t(labels[r]) * kDim]; + double best = std::numeric_limits::max(); + for (int64_t c = 0; c < kClusters; c++) { + best = std::min(best, sq_dist(row, ¢roids[c * kDim], kDim)); + } + const double own_dist = sq_dist(row, own, kDim); + ASSERT_LE(own_dist, best + 1e-3 * (sq_norm(row, kDim) + sq_norm(own, kDim))) << "row " << r; + host_inertia += own_dist; + } + EXPECT_NEAR(inertia, host_inertia, 1e-4 * host_inertia); + + std::vector sums(kClusters * kDim, 0.0); + std::vector sizes(kClusters, 0); + for (int64_t r = 0; r < kRows; r++) { + sizes[labels[r]]++; + for (int64_t j = 0; j < kDim; j++) { + sums[int64_t(labels[r]) * kDim + j] += h_X[r * kDim + j]; + } + } + for (int64_t c = 0; c < kClusters; c++) { + if (sizes[c] == 0) { continue; } + for (int64_t j = 0; j < kDim; j++) { + ASSERT_NEAR(centroids[c * kDim + j], sums[c * kDim + j] / sizes[c], 1e-4) + << "centroid " << c; + } + } + + // predict on the returned centroids gives fit_predict's labels. + auto d_centroids_fp = raft::make_device_matrix(handle, kClusters, kDim); + auto d_labels = raft::make_device_vector(handle, kRows); + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + raft::copy(d_centroids_fp.data_handle(), centroids.data(), centroids.size(), stream); + cuvs::cluster::kmeans::experimental::predict( + handle, + cuvs::cluster::kmeans::experimental::superkmeans_params{}, + raft::make_const_mdspan(X.view()), + raft::make_const_mdspan(d_centroids_fp.view()), + d_labels.view()); + std::vector predicted(kRows); + raft::copy(predicted.data(), d_labels.data_handle(), kRows, stream); + raft::resource::sync_stream(handle); + EXPECT_EQ(predicted, labels); + + // fit runs the same iterations; its centroids may differ in the last bits (atomic sums). + cuvs::cluster::kmeans::experimental::superkmeans_params params; + params.n_iters = n_iters; + auto d_centroids = raft::make_device_matrix(handle, kClusters, kDim); + float fit_inertia = 0; + cuvs::cluster::kmeans::experimental::fit(handle, + params, + raft::make_const_mdspan(X.view()), + d_centroids.view(), + raft::make_host_scalar_view(&fit_inertia)); + EXPECT_NEAR(fit_inertia, inertia, 1e-3 * inertia); + } +} + +/* Exercise pruned assignment with whole dimension tiles and a partial tail. The pruning bound + * is a heuristic without a rotation, so compare with Lloyd's from the same start rows rather than + * exactly. The labels and inertia come from a full assignment. + */ +class KmeansSuperKMeansPruned : public ::testing::TestWithParam {}; + +TEST_P(KmeansSuperKMeansPruned, Pruned) +{ + // Multiple row and centroid batches, with partial final batches and several tail dimension tiles + constexpr int64_t n_rows = 300000; + const int64_t dim = GetParam(); + constexpr int64_t n_clusters = 2200; + constexpr uint32_t n_iters = 8; + constexpr uint64_t seed = 42; + raft::handle_t handle; + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + + auto blobs = raft::make_device_matrix(handle, n_rows, dim); + auto blob_labels = raft::make_device_vector(handle, n_rows); + raft::random::make_blobs(blobs.data_handle(), + blob_labels.data_handle(), + n_rows, + dim, + n_clusters, + stream, + true, + nullptr, + nullptr, + 1.0f, + true, + -2.0f, + 2.0f, + uint64_t{1234}); + auto X_half = raft::make_device_matrix(handle, n_rows, dim); + raft::linalg::unaryOp( + X_half.data_handle(), blobs.data_handle(), n_rows * dim, raft::cast_op(), stream); + // Lloyd's runs on the half values too, so both data types are compared with the same reference + raft::linalg::unaryOp( + blobs.data_handle(), X_half.data_handle(), n_rows * dim, raft::cast_op(), stream); + auto X = raft::make_const_mdspan(blobs.view()); + + // Lloyd's from SuperKMeans' start rows (its own Random init draws other rows for the same seed) + auto centroids = raft::make_device_matrix(handle, n_clusters, dim); + raft::matrix::sample_rows(handle, raft::random::RngState(seed), X, centroids.view()); + cuvs::cluster::kmeans::params lloyd; + lloyd.n_clusters = n_clusters; + lloyd.init = cuvs::cluster::kmeans::params::Array; + lloyd.max_iter = n_iters; + lloyd.tol = 1e-30; // must be positive; this small, all iterations run + float lloyd_inertia = 0; + int64_t lloyd_iters = 0; + cuvs::cluster::kmeans::fit(handle, + lloyd, + X, + std::nullopt, + centroids.view(), + raft::make_host_scalar_view(&lloyd_inertia), + raft::make_host_scalar_view(&lloyd_iters)); + + cuvs::cluster::kmeans::experimental::superkmeans_params params; + params.n_iters = n_iters; + params.rng_state = raft::random::RngState(seed); + auto labels = raft::make_device_vector(handle, n_rows); + auto check = [&](auto data) { + float inertia = 0; + cuvs::cluster::kmeans::experimental::fit_predict( + handle, params, data, centroids.view(), labels.view(), raft::make_host_scalar_view(&inertia)); + EXPECT_NEAR(inertia, lloyd_inertia, 0.01 * lloyd_inertia); + }; + + for (bool pooled : {false, true}) { + SCOPED_TRACE(pooled ? "two streams" : "one stream"); + // With a stream pool the row batches alternate between the main stream and the pool's + if (pooled) { + raft::resource::set_cuda_stream_pool(handle, std::make_shared(1)); + } + try { + check(X); + check(raft::make_const_mdspan(X_half.view())); + } catch (const raft::logic_error& e) { + if (std::string{e.what()}.find("SuperKMeans pruning kernel is unavailable") != + std::string::npos) { + GTEST_SKIP() << e.what(); + } + throw; + } + } +} + +INSTANTIATE_TEST_SUITE_P(DimensionTails, + KmeansSuperKMeansPruned, + ::testing::Values(kPrunedDim, + kPrunedDim + kRowAlignmentBytes / sizeof(half))); + +TEST(KmeansSuperKMeans, AlignmentRequirements) +{ + constexpr int64_t n_rows = 1024, n_clusters = 300; + for (int64_t dim : {1024, 1280}) { + raft::handle_t handle; + auto stream = raft::resource::get_cuda_stream(handle); + auto check = [&](auto value) { + using T = decltype(value); + auto storage = raft::make_device_vector(handle, n_rows * dim + 1); + auto centers = raft::make_device_vector(handle, n_clusters * dim + 1); + auto labels = raft::make_device_vector(handle, n_rows); + std::vector host(n_rows * dim); + for (int64_t i = 0; i < n_rows * dim; ++i) { + host[i] = static_cast(std::sin(static_cast(i) * 0.13f)); + } + cuvs::cluster::kmeans::experimental::superkmeans_params params; + params.n_iters = 3; + for (auto offsets : {std::pair{1, 0}, std::pair{0, 1}, std::pair{1, 1}}) { + SCOPED_TRACE(sizeof(T)); + SCOPED_TRACE(offsets.first); + SCOPED_TRACE(offsets.second); + auto X = raft::make_device_matrix_view( + storage.data_handle() + offsets.first, n_rows, dim); + auto C = raft::make_device_matrix_view( + centers.data_handle() + offsets.second, n_clusters, dim); + raft::copy(storage.data_handle() + offsets.first, host.data(), host.size(), stream); + float inertia = 0; + if (dim > 1024) { + EXPECT_THROW( + cuvs::cluster::kmeans::experimental::fit_predict( + handle, params, X, C, labels.view(), raft::make_host_scalar_view(&inertia)), + raft::logic_error); + continue; + } + cuvs::cluster::kmeans::experimental::fit_predict( + handle, params, X, C, labels.view(), raft::make_host_scalar_view(&inertia)); + std::vector host_centers(n_clusters * dim); + std::vector host_labels(n_rows); + raft::copy(host_centers.data(), C.data_handle(), host_centers.size(), stream); + raft::copy(host_labels.data(), labels.data_handle(), n_rows, stream); + raft::resource::sync_stream(handle); + double expected = 0; + for (int64_t r = 0; r < n_rows; ++r) { + ASSERT_LT(host_labels[r], n_clusters); + for (int64_t j = 0; j < dim; ++j) { + const double diff = static_cast(host[r * dim + j]) - + host_centers[int64_t{host_labels[r]} * dim + j]; + expected += diff * diff; + } + } + EXPECT_TRUE(std::isfinite(inertia)); + EXPECT_NEAR(inertia, expected, 1e-4 * expected); + } + }; + check(float{}); + check(half{}); + } +} + +TEST(KmeansSuperKMeans, UntiledDimensionFallback) +{ + raft::handle_t handle; + cuvs::cluster::kmeans::experimental::superkmeans_params params; + params.n_iters = 3; + auto X_odd = raft::make_device_matrix(handle, 1000, 384); + auto centroids = raft::make_device_matrix(handle, 300, 384); + RAFT_CUDA_TRY(cudaMemsetAsync(X_odd.data_handle(), + 0, + X_odd.size() * sizeof(float), + raft::resource::get_cuda_stream(handle).get())); + float inertia = -1; + cuvs::cluster::kmeans::experimental::fit(handle, + params, + raft::make_const_mdspan(X_odd.view()), + centroids.view(), + raft::make_host_scalar_view(&inertia)); + EXPECT_EQ(inertia, 0); +} + +TEST(KmeansSuperKMeans, RejectsUnsupported) +{ + raft::handle_t handle; + cuvs::cluster::kmeans::experimental::superkmeans_params params; + auto X = raft::make_device_matrix(handle, 1000, 256); + auto small_centroids = raft::make_device_matrix(handle, 8, 256); + params.metric = cuvs::distance::DistanceType::InnerProduct; + EXPECT_THROW(cuvs::cluster::kmeans::experimental::fit( + handle, params, raft::make_const_mdspan(X.view()), small_centroids.view()), + raft::logic_error); + + params.metric = cuvs::distance::DistanceType::L2Expanded; + constexpr int64_t invalid_stride_dim = kPrunedDim + 1; + auto X_odd = raft::make_device_matrix(handle, 1000, invalid_stride_dim); + auto C_odd = raft::make_device_matrix(handle, 300, invalid_stride_dim); + EXPECT_THROW(cuvs::cluster::kmeans::experimental::fit( + handle, params, raft::make_const_mdspan(X_odd.view()), C_odd.view()), + raft::logic_error); + auto too_many = raft::make_device_matrix(handle, 1001, 256); + EXPECT_THROW(cuvs::cluster::kmeans::experimental::fit( + handle, params, raft::make_const_mdspan(X.view()), too_many.view()), + raft::logic_error); +} + +} // namespace cuvs diff --git a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md index ed175c039d..1dc0df18ef 100644 --- a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md +++ b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md @@ -1310,3 +1310,201 @@ float tol = 1e-3); **Returns** `void` + +## Experimental SuperKMeans APIs + + +### cluster::kmeans::experimental::superkmeans_params + +Simple object to specify hyper-parameters to the SuperKMeans algorithm. + +SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every row with the fused 1-NN over all dimensions; later iterations compute partial distances over the first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the ADSampling bound, without its random rotation), and finish the remaining dimensions only for the survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: row strides must be divisible by 16 bytes, dataset and centroid pointers must be 16-byte aligned, and the build and device must provide the pruning kernel. Inputs that violate these requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every iteration uses full nearest-centroid assignment (Lloyd's algorithm). + +The number of clusters is taken from the `centroids` argument. The start centroids are rows sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run (there is no convergence test), and an empty cluster keeps its previous centroid. + +Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 tensor cores for both float and half input; surviving candidates are refined in FP32 over subsequent tiles of up to 256 dimensions, with further pruning between tiles. + +Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional inertia are computed with a full assignment against the final centroids. + +The following metrics are currently supported: + +- L2Expanded + +```cpp +struct superkmeans_params : base_params { + uint32_t n_iters; + raft::random::RngState rng_state; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `n_iters` | `uint32_t` | Number of training iterations (default 10). | +| `rng_state` | `raft::random::RngState` | Seed to the random number generator (the start rows); default seed 0. | + + +### cluster::kmeans::experimental::fit + +Find clusters with the SuperKMeans algorithm. + +```cpp +void fit(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::experimental::fit` + +Find clusters with the SuperKMeans algorithm. + +```cpp +void fit(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +std::optional> inertia = std::nullopt); +``` + +With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the centroids, which are accumulated and returned in FP32. + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + + +### cluster::kmeans::experimental::predict + +Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + +```cpp +void predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | +| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::experimental::predict` + +Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + +```cpp +void predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | +| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | + +**Returns** + +`void` + + +### cluster::kmeans::experimental::fit_predict + +Compute SuperKMeans clusters and predict the cluster index of each sample. + +```cpp +void fit_predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::experimental::fit_predict` + +Compute SuperKMeans clusters and predict the cluster index of each sample. + +```cpp +void fit_predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` diff --git a/thirdparty/LICENSES/LICENSE.superkmeans b/thirdparty/LICENSES/LICENSE.superkmeans new file mode 100644 index 0000000000..b70483dcf7 --- /dev/null +++ b/thirdparty/LICENSES/LICENSE.superkmeans @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.