From 8b35602173a66834a4511fc38ebd3e63ab1264f4 Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 25 Oct 2026 19:14:14 -0700 Subject: [PATCH 01/39] Add SuperKMeans k-means API with an unpruned fallback superkmeans_params and fit / fit_predict overloads for float and half input. Without pruning (dim below one 256-dim tile, or at most 256 clusters) fit runs Lloyd's iterations from SuperKMeans' random start rows; the pruned path is not implemented yet and fails. Co-Authored-By: Claude Opus 5.5 --- cpp/CMakeLists.txt | 1 + cpp/include/cuvs/cluster/kmeans.hpp | 198 +++++++++++++++ cpp/src/cluster/detail/superkmeans.cuh | 176 +++++++++++++ cpp/src/cluster/kmeans_superkmeans.cu | 68 +++++ cpp/tests/CMakeLists.txt | 1 + cpp/tests/cluster/kmeans_superkmeans.cu | 316 ++++++++++++++++++++++++ 6 files changed, 760 insertions(+) create mode 100644 cpp/src/cluster/detail/superkmeans.cuh create mode 100644 cpp/src/cluster/kmeans_superkmeans.cu create mode 100644 cpp/tests/cluster/kmeans_superkmeans.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 78bde6f657..15f48c87bf 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1456,6 +1456,7 @@ if(NOT BUILD_CPU_ONLY) src/cluster/kmeans_balanced_fit_predict_int8.cu src/cluster/kmeans_balanced_predict_int8.cu src/cluster/kmeans_balanced_predict_uint8.cu + src/cluster/kmeans_superkmeans.cu src/cluster/kmeans_transform_double.cu src/cluster/kmeans_transform_float.cu src/cluster/single_linkage_float.cu diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index 16de840e81..cb804e345f 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -231,6 +231,33 @@ struct balanced_params : base_params { bool is_packed_binary = false; }; +/** + * Simple object to specify hyper-parameters to the SuperKMeans algorithm. + * + * SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every + * row with the fused 1-NN over all dimensions; later iterations compute partial distances over the + * first 256 dimensions, discard centroids that cannot beat a row's current one + * (ADSampling bound), and finish the remaining dimensions only for the survivors. + * + * The number of clusters is taken from the `centroids` argument. The start centroids are rows + * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run + * (there is no convergence test), and an empty cluster keeps its previous centroid. + * + * The following metrics are currently supported: + * - L2Expanded + */ +struct superkmeans_params : base_params { + /** + * Number of training iterations. + */ + uint32_t n_iters = 10; + + /** + * Seed to the random number generator (the start rows). + */ + raft::random::RngState rng_state{0}; +}; + /** * @brief Type of k-means algorithm. */ @@ -770,6 +797,63 @@ void fit(const raft::resources& handle, raft::device_matrix_view centroids, std::optional> inertia = std::nullopt); +/** + * @brief Find clusters with the SuperKMeans algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::superkmeans_params params; + * int64_t n_features = 1024, n_clusters = 4096; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * centroids.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia = std::nullopt); + +/** + * @brief Find clusters with the SuperKMeans algorithm. + * + * With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the + * centroids, which are accumulated and returned in FP32. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia = std::nullopt); + /** * @brief Predict the closest cluster each sample in X belongs to. * @@ -1281,6 +1365,58 @@ void predict(const raft::resources& handle, raft::device_matrix_view centroids, raft::device_vector_view labels); +/** + * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::superkmeans_params params; + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * centroids, + * labels.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + /** * @brief Compute k-means clustering and predicts cluster index for each sample * in the input. @@ -1589,6 +1725,68 @@ void fit_predict(const raft::resources& handle, raft::device_matrix_view centroids, raft::device_vector_view labels); +/** + * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::superkmeans_params params; + * int64_t n_features = 1024, n_clusters = 4096; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::fit_predict(handle, + * params, + * X, + * centroids.view(), + * labels.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia = std::nullopt); + +/** + * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia = std::nullopt); + /** * @brief Transform X to a cluster-distance space. * diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh new file mode 100644 index 0000000000..ccf6c89a01 --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "../../neighbors/detail/ann_utils.cuh" +#include "../kmeans_balanced.cuh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace cuvs::cluster::kmeans::detail::superkmeans { + +// Pruning works on whole dimension tiles and only pays off with enough centroids to discard; below +// that every iteration assigns with the full 1-NN. +constexpr int64_t kDimTileSize = 256; +constexpr int64_t kMinPrunedK = 256; + +inline bool prunes(int64_t n_clusters, int64_t dim) +{ + return dim >= kDimTileSize && n_clusters > kMinPrunedK; +} + +/** Sum of squared distances of the rows to their assigned centroids. */ +template +auto inertia(const raft::resources& handle, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) -> float +{ + const int64_t dim = X.extent(1); + auto dist = raft::make_device_vector(handle, X.extent(0)); + const T* x = X.data_handle(); + const float* c = centroids.data_handle(); + const uint32_t* l = labels.data_handle(); + raft::linalg::map_offset(handle, dist.view(), [=] __device__(int64_t row) { + const float* center = c + static_cast(l[row]) * dim; + float sum = 0; + for (int64_t j = 0; j < dim; j++) { + const float diff = static_cast(x[row * dim + j]) - center[j]; + sum += diff * diff; + } + return sum; + }); + return thrust::reduce(raft::resource::get_thrust_policy(handle), + dist.data_handle(), + dist.data_handle() + dist.size(), + 0.0f); +} + +template +void validate(const cuvs::cluster::kmeans::superkmeans_params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids) +{ + RAFT_EXPECTS(params.metric == cuvs::distance::DistanceType::L2Expanded, + "SuperKMeans supports only the L2Expanded metric"); + RAFT_EXPECTS(centroids.extent(0) > 0, "The number of centroids must be strictly positive"); + RAFT_EXPECTS(centroids.extent(1) == X.extent(1), + "Number of features in dataset and centroids are different"); +} + +/** Each row's closest centroid over all dims. */ +template +void predict(const raft::resources& handle, + const cuvs::cluster::kmeans::superkmeans_params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) +{ + validate(params, X, centroids); + RAFT_EXPECTS(labels.extent(0) == X.extent(0), + "Number of rows in dataset and labels are different"); + cuvs::cluster::kmeans::balanced_params assign_params; + assign_params.metric = params.metric; + cuvs::cluster::kmeans_balanced::predict(handle, + assign_params, + X, + centroids, + labels, + cuvs::spatial::knn::detail::utils::mapping{}); +} + +/** + * Lloyd's iterations from rows sampled at random, a fixed number of them. Each iteration assigns + * every row (the full 1-NN when pruning does not apply) and sets each centroid to the mean of its + * rows; an empty cluster keeps its centroid. `labels` and `inertia`, when given, are for the + * returned centroids. + */ +template +void fit(const raft::resources& handle, + const cuvs::cluster::kmeans::superkmeans_params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> labels, + std::optional> inertia_out) +{ + const int64_t n_rows = X.extent(0); + const int64_t dim = X.extent(1); + const int64_t n_clusters = centroids.extent(0); + validate(params, X, raft::make_const_mdspan(centroids)); + RAFT_EXPECTS(n_rows >= n_clusters, "SuperKMeans needs at least as many rows as clusters"); + RAFT_EXPECTS(!labels.has_value() || labels->extent(0) == n_rows, + "Number of rows in dataset and labels are different"); + RAFT_EXPECTS(params.n_iters > 0, "SuperKMeans needs at least one iteration"); + if (prunes(n_clusters, dim)) { + RAFT_FAIL("SuperKMeans with pruning (dim >= %ld and n_clusters > %ld) is not implemented yet", + kDimTileSize, + kMinPrunedK); + } + + auto stream = raft::resource::get_cuda_stream(handle); + auto mapping_op = cuvs::spatial::knn::detail::utils::mapping{}; + + // The start centroids: rows sampled at random, the same draw as kmeans::fit's Random init. + raft::random::RngState random_state(params.rng_state.seed); + if constexpr (std::is_same_v) { + raft::matrix::sample_rows(handle, random_state, X, centroids); + } else { + auto start = raft::make_device_matrix(handle, n_clusters, dim); + raft::matrix::sample_rows(handle, random_state, X, start.view()); + raft::linalg::map(handle, centroids, mapping_op, raft::make_const_mdspan(start.view())); + } + + auto assigned = raft::make_device_vector(handle, n_rows); + auto sizes = raft::make_device_vector(handle, n_clusters); + auto means = raft::make_device_matrix(handle, n_clusters, dim); + auto assign = [&]() { + predict(handle, params, X, raft::make_const_mdspan(centroids), assigned.view()); + }; + + for (uint32_t iter = 0; iter < params.n_iters; iter++) { + assign(); + cuvs::cluster::kmeans_balanced::helpers::calc_centers_and_sizes( + handle, + X, + raft::make_const_mdspan(assigned.view()), + means.view(), + sizes.view(), + true, + mapping_op); + const float* m = means.data_handle(); + const float* c = centroids.data_handle(); + const uint32_t* z = sizes.data_handle(); + raft::linalg::map_offset( + handle, centroids, [=] __device__(int64_t i) { return z[i / dim] ? m[i] : c[i]; }); + } + + if (labels.has_value() || inertia_out.has_value()) { + assign(); + if (inertia_out.has_value()) { + *inertia_out->data_handle() = inertia( + handle, X, raft::make_const_mdspan(centroids), raft::make_const_mdspan(assigned.view())); + } + if (labels.has_value()) { + raft::copy(labels->data_handle(), assigned.data_handle(), n_rows, stream); + } + } +} + +} // namespace cuvs::cluster::kmeans::detail::superkmeans diff --git a/cpp/src/cluster/kmeans_superkmeans.cu b/cpp/src/cluster/kmeans_superkmeans.cu new file mode 100644 index 0000000000..00690dd6e6 --- /dev/null +++ b/cpp/src/cluster/kmeans_superkmeans.cu @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "detail/superkmeans.cuh" + +#include + +namespace cuvs::cluster::kmeans { + +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, std::nullopt, inertia); +} + +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, std::nullopt, inertia); +} + +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) +{ + detail::superkmeans::predict(handle, params, X, centroids, labels); +} + +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels) +{ + detail::superkmeans::predict(handle, params, X, centroids, labels); +} + +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, labels, inertia); +} + +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia) +{ + detail::superkmeans::fit(handle, params, X, centroids, labels, inertia); +} + +} // namespace cuvs::cluster::kmeans diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 9f78e47eed..e569e44ef4 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -134,6 +134,7 @@ ConfigureTest( NAME CLUSTER_TEST PATH cluster/kmeans.cu cluster/kmeans_balanced.cu + cluster/kmeans_superkmeans.cu cluster/kmeans_find_k.cu cluster/gmm.cu cluster/kmeans_predict_batching.cu diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu new file mode 100644 index 0000000000..cfdac29c75 --- /dev/null +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -0,0 +1,316 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "../test_utils.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include + +namespace cuvs { + +namespace { + +constexpr int64_t kRows = 4000; +constexpr int64_t kDim = 200; // below one dimension tile: no pruning +constexpr int64_t kClusters = 300; + +double sq_dist(const float* a, const float* b, int64_t dim) +{ + double s = 0; + for (int64_t j = 0; j < dim; j++) { + const double d = double(a[j]) - double(b[j]); + s += d * d; + } + return s; +} + +double sq_norm(const float* a, int64_t dim) +{ + double s = 0; + for (int64_t j = 0; j < dim; j++) { + s += double(a[j]) * a[j]; + } + return s; +} + +/** + * Host SuperKMeans without pruning: Lloyd's from the given start centroids, an empty cluster + * keeping its centroid. + */ +void reference_superkmeans(const std::vector& X, + int64_t n_rows, + int64_t dim, + int64_t n_clusters, + uint32_t n_iters, + std::vector start, + std::vector& centroids, + std::vector& labels, + double& inertia) +{ + centroids = std::move(start); + labels.resize(n_rows); + auto assign = [&]() { + inertia = 0; + for (int64_t r = 0; r < n_rows; r++) { + double best = std::numeric_limits::max(); + for (int64_t c = 0; c < n_clusters; c++) { + const double d = sq_dist(&X[r * dim], ¢roids[c * dim], dim); + if (d < best) { + best = d; + labels[r] = c; + } + } + inertia += best; + } + }; + std::vector sums(n_clusters * dim); + std::vector sizes(n_clusters); + for (uint32_t iter = 0; iter < n_iters; iter++) { + assign(); + std::fill(sums.begin(), sums.end(), 0.0); + std::fill(sizes.begin(), sizes.end(), 0u); + for (int64_t r = 0; r < n_rows; r++) { + sizes[labels[r]]++; + for (int64_t j = 0; j < dim; j++) { + sums[labels[r] * dim + j] += X[r * dim + j]; + } + } + for (int64_t c = 0; c < n_clusters; c++) { + if (sizes[c] == 0) { continue; } + for (int64_t j = 0; j < dim; j++) { + centroids[c * dim + j] = float(sums[c * dim + j] / sizes[c]); + } + } + } + assign(); +} + +/** Well separated blobs as T, and the same values as float on the host. */ +template +auto make_data(raft::handle_t& handle, std::vector& host) +{ + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + auto blobs = raft::make_device_matrix(handle, kRows, kDim); + auto blob_labels = raft::make_device_vector(handle, kRows); + raft::random::make_blobs(blobs.data_handle(), + blob_labels.data_handle(), + kRows, + kDim, + kClusters, + stream, + true, + nullptr, + nullptr, + 0.1f, + true, + -1.0f, + 1.0f, + uint64_t{1234}); + auto X = raft::make_device_matrix(handle, kRows, kDim); + raft::linalg::unaryOp( + X.data_handle(), blobs.data_handle(), kRows * kDim, raft::cast_op(), stream); + raft::linalg::unaryOp( + blobs.data_handle(), X.data_handle(), kRows * kDim, raft::cast_op(), stream); + host.resize(kRows * kDim); + raft::copy(host.data(), blobs.data_handle(), host.size(), stream); + raft::resource::sync_stream(handle); + return X; +} + +/** The start centroids SuperKMeans draws for `seed`: rows sampled as by kmeans::fit's Random init. + */ +std::vector start_centroids(raft::handle_t& handle, + const std::vector& host, + uint64_t seed) +{ + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + auto X = raft::make_device_matrix(handle, kRows, kDim); + auto start = raft::make_device_matrix(handle, kClusters, kDim); + raft::copy(X.data_handle(), host.data(), host.size(), stream); + raft::matrix::sample_rows( + handle, raft::random::RngState(seed), raft::make_const_mdspan(X.view()), start.view()); + std::vector out(kClusters * kDim); + raft::copy(out.data(), start.data_handle(), out.size(), stream); + raft::resource::sync_stream(handle); + return out; +} + +template +void fit_predict(raft::handle_t& handle, + raft::device_matrix_view X, + uint32_t n_iters, + std::vector& centroids, + std::vector& labels, + float& inertia) +{ + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + cuvs::cluster::kmeans::superkmeans_params params; + params.n_iters = n_iters; + auto d_centroids = raft::make_device_matrix(handle, kClusters, kDim); + auto d_labels = raft::make_device_vector(handle, kRows); + cuvs::cluster::kmeans::fit_predict( + handle, params, X, d_centroids.view(), d_labels.view(), raft::make_host_scalar_view(&inertia)); + centroids.resize(kClusters * kDim); + labels.resize(kRows); + raft::copy(centroids.data(), d_centroids.data_handle(), centroids.size(), stream); + raft::copy(labels.data(), d_labels.data_handle(), labels.size(), stream); + raft::resource::sync_stream(handle); +} + +} // namespace + +/* Without pruning SuperKMeans is Lloyd's algorithm from random start rows. The GPU assignment + * rounds its GEMM operands to TF32, so a run on float data drifts from an exact reference once + * near-ties go the other way: the test compares against the reference where that cannot happen, + * and checks the invariants of Lloyd's output otherwise. + */ +TEST(KmeansSuperKMeans, Fallback) +{ + raft::handle_t handle; + + // Half values are exact in TF32 and one iteration assigns against the start rows, so the start + // rows, the assignment and the update must match the reference. + { + std::vector h_X, centroids, ref_centroids; + std::vector labels, ref_labels; + float inertia; + double ref_inertia; + auto X = make_data(handle, h_X); + fit_predict(handle, raft::make_const_mdspan(X.view()), 1, centroids, labels, inertia); + reference_superkmeans( + h_X, + kRows, + kDim, + kClusters, + 1, + start_centroids(handle, h_X, cuvs::cluster::kmeans::superkmeans_params{}.rng_state.seed), + ref_centroids, + ref_labels, + ref_inertia); + for (size_t i = 0; i < centroids.size(); i++) { + ASSERT_NEAR(centroids[i], ref_centroids[i], 1e-5f) << "centroid " << i / kDim; + } + int64_t same_label = 0; + for (int64_t r = 0; r < kRows; r++) { + same_label += labels[r] == ref_labels[r]; + } + EXPECT_GE(double(same_label) / kRows, 0.999); + EXPECT_NEAR(inertia, ref_inertia, 1e-4 * ref_inertia); + } + + // Float data, several iterations: each row's label is its nearest centroid (up to the TF32 + // rounding of the distances), the inertia is that of the labels, and each centroid is the mean of + // its rows. + { + constexpr uint32_t n_iters = 5; + std::vector h_X, centroids; + std::vector labels; + float inertia; + auto X = make_data(handle, h_X); + fit_predict( + handle, raft::make_const_mdspan(X.view()), n_iters, centroids, labels, inertia); + + double host_inertia = 0; + for (int64_t r = 0; r < kRows; r++) { + const float* row = &h_X[r * kDim]; + const float* own = ¢roids[int64_t(labels[r]) * kDim]; + double best = std::numeric_limits::max(); + for (int64_t c = 0; c < kClusters; c++) { + best = std::min(best, sq_dist(row, ¢roids[c * kDim], kDim)); + } + const double own_dist = sq_dist(row, own, kDim); + ASSERT_LE(own_dist, best + 1e-3 * (sq_norm(row, kDim) + sq_norm(own, kDim))) << "row " << r; + host_inertia += own_dist; + } + EXPECT_NEAR(inertia, host_inertia, 1e-4 * host_inertia); + + std::vector sums(kClusters * kDim, 0.0); + std::vector sizes(kClusters, 0); + for (int64_t r = 0; r < kRows; r++) { + sizes[labels[r]]++; + for (int64_t j = 0; j < kDim; j++) { + sums[int64_t(labels[r]) * kDim + j] += h_X[r * kDim + j]; + } + } + for (int64_t c = 0; c < kClusters; c++) { + if (sizes[c] == 0) { continue; } + for (int64_t j = 0; j < kDim; j++) { + ASSERT_NEAR(centroids[c * kDim + j], sums[c * kDim + j] / sizes[c], 1e-4) + << "centroid " << c; + } + } + + // predict on the returned centroids gives fit_predict's labels. + auto d_centroids_fp = raft::make_device_matrix(handle, kClusters, kDim); + auto d_labels = raft::make_device_vector(handle, kRows); + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + raft::copy(d_centroids_fp.data_handle(), centroids.data(), centroids.size(), stream); + cuvs::cluster::kmeans::predict(handle, + cuvs::cluster::kmeans::superkmeans_params{}, + raft::make_const_mdspan(X.view()), + raft::make_const_mdspan(d_centroids_fp.view()), + d_labels.view()); + std::vector predicted(kRows); + raft::copy(predicted.data(), d_labels.data_handle(), kRows, stream); + raft::resource::sync_stream(handle); + EXPECT_EQ(predicted, labels); + + // fit runs the same iterations; its centroids may differ in the last bits (atomic sums). + cuvs::cluster::kmeans::superkmeans_params params; + params.n_iters = n_iters; + auto d_centroids = raft::make_device_matrix(handle, kClusters, kDim); + float fit_inertia = 0; + cuvs::cluster::kmeans::fit(handle, + params, + raft::make_const_mdspan(X.view()), + d_centroids.view(), + raft::make_host_scalar_view(&fit_inertia)); + EXPECT_NEAR(fit_inertia, inertia, 1e-3 * inertia); + } +} + +TEST(KmeansSuperKMeans, RejectsUnsupported) +{ + raft::handle_t handle; + cuvs::cluster::kmeans::superkmeans_params params; + auto X = raft::make_device_matrix(handle, 1000, 256); + auto centroids = raft::make_device_matrix(handle, 300, 256); + // Pruned configurations are not implemented yet. + EXPECT_THROW( + cuvs::cluster::kmeans::fit(handle, params, raft::make_const_mdspan(X.view()), centroids.view()), + raft::logic_error); + + auto small_centroids = raft::make_device_matrix(handle, 8, 256); + params.metric = cuvs::distance::DistanceType::InnerProduct; + EXPECT_THROW(cuvs::cluster::kmeans::fit( + handle, params, raft::make_const_mdspan(X.view()), small_centroids.view()), + raft::logic_error); + + params.metric = cuvs::distance::DistanceType::L2Expanded; + auto too_many = raft::make_device_matrix(handle, 1001, 256); + EXPECT_THROW( + cuvs::cluster::kmeans::fit(handle, params, raft::make_const_mdspan(X.view()), too_many.view()), + raft::logic_error); +} + +} // namespace cuvs From e83a28f02c0928daae4379fe803527129af90883 Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 25 Oct 2026 19:24:29 -0700 Subject: [PATCH 02/39] Add SuperKMeans' fused first-filter cuTile kernel The partial GEMM over the first d' dims fused with the first pruning filter, from SuperKMeans' GPU k-means: per (row, centroid tile) it keeps the centroids whose partial distance is under the row's bound and packs them into fixed segments. FP16 only, as in SuperKMeans, where float data is filtered on half copies. Exported next to the fused 1-NN kernels, with a planner and launcher; not used yet. Co-Authored-By: Claude Opus 5.5 --- cpp/CMakeLists.txt | 23 +++ .../superkmeans/fused_filter_fragments.hpp | 16 ++ .../superkmeans/cutile/export_fused_filter.py | 125 ++++++++++++ .../cutile/fused_filter_cutile_matrix.json | 18 ++ .../superkmeans/cutile/fused_filter_kernel.py | 111 +++++++++++ .../superkmeans/cutile/fused_filter_tile.cu | 182 ++++++++++++++++++ .../superkmeans/cutile/fused_filter_tile.hpp | 61 ++++++ 7 files changed, 536 insertions(+) create mode 100644 cpp/include/cuvs/detail/jit_lto/superkmeans/fused_filter_fragments.hpp create mode 100644 cpp/src/cluster/detail/superkmeans/cutile/export_fused_filter.py create mode 100644 cpp/src/cluster/detail/superkmeans/cutile/fused_filter_cutile_matrix.json create mode 100644 cpp/src/cluster/detail/superkmeans/cutile/fused_filter_kernel.py create mode 100644 cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu create mode 100644 cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 15f48c87bf..9daa365b51 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1234,6 +1234,26 @@ if(NOT BUILD_CPU_ONLY) "${cutile_fused_1nn_tiles}" @ONLY ) list(APPEND cutile_fused_1nn_files "${cutile_fused_1nn_tiles}") + + set(superkmeans_cutile_dir "${CMAKE_CURRENT_SOURCE_DIR}/src/cluster/detail/superkmeans/cutile") + set(cutile_superkmeans_generated_dir + "${CMAKE_CURRENT_BINARY_DIR}/generated_kernels/cluster/superkmeans/cutile" + ) + generate_cutile_kernels( + cutile_superkmeans_files + KERNEL_DIR "${superkmeans_cutile_dir}" + KERNEL_BASENAME "superkmeans_fused_filter" + KERNEL_PYTHON "fused_filter_kernel.py" + EXPORT_SCRIPT "export_fused_filter.py" + OUTPUT_DIRECTORY "${cutile_superkmeans_generated_dir}" + MATRIX_JSON_FILE "${superkmeans_cutile_dir}/fused_filter_cutile_matrix.json" + PYTHON "${cutile_python}" BIN2C "${cutile_bin2c}" + FRAGMENT_TAG_FORMAT_CUBIN + "cuvs::cluster::kmeans::detail::superkmeans::fragment_tag_fused_filter_cubin" + FRAGMENT_TAG_HEADER_FILES + "" + "" "" + ) endif() # Note that this matrix contains an `arch_includes` placeholder, since we don't currently have a # way to do an item-wise transform on a list after computing the matrix product and before @@ -1572,6 +1592,8 @@ if(NOT BUILD_CPU_ONLY) cuvs_objs PRIVATE ${cutile_fused_1nn_files} src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu ${cutile_smoke_files} + ${cutile_superkmeans_files} + src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu ) endif() @@ -1621,6 +1643,7 @@ if(NOT BUILD_CPU_ONLY) INTERFACE "$" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" "${CMAKE_CURRENT_BINARY_DIR}/src" "${cutile_fused_1nn_generated_dir}" "${cutile_smoke_generated_dir}" + "${cutile_superkmeans_generated_dir}" ) # Endian detection diff --git a/cpp/include/cuvs/detail/jit_lto/superkmeans/fused_filter_fragments.hpp b/cpp/include/cuvs/detail/jit_lto/superkmeans/fused_filter_fragments.hpp new file mode 100644 index 0000000000..f4dd65c93a --- /dev/null +++ b/cpp/include/cuvs/detail/jit_lto/superkmeans/fused_filter_fragments.hpp @@ -0,0 +1,16 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +namespace cuvs::cluster::kmeans::detail::superkmeans { + +template +struct fragment_tag_fused_filter_cubin { + static constexpr int cc_major = ArchTag::cc_major; + static constexpr int cc_minor = ArchTag::cc_minor; +}; + +} // namespace cuvs::cluster::kmeans::detail::superkmeans diff --git a/cpp/src/cluster/detail/superkmeans/cutile/export_fused_filter.py b/cpp/src/cluster/detail/superkmeans/cutile/export_fused_filter.py new file mode 100644 index 0000000000..6f7cbf7ed9 --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans/cutile/export_fused_filter.py @@ -0,0 +1,125 @@ +# ============================================================================= +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ============================================================================= +"""Export SuperKMeans' fused first-filter cuTile kernel to cubin.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import cuda.tile as ct +from cuda.tile.compilation import ( + ArrayConstraint, + CallingConvention, + ConstantConstraint, + KernelSignature, + ScalarConstraint, + export_kernel, +) + +# CI enables Python safe-path mode, so the script directory is not guaranteed +# to be importable even when this file is executed directly. +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from fused_filter_kernel import kernel_symbol, make_kernel # noqa: E402 + + +def _matrix(): + """float16, row-major, unit last stride, 16-byte aligned base and row pitch.""" + return ArrayConstraint( + ct.float16, + ndim=2, + index_dtype=ct.int32, + stride_lower_bound_incl=(0, None), + alias_groups=("read_only_inputs",), + may_alias_internally=False, + stride_constant=(None, 1), + stride_divisible_by=(8, 1), + base_addr_divisible_by=16, + ) + + +def _vector(elem_dtype, alias_groups=()): + return ArrayConstraint( + elem_dtype, + ndim=1, + index_dtype=ct.int32, + stride_lower_bound_incl=(None,), + alias_groups=alias_groups, + may_alias_internally=False, + stride_constant=(1,), + base_addr_divisible_by=16, + ) + + +def _segments(): + return ArrayConstraint( + ct.int32, + ndim=2, + index_dtype=ct.int32, + stride_lower_bound_incl=(0, None), + alias_groups=(), + may_alias_internally=False, + stride_constant=(None, 1), + base_addr_divisible_by=16, + ) + + +def _signature(data_type: str, tile_m: int, tile_n: int, tile_k: int): + # Argument order of the C++ launch: each array as (pointer, shape..., stride...), then the + # scalars; the tile sizes are compiled in. + read_only = ("read_only_inputs",) + return KernelSignature( + parameters=[ + _matrix(), # rows (n_rows, d) + _matrix(), # centroids (n_centroids, d) + _vector(ct.float32, read_only), # row_norms: ||row||^2 over d' + _vector(ct.float32, read_only), # centroid_norms: ||centroid||^2 over d' + _vector(ct.float32, read_only), # best: each row's current distance + _vector(ct.int64), # packed survivors + _segments(), # seg_count (n_rows, n_col_tiles) + ScalarConstraint(ct.int32), # n_rows + ScalarConstraint(ct.int32), # n_centroids + ScalarConstraint(ct.float32), # ratio + ScalarConstraint(ct.int32), # d' + ConstantConstraint(tile_m), + ConstantConstraint(tile_n), + ConstantConstraint(tile_k), + ], + calling_convention=CallingConvention.cutile_python_v1(), + ).with_symbol(kernel_symbol(data_type)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output_file", type=Path) + parser.add_argument("--format", choices=("cubin",), default="cubin") + parser.add_argument("--data-type", choices=("half",), required=True) + parser.add_argument("--metric", choices=("l2",), required=True) + parser.add_argument("--index-type", choices=("int32",), required=True) + parser.add_argument("--tile-m", type=int, required=True) + parser.add_argument("--tile-n", type=int, required=True) + parser.add_argument("--tile-k", type=int, required=True) + parser.add_argument("--gpu-code", required=True) + parser.add_argument("--occupancy", type=int) + args = parser.parse_args() + + export_kernel( + kernel=make_kernel(gpu_code=args.gpu_code, occupancy=args.occupancy), + signatures=[ + _signature(args.data_type, args.tile_m, args.tile_n, args.tile_k) + ], + output_file=str(args.output_file), + gpu_code=args.gpu_code, + output_format=args.format, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_cutile_matrix.json b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_cutile_matrix.json new file mode 100644 index 0000000000..92427d264c --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_cutile_matrix.json @@ -0,0 +1,18 @@ +[ + { + "_data": [ + {"data_type": "half", "data_abbrev": "h"} + ], + "_metric": [{"metric": "l2", "metric_abbrev": "l2"}], + "_index": [{"index_type": "int32", "index_abbrev": "i32"}], + "_tile": [{"tile_m": 128, "tile_n": 128, "tile_k": 64, "occupancy": 2}], + "_export": [ + {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_80", "arch_tag": "cutile_arch_8_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, + {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_86", "arch_tag": "cutile_arch_8_6", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, + {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_90", "arch_tag": "cutile_arch_9_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, + {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_100", "arch_tag": "cutile_arch_10_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, + {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_107", "arch_tag": "cutile_arch_10_7", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, + {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_120", "arch_tag": "cutile_arch_12_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"} + ] + } +] diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_kernel.py b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_kernel.py new file mode 100644 index 0000000000..bdd5c266ba --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_kernel.py @@ -0,0 +1,111 @@ +# ============================================================================= +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ============================================================================= +"""SuperKMeans' first pruning filter: the partial GEMM over the first d' dims fused with the filter. + +Each CTA owns TM rows and loops over the centroid tiles (TN wide). For each it computes the partial +distance ||row||^2 + ||centroid||^2 - 2 row.centroid over the first d' dims (FP16 tensor cores) and +keeps the pairs under the row's threshold best[row] * ratio, where best is the row's +distance to its current centroid. + +Output, for n_rows rows against n_centroids centroids (n_col_tiles = ceil(n_centroids / TN)): + + packed int64, one segment of TN slots per (row, centroid tile), at (row * n_col_tiles + tile) * TN: + slots 0 .. count-1 that tile's survivors for that row, in centroid order + slots count .. TN-1 not written + a survivor: (centroid index) << 32 | (the partial distance's float32 bits) + seg_count int32 (n_rows, n_col_tiles): count, how many slots of each segment are filled + +A segment holds a whole tile, so it cannot overflow, and a survivor's slot is just its rank among +the row's survivors in that tile: no prefix sum across rows. + +d' is a runtime argument and must be a multiple of TK. +""" + +import cuda.tile as ct + +ConstInt = ct.Constant[int] + +# A survivor packed in one int64: the centroid index in the high 32 bits, the partial distance's +# float32 bits in the low 32. One scatter per survivor is the costly part of the kernel; separate +# 32-bit scatters per field are much slower. +ID_SHIFT = 32 +LOW_32_BITS = (1 << 32) - 1 + + +def kernel_symbol(data_type: str) -> str: + return {"half": "superkmeans_fused_filter_h"}[data_type] + + +def make_kernel(*, gpu_code: str, occupancy: int | None = None): + kernel_options = {} + if occupancy is not None: + kernel_options["occupancy"] = ct.ByTarget(**{gpu_code: occupancy}) + + @ct.kernel(**kernel_options) + def fused_filter( + rows, + centroids, + row_norms, + centroid_norms, + best, + packed, + seg_count, + n_rows, + n_centroids, + ratio, + dp, + tm: ConstInt, + tn: ConstInt, + tk: ConstInt, + ): + row_tile = ct.bid(0) + zero_pad = ct.PaddingMode.ZERO + n_col_tiles = ct.num_tiles(centroids, axis=0, shape=(tn, tk)) + row_ids = row_tile * tm + ct.arange(tm, dtype=ct.int32)[:, None] + row_norm = ct.load( + row_norms, index=(row_tile,), shape=(tm,), padding_mode=zero_pad + )[:, None] + threshold = ( + ct.load(best, index=(row_tile,), shape=(tm,), padding_mode=zero_pad)[ + :, None + ] + * ratio + ) + for centroid_tile in range(n_col_tiles): + acc = ct.full((tm, tn), 0, dtype=ct.float32) + for kk in range(dp // tk): + a = ct.load( + rows, index=(row_tile, kk), shape=(tm, tk), padding_mode=zero_pad + ) + b = ct.load( + centroids, + index=(kk, centroid_tile), + shape=(tk, tn), + padding_mode=zero_pad, + order=(1, 0), + ) + acc = ct.mma(a, b, acc) + centroid_norm = ct.load( + centroid_norms, + index=(centroid_tile,), + shape=(tn,), + padding_mode=zero_pad, + )[None, :] + dist = -2.0 * acc + row_norm + centroid_norm + centroid_ids = centroid_tile * tn + ct.arange(tn, dtype=ct.int32)[None, :] + keep = (dist < threshold) & (centroid_ids < n_centroids) & (row_ids < n_rows) + # Slot: the start of the (row, tile) segment plus the survivor's rank in the row + keep_i = keep.astype(ct.int32) + per_row = ct.sum(keep_i, axis=1, keepdims=True) + slot = (row_ids * n_col_tiles + centroid_tile) * tn + ct.cumsum(keep_i, 1) - 1 + # The distance's bits are masked: widening a negative int32 (a negative float, e.g. a + # rounded-down 0) to int64 would sign-extend into the index + index = ct.broadcast_to(centroid_ids, (tm, tn)).astype(ct.int64) + dist_bits = ct.bitcast(dist, ct.int32).astype(ct.int64) & LOW_32_BITS + value = ct.bitwise_lshift(index, ID_SHIFT) | dist_bits + ct.scatter(packed, slot, value, mask=keep) + ct.store(seg_count, index=(row_tile, centroid_tile), tile=per_row) + + return fused_filter diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu new file mode 100644 index 0000000000..58b24cef6c --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu @@ -0,0 +1,182 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "fused_filter_tile.hpp" + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#include + +namespace cuvs::cluster::kmeans::detail::superkmeans { + +namespace { + +/** Must match kernel_symbol() in fused_filter_kernel.py. */ +constexpr const char* kFusedFilterEntrypoint = "superkmeans_fused_filter_h"; + +struct FusedFilterTilePlanner : cuvs::detail::jit_lto::TileAlgorithmPlanner { + using DataTag = cuvs::neighbors::detail::tag_h; + + inline static cuvs::detail::jit_lto::TileLauncherCache launcher_cache{}; + + FusedFilterTilePlanner() : TileAlgorithmPlanner(kFusedFilterEntrypoint, launcher_cache) + { + using namespace cuvs::detail::jit_lto; + add_static_fragment>(); + add_static_fragment>(); + add_static_fragment>(); + add_static_fragment>(); +#if CUVS_CUTILE_SM107_ENABLED + add_static_fragment>(); +#endif + add_static_fragment>(); + } +}; + +} // namespace + +auto fused_filter_tile_config() -> std::optional +{ + FusedFilterTilePlanner planner; + if (planner.try_get_launcher() == nullptr) { return std::nullopt; } + const auto cfg = planner.tile_config(); + return fused_filter_tile{cfg.tile_m, cfg.tile_n, cfg.tile_k}; +} + +void launch_fused_filter(raft::resources const& handle, + const half* rows, + int n_rows, + const half* centroids, + int n_centroids, + int dim, + const float* row_norms, + const float* centroid_norms, + const float* best, + float ratio, + int partial_dim, + int64_t* packed, + int* seg_count) +{ + FusedFilterTilePlanner planner; + auto launcher = planner.try_get_launcher(); + RAFT_EXPECTS(launcher != nullptr, "SuperKMeans fused filter: no cuTile cubin for this device"); + const auto tile = planner.tile_config(); + + constexpr int row_pitch_divisor = 16 / sizeof(half); + RAFT_EXPECTS(n_rows > 0 && n_centroids > 0, "SuperKMeans fused filter: empty input"); + RAFT_EXPECTS(dim % row_pitch_divisor == 0, + "SuperKMeans fused filter: the row pitch must be a multiple of 16 bytes"); + RAFT_EXPECTS(partial_dim > 0 && partial_dim <= dim && partial_dim % tile.tile_k == 0, + "SuperKMeans fused filter: partial_dim must be a positive multiple of %d, at most " + "dim", + tile.tile_k); + for (const void* ptr : {static_cast(rows), + static_cast(centroids), + static_cast(row_norms), + static_cast(centroid_norms), + static_cast(best), + static_cast(packed), + static_cast(seg_count)}) { + RAFT_EXPECTS(reinterpret_cast(ptr) % 16 == 0, + "SuperKMeans fused filter: buffers must be 16-byte aligned"); + } + const int n_col_tiles = (n_centroids + tile.tile_n - 1) / tile.tile_n; + const int64_t packed_size = int64_t{n_rows} * n_col_tiles * tile.tile_n; + RAFT_EXPECTS(packed_size <= std::numeric_limits::max(), + "SuperKMeans fused filter: too many rows x centroids in one call"); + + using fused_filter_kernel_t = void(void*, + int, + int, + int, + int, + void*, + int, + int, + int, + int, + void*, + int, + int, + void*, + int, + int, + void*, + int, + int, + void*, + int, + int, + void*, + int, + int, + int, + int, + int, + int, + float, + int); + void* rows_ptr = const_cast(rows); + void* centroids_ptr = const_cast(centroids); + void* row_norms_ptr = const_cast(row_norms); + void* centroid_norms_ptr = const_cast(centroid_norms); + void* best_ptr = const_cast(best); + void* packed_ptr = packed; + void* seg_count_ptr = seg_count; + const int packed_len = static_cast(packed_size); + const int unit = 1; + + const auto stream = raft::resource::get_cuda_stream(handle); + dim3 grid((n_rows + tile.tile_m - 1) / tile.tile_m, 1, 1); + dim3 block(1, 1, 1); + // Each array as (pointer, shape..., stride...), then the scalars; the tile sizes are compiled in. + launcher->template dispatch(stream.get(), + grid, + block, + 0, + rows_ptr, + int{n_rows}, + int{dim}, + int{dim}, + int{unit}, + centroids_ptr, + int{n_centroids}, + int{dim}, + int{dim}, + int{unit}, + row_norms_ptr, + int{n_rows}, + int{unit}, + centroid_norms_ptr, + int{n_centroids}, + int{unit}, + best_ptr, + int{n_rows}, + int{unit}, + packed_ptr, + int{packed_len}, + int{unit}, + seg_count_ptr, + int{n_rows}, + int{n_col_tiles}, + int{n_col_tiles}, + int{unit}, + int{n_rows}, + int{n_centroids}, + float{ratio}, + int{partial_dim}); + RAFT_CUDA_TRY(cudaGetLastError()); +} + +} // namespace cuvs::cluster::kmeans::detail::superkmeans diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp new file mode 100644 index 0000000000..d3c43368ec --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include + +#include +#include + +namespace cuvs::cluster::kmeans::detail::superkmeans { + +struct fused_filter_tile { + int tile_m; + int tile_n; + int tile_k; +}; + +/** The tile of the fused first-filter cubin for this device, or nullopt when there is none. */ +auto fused_filter_tile_config() -> std::optional; + +/** + * SuperKMeans' first pruning filter: the partial distances of rows to centroids over the first + * `partial_dim` dims, keeping the pairs below `best[row] * ratio`. FP16 only: the caller filters + * float data on half copies of its first dims. + * + * Survivors go to `packed`, one segment of tile_n slots per (row, centroid tile) at + * (row * n_col_tiles + tile) * tile_n, each `centroid << 32 | float bits of the partial distance`, + * in centroid order; `seg_count` [n_rows, n_col_tiles] holds how many slots of each segment are + * filled. n_col_tiles = ceil(n_centroids / tile_n). + * + * @param rows [n_rows, dim] row-major, 16-byte aligned, 16-byte row pitch + * @param centroids [n_centroids, dim] same layout as rows + * @param row_norms [n_rows] squared norms over the first partial_dim dims + * @param centroid_norms [n_centroids] squared norms over the first partial_dim dims + * @param best [n_rows] each row's distance to its current centroid + * @param ratio the pruning bound's scale at partial_dim + * @param partial_dim a multiple of tile_k, at most dim + * @param packed [n_rows * n_col_tiles * tile_n] survivors + * @param seg_count [n_rows, n_col_tiles] survivors per segment + */ +void launch_fused_filter(raft::resources const& handle, + const half* rows, + int n_rows, + const half* centroids, + int n_centroids, + int dim, + const float* row_norms, + const float* centroid_norms, + const float* best, + float ratio, + int partial_dim, + int64_t* packed, + int* seg_count); + +} // namespace cuvs::cluster::kmeans::detail::superkmeans From 19b3fb42ba3a349fb5fe7a3093c83ecd69da6e5e Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 25 Oct 2026 23:22:27 -0700 Subject: [PATCH 03/39] Add SuperKMeans' pruned assignment After the first iteration, each row starts from its current centroid: the fused first filter over the first 256 dims (FP16; float rows are filtered on half copies), then a pruning kernel ported from SuperKMeans that finishes the tail dims for the survivors with ADSampling re-filters and keeps the closest. Rows and centroids go in fixed-size batches; d' stays at 256 and there is no rotation, so the bound is a heuristic. Builds or devices without the filter's cubin assign in full. Co-Authored-By: Claude Opus 5.5 --- cpp/include/cuvs/cluster/kmeans.hpp | 6 +- cpp/src/cluster/detail/superkmeans.cuh | 217 +++++++++-- cpp/src/cluster/detail/superkmeans/prune.cuh | 357 +++++++++++++++++++ cpp/tests/cluster/kmeans_superkmeans.cu | 92 ++++- 4 files changed, 633 insertions(+), 39 deletions(-) create mode 100644 cpp/src/cluster/detail/superkmeans/prune.cuh diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index cb804e345f..52993cb9c7 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -236,8 +236,10 @@ struct balanced_params : base_params { * * SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every * row with the fused 1-NN over all dimensions; later iterations compute partial distances over the - * first 256 dimensions, discard centroids that cannot beat a row's current one - * (ADSampling bound), and finish the remaining dimensions only for the survivors. + * first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the + * ADSampling bound, without its random rotation), and finish the remaining dimensions only for the + * survivors. Pruning applies with more than 256 clusters and at least 256 dimensions, which must + * then be a multiple of 256; otherwise every iteration assigns with the full 1-NN. * * The number of clusters is taken from the `centroids` argument. The start centroids are rows * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index ccf6c89a01..41385ceaa3 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -6,20 +6,20 @@ #include "../../neighbors/detail/ann_utils.cuh" #include "../kmeans_balanced.cuh" +#include "superkmeans/cutile/fused_filter_tile.hpp" +#include "superkmeans/prune.cuh" #include #include #include #include -#include #include #include +#include #include #include #include -#include - #include #include #include @@ -28,8 +28,7 @@ namespace cuvs::cluster::kmeans::detail::superkmeans { // Pruning works on whole dimension tiles and only pays off with enough centroids to discard; below // that every iteration assigns with the full 1-NN. -constexpr int64_t kDimTileSize = 256; -constexpr int64_t kMinPrunedK = 256; +constexpr int64_t kMinPrunedK = 256; inline bool prunes(int64_t n_clusters, int64_t dim) { @@ -43,24 +42,22 @@ auto inertia(const raft::resources& handle, raft::device_matrix_view centroids, raft::device_vector_view labels) -> float { - const int64_t dim = X.extent(1); - auto dist = raft::make_device_vector(handle, X.extent(0)); - const T* x = X.data_handle(); - const float* c = centroids.data_handle(); - const uint32_t* l = labels.data_handle(); - raft::linalg::map_offset(handle, dist.view(), [=] __device__(int64_t row) { - const float* center = c + static_cast(l[row]) * dim; - float sum = 0; - for (int64_t j = 0; j < dim; j++) { - const float diff = static_cast(x[row * dim + j]) - center[j]; - sum += diff * diff; - } - return sum; - }); - return thrust::reduce(raft::resource::get_thrust_policy(handle), - dist.data_handle(), - dist.data_handle() + dist.size(), - 0.0f); + auto dist = raft::make_device_vector(handle, X.extent(0)); + distance_to_assigned(handle, + X.data_handle(), + X.extent(0), + centroids.data_handle(), + X.extent(1), + labels.data_handle(), + dist.data_handle()); + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + auto sum = raft::make_device_scalar(handle, 0.0f); + raft::linalg::mapThenSumReduce( + sum.data_handle(), dist.size(), raft::identity_op{}, stream, dist.data_handle()); + float out = 0; + raft::copy(&out, sum.data_handle(), 1, stream); + raft::resource::sync_stream(handle); + return out; } template @@ -96,6 +93,157 @@ void predict(const raft::resources& handle, cuvs::spatial::knn::detail::utils::mapping{}); } +/** The fused first filter's tile on this device, or nullopt when the build or device has none. */ +inline auto filter_tile() -> std::optional +{ +#if CUVS_CUTILE_ENABLED + return fused_filter_tile_config(); +#else + return std::nullopt; +#endif +} + +/** The first `cols` dims of n rows (row stride `stride`) as half, packed (row stride `cols`). */ +template +void copy_first_columns( + const raft::resources& handle, const S* in, int64_t n, int64_t stride, int64_t cols, half* out) +{ + raft::linalg::map_offset( + handle, raft::make_device_vector_view(out, n * cols), [=] __device__(int64_t i) { + return static_cast(in[i / cols * stride + i % cols]); + }); +} + +/** + * The pruned assignment: from each row's current centroid, the fused partial GEMM over the first + * kDimTileSize dims with the first filter, then the pruning kernel on the survivors' tail dims. + * Rows go in batches of kRowBatchSize against centroid batches of kCentroidBatchSize. The filter + * runs on FP16: float rows are filtered on half copies of their first dims, and the centroids the + * kernels read are T, a half copy of the float centroids for half rows. + */ +template +class pruned_assignment { + public: + pruned_assignment(const raft::resources& handle, + raft::device_matrix_view X, + int64_t n_clusters, + fused_filter_tile tile) + : handle_(handle), + X_(X), + n_rows_(X.extent(0)), + dim_(X.extent(1)), + n_clusters_(n_clusters), + filter_dim_(std::is_same_v ? dim_ : kDimTileSize), + tile_(tile), + n_col_tiles_(raft::ceildiv(kCentroidBatchSize, tile.tile_n)), + ratios_(raft::make_device_vector(handle, dim_ + 1)), + row_norms_(raft::make_device_vector(handle, n_rows_)), + centroid_norms_(raft::make_device_vector(handle, n_clusters)), + dist_(raft::make_device_vector(handle, n_rows_)), + filter_rows_(raft::make_device_matrix( + handle, std::is_same_v ? 0 : n_rows_, filter_dim_)), + filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), + packed_(raft::make_device_vector( + handle, std::min(n_rows_, kRowBatchSize) * n_col_tiles_ * tile.tile_n)), + seg_count_(raft::make_device_vector( + handle, std::min(n_rows_, kRowBatchSize) * n_col_tiles_)) + { + const auto ratios = adsampling_ratios(dim_); + ratio_ = ratios[kDimTileSize]; + raft::copy( + ratios_.data_handle(), ratios.data(), ratios.size(), raft::resource::get_cuda_stream(handle)); + squared_norms(handle, X.data_handle(), n_rows_, dim_, kDimTileSize, row_norms_.data_handle()); + if constexpr (!std::is_same_v) { + copy_columns(X.data_handle(), n_rows_, filter_rows_.view()); + } + // The host ratios are released when the copy is done + raft::resource::sync_stream(handle); + } + + /** Updates `labels`, each row's centroid from the previous assignment, for `centroids`. */ + void assign(raft::device_matrix_view centroids, + raft::device_vector_view labels) + { +#if CUVS_CUTILE_ENABLED + // The filter's FP16 copy of the centroids' first dims; for half rows, all dims, which is also + // the copy the other kernels read + copy_columns(centroids.data_handle(), n_clusters_, filter_centroids_.view()); + const T* kernel_centroids = nullptr; + const half* filter_rows = nullptr; + if constexpr (std::is_same_v) { + kernel_centroids = filter_centroids_.data_handle(); + filter_rows = X_.data_handle(); + } else { + kernel_centroids = centroids.data_handle(); + filter_rows = filter_rows_.data_handle(); + } + squared_norms( + handle_, kernel_centroids, n_clusters_, dim_, kDimTileSize, centroid_norms_.data_handle()); + + for (int64_t i = 0; i < n_rows_; i += kRowBatchSize) { + const int64_t n_rows = std::min(kRowBatchSize, n_rows_ - i); + const T* rows = X_.data_handle() + i * dim_; + uint32_t* row_labels = labels.data_handle() + i; + float* row_dist = dist_.data_handle() + i; + // Each row's bound: its distance to its current centroid + distance_to_assigned(handle_, rows, n_rows, kernel_centroids, dim_, row_labels, row_dist); + for (int64_t j = 0; j < n_clusters_; j += kCentroidBatchSize) { + const int64_t n_centroids = std::min(kCentroidBatchSize, n_clusters_ - j); + launch_fused_filter(handle_, + filter_rows + i * filter_dim_, + static_cast(n_rows), + filter_centroids_.data_handle() + j * filter_dim_, + static_cast(n_centroids), + static_cast(filter_dim_), + row_norms_.data_handle() + i, + centroid_norms_.data_handle() + j, + row_dist, + ratio_, + static_cast(kDimTileSize), + packed_.data_handle(), + seg_count_.data_handle()); + prune(handle_, + rows, + n_rows, + kernel_centroids + j * dim_, + static_cast(j), + dim_, + kDimTileSize, + ratios_.data_handle(), + first_filter_lists{ + packed_.data_handle(), + seg_count_.data_handle(), + static_cast(raft::ceildiv(n_centroids, tile_.tile_n)), + static_cast(tile_.tile_n)}, + row_labels, + row_dist); + } + } +#else + RAFT_FAIL("SuperKMeans pruning needs a build with cuTile"); +#endif + } + + private: + /** The first filter_dim_ dims of n rows as half, packed into `out`. */ + template + void copy_columns(const S* in, int64_t n, raft::device_matrix_view out) + { + copy_first_columns(handle_, in, n, dim_, filter_dim_, out.data_handle()); + } + + const raft::resources& handle_; + raft::device_matrix_view X_; + int64_t n_rows_, dim_, n_clusters_, filter_dim_; + fused_filter_tile tile_; + int64_t n_col_tiles_; + float ratio_; + raft::device_vector ratios_, row_norms_, centroid_norms_, dist_; + raft::device_matrix filter_rows_, filter_centroids_; + raft::device_vector packed_; + raft::device_vector seg_count_; +}; + /** * Lloyd's iterations from rows sampled at random, a fixed number of them. Each iteration assigns * every row (the full 1-NN when pruning does not apply) and sets each centroid to the mean of its @@ -118,16 +266,18 @@ void fit(const raft::resources& handle, RAFT_EXPECTS(!labels.has_value() || labels->extent(0) == n_rows, "Number of rows in dataset and labels are different"); RAFT_EXPECTS(params.n_iters > 0, "SuperKMeans needs at least one iteration"); - if (prunes(n_clusters, dim)) { - RAFT_FAIL("SuperKMeans with pruning (dim >= %ld and n_clusters > %ld) is not implemented yet", - kDimTileSize, - kMinPrunedK); - } + RAFT_EXPECTS(!prunes(n_clusters, dim) || dim % kDimTileSize == 0, + "SuperKMeans with pruning (n_clusters > %ld) needs dim to be a multiple of %u", + kMinPrunedK, + kDimTileSize); + // Without the filter's kernels on this build or device, every iteration assigns in full + const auto tile = prunes(n_clusters, dim) ? filter_tile() : std::nullopt; + const bool pruned = tile.has_value(); auto stream = raft::resource::get_cuda_stream(handle); auto mapping_op = cuvs::spatial::knn::detail::utils::mapping{}; - // The start centroids: rows sampled at random, the same draw as kmeans::fit's Random init. + // The start centroids: rows sampled at random raft::random::RngState random_state(params.rng_state.seed); if constexpr (std::is_same_v) { raft::matrix::sample_rows(handle, random_state, X, centroids); @@ -144,8 +294,15 @@ void fit(const raft::resources& handle, predict(handle, params, X, raft::make_const_mdspan(centroids), assigned.view()); }; + std::optional> pruned_assign; + if (pruned) { pruned_assign.emplace(handle, X, n_clusters, *tile); } + for (uint32_t iter = 0; iter < params.n_iters; iter++) { - assign(); + if (pruned_assign.has_value() && iter > 0) { + pruned_assign->assign(raft::make_const_mdspan(centroids), assigned.view()); + } else { + assign(); + } cuvs::cluster::kmeans_balanced::helpers::calc_centers_and_sizes( handle, X, diff --git a/cpp/src/cluster/detail/superkmeans/prune.cuh b/cpp/src/cluster/detail/superkmeans/prune.cuh new file mode 100644 index 0000000000..7742396abb --- /dev/null +++ b/cpp/src/cluster/detail/superkmeans/prune.cuh @@ -0,0 +1,357 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace cuvs::cluster::kmeans::detail::superkmeans { + +// Dims between the survivors' re-filters, and d' of the first filter +constexpr uint32_t kDimTileSize = 256; +// Centroids per batch: a row's survivor list of one batch lives in the pruning kernel's shared +// memory +constexpr uint32_t kCentroidBatchSize = 1024; +// Rows per batch: bounds the first filter's survivor lists (row batch x centroid batch slots) +constexpr int64_t kRowBatchSize = 128 * 1024; +// Survivors whose loads are batched over a dim tile +constexpr uint32_t kSurvivorTileSize = 16; +// fused_filter_kernel.py's ID_SHIFT +constexpr int kSurvivorIdShift = 32; + +constexpr uint32_t kWarpSize = 32; +constexpr uint32_t kFullMask = 0xffffffff; + +using survivor_id_t = std::conditional_t<(kCentroidBatchSize <= 65536), uint16_t, uint32_t>; + +/** + * ADSampling's pruning ratios: ratios[i] = (i / d) * (1 + eps0 / sqrt(i))^2 for i dims seen, + * i = 0..d (1 at both ends). A partial distance over the first i dims is compared against + * ratios[i] times the row's current best distance. The ratios assume every dim carries the same + * share of a distance, which ADSampling ensures with a random rotation of the data; without one + * they are a heuristic. + */ +inline auto adsampling_ratios(uint32_t dim, float eps0 = 1.5f) -> std::vector +{ + std::vector ratios(dim + 1); + for (uint32_t i = 0; i <= dim; i++) { + if (i == 0 || i == dim) { + ratios[i] = 1.0f; + continue; + } + const double f = 1.0 + eps0 / std::sqrt(static_cast(i)); + ratios[i] = static_cast(static_cast(i) / dim * f * f); + } + return ratios; +} + +/** + * Sums N independent per-lane values across the warp (valid in the first lane), the N shuffles of + * each step issued together so they overlap. + */ +template +__device__ __forceinline__ void warp_reduce_sum_n(T* values) +{ + T shifted[N]; +#pragma unroll + for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { +#pragma unroll + for (uint32_t i = 0; i < N; i++) { + shifted[i] = __shfl_down_sync(kFullMask, values[i], offset); + } + asm volatile("" ::: "memory"); +#pragma unroll + for (uint32_t i = 0; i < N; i++) { + values[i] += shifted[i]; + } + } +} + +/** Squared norm of each of n rows (row stride `stride`) over their first `cols` dims; a warp per + * row. */ +template +RAFT_KERNEL squared_norms_kernel( + const T* __restrict__ rows, int64_t n, int64_t stride, int64_t cols, float* __restrict__ out) +{ + const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; + if (row >= n) { return; } + float s = 0.f; + for (int64_t j = threadIdx.x % kWarpSize; j < cols; j += kWarpSize) { + const float v = static_cast(rows[row * stride + j]); + s += v * v; + } + s = raft::warpReduce(s); + if (threadIdx.x % kWarpSize == 0) { out[row] = s; } +} + +template +void squared_norms( + raft::resources const& handle, const T* rows, int64_t n, int64_t stride, int64_t cols, float* out) +{ + constexpr uint32_t kThreads = 256; + const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); + squared_norms_kernel<<>>( + rows, n, stride, cols, out); + RAFT_CUDA_TRY(cudaPeekAtLastError()); +} + +/** Each row's squared distance to its assigned centroid over all dims; a warp per row. */ +template +RAFT_KERNEL distance_to_assigned_kernel(const T* __restrict__ rows, + int64_t n, + const C* __restrict__ centroids, + int64_t dim, + const uint32_t* __restrict__ labels, + float* __restrict__ dist) +{ + const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; + if (row >= n) { return; } + const T* a = rows + row * dim; + const C* b = centroids + static_cast(labels[row]) * dim; + float distance = 0.f; + for (int64_t j = threadIdx.x % kWarpSize; j < dim; j += kWarpSize) { + const float diff = static_cast(a[j]) - static_cast(b[j]); + distance += diff * diff; + } + distance = raft::warpReduce(distance); + if (threadIdx.x % kWarpSize == 0) { dist[row] = distance; } +} + +template +void distance_to_assigned(raft::resources const& handle, + const T* rows, + int64_t n, + const C* centroids, + int64_t dim, + const uint32_t* labels, + float* dist) +{ + constexpr uint32_t kThreads = 256; + const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); + distance_to_assigned_kernel<<>>( + rows, n, centroids, dim, labels, dist); + RAFT_CUDA_TRY(cudaPeekAtLastError()); +} + +/** + * The first filter's survivor lists of one row batch x centroid batch (see launch_fused_filter): + * per (row, centroid tile) a segment of tile_n slots in `packed`, its count in `seg_count`. + */ +struct first_filter_lists { + const int64_t* packed; + const int32_t* seg_count; + uint32_t n_col_tiles; + uint32_t tile_n; +}; + +/** + * Distances over one dim tile between the row (`row_tile`, lane l holding dims l + 32 j) and N of + * its survivors, added to `distances` by lane 0. The N centroids' loads are issued back to back. + */ +template +__device__ __forceinline__ void add_tile_distances(const float (&row_tile)[Dims / kWarpSize], + const T* __restrict__ centroids, + uint32_t stride, + const survivor_id_t* __restrict__ ids, + float* __restrict__ distances) +{ + const uint32_t lane = threadIdx.x % kWarpSize; + float partial[N]; + for (uint32_t i = 0; i < N; i++) { + const T* centroid = centroids + static_cast(ids[i]) * stride; + partial[i] = 0; +#pragma unroll + for (uint32_t j = 0; j < Dims / kWarpSize; j++) { + const float diff = row_tile[j] - static_cast(centroid[lane + j * kWarpSize]); + partial[i] = fmaf(diff, diff, partial[i]); + } + } + warp_reduce_sum_n(partial); + if (lane == 0) { + for (uint32_t i = 0; i < N; i++) { + distances[i] += partial[i]; + } + } +} + +/** Calls f(integral_constant), then with N / 2, ..., 1. */ +template +__device__ __forceinline__ void for_each_halving(F&& f) +{ + f(std::integral_constant{}); + if constexpr (N > 1) { for_each_halving(f); } +} + +/** A row's first-filter survivors, in centroid order, into the warp's list; returns how many. */ +__device__ inline uint32_t load_survivors(first_filter_lists lists, + int64_t row, + survivor_id_t* __restrict__ ids, + float* __restrict__ distances) +{ + const uint32_t lane = threadIdx.x % kWarpSize; + const int64_t* segments = lists.packed + row * lists.n_col_tiles * lists.tile_n; + const int32_t* counts = lists.seg_count + row * lists.n_col_tiles; + uint32_t n = 0; + for (uint32_t t = 0; t < lists.n_col_tiles; t++) { + const auto count = static_cast(counts[t]); + for (uint32_t j = lane; j < count; j += kWarpSize) { + const int64_t value = segments[t * lists.tile_n + j]; + ids[n + j] = static_cast(value >> kSurvivorIdShift); + distances[n + j] = __int_as_float(static_cast(value)); + } + n += count; + } + __syncwarp(); + return n; +} + +/** Keeps the n survivors under `threshold`, compacting the list in place; returns how many. */ +__device__ inline uint32_t refilter(uint32_t n, + float threshold, + survivor_id_t* __restrict__ ids, + float* __restrict__ distances) +{ + const uint32_t lane = threadIdx.x % kWarpSize; + const uint32_t lanes_below = (1u << lane) - 1; + uint32_t kept = 0; + for (uint32_t base = 0; base < n; base += kWarpSize) { + const uint32_t i = base + lane; + uint32_t id = 0; + float distance = 0.f; + bool keep = false; + if (i < n) { + id = ids[i]; + distance = distances[i]; + keep = distance < threshold; + } + const uint32_t ballot = __ballot_sync(kFullMask, keep); + __syncwarp(); // all lanes have read their entry before the compaction writes + if (keep) { + const uint32_t slot = kept + __popc(ballot & lanes_below); + ids[slot] = static_cast(id); + distances[slot] = distance; + } + kept += __popc(ballot); + } + return kept; +} + +/** + * A warp per block and row, against one centroid batch: the row's first-filter survivors (partial + * distances over the first partial_dim dims), the tail dims in dim tiles with a re-filter after + * each but the last, then the argmin, kept if closer than the row's current centroid. + */ +template +RAFT_KERNEL prune_kernel(const T* __restrict__ rows, + const T* __restrict__ centroids, + uint32_t centroid_offset, + uint32_t dim, + uint32_t partial_dim, + const float* __restrict__ ratios, + first_filter_lists lists, + uint32_t* __restrict__ labels, + float* __restrict__ dist) +{ + __shared__ survivor_id_t ids[kCentroidBatchSize]; + __shared__ float distances[kCentroidBatchSize]; + const uint32_t lane = threadIdx.x; + const int64_t row = blockIdx.x; + const T* row_values = rows + row * dim; + uint32_t best_index = labels[row]; + float best = dist[row]; + const float bound = best; + + uint32_t n_left = load_survivors(lists, row, ids, distances); + if (n_left == 0 || (n_left == 1 && centroid_offset + ids[0] == best_index)) { return; } + + for (uint32_t d = partial_dim; d < dim && n_left; d += kDimTileSize) { + float row_tile[kDimTileSize / kWarpSize]; +#pragma unroll + for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { + row_tile[j] = static_cast(row_values[d + lane + j * kWarpSize]); + } + // Survivor tiles of kSurvivorTileSize, then halving sizes down to 1 for the ones left + uint32_t i = 0; + for_each_halving([&](auto tile) { + constexpr uint32_t kTile = decltype(tile)::value; + for (; i + kTile <= n_left; i += kTile) { + add_tile_distances( + row_tile, centroids + d, dim, ids + i, distances + i); + } + }); + __syncwarp(); // lane 0's distance updates are visible to the warp + // After the last tile the distances are complete: the argmin compares them with the best + if (d + kDimTileSize < dim) { + n_left = refilter(n_left, bound * ratios[d + kDimTileSize], ids, distances); + } + } + + for (uint32_t i = lane; i < n_left; i += kWarpSize) { + if (distances[i] < best) { + best = distances[i]; + best_index = centroid_offset + ids[i]; + } + } + for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { + const uint32_t other_index = __shfl_down_sync(kFullMask, best_index, offset); + const float other = __shfl_down_sync(kFullMask, best, offset); + if (other < best) { + best = other; + best_index = other_index; + } + } + if (lane == 0) { + labels[row] = best_index; + dist[row] = best; + } +} + +/** + * The pruning kernel for n_rows rows against one centroid batch (its first centroid + * centroid_offset), after the fused first filter wrote `lists`. `labels` and `dist` hold each + * row's current centroid and distance, and get the closer survivor if there is one. + */ +template +void prune(raft::resources const& handle, + const T* rows, + int64_t n_rows, + const T* centroids, + uint32_t centroid_offset, + int64_t dim, + uint32_t partial_dim, + const float* ratios, + first_filter_lists lists, + uint32_t* labels, + float* dist) +{ + RAFT_EXPECTS(partial_dim <= dim && (dim - partial_dim) % kDimTileSize == 0, + "SuperKMeans pruning needs dim - partial_dim to be a multiple of %u", + kDimTileSize); + prune_kernel<<>>( + rows, + centroids, + centroid_offset, + static_cast(dim), + partial_dim, + ratios, + lists, + labels, + dist); + RAFT_CUDA_TRY(cudaPeekAtLastError()); +} + +} // namespace cuvs::cluster::kmeans::detail::superkmeans diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index cfdac29c75..abc286c94e 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -137,7 +137,7 @@ auto make_data(raft::handle_t& handle, std::vector& host) return X; } -/** The start centroids SuperKMeans draws for `seed`: rows sampled as by kmeans::fit's Random init. +/** The start centroids SuperKMeans draws for `seed`: rows sampled with raft::matrix::sample_rows. */ std::vector start_centroids(raft::handle_t& handle, const std::vector& host, @@ -289,17 +289,95 @@ TEST(KmeansSuperKMeans, Fallback) } } +/* With more than 256 clusters and a multiple of 256 dims, the iterations after the first prune + * (on a build and device with the filter's kernels; otherwise they assign in full). The pruning + * bound is a heuristic without a rotation, so the result is compared with Lloyd's from the same + * start rows rather than exactly. The labels and inertia come from a full assignment. + */ +TEST(KmeansSuperKMeans, Pruned) +{ + // Two centroid batches, the second partial, and two tail dim tiles (one re-filter) + constexpr int64_t n_rows = 20000; + constexpr int64_t dim = 768; + constexpr int64_t n_clusters = 1100; + constexpr uint32_t n_iters = 8; + constexpr uint64_t seed = 42; + raft::handle_t handle; + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + + auto blobs = raft::make_device_matrix(handle, n_rows, dim); + auto blob_labels = raft::make_device_vector(handle, n_rows); + raft::random::make_blobs(blobs.data_handle(), + blob_labels.data_handle(), + n_rows, + dim, + n_clusters, + stream, + true, + nullptr, + nullptr, + 1.0f, + true, + -2.0f, + 2.0f, + uint64_t{1234}); + auto X_half = raft::make_device_matrix(handle, n_rows, dim); + raft::linalg::unaryOp( + X_half.data_handle(), blobs.data_handle(), n_rows * dim, raft::cast_op(), stream); + // Lloyd's runs on the half values too, so both data types are compared with the same reference + raft::linalg::unaryOp( + blobs.data_handle(), X_half.data_handle(), n_rows * dim, raft::cast_op(), stream); + auto X = raft::make_const_mdspan(blobs.view()); + + // Lloyd's from SuperKMeans' start rows (its own Random init draws other rows for the same seed) + auto centroids = raft::make_device_matrix(handle, n_clusters, dim); + raft::matrix::sample_rows(handle, raft::random::RngState(seed), X, centroids.view()); + cuvs::cluster::kmeans::params lloyd; + lloyd.n_clusters = n_clusters; + lloyd.init = cuvs::cluster::kmeans::params::Array; + lloyd.max_iter = n_iters; + lloyd.tol = 1e-30; // must be positive; this small, all iterations run + float lloyd_inertia = 0; + int64_t lloyd_iters = 0; + cuvs::cluster::kmeans::fit(handle, + lloyd, + X, + std::nullopt, + centroids.view(), + raft::make_host_scalar_view(&lloyd_inertia), + raft::make_host_scalar_view(&lloyd_iters)); + + cuvs::cluster::kmeans::superkmeans_params params; + params.n_iters = n_iters; + params.rng_state = raft::random::RngState(seed); + auto labels = raft::make_device_vector(handle, n_rows); + float inertia = 0; + cuvs::cluster::kmeans::fit_predict( + handle, params, X, centroids.view(), labels.view(), raft::make_host_scalar_view(&inertia)); + EXPECT_NEAR(inertia, lloyd_inertia, 0.01 * lloyd_inertia); + + float half_inertia = 0; + cuvs::cluster::kmeans::fit_predict(handle, + params, + raft::make_const_mdspan(X_half.view()), + centroids.view(), + labels.view(), + raft::make_host_scalar_view(&half_inertia)); + EXPECT_NEAR(half_inertia, lloyd_inertia, 0.01 * lloyd_inertia); +} + TEST(KmeansSuperKMeans, RejectsUnsupported) { raft::handle_t handle; cuvs::cluster::kmeans::superkmeans_params params; - auto X = raft::make_device_matrix(handle, 1000, 256); - auto centroids = raft::make_device_matrix(handle, 300, 256); - // Pruned configurations are not implemented yet. - EXPECT_THROW( - cuvs::cluster::kmeans::fit(handle, params, raft::make_const_mdspan(X.view()), centroids.view()), - raft::logic_error); + // Pruning works on whole 256-dim tiles. + auto X_odd = raft::make_device_matrix(handle, 1000, 384); + auto centroids = raft::make_device_matrix(handle, 300, 384); + EXPECT_THROW(cuvs::cluster::kmeans::fit( + handle, params, raft::make_const_mdspan(X_odd.view()), centroids.view()), + raft::logic_error); + auto X = raft::make_device_matrix(handle, 1000, 256); auto small_centroids = raft::make_device_matrix(handle, 8, 256); params.metric = cuvs::distance::DistanceType::InnerProduct; EXPECT_THROW(cuvs::cluster::kmeans::fit( From 443bb05b03c2123d747d1e1e55e9eb67de555e2c Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 26 Oct 2026 01:18:56 -0700 Subject: [PATCH 04/39] Order SuperKMeans' pruning by survivor count (LPT) Within a row batch the pruning kernel takes the rows with the most first-filter survivors in the previous iteration first, as SuperKMeans does: the kernel counts each row's survivors and an approximate segmented radix sort of the counts orders the next call's rows. The smoke test now has several row batches. Co-Authored-By: Claude Opus 5.5 --- cpp/src/cluster/detail/superkmeans.cuh | 103 +++++++++++++++++-- cpp/src/cluster/detail/superkmeans/prune.cuh | 22 ++-- cpp/tests/cluster/kmeans_superkmeans.cu | 5 +- 3 files changed, 113 insertions(+), 17 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 41385ceaa3..8443346ec9 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,10 @@ #include #include +#include + +#include + #include #include #include @@ -114,12 +119,29 @@ void copy_first_columns( }); } +/** Each row's id within its row batch, and the row batches' bounds (n_row_batches + 1). */ +inline void row_batch_layout(const raft::resources& handle, + int64_t n_rows, + raft::device_vector_view local_ids, + raft::device_vector_view batch_offsets) +{ + raft::linalg::map_offset(handle, local_ids, [] __device__(int64_t i) { + return static_cast(i % kRowBatchSize); + }); + raft::linalg::map_offset(handle, batch_offsets, [n_rows] __device__(int64_t b) { + return b * kRowBatchSize < n_rows ? b * kRowBatchSize : n_rows; + }); +} + /** * The pruned assignment: from each row's current centroid, the fused partial GEMM over the first * kDimTileSize dims with the first filter, then the pruning kernel on the survivors' tail dims. * Rows go in batches of kRowBatchSize against centroid batches of kCentroidBatchSize. The filter * runs on FP16: float rows are filtered on half copies of their first dims, and the centroids the * kernels read are T, a half copy of the float centroids for half rows. + * + * Within a row batch the pruning kernel's blocks take the heaviest rows first (LPT): the rows + * ordered by their first-filter survivors in the previous call. */ template class pruned_assignment { @@ -135,11 +157,17 @@ class pruned_assignment { n_clusters_(n_clusters), filter_dim_(std::is_same_v ? dim_ : kDimTileSize), tile_(tile), + n_row_batches_(raft::ceildiv(n_rows_, kRowBatchSize)), n_col_tiles_(raft::ceildiv(kCentroidBatchSize, tile.tile_n)), ratios_(raft::make_device_vector(handle, dim_ + 1)), row_norms_(raft::make_device_vector(handle, n_rows_)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), dist_(raft::make_device_vector(handle, n_rows_)), + survivors_(raft::make_device_vector(handle, n_rows_)), + row_order_(raft::make_device_vector(handle, n_rows_)), + local_ids_(raft::make_device_vector(handle, n_rows_)), + sorted_keys_(raft::make_device_vector(handle, n_rows_)), + batch_offsets_(raft::make_device_vector(handle, n_row_batches_ + 1)), filter_rows_(raft::make_device_matrix( handle, std::is_same_v ? 0 : n_rows_, filter_dim_)), filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), @@ -156,15 +184,22 @@ class pruned_assignment { if constexpr (!std::is_same_v) { copy_columns(X.data_handle(), n_rows_, filter_rows_.view()); } + // LPT's sort: each row's id within its batch, and the batches' bounds + row_batch_layout(handle, n_rows_, local_ids_.view(), batch_offsets_.view()); // The host ratios are released when the copy is done raft::resource::sync_stream(handle); } - /** Updates `labels`, each row's centroid from the previous assignment, for `centroids`. */ + /** + * Updates `labels`, each row's centroid from the previous assignment, for `centroids`. + * `order_next`: another call follows, so order the rows for it. + */ void assign(raft::device_matrix_view centroids, - raft::device_vector_view labels) + raft::device_vector_view labels, + bool order_next) { #if CUVS_CUTILE_ENABLED + auto stream = raft::resource::get_cuda_stream(handle_).get(); // The filter's FP16 copy of the centroids' first dims; for half rows, all dims, which is also // the copy the other kernels read copy_columns(centroids.data_handle(), n_clusters_, filter_centroids_.view()); @@ -179,12 +214,15 @@ class pruned_assignment { } squared_norms( handle_, kernel_centroids, n_clusters_, dim_, kDimTileSize, centroid_norms_.data_handle()); + RAFT_CUDA_TRY( + cudaMemsetAsync(survivors_.data_handle(), 0, survivors_.size() * sizeof(uint32_t), stream)); for (int64_t i = 0; i < n_rows_; i += kRowBatchSize) { - const int64_t n_rows = std::min(kRowBatchSize, n_rows_ - i); - const T* rows = X_.data_handle() + i * dim_; - uint32_t* row_labels = labels.data_handle() + i; - float* row_dist = dist_.data_handle() + i; + const int64_t n_rows = std::min(kRowBatchSize, n_rows_ - i); + const T* rows = X_.data_handle() + i * dim_; + uint32_t* row_labels = labels.data_handle() + i; + float* row_dist = dist_.data_handle() + i; + const uint32_t* order = has_row_order_ ? row_order_.data_handle() + i : nullptr; // Each row's bound: its distance to its current centroid distance_to_assigned(handle_, rows, n_rows, kernel_centroids, dim_, row_labels, row_dist); for (int64_t j = 0; j < n_clusters_; j += kCentroidBatchSize) { @@ -205,6 +243,7 @@ class pruned_assignment { prune(handle_, rows, n_rows, + order, kernel_centroids + j * dim_, static_cast(j), dim_, @@ -216,9 +255,14 @@ class pruned_assignment { static_cast(raft::ceildiv(n_centroids, tile_.tile_n)), static_cast(tile_.tile_n)}, row_labels, - row_dist); + row_dist, + survivors_.data_handle() + i); } } + if (order_next) { + order_rows_by_survivors(stream); + has_row_order_ = true; + } #else RAFT_FAIL("SuperKMeans pruning needs a build with cuTile"); #endif @@ -232,13 +276,52 @@ class pruned_assignment { copy_first_columns(handle_, in, n, dim_, filter_dim_, out.data_handle()); } + /** + * Each row batch's local row ids ordered by survivor count, descending, into row_order_. + * Approximate: one radix pass over the counts' top 8 significant bits (the counts are at most + * n_clusters), rows of a bucket keeping their order. + */ + void order_rows_by_survivors(cudaStream_t stream) + { + int end_bit = 1; + while (end_bit < 32 && (int64_t{1} << end_bit) <= n_clusters_) { + end_bit++; + } + const int begin_bit = std::max(0, end_bit - 8); + auto sort = [&](void* temp, size_t& temp_bytes) { + RAFT_CUDA_TRY( + cub::DeviceSegmentedRadixSort::SortPairsDescending(temp, + temp_bytes, + survivors_.data_handle(), + sorted_keys_.data_handle(), + local_ids_.data_handle(), + row_order_.data_handle(), + n_rows_, + n_row_batches_, + batch_offsets_.data_handle(), + batch_offsets_.data_handle() + 1, + begin_bit, + end_bit, + stream)); + }; + size_t temp_bytes = 0; + sort(nullptr, temp_bytes); + rmm::device_uvector temp( + temp_bytes, stream, raft::resource::get_workspace_resource_ref(handle_)); + sort(temp.data(), temp_bytes); + } + const raft::resources& handle_; raft::device_matrix_view X_; int64_t n_rows_, dim_, n_clusters_, filter_dim_; fused_filter_tile tile_; + int64_t n_row_batches_; int64_t n_col_tiles_; - float ratio_; + float ratio_ = 1.0f; + bool has_row_order_ = false; raft::device_vector ratios_, row_norms_, centroid_norms_, dist_; + raft::device_vector survivors_, row_order_, local_ids_, sorted_keys_; + raft::device_vector batch_offsets_; raft::device_matrix filter_rows_, filter_centroids_; raft::device_vector packed_; raft::device_vector seg_count_; @@ -299,7 +382,9 @@ void fit(const raft::resources& handle, for (uint32_t iter = 0; iter < params.n_iters; iter++) { if (pruned_assign.has_value() && iter > 0) { - pruned_assign->assign(raft::make_const_mdspan(centroids), assigned.view()); + // The last pruned call orders no rows: the final labels come from a full assignment + pruned_assign->assign( + raft::make_const_mdspan(centroids), assigned.view(), iter + 1 < params.n_iters); } else { assign(); } diff --git a/cpp/src/cluster/detail/superkmeans/prune.cuh b/cpp/src/cluster/detail/superkmeans/prune.cuh index 7742396abb..a71a70ac15 100644 --- a/cpp/src/cluster/detail/superkmeans/prune.cuh +++ b/cpp/src/cluster/detail/superkmeans/prune.cuh @@ -253,10 +253,13 @@ __device__ inline uint32_t refilter(uint32_t n, /** * A warp per block and row, against one centroid batch: the row's first-filter survivors (partial * distances over the first partial_dim dims), the tail dims in dim tiles with a re-filter after - * each but the last, then the argmin, kept if closer than the row's current centroid. + * each but the last, then the argmin, kept if closer than the row's current centroid. Block b + * handles row row_order[b] when given (heaviest rows first), and each row's first-filter survivors + * are added to `survivors`. */ template RAFT_KERNEL prune_kernel(const T* __restrict__ rows, + const uint32_t* __restrict__ row_order, const T* __restrict__ centroids, uint32_t centroid_offset, uint32_t dim, @@ -264,18 +267,20 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const float* __restrict__ ratios, first_filter_lists lists, uint32_t* __restrict__ labels, - float* __restrict__ dist) + float* __restrict__ dist, + uint32_t* __restrict__ survivors) { __shared__ survivor_id_t ids[kCentroidBatchSize]; __shared__ float distances[kCentroidBatchSize]; const uint32_t lane = threadIdx.x; - const int64_t row = blockIdx.x; + const int64_t row = row_order != nullptr ? row_order[blockIdx.x] : blockIdx.x; const T* row_values = rows + row * dim; uint32_t best_index = labels[row]; float best = dist[row]; const float bound = best; uint32_t n_left = load_survivors(lists, row, ids, distances); + if (lane == 0) { survivors[row] += n_left; } if (n_left == 0 || (n_left == 1 && centroid_offset + ids[0] == best_index)) { return; } for (uint32_t d = partial_dim; d < dim && n_left; d += kDimTileSize) { @@ -323,12 +328,14 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, /** * The pruning kernel for n_rows rows against one centroid batch (its first centroid * centroid_offset), after the fused first filter wrote `lists`. `labels` and `dist` hold each - * row's current centroid and distance, and get the closer survivor if there is one. + * row's current centroid and distance, and get the closer survivor if there is one; `survivors` + * gets each row's first-filter survivors added. row_order (optional): the rows' order. */ template void prune(raft::resources const& handle, const T* rows, int64_t n_rows, + const uint32_t* row_order, const T* centroids, uint32_t centroid_offset, int64_t dim, @@ -336,13 +343,15 @@ void prune(raft::resources const& handle, const float* ratios, first_filter_lists lists, uint32_t* labels, - float* dist) + float* dist, + uint32_t* survivors) { RAFT_EXPECTS(partial_dim <= dim && (dim - partial_dim) % kDimTileSize == 0, "SuperKMeans pruning needs dim - partial_dim to be a multiple of %u", kDimTileSize); prune_kernel<<>>( rows, + row_order, centroids, centroid_offset, static_cast(dim), @@ -350,7 +359,8 @@ void prune(raft::resources const& handle, ratios, lists, labels, - dist); + dist, + survivors); RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index abc286c94e..1f312c1f7d 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -296,8 +296,9 @@ TEST(KmeansSuperKMeans, Fallback) */ TEST(KmeansSuperKMeans, Pruned) { - // Two centroid batches, the second partial, and two tail dim tiles (one re-filter) - constexpr int64_t n_rows = 20000; + // Three row batches, the last partial; two centroid batches, the second partial; and two tail + // dim tiles (one re-filter) + constexpr int64_t n_rows = 300000; constexpr int64_t dim = 768; constexpr int64_t n_clusters = 1100; constexpr uint32_t n_iters = 8; From 669fcc1f69954330a6882ad6e09251c94bca958f Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 26 Oct 2026 01:19:15 -0700 Subject: [PATCH 05/39] Run SuperKMeans' row batches on the handle's stream pool With a stream pool on the handle, the pruned assignment's row batches round-robin over up to two of its streams, each with its own survivor lists; without one they stay on the main stream. The kernels and the filter launcher take a stream. The smoke test runs once with a two-stream pool. Co-Authored-By: Claude Opus 5.5 --- cpp/src/cluster/detail/superkmeans.cuh | 110 ++++++++++++------ .../superkmeans/cutile/fused_filter_tile.cu | 10 +- .../superkmeans/cutile/fused_filter_tile.hpp | 9 +- cpp/src/cluster/detail/superkmeans/prune.cuh | 45 ++++--- cpp/tests/cluster/kmeans_superkmeans.cu | 15 +++ 5 files changed, 116 insertions(+), 73 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 8443346ec9..d9208a09a3 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -26,8 +27,10 @@ #include #include +#include #include #include +#include namespace cuvs::cluster::kmeans::detail::superkmeans { @@ -47,16 +50,16 @@ auto inertia(const raft::resources& handle, raft::device_matrix_view centroids, raft::device_vector_view labels) -> float { - auto dist = raft::make_device_vector(handle, X.extent(0)); - distance_to_assigned(handle, - X.data_handle(), + auto dist = raft::make_device_vector(handle, X.extent(0)); + cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); + distance_to_assigned(X.data_handle(), X.extent(0), centroids.data_handle(), X.extent(1), labels.data_handle(), - dist.data_handle()); - cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); - auto sum = raft::make_device_scalar(handle, 0.0f); + dist.data_handle(), + stream); + auto sum = raft::make_device_scalar(handle, 0.0f); raft::linalg::mapThenSumReduce( sum.data_handle(), dist.size(), raft::identity_op{}, stream, dist.data_handle()); float out = 0; @@ -136,9 +139,10 @@ inline void row_batch_layout(const raft::resources& handle, /** * The pruned assignment: from each row's current centroid, the fused partial GEMM over the first * kDimTileSize dims with the first filter, then the pruning kernel on the survivors' tail dims. - * Rows go in batches of kRowBatchSize against centroid batches of kCentroidBatchSize. The filter - * runs on FP16: float rows are filtered on half copies of their first dims, and the centroids the - * kernels read are T, a half copy of the float centroids for half rows. + * Rows go in batches of kRowBatchSize against centroid batches of kCentroidBatchSize, the row + * batches round-robin over up to kMaxStreams streams of the handle's stream pool (the main stream + * without one). The filter runs on FP16: float rows are filtered on half copies of their first + * dims, and the centroids the kernels read are T, a half copy of the float centroids for half rows. * * Within a row batch the pruning kernel's blocks take the heaviest rows first (LPT): the rows * ordered by their first-filter survivors in the previous call. @@ -158,7 +162,6 @@ class pruned_assignment { filter_dim_(std::is_same_v ? dim_ : kDimTileSize), tile_(tile), n_row_batches_(raft::ceildiv(n_rows_, kRowBatchSize)), - n_col_tiles_(raft::ceildiv(kCentroidBatchSize, tile.tile_n)), ratios_(raft::make_device_vector(handle, dim_ + 1)), row_norms_(raft::make_device_vector(handle, n_rows_)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), @@ -170,17 +173,27 @@ class pruned_assignment { batch_offsets_(raft::make_device_vector(handle, n_row_batches_ + 1)), filter_rows_(raft::make_device_matrix( handle, std::is_same_v ? 0 : n_rows_, filter_dim_)), - filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), - packed_(raft::make_device_vector( - handle, std::min(n_rows_, kRowBatchSize) * n_col_tiles_ * tile.tile_n)), - seg_count_(raft::make_device_vector( - handle, std::min(n_rows_, kRowBatchSize) * n_col_tiles_)) + filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)) { + auto stream = raft::resource::get_cuda_stream(handle).get(); + const auto pool_size = raft::resource::is_stream_pool_initialized(handle) + ? static_cast(raft::resource::get_stream_pool_size(handle)) + : 0; + n_streams_ = std::max(1, std::min({pool_size, kMaxStreams, n_row_batches_})); + // The first filter's survivor lists, one pair per stream, sized for a whole batch + const int64_t batch_rows = std::min(n_rows_, kRowBatchSize); + const int64_t n_col_tiles = raft::ceildiv(kCentroidBatchSize, tile.tile_n); + for (int64_t s = 0; s < n_streams_; s++) { + packed_.push_back( + raft::make_device_vector(handle, batch_rows * n_col_tiles * tile.tile_n)); + seg_count_.push_back( + raft::make_device_vector(handle, batch_rows * n_col_tiles)); + } + const auto ratios = adsampling_ratios(dim_); ratio_ = ratios[kDimTileSize]; - raft::copy( - ratios_.data_handle(), ratios.data(), ratios.size(), raft::resource::get_cuda_stream(handle)); - squared_norms(handle, X.data_handle(), n_rows_, dim_, kDimTileSize, row_norms_.data_handle()); + raft::copy(ratios_.data_handle(), ratios.data(), ratios.size(), stream); + squared_norms(X.data_handle(), n_rows_, dim_, kDimTileSize, row_norms_.data_handle(), stream); if constexpr (!std::is_same_v) { copy_columns(X.data_handle(), n_rows_, filter_rows_.view()); } @@ -199,7 +212,7 @@ class pruned_assignment { bool order_next) { #if CUVS_CUTILE_ENABLED - auto stream = raft::resource::get_cuda_stream(handle_).get(); + auto main_stream = raft::resource::get_cuda_stream(handle_).get(); // The filter's FP16 copy of the centroids' first dims; for half rows, all dims, which is also // the copy the other kernels read copy_columns(centroids.data_handle(), n_clusters_, filter_centroids_.view()); @@ -212,23 +225,31 @@ class pruned_assignment { kernel_centroids = centroids.data_handle(); filter_rows = filter_rows_.data_handle(); } - squared_norms( - handle_, kernel_centroids, n_clusters_, dim_, kDimTileSize, centroid_norms_.data_handle()); - RAFT_CUDA_TRY( - cudaMemsetAsync(survivors_.data_handle(), 0, survivors_.size() * sizeof(uint32_t), stream)); + squared_norms(kernel_centroids, + n_clusters_, + dim_, + kDimTileSize, + centroid_norms_.data_handle(), + main_stream); + RAFT_CUDA_TRY(cudaMemsetAsync( + survivors_.data_handle(), 0, survivors_.size() * sizeof(uint32_t), main_stream)); + const bool pooled = n_streams_ > 1; + if (pooled) { raft::resource::wait_stream_pool_on_stream(handle_); } - for (int64_t i = 0; i < n_rows_; i += kRowBatchSize) { + for (int64_t b = 0; b < n_row_batches_; b++) { + const int64_t i = b * kRowBatchSize; const int64_t n_rows = std::min(kRowBatchSize, n_rows_ - i); + const int64_t s = b % n_streams_; + cudaStream_t stream = pooled ? stream_of(s) : main_stream; const T* rows = X_.data_handle() + i * dim_; uint32_t* row_labels = labels.data_handle() + i; float* row_dist = dist_.data_handle() + i; const uint32_t* order = has_row_order_ ? row_order_.data_handle() + i : nullptr; // Each row's bound: its distance to its current centroid - distance_to_assigned(handle_, rows, n_rows, kernel_centroids, dim_, row_labels, row_dist); + distance_to_assigned(rows, n_rows, kernel_centroids, dim_, row_labels, row_dist, stream); for (int64_t j = 0; j < n_clusters_; j += kCentroidBatchSize) { const int64_t n_centroids = std::min(kCentroidBatchSize, n_clusters_ - j); - launch_fused_filter(handle_, - filter_rows + i * filter_dim_, + launch_fused_filter(filter_rows + i * filter_dim_, static_cast(n_rows), filter_centroids_.data_handle() + j * filter_dim_, static_cast(n_centroids), @@ -238,10 +259,10 @@ class pruned_assignment { row_dist, ratio_, static_cast(kDimTileSize), - packed_.data_handle(), - seg_count_.data_handle()); - prune(handle_, - rows, + packed_[s].data_handle(), + seg_count_[s].data_handle(), + stream); + prune(rows, n_rows, order, kernel_centroids + j * dim_, @@ -250,17 +271,23 @@ class pruned_assignment { kDimTileSize, ratios_.data_handle(), first_filter_lists{ - packed_.data_handle(), - seg_count_.data_handle(), + packed_[s].data_handle(), + seg_count_[s].data_handle(), static_cast(raft::ceildiv(n_centroids, tile_.tile_n)), static_cast(tile_.tile_n)}, row_labels, row_dist, - survivors_.data_handle() + i); + survivors_.data_handle() + i, + stream); } } + if (pooled) { + std::vector used(n_streams_); + std::iota(used.begin(), used.end(), std::size_t{0}); + raft::resource::sync_stream_pool(handle_, used); + } if (order_next) { - order_rows_by_survivors(stream); + order_rows_by_survivors(main_stream); has_row_order_ = true; } #else @@ -269,6 +296,11 @@ class pruned_assignment { } private: + cudaStream_t stream_of(int64_t s) const + { + return raft::resource::get_stream_from_stream_pool(handle_, s).get(); + } + /** The first filter_dim_ dims of n rows as half, packed into `out`. */ template void copy_columns(const S* in, int64_t n, raft::device_matrix_view out) @@ -311,20 +343,22 @@ class pruned_assignment { sort(temp.data(), temp_bytes); } + static constexpr int64_t kMaxStreams = 2; + const raft::resources& handle_; raft::device_matrix_view X_; int64_t n_rows_, dim_, n_clusters_, filter_dim_; fused_filter_tile tile_; int64_t n_row_batches_; - int64_t n_col_tiles_; + int64_t n_streams_ = 1; float ratio_ = 1.0f; bool has_row_order_ = false; raft::device_vector ratios_, row_norms_, centroid_norms_, dist_; raft::device_vector survivors_, row_order_, local_ids_, sorted_keys_; raft::device_vector batch_offsets_; raft::device_matrix filter_rows_, filter_centroids_; - raft::device_vector packed_; - raft::device_vector seg_count_; + std::vector> packed_; + std::vector> seg_count_; }; /** diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu index 58b24cef6c..0eda842580 100644 --- a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu +++ b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu @@ -11,7 +11,6 @@ #include #include -#include #include #include @@ -54,8 +53,7 @@ auto fused_filter_tile_config() -> std::optional return fused_filter_tile{cfg.tile_m, cfg.tile_n, cfg.tile_k}; } -void launch_fused_filter(raft::resources const& handle, - const half* rows, +void launch_fused_filter(const half* rows, int n_rows, const half* centroids, int n_centroids, @@ -66,7 +64,8 @@ void launch_fused_filter(raft::resources const& handle, float ratio, int partial_dim, int64_t* packed, - int* seg_count) + int* seg_count, + cudaStream_t stream) { FusedFilterTilePlanner planner; auto launcher = planner.try_get_launcher(); @@ -137,11 +136,10 @@ void launch_fused_filter(raft::resources const& handle, const int packed_len = static_cast(packed_size); const int unit = 1; - const auto stream = raft::resource::get_cuda_stream(handle); dim3 grid((n_rows + tile.tile_m - 1) / tile.tile_m, 1, 1); dim3 block(1, 1, 1); // Each array as (pointer, shape..., stride...), then the scalars; the tile sizes are compiled in. - launcher->template dispatch(stream.get(), + launcher->template dispatch(stream, grid, block, 0, diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp index d3c43368ec..b3f39da3e2 100644 --- a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp +++ b/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp @@ -5,8 +5,8 @@ #pragma once +#include #include -#include #include @@ -43,9 +43,9 @@ auto fused_filter_tile_config() -> std::optional; * @param partial_dim a multiple of tile_k, at most dim * @param packed [n_rows * n_col_tiles * tile_n] survivors * @param seg_count [n_rows, n_col_tiles] survivors per segment + * @param stream the stream to launch on */ -void launch_fused_filter(raft::resources const& handle, - const half* rows, +void launch_fused_filter(const half* rows, int n_rows, const half* centroids, int n_centroids, @@ -56,6 +56,7 @@ void launch_fused_filter(raft::resources const& handle, float ratio, int partial_dim, int64_t* packed, - int* seg_count); + int* seg_count, + cudaStream_t stream); } // namespace cuvs::cluster::kmeans::detail::superkmeans diff --git a/cpp/src/cluster/detail/superkmeans/prune.cuh b/cpp/src/cluster/detail/superkmeans/prune.cuh index a71a70ac15..7ac6c1c634 100644 --- a/cpp/src/cluster/detail/superkmeans/prune.cuh +++ b/cpp/src/cluster/detail/superkmeans/prune.cuh @@ -97,12 +97,11 @@ RAFT_KERNEL squared_norms_kernel( template void squared_norms( - raft::resources const& handle, const T* rows, int64_t n, int64_t stride, int64_t cols, float* out) + const T* rows, int64_t n, int64_t stride, int64_t cols, float* out, cudaStream_t stream) { constexpr uint32_t kThreads = 256; const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); - squared_norms_kernel<<>>( - rows, n, stride, cols, out); + squared_norms_kernel<<>>(rows, n, stride, cols, out); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -129,20 +128,17 @@ RAFT_KERNEL distance_to_assigned_kernel(const T* __restrict__ rows, } template -void distance_to_assigned(raft::resources const& handle, - const T* rows, +void distance_to_assigned(const T* rows, int64_t n, const C* centroids, int64_t dim, const uint32_t* labels, - float* dist) + float* dist, + cudaStream_t stream) { constexpr uint32_t kThreads = 256; const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); - distance_to_assigned_kernel<<>>( + distance_to_assigned_kernel<<>>( rows, n, centroids, dim, labels, dist); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -332,8 +328,7 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, * gets each row's first-filter survivors added. row_order (optional): the rows' order. */ template -void prune(raft::resources const& handle, - const T* rows, +void prune(const T* rows, int64_t n_rows, const uint32_t* row_order, const T* centroids, @@ -344,23 +339,23 @@ void prune(raft::resources const& handle, first_filter_lists lists, uint32_t* labels, float* dist, - uint32_t* survivors) + uint32_t* survivors, + cudaStream_t stream) { RAFT_EXPECTS(partial_dim <= dim && (dim - partial_dim) % kDimTileSize == 0, "SuperKMeans pruning needs dim - partial_dim to be a multiple of %u", kDimTileSize); - prune_kernel<<>>( - rows, - row_order, - centroids, - centroid_offset, - static_cast(dim), - partial_dim, - ratios, - lists, - labels, - dist, - survivors); + prune_kernel<<>>(rows, + row_order, + centroids, + centroid_offset, + static_cast(dim), + partial_dim, + ratios, + lists, + labels, + dist, + survivors); RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index 1f312c1f7d..94f6922508 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -11,18 +11,22 @@ #include #include #include +#include #include #include #include #include #include +#include + #include #include #include #include +#include #include #include @@ -357,6 +361,17 @@ TEST(KmeansSuperKMeans, Pruned) handle, params, X, centroids.view(), labels.view(), raft::make_host_scalar_view(&inertia)); EXPECT_NEAR(inertia, lloyd_inertia, 0.01 * lloyd_inertia); + // With a stream pool the row batches run on two streams + raft::resource::set_cuda_stream_pool(handle, std::make_shared(2)); + float pooled_inertia = 0; + cuvs::cluster::kmeans::fit_predict(handle, + params, + X, + centroids.view(), + labels.view(), + raft::make_host_scalar_view(&pooled_inertia)); + EXPECT_NEAR(pooled_inertia, lloyd_inertia, 0.01 * lloyd_inertia); + float half_inertia = 0; cuvs::cluster::kmeans::fit_predict(handle, params, From 844f2f0743c74d704a6ca5f7149e097c0522e8f7 Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 26 Oct 2026 03:30:45 -0700 Subject: [PATCH 06/39] Move SuperKMeans' pruned assignment into a pruned 1-NN distance cuvs::distance::pruned_top_1_nn mirrors top_1_nn's interface (output, x, y, norms, m, n, k, params, workspace) for the approximate, warm-started 1-NN; the fused filter and pruning kernels move to distance/detail. Co-Authored-By: Claude Opus 5.5 --- cpp/CMakeLists.txt | 28 +- .../fused_filter_fragments.hpp | 4 +- cpp/src/cluster/detail/superkmeans.cuh | 285 ++++------------ .../cutile/export_fused_filter.py | 2 +- .../cutile/fused_filter_cutile_matrix.json | 0 .../cutile/fused_filter_kernel.py | 4 +- .../cutile/fused_filter_tile.cu | 20 +- .../cutile/fused_filter_tile.hpp | 6 +- .../detail/pruned_top_1_nn}/prune.cuh | 6 +- cpp/src/distance/pruned_top_1_nn.cuh | 318 ++++++++++++++++++ 10 files changed, 418 insertions(+), 255 deletions(-) rename cpp/include/cuvs/detail/jit_lto/{superkmeans => pruned_top_1_nn}/fused_filter_fragments.hpp (75%) rename cpp/src/{cluster/detail/superkmeans => distance/detail/pruned_top_1_nn}/cutile/export_fused_filter.py (98%) rename cpp/src/{cluster/detail/superkmeans => distance/detail/pruned_top_1_nn}/cutile/fused_filter_cutile_matrix.json (100%) rename cpp/src/{cluster/detail/superkmeans => distance/detail/pruned_top_1_nn}/cutile/fused_filter_kernel.py (96%) rename cpp/src/{cluster/detail/superkmeans => distance/detail/pruned_top_1_nn}/cutile/fused_filter_tile.cu (92%) rename cpp/src/{cluster/detail/superkmeans => distance/detail/pruned_top_1_nn}/cutile/fused_filter_tile.hpp (91%) rename cpp/src/{cluster/detail/superkmeans => distance/detail/pruned_top_1_nn}/prune.cuh (98%) create mode 100644 cpp/src/distance/pruned_top_1_nn.cuh diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9daa365b51..99186a0d67 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1235,23 +1235,25 @@ if(NOT BUILD_CPU_ONLY) ) list(APPEND cutile_fused_1nn_files "${cutile_fused_1nn_tiles}") - set(superkmeans_cutile_dir "${CMAKE_CURRENT_SOURCE_DIR}/src/cluster/detail/superkmeans/cutile") - set(cutile_superkmeans_generated_dir - "${CMAKE_CURRENT_BINARY_DIR}/generated_kernels/cluster/superkmeans/cutile" + set(pruned_1nn_cutile_dir + "${CMAKE_CURRENT_SOURCE_DIR}/src/distance/detail/pruned_top_1_nn/cutile" + ) + set(cutile_pruned_1nn_generated_dir + "${CMAKE_CURRENT_BINARY_DIR}/generated_kernels/distance/pruned_1nn/cutile" ) generate_cutile_kernels( - cutile_superkmeans_files - KERNEL_DIR "${superkmeans_cutile_dir}" - KERNEL_BASENAME "superkmeans_fused_filter" + cutile_pruned_1nn_files + KERNEL_DIR "${pruned_1nn_cutile_dir}" + KERNEL_BASENAME "pruned_1nn_fused_filter" KERNEL_PYTHON "fused_filter_kernel.py" EXPORT_SCRIPT "export_fused_filter.py" - OUTPUT_DIRECTORY "${cutile_superkmeans_generated_dir}" - MATRIX_JSON_FILE "${superkmeans_cutile_dir}/fused_filter_cutile_matrix.json" + OUTPUT_DIRECTORY "${cutile_pruned_1nn_generated_dir}" + MATRIX_JSON_FILE "${pruned_1nn_cutile_dir}/fused_filter_cutile_matrix.json" PYTHON "${cutile_python}" BIN2C "${cutile_bin2c}" FRAGMENT_TAG_FORMAT_CUBIN - "cuvs::cluster::kmeans::detail::superkmeans::fragment_tag_fused_filter_cubin" + "cuvs::distance::detail::pruned_nn::fragment_tag_fused_filter_cubin" FRAGMENT_TAG_HEADER_FILES - "" + "" "" "" ) endif() @@ -1592,8 +1594,8 @@ if(NOT BUILD_CPU_ONLY) cuvs_objs PRIVATE ${cutile_fused_1nn_files} src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu ${cutile_smoke_files} - ${cutile_superkmeans_files} - src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu + ${cutile_pruned_1nn_files} + src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu ) endif() @@ -1643,7 +1645,7 @@ if(NOT BUILD_CPU_ONLY) INTERFACE "$" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" "${CMAKE_CURRENT_BINARY_DIR}/src" "${cutile_fused_1nn_generated_dir}" "${cutile_smoke_generated_dir}" - "${cutile_superkmeans_generated_dir}" + "${cutile_pruned_1nn_generated_dir}" ) # Endian detection diff --git a/cpp/include/cuvs/detail/jit_lto/superkmeans/fused_filter_fragments.hpp b/cpp/include/cuvs/detail/jit_lto/pruned_top_1_nn/fused_filter_fragments.hpp similarity index 75% rename from cpp/include/cuvs/detail/jit_lto/superkmeans/fused_filter_fragments.hpp rename to cpp/include/cuvs/detail/jit_lto/pruned_top_1_nn/fused_filter_fragments.hpp index f4dd65c93a..c678409dd1 100644 --- a/cpp/include/cuvs/detail/jit_lto/superkmeans/fused_filter_fragments.hpp +++ b/cpp/include/cuvs/detail/jit_lto/pruned_top_1_nn/fused_filter_fragments.hpp @@ -5,7 +5,7 @@ #pragma once -namespace cuvs::cluster::kmeans::detail::superkmeans { +namespace cuvs::distance::detail::pruned_nn { template struct fragment_tag_fused_filter_cubin { @@ -13,4 +13,4 @@ struct fragment_tag_fused_filter_cubin { static constexpr int cc_minor = ArchTag::cc_minor; }; -} // namespace cuvs::cluster::kmeans::detail::superkmeans +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index d9208a09a3..15d7fc1317 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -4,17 +4,14 @@ */ #pragma once +#include "../../distance/pruned_top_1_nn.cuh" #include "../../neighbors/detail/ann_utils.cuh" #include "../kmeans_balanced.cuh" -#include "superkmeans/cutile/fused_filter_tile.hpp" -#include "superkmeans/prune.cuh" #include #include #include #include -#include -#include #include #include #include @@ -22,25 +19,21 @@ #include #include -#include - -#include - #include -#include #include #include -#include namespace cuvs::cluster::kmeans::detail::superkmeans { +namespace pruned_nn = cuvs::distance::detail::pruned_nn; + // Pruning works on whole dimension tiles and only pays off with enough centroids to discard; below // that every iteration assigns with the full 1-NN. constexpr int64_t kMinPrunedK = 256; inline bool prunes(int64_t n_clusters, int64_t dim) { - return dim >= kDimTileSize && n_clusters > kMinPrunedK; + return dim >= pruned_nn::kDimTileSize && n_clusters > kMinPrunedK; } /** Sum of squared distances of the rows to their assigned centroids. */ @@ -52,13 +45,13 @@ auto inertia(const raft::resources& handle, { auto dist = raft::make_device_vector(handle, X.extent(0)); cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); - distance_to_assigned(X.data_handle(), - X.extent(0), - centroids.data_handle(), - X.extent(1), - labels.data_handle(), - dist.data_handle(), - stream); + pruned_nn::distance_to_assigned(X.data_handle(), + X.extent(0), + centroids.data_handle(), + X.extent(1), + labels.data_handle(), + dist.data_handle(), + stream); auto sum = raft::make_device_scalar(handle, 0.0f); raft::linalg::mapThenSumReduce( sum.data_handle(), dist.size(), raft::identity_op{}, stream, dist.data_handle()); @@ -101,16 +94,6 @@ void predict(const raft::resources& handle, cuvs::spatial::knn::detail::utils::mapping{}); } -/** The fused first filter's tile on this device, or nullopt when the build or device has none. */ -inline auto filter_tile() -> std::optional -{ -#if CUVS_CUTILE_ENABLED - return fused_filter_tile_config(); -#else - return std::nullopt; -#endif -} - /** The first `cols` dims of n rows (row stride `stride`) as half, packed (row stride `cols`). */ template void copy_first_columns( @@ -122,83 +105,46 @@ void copy_first_columns( }); } -/** Each row's id within its row batch, and the row batches' bounds (n_row_batches + 1). */ -inline void row_batch_layout(const raft::resources& handle, - int64_t n_rows, - raft::device_vector_view local_ids, - raft::device_vector_view batch_offsets) -{ - raft::linalg::map_offset(handle, local_ids, [] __device__(int64_t i) { - return static_cast(i % kRowBatchSize); - }); - raft::linalg::map_offset(handle, batch_offsets, [n_rows] __device__(int64_t b) { - return b * kRowBatchSize < n_rows ? b * kRowBatchSize : n_rows; - }); -} - /** - * The pruned assignment: from each row's current centroid, the fused partial GEMM over the first - * kDimTileSize dims with the first filter, then the pruning kernel on the survivors' tail dims. - * Rows go in batches of kRowBatchSize against centroid batches of kCentroidBatchSize, the row - * batches round-robin over up to kMaxStreams streams of the handle's stream pool (the main stream - * without one). The filter runs on FP16: float rows are filtered on half copies of their first - * dims, and the centroids the kernels read are T, a half copy of the float centroids for half rows. - * - * Within a row batch the pruning kernel's blocks take the heaviest rows first (LPT): the rows - * ordered by their first-filter survivors in the previous call. + * The pruned assignment (cuvs::distance::pruned_top_1_nn) and what it keeps across iterations: the + * rows' norms and FP16 copy, the pruning ratios, the row order, and the workspace. The centroids + * the 1-NN reads are T: for half rows, a half copy of the float centroids, which is also the + * filter's. */ template class pruned_assignment { public: pruned_assignment(const raft::resources& handle, raft::device_matrix_view X, - int64_t n_clusters, - fused_filter_tile tile) + int64_t n_clusters) : handle_(handle), X_(X), n_rows_(X.extent(0)), dim_(X.extent(1)), n_clusters_(n_clusters), - filter_dim_(std::is_same_v ? dim_ : kDimTileSize), - tile_(tile), - n_row_batches_(raft::ceildiv(n_rows_, kRowBatchSize)), + filter_dim_(std::is_same_v ? dim_ : pruned_nn::kDimTileSize), ratios_(raft::make_device_vector(handle, dim_ + 1)), row_norms_(raft::make_device_vector(handle, n_rows_)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), dist_(raft::make_device_vector(handle, n_rows_)), - survivors_(raft::make_device_vector(handle, n_rows_)), row_order_(raft::make_device_vector(handle, n_rows_)), - local_ids_(raft::make_device_vector(handle, n_rows_)), - sorted_keys_(raft::make_device_vector(handle, n_rows_)), - batch_offsets_(raft::make_device_vector(handle, n_row_batches_ + 1)), filter_rows_(raft::make_device_matrix( handle, std::is_same_v ? 0 : n_rows_, filter_dim_)), - filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)) + filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), + workspace_(raft::make_device_vector( + handle, cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_))) { - auto stream = raft::resource::get_cuda_stream(handle).get(); - const auto pool_size = raft::resource::is_stream_pool_initialized(handle) - ? static_cast(raft::resource::get_stream_pool_size(handle)) - : 0; - n_streams_ = std::max(1, std::min({pool_size, kMaxStreams, n_row_batches_})); - // The first filter's survivor lists, one pair per stream, sized for a whole batch - const int64_t batch_rows = std::min(n_rows_, kRowBatchSize); - const int64_t n_col_tiles = raft::ceildiv(kCentroidBatchSize, tile.tile_n); - for (int64_t s = 0; s < n_streams_; s++) { - packed_.push_back( - raft::make_device_vector(handle, batch_rows * n_col_tiles * tile.tile_n)); - seg_count_.push_back( - raft::make_device_vector(handle, batch_rows * n_col_tiles)); - } - - const auto ratios = adsampling_ratios(dim_); - ratio_ = ratios[kDimTileSize]; + auto stream = raft::resource::get_cuda_stream(handle).get(); + const auto ratios = pruned_nn::adsampling_ratios(dim_); + params_.ratios = ratios_.data_handle(); + params_.first_ratio = ratios[pruned_nn::kDimTileSize]; raft::copy(ratios_.data_handle(), ratios.data(), ratios.size(), stream); - squared_norms(X.data_handle(), n_rows_, dim_, kDimTileSize, row_norms_.data_handle(), stream); + pruned_nn::squared_norms( + X.data_handle(), n_rows_, dim_, pruned_nn::kDimTileSize, row_norms_.data_handle(), stream); if constexpr (!std::is_same_v) { - copy_columns(X.data_handle(), n_rows_, filter_rows_.view()); + copy_first_columns( + handle, X.data_handle(), n_rows_, dim_, filter_dim_, filter_rows_.data_handle()); } - // LPT's sort: each row's id within its batch, and the batches' bounds - row_batch_layout(handle, n_rows_, local_ids_.view(), batch_offsets_.view()); // The host ratios are released when the copy is done raft::resource::sync_stream(handle); } @@ -211,154 +157,52 @@ class pruned_assignment { raft::device_vector_view labels, bool order_next) { -#if CUVS_CUTILE_ENABLED - auto main_stream = raft::resource::get_cuda_stream(handle_).get(); - // The filter's FP16 copy of the centroids' first dims; for half rows, all dims, which is also - // the copy the other kernels read - copy_columns(centroids.data_handle(), n_clusters_, filter_centroids_.view()); + auto stream = raft::resource::get_cuda_stream(handle_).get(); + copy_first_columns(handle_, + centroids.data_handle(), + n_clusters_, + dim_, + filter_dim_, + filter_centroids_.data_handle()); const T* kernel_centroids = nullptr; - const half* filter_rows = nullptr; if constexpr (std::is_same_v) { kernel_centroids = filter_centroids_.data_handle(); - filter_rows = X_.data_handle(); } else { kernel_centroids = centroids.data_handle(); - filter_rows = filter_rows_.data_handle(); - } - squared_norms(kernel_centroids, - n_clusters_, - dim_, - kDimTileSize, - centroid_norms_.data_handle(), - main_stream); - RAFT_CUDA_TRY(cudaMemsetAsync( - survivors_.data_handle(), 0, survivors_.size() * sizeof(uint32_t), main_stream)); - const bool pooled = n_streams_ > 1; - if (pooled) { raft::resource::wait_stream_pool_on_stream(handle_); } - - for (int64_t b = 0; b < n_row_batches_; b++) { - const int64_t i = b * kRowBatchSize; - const int64_t n_rows = std::min(kRowBatchSize, n_rows_ - i); - const int64_t s = b % n_streams_; - cudaStream_t stream = pooled ? stream_of(s) : main_stream; - const T* rows = X_.data_handle() + i * dim_; - uint32_t* row_labels = labels.data_handle() + i; - float* row_dist = dist_.data_handle() + i; - const uint32_t* order = has_row_order_ ? row_order_.data_handle() + i : nullptr; - // Each row's bound: its distance to its current centroid - distance_to_assigned(rows, n_rows, kernel_centroids, dim_, row_labels, row_dist, stream); - for (int64_t j = 0; j < n_clusters_; j += kCentroidBatchSize) { - const int64_t n_centroids = std::min(kCentroidBatchSize, n_clusters_ - j); - launch_fused_filter(filter_rows + i * filter_dim_, - static_cast(n_rows), - filter_centroids_.data_handle() + j * filter_dim_, - static_cast(n_centroids), - static_cast(filter_dim_), - row_norms_.data_handle() + i, - centroid_norms_.data_handle() + j, - row_dist, - ratio_, - static_cast(kDimTileSize), - packed_[s].data_handle(), - seg_count_[s].data_handle(), - stream); - prune(rows, - n_rows, - order, - kernel_centroids + j * dim_, - static_cast(j), - dim_, - kDimTileSize, - ratios_.data_handle(), - first_filter_lists{ - packed_[s].data_handle(), - seg_count_[s].data_handle(), - static_cast(raft::ceildiv(n_centroids, tile_.tile_n)), - static_cast(tile_.tile_n)}, - row_labels, - row_dist, - survivors_.data_handle() + i, - stream); - } } - if (pooled) { - std::vector used(n_streams_); - std::iota(used.begin(), used.end(), std::size_t{0}); - raft::resource::sync_stream_pool(handle_, used); - } - if (order_next) { - order_rows_by_survivors(main_stream); - has_row_order_ = true; - } -#else - RAFT_FAIL("SuperKMeans pruning needs a build with cuTile"); -#endif + pruned_nn::squared_norms(kernel_centroids, + n_clusters_, + dim_, + pruned_nn::kDimTileSize, + centroid_norms_.data_handle(), + stream); + params_.next_row_order = order_next ? row_order_.data_handle() : nullptr; + cuvs::distance::pruned_top_1_nn(handle_, + {labels.data_handle(), dist_.data_handle()}, + X_.data_handle(), + kernel_centroids, + filter_rows_.data_handle(), + filter_centroids_.data_handle(), + row_norms_.data_handle(), + centroid_norms_.data_handle(), + n_rows_, + n_clusters_, + dim_, + params_, + workspace_.data_handle(), + workspace_.size()); + if (order_next) { params_.row_order = row_order_.data_handle(); } } private: - cudaStream_t stream_of(int64_t s) const - { - return raft::resource::get_stream_from_stream_pool(handle_, s).get(); - } - - /** The first filter_dim_ dims of n rows as half, packed into `out`. */ - template - void copy_columns(const S* in, int64_t n, raft::device_matrix_view out) - { - copy_first_columns(handle_, in, n, dim_, filter_dim_, out.data_handle()); - } - - /** - * Each row batch's local row ids ordered by survivor count, descending, into row_order_. - * Approximate: one radix pass over the counts' top 8 significant bits (the counts are at most - * n_clusters), rows of a bucket keeping their order. - */ - void order_rows_by_survivors(cudaStream_t stream) - { - int end_bit = 1; - while (end_bit < 32 && (int64_t{1} << end_bit) <= n_clusters_) { - end_bit++; - } - const int begin_bit = std::max(0, end_bit - 8); - auto sort = [&](void* temp, size_t& temp_bytes) { - RAFT_CUDA_TRY( - cub::DeviceSegmentedRadixSort::SortPairsDescending(temp, - temp_bytes, - survivors_.data_handle(), - sorted_keys_.data_handle(), - local_ids_.data_handle(), - row_order_.data_handle(), - n_rows_, - n_row_batches_, - batch_offsets_.data_handle(), - batch_offsets_.data_handle() + 1, - begin_bit, - end_bit, - stream)); - }; - size_t temp_bytes = 0; - sort(nullptr, temp_bytes); - rmm::device_uvector temp( - temp_bytes, stream, raft::resource::get_workspace_resource_ref(handle_)); - sort(temp.data(), temp_bytes); - } - - static constexpr int64_t kMaxStreams = 2; - const raft::resources& handle_; raft::device_matrix_view X_; int64_t n_rows_, dim_, n_clusters_, filter_dim_; - fused_filter_tile tile_; - int64_t n_row_batches_; - int64_t n_streams_ = 1; - float ratio_ = 1.0f; - bool has_row_order_ = false; + cuvs::distance::PrunedTop1nnParams params_; raft::device_vector ratios_, row_norms_, centroid_norms_, dist_; - raft::device_vector survivors_, row_order_, local_ids_, sorted_keys_; - raft::device_vector batch_offsets_; + raft::device_vector row_order_; raft::device_matrix filter_rows_, filter_centroids_; - std::vector> packed_; - std::vector> seg_count_; + raft::device_vector workspace_; }; /** @@ -383,13 +227,12 @@ void fit(const raft::resources& handle, RAFT_EXPECTS(!labels.has_value() || labels->extent(0) == n_rows, "Number of rows in dataset and labels are different"); RAFT_EXPECTS(params.n_iters > 0, "SuperKMeans needs at least one iteration"); - RAFT_EXPECTS(!prunes(n_clusters, dim) || dim % kDimTileSize == 0, + RAFT_EXPECTS(!prunes(n_clusters, dim) || dim % pruned_nn::kDimTileSize == 0, "SuperKMeans with pruning (n_clusters > %ld) needs dim to be a multiple of %u", kMinPrunedK, - kDimTileSize); + pruned_nn::kDimTileSize); // Without the filter's kernels on this build or device, every iteration assigns in full - const auto tile = prunes(n_clusters, dim) ? filter_tile() : std::nullopt; - const bool pruned = tile.has_value(); + const bool pruned = prunes(n_clusters, dim) && cuvs::distance::is_pruned_top_1_nn_available(dim); auto stream = raft::resource::get_cuda_stream(handle); auto mapping_op = cuvs::spatial::knn::detail::utils::mapping{}; @@ -412,7 +255,7 @@ void fit(const raft::resources& handle, }; std::optional> pruned_assign; - if (pruned) { pruned_assign.emplace(handle, X, n_clusters, *tile); } + if (pruned) { pruned_assign.emplace(handle, X, n_clusters); } for (uint32_t iter = 0; iter < params.n_iters; iter++) { if (pruned_assign.has_value() && iter > 0) { diff --git a/cpp/src/cluster/detail/superkmeans/cutile/export_fused_filter.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py similarity index 98% rename from cpp/src/cluster/detail/superkmeans/cutile/export_fused_filter.py rename to cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py index 6f7cbf7ed9..2d67c1977a 100644 --- a/cpp/src/cluster/detail/superkmeans/cutile/export_fused_filter.py +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # ============================================================================= -"""Export SuperKMeans' fused first-filter cuTile kernel to cubin.""" +"""Export the pruned 1-NN's fused first-filter cuTile kernel to cubin.""" from __future__ import annotations diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_cutile_matrix.json b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json similarity index 100% rename from cpp/src/cluster/detail/superkmeans/cutile/fused_filter_cutile_matrix.json rename to cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_kernel.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py similarity index 96% rename from cpp/src/cluster/detail/superkmeans/cutile/fused_filter_kernel.py rename to cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py index bdd5c266ba..2c82addd0d 100644 --- a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_kernel.py +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # ============================================================================= -"""SuperKMeans' first pruning filter: the partial GEMM over the first d' dims fused with the filter. +"""The pruned 1-NN's first filter: the partial GEMM over the first d' dims fused with the filter. Each CTA owns TM rows and loops over the centroid tiles (TN wide). For each it computes the partial distance ||row||^2 + ||centroid||^2 - 2 row.centroid over the first d' dims (FP16 tensor cores) and @@ -35,7 +35,7 @@ def kernel_symbol(data_type: str) -> str: - return {"half": "superkmeans_fused_filter_h"}[data_type] + return {"half": "pruned_1nn_fused_filter_h"}[data_type] def make_kernel(*, gpu_code: str, occupancy: int | None = None): diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu similarity index 92% rename from cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu rename to cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu index 0eda842580..9b4b16fceb 100644 --- a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.cu +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include @@ -17,12 +17,12 @@ #include -namespace cuvs::cluster::kmeans::detail::superkmeans { +namespace cuvs::distance::detail::pruned_nn { namespace { /** Must match kernel_symbol() in fused_filter_kernel.py. */ -constexpr const char* kFusedFilterEntrypoint = "superkmeans_fused_filter_h"; +constexpr const char* kFusedFilterEntrypoint = "pruned_1nn_fused_filter_h"; struct FusedFilterTilePlanner : cuvs::detail::jit_lto::TileAlgorithmPlanner { using DataTag = cuvs::neighbors::detail::tag_h; @@ -69,15 +69,15 @@ void launch_fused_filter(const half* rows, { FusedFilterTilePlanner planner; auto launcher = planner.try_get_launcher(); - RAFT_EXPECTS(launcher != nullptr, "SuperKMeans fused filter: no cuTile cubin for this device"); + RAFT_EXPECTS(launcher != nullptr, "Pruned 1-NN fused filter: no cuTile cubin for this device"); const auto tile = planner.tile_config(); constexpr int row_pitch_divisor = 16 / sizeof(half); - RAFT_EXPECTS(n_rows > 0 && n_centroids > 0, "SuperKMeans fused filter: empty input"); + RAFT_EXPECTS(n_rows > 0 && n_centroids > 0, "Pruned 1-NN fused filter: empty input"); RAFT_EXPECTS(dim % row_pitch_divisor == 0, - "SuperKMeans fused filter: the row pitch must be a multiple of 16 bytes"); + "Pruned 1-NN fused filter: the row pitch must be a multiple of 16 bytes"); RAFT_EXPECTS(partial_dim > 0 && partial_dim <= dim && partial_dim % tile.tile_k == 0, - "SuperKMeans fused filter: partial_dim must be a positive multiple of %d, at most " + "Pruned 1-NN fused filter: partial_dim must be a positive multiple of %d, at most " "dim", tile.tile_k); for (const void* ptr : {static_cast(rows), @@ -88,12 +88,12 @@ void launch_fused_filter(const half* rows, static_cast(packed), static_cast(seg_count)}) { RAFT_EXPECTS(reinterpret_cast(ptr) % 16 == 0, - "SuperKMeans fused filter: buffers must be 16-byte aligned"); + "Pruned 1-NN fused filter: buffers must be 16-byte aligned"); } const int n_col_tiles = (n_centroids + tile.tile_n - 1) / tile.tile_n; const int64_t packed_size = int64_t{n_rows} * n_col_tiles * tile.tile_n; RAFT_EXPECTS(packed_size <= std::numeric_limits::max(), - "SuperKMeans fused filter: too many rows x centroids in one call"); + "Pruned 1-NN fused filter: too many rows x centroids in one call"); using fused_filter_kernel_t = void(void*, int, @@ -177,4 +177,4 @@ void launch_fused_filter(const half* rows, RAFT_CUDA_TRY(cudaGetLastError()); } -} // namespace cuvs::cluster::kmeans::detail::superkmeans +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp similarity index 91% rename from cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp rename to cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp index b3f39da3e2..b7abab3ede 100644 --- a/cpp/src/cluster/detail/superkmeans/cutile/fused_filter_tile.hpp +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp @@ -13,7 +13,7 @@ #include #include -namespace cuvs::cluster::kmeans::detail::superkmeans { +namespace cuvs::distance::detail::pruned_nn { struct fused_filter_tile { int tile_m; @@ -25,7 +25,7 @@ struct fused_filter_tile { auto fused_filter_tile_config() -> std::optional; /** - * SuperKMeans' first pruning filter: the partial distances of rows to centroids over the first + * The pruned 1-NN's first filter: the partial distances of rows to centroids over the first * `partial_dim` dims, keeping the pairs below `best[row] * ratio`. FP16 only: the caller filters * float data on half copies of its first dims. * @@ -59,4 +59,4 @@ void launch_fused_filter(const half* rows, int* seg_count, cudaStream_t stream); -} // namespace cuvs::cluster::kmeans::detail::superkmeans +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/cluster/detail/superkmeans/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh similarity index 98% rename from cpp/src/cluster/detail/superkmeans/prune.cuh rename to cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 7ac6c1c634..f323b716f6 100644 --- a/cpp/src/cluster/detail/superkmeans/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -16,7 +16,7 @@ #include #include -namespace cuvs::cluster::kmeans::detail::superkmeans { +namespace cuvs::distance::detail::pruned_nn { // Dims between the survivors' re-filters, and d' of the first filter constexpr uint32_t kDimTileSize = 256; @@ -343,7 +343,7 @@ void prune(const T* rows, cudaStream_t stream) { RAFT_EXPECTS(partial_dim <= dim && (dim - partial_dim) % kDimTileSize == 0, - "SuperKMeans pruning needs dim - partial_dim to be a multiple of %u", + "Pruned 1-NN needs dim - partial_dim to be a multiple of %u", kDimTileSize); prune_kernel<<>>(rows, row_order, @@ -359,4 +359,4 @@ void prune(const T* rows, RAFT_CUDA_TRY(cudaPeekAtLastError()); } -} // namespace cuvs::cluster::kmeans::detail::superkmeans +} // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh new file mode 100644 index 0000000000..69e0ac9ddc --- /dev/null +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -0,0 +1,318 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include "detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp" +#include "detail/pruned_top_1_nn/prune.cuh" +#include "top_1_nn.cuh" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace cuvs::distance { + +/** The pruned 1-NN's bounds, and the order its pruning kernel takes the rows in. */ +struct PrunedTop1nnParams { + /** [k + 1] on device: ratios[i] scales a row's bound after i dims (see + * detail::pruned_nn::adsampling_ratios). */ + const float* ratios = nullptr; + /** ratios[detail::pruned_nn::kDimTileSize] on host: the first filter's. */ + float first_ratio = 1.0f; + /** [m], optional: each row batch's rows in the order to prune them, heaviest first. */ + const uint32_t* row_order = nullptr; + /** [m], optional: the order for the next call, from this call's first-filter survivors. May be + * row_order. */ + uint32_t* next_row_order = nullptr; +}; + +namespace detail::pruned_nn { + +constexpr int64_t kMaxStreams = 2; +constexpr std::size_t kAlignment = 256; + +/** The fused first filter's tile on this device, or nullopt when the build or device has none. */ +inline auto filter_tile() -> std::optional +{ +#if CUVS_CUTILE_ENABLED + return fused_filter_tile_config(); +#else + return std::nullopt; +#endif +} + +/** The workspace's buffers, as byte offsets, for m rows on the handle's streams. */ +struct workspace_layout { + int64_t n_row_batches; + int64_t n_streams; + fused_filter_tile tile; + std::size_t survivors, sorted_keys, local_ids, batch_offsets, sort_temp, sort_temp_bytes; + std::vector packed, seg_count; + std::size_t total_bytes; + + workspace_layout(raft::resources const& handle, int64_t m) + { + const auto t = filter_tile(); + RAFT_EXPECTS(t.has_value(), "Pruned 1-NN needs the fused filter's cuTile kernel"); + tile = *t; + n_row_batches = raft::ceildiv(m, kRowBatchSize); + const auto pool_size = raft::resource::is_stream_pool_initialized(handle) + ? static_cast(raft::resource::get_stream_pool_size(handle)) + : 0; + n_streams = std::max(1, std::min({pool_size, kMaxStreams, n_row_batches})); + + std::size_t end = 0; + auto take = [&](std::size_t bytes) { + const std::size_t offset = raft::round_up_safe(end, kAlignment); + end = offset + bytes; + return offset; + }; + survivors = take(m * sizeof(uint32_t)); + sorted_keys = take(m * sizeof(uint32_t)); + local_ids = take(m * sizeof(uint32_t)); + batch_offsets = take((n_row_batches + 1) * sizeof(int64_t)); + RAFT_CUDA_TRY( + cub::DeviceSegmentedRadixSort::SortPairsDescending(nullptr, + sort_temp_bytes, + static_cast(nullptr), + static_cast(nullptr), + static_cast(nullptr), + static_cast(nullptr), + m, + n_row_batches, + static_cast(nullptr), + static_cast(nullptr))); + sort_temp = take(sort_temp_bytes); + // The first filter's survivor lists, one pair per stream, sized for a whole batch + const int64_t batch_rows = std::min(m, kRowBatchSize); + const int64_t n_col_tiles = raft::ceildiv(kCentroidBatchSize, tile.tile_n); + for (int64_t s = 0; s < n_streams; s++) { + packed.push_back(take(batch_rows * n_col_tiles * tile.tile_n * sizeof(int64_t))); + seg_count.push_back(take(batch_rows * n_col_tiles * sizeof(int32_t))); + } + total_bytes = end; + } +}; + +/** Each row's id within its row batch, and the row batches' bounds (n_row_batches + 1). */ +inline void row_batch_layout(raft::resources const& handle, + int64_t m, + raft::device_vector_view local_ids, + raft::device_vector_view batch_offsets) +{ + raft::linalg::map_offset(handle, local_ids, [] __device__(int64_t i) { + return static_cast(i % kRowBatchSize); + }); + raft::linalg::map_offset(handle, batch_offsets, [m] __device__(int64_t b) { + return b * kRowBatchSize < m ? b * kRowBatchSize : m; + }); +} + +/** + * Each row batch's local row ids ordered by survivor count, descending, into `row_order`. + * Approximate: one radix pass over the counts' top 8 significant bits (the counts are at most n), + * rows of a bucket keeping their order. + */ +inline void order_rows_by_survivors(raft::resources const& handle, + const workspace_layout& layout, + char* workspace, + int64_t m, + int64_t n, + uint32_t* row_order) +{ + auto* local_ids = reinterpret_cast(workspace + layout.local_ids); + auto* batch_offsets = reinterpret_cast(workspace + layout.batch_offsets); + row_batch_layout(handle, + m, + raft::make_device_vector_view(local_ids, m), + raft::make_device_vector_view(batch_offsets, layout.n_row_batches + 1)); + int end_bit = 1; + while (end_bit < 32 && (int64_t{1} << end_bit) <= n) { + end_bit++; + } + const int begin_bit = std::max(0, end_bit - 8); + std::size_t temp_size = layout.sort_temp_bytes; + RAFT_CUDA_TRY(cub::DeviceSegmentedRadixSort::SortPairsDescending( + workspace + layout.sort_temp, + temp_size, + reinterpret_cast(workspace + layout.survivors), + reinterpret_cast(workspace + layout.sorted_keys), + local_ids, + row_order, + m, + layout.n_row_batches, + batch_offsets, + batch_offsets + 1, + begin_bit, + end_bit, + raft::resource::get_cuda_stream(handle).get())); +} + +} // namespace detail::pruned_nn + +/** + * Whether pruned_top_1_nn runs for k features on this build and device: it needs the fused + * filter's cuTile kernel, and k a positive multiple of detail::pruned_nn::kDimTileSize. + */ +inline bool is_pruned_top_1_nn_available(int64_t k) +{ + return k > 0 && k % detail::pruned_nn::kDimTileSize == 0 && + detail::pruned_nn::filter_tile().has_value(); +} + +/** The workspace pruned_top_1_nn needs for m rows on `handle` (the same handle as the call). */ +inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, int64_t m) +{ + return detail::pruned_nn::workspace_layout(handle, m).total_bytes; +} + +/** + * Updates each row of x to its nearest row of y in L2, starting from its current one. Not exact: + * from the distance to the current neighbor (the bound), a fused partial GEMM over the first + * detail::pruned_nn::kDimTileSize dims (FP16) keeps the candidates under the bound times + * params.first_ratio; the pruning kernel adds the candidates' tail dims a tile at a time, keeping + * those under the bound times params.ratios[dims so far], and takes the closest left if it beats + * the current neighbor. A neighbor pruned away early can be missed. + * + * Rows go in batches of kRowBatchSize against batches of kCentroidBatchSize rows of y. The row + * batches round-robin over up to two streams of the handle's stream pool (the main stream without + * one), which wait on and are synchronized with the main stream. + * + * @param output in: each row's current neighbor (nearest_idx); out: its nearest neighbor and + * squared distance + * @param x [m, k] row-major + * @param y [n, k] row-major + * @param x_fp16 [m, kDimTileSize]: x's first dims in FP16, for float; ignored for half + * @param y_fp16 [n, kDimTileSize]: y's first dims in FP16, for float; ignored for half + * @param xn [m] squared norms of x over the first kDimTileSize dims + * @param yn [n] squared norms of y over the first kDimTileSize dims + * @param workspace pruned_top_1_nn_workspace_size(handle, m) bytes, 256-byte aligned + */ +template +void pruned_top_1_nn(raft::resources const& handle, + Top1nnOutput output, + const DataT* x, + const DataT* y, + const half* x_fp16, + const half* y_fp16, + const float* xn, + const float* yn, + int64_t m, + int64_t n, + int64_t k, + const PrunedTop1nnParams& params, + void* workspace, + std::size_t workspace_bytes) +{ + static_assert(std::is_same_v || std::is_same_v, + "Pruned 1-NN supports float and half"); +#if CUVS_CUTILE_ENABLED + using namespace detail::pruned_nn; + constexpr bool kHalf = std::is_same_v; + RAFT_EXPECTS(is_pruned_top_1_nn_available(k), + "Pruned 1-NN needs the fused filter's cuTile kernel and k a multiple of %u", + kDimTileSize); + RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); + RAFT_EXPECTS(params.ratios != nullptr, "Pruned 1-NN needs the pruning ratios"); + RAFT_EXPECTS(kHalf || (x_fp16 != nullptr && y_fp16 != nullptr), + "Pruned 1-NN on float needs FP16 copies of x's and y's first dims"); + const workspace_layout layout(handle, m); + RAFT_EXPECTS(workspace_bytes >= layout.total_bytes, "Pruned 1-NN's workspace is too small"); + auto* ws = static_cast(workspace); + + // The filter reads FP16 rows: half data whole, float data's FP16 copies of its first dims + const int64_t filter_dim = kHalf ? k : kDimTileSize; + const half* x_filter = nullptr; + const half* y_filter = nullptr; + if constexpr (kHalf) { + x_filter = x; + y_filter = y; + } else { + x_filter = x_fp16; + y_filter = y_fp16; + } + auto main_stream = raft::resource::get_cuda_stream(handle).get(); + auto* survivors = reinterpret_cast(ws + layout.survivors); + RAFT_CUDA_TRY(cudaMemsetAsync(survivors, 0, m * sizeof(uint32_t), main_stream)); + const bool pooled = layout.n_streams > 1; + if (pooled) { raft::resource::wait_stream_pool_on_stream(handle); } + + for (int64_t b = 0; b < layout.n_row_batches; b++) { + const int64_t i = b * kRowBatchSize; + const int64_t n_rows = std::min(kRowBatchSize, m - i); + const int64_t s = b % layout.n_streams; + cudaStream_t stream = + pooled ? raft::resource::get_stream_from_stream_pool(handle, s).get() : main_stream; + const DataT* rows = x + i * k; + uint32_t* row_labels = output.nearest_idx + i; + float* row_dist = output.nearest_dist + i; + const uint32_t* order = params.row_order != nullptr ? params.row_order + i : nullptr; + auto* packed = reinterpret_cast(ws + layout.packed[s]); + auto* seg_count = reinterpret_cast(ws + layout.seg_count[s]); + // Each row's bound: its distance to its current neighbor + distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream); + for (int64_t j = 0; j < n; j += kCentroidBatchSize) { + const int64_t n_centroids = std::min(kCentroidBatchSize, n - j); + launch_fused_filter(x_filter + i * filter_dim, + static_cast(n_rows), + y_filter + j * filter_dim, + static_cast(n_centroids), + static_cast(filter_dim), + xn + i, + yn + j, + row_dist, + params.first_ratio, + static_cast(kDimTileSize), + packed, + seg_count, + stream); + prune(rows, + n_rows, + order, + y + j * k, + static_cast(j), + k, + kDimTileSize, + params.ratios, + first_filter_lists{ + packed, + seg_count, + static_cast(raft::ceildiv(n_centroids, layout.tile.tile_n)), + static_cast(layout.tile.tile_n)}, + row_labels, + row_dist, + survivors + i, + stream); + } + } + if (pooled) { + std::vector used(layout.n_streams); + std::iota(used.begin(), used.end(), std::size_t{0}); + raft::resource::sync_stream_pool(handle, used); + } + if (params.next_row_order != nullptr) { + order_rows_by_survivors(handle, layout, ws, m, n, params.next_row_order); + } +#else + RAFT_FAIL("Pruned 1-NN needs a build with cuTile"); +#endif +} + +} // namespace cuvs::distance From 4bf9ccc8369f2928a19391a0aae72585d62cb337 Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 28 Oct 2026 00:14:52 -0700 Subject: [PATCH 07/39] Prune survivor lists in place, out of shared memory The pruning kernel compacts each row's survivors in its region of the first filter's lists, with only the first 32 entries in shared memory, so centroid batches grow to 2048 without lowering occupancy. Co-Authored-By: Claude Opus 5.5 --- .../distance/detail/pruned_top_1_nn/prune.cuh | 232 ++++++++++-------- cpp/tests/cluster/kmeans_superkmeans.cu | 2 +- 2 files changed, 131 insertions(+), 103 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index f323b716f6..a03a2e8607 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -20,9 +20,8 @@ namespace cuvs::distance::detail::pruned_nn { // Dims between the survivors' re-filters, and d' of the first filter constexpr uint32_t kDimTileSize = 256; -// Centroids per batch: a row's survivor list of one batch lives in the pruning kernel's shared -// memory -constexpr uint32_t kCentroidBatchSize = 1024; +// Centroids per batch: one fused filter and pruning launch per row batch and centroid batch +constexpr uint32_t kCentroidBatchSize = 2048; // Rows per batch: bounds the first filter's survivor lists (row batch x centroid batch slots) constexpr int64_t kRowBatchSize = 128 * 1024; // Survivors whose loads are batched over a dim tile @@ -33,8 +32,6 @@ constexpr int kSurvivorIdShift = 32; constexpr uint32_t kWarpSize = 32; constexpr uint32_t kFullMask = 0xffffffff; -using survivor_id_t = std::conditional_t<(kCentroidBatchSize <= 65536), uint16_t, uint32_t>; - /** * ADSampling's pruning ratios: ratios[i] = (i / d) * (1 + eps0 / sqrt(i))^2 for i dims seen, * i = 0..d (1 at both ends). A partial distance over the first i dims is compared against @@ -57,8 +54,8 @@ inline auto adsampling_ratios(uint32_t dim, float eps0 = 1.5f) -> std::vector __device__ __forceinline__ void warp_reduce_sum_n(T* values) @@ -68,7 +65,7 @@ __device__ __forceinline__ void warp_reduce_sum_n(T* values) for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { #pragma unroll for (uint32_t i = 0; i < N; i++) { - shifted[i] = __shfl_down_sync(kFullMask, values[i], offset); + shifted[i] = __shfl_xor_sync(kFullMask, values[i], offset); } asm volatile("" ::: "memory"); #pragma unroll @@ -145,31 +142,37 @@ void distance_to_assigned(const T* rows, /** * The first filter's survivor lists of one row batch x centroid batch (see launch_fused_filter): - * per (row, centroid tile) a segment of tile_n slots in `packed`, its count in `seg_count`. + * per (row, centroid tile) a segment of tile_n slots in `packed`, its count in `seg_count`. The + * pruning kernel compacts a row's survivors in place in the row's segments. */ struct first_filter_lists { - const int64_t* packed; + int64_t* packed; const int32_t* seg_count; uint32_t n_col_tiles; uint32_t tile_n; }; /** - * Distances over one dim tile between the row (`row_tile`, lane l holding dims l + 32 j) and N of - * its survivors, added to `distances` by lane 0. The N centroids' loads are issued back to back. + * Distances over one dim tile between the row (`row_tile`, lane l holding dims l + 32 j) and the N + * survivors at window positions [first, first + N), their ids held one per lane in `ids`; the lane + * holding each survivor adds its distance to `distance`. The N centroids' loads are issued back to + * back. */ template __device__ __forceinline__ void add_tile_distances(const float (&row_tile)[Dims / kWarpSize], const T* __restrict__ centroids, uint32_t stride, - const survivor_id_t* __restrict__ ids, - float* __restrict__ distances) + uint32_t ids, + uint32_t first, + float& distance) { const uint32_t lane = threadIdx.x % kWarpSize; float partial[N]; +#pragma unroll for (uint32_t i = 0; i < N; i++) { - const T* centroid = centroids + static_cast(ids[i]) * stride; - partial[i] = 0; + const T* centroid = + centroids + static_cast(__shfl_sync(kFullMask, ids, first + i)) * stride; + partial[i] = 0; #pragma unroll for (uint32_t j = 0; j < Dims / kWarpSize; j++) { const float diff = row_tile[j] - static_cast(centroid[lane + j * kWarpSize]); @@ -177,10 +180,9 @@ __device__ __forceinline__ void add_tile_distances(const float (&row_tile)[Dims } } warp_reduce_sum_n(partial); - if (lane == 0) { - for (uint32_t i = 0; i < N; i++) { - distances[i] += partial[i]; - } +#pragma unroll + for (uint32_t i = 0; i < N; i++) { + if (lane == first + i) { distance += partial[i]; } } } @@ -192,22 +194,35 @@ __device__ __forceinline__ void for_each_halving(F&& f) if constexpr (N > 1) { for_each_halving(f); } } -/** A row's first-filter survivors, in centroid order, into the warp's list; returns how many. */ -__device__ inline uint32_t load_survivors(first_filter_lists lists, - int64_t row, - survivor_id_t* __restrict__ ids, - float* __restrict__ distances) +__device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) { - const uint32_t lane = threadIdx.x % kWarpSize; - const int64_t* segments = lists.packed + row * lists.n_col_tiles * lists.tile_n; - const int32_t* counts = lists.seg_count + row * lists.n_col_tiles; - uint32_t n = 0; - for (uint32_t t = 0; t < lists.n_col_tiles; t++) { + return (static_cast(id) << kSurvivorIdShift) | + static_cast(__float_as_int(distance)); +} + +/** + * A row's first-filter survivors, from its segments (`list`, the row's region), compacted to + * positions [0, n) in centroid order and written with put(position, entry). In place: a chunk's + * entries are all read before any is written, and a write never passes the read position. + * Returns n. + */ +template +__device__ __forceinline__ uint32_t compact_survivors(const int64_t* list, + const int32_t* __restrict__ counts, + uint32_t n_col_tiles, + uint32_t tile_n, + Put&& put) +{ + const uint32_t lane = threadIdx.x % kWarpSize; + uint32_t n = 0; + for (uint32_t t = 0; t < n_col_tiles; t++) { const auto count = static_cast(counts[t]); - for (uint32_t j = lane; j < count; j += kWarpSize) { - const int64_t value = segments[t * lists.tile_n + j]; - ids[n + j] = static_cast(value >> kSurvivorIdShift); - distances[n + j] = __int_as_float(static_cast(value)); + for (uint32_t j0 = 0; j0 < count; j0 += kWarpSize) { + const uint32_t j = j0 + lane; + int64_t value = 0; + if (j < count) { value = list[t * tile_n + j]; } + __syncwarp(); + if (j < count && (n + j < kWarpSize || n != t * tile_n)) { put(n + j, value); } } n += count; } @@ -215,96 +230,109 @@ __device__ inline uint32_t load_survivors(first_filter_lists lists, return n; } -/** Keeps the n survivors under `threshold`, compacting the list in place; returns how many. */ -__device__ inline uint32_t refilter(uint32_t n, - float threshold, - survivor_id_t* __restrict__ ids, - float* __restrict__ distances) -{ - const uint32_t lane = threadIdx.x % kWarpSize; - const uint32_t lanes_below = (1u << lane) - 1; - uint32_t kept = 0; - for (uint32_t base = 0; base < n; base += kWarpSize) { - const uint32_t i = base + lane; - uint32_t id = 0; - float distance = 0.f; - bool keep = false; - if (i < n) { - id = ids[i]; - distance = distances[i]; - keep = distance < threshold; - } - const uint32_t ballot = __ballot_sync(kFullMask, keep); - __syncwarp(); // all lanes have read their entry before the compaction writes - if (keep) { - const uint32_t slot = kept + __popc(ballot & lanes_below); - ids[slot] = static_cast(id); - distances[slot] = distance; - } - kept += __popc(ballot); - } - return kept; -} - /** * A warp per block and row, against one centroid batch: the row's first-filter survivors (partial * distances over the first partial_dim dims), the tail dims in dim tiles with a re-filter after * each but the last, then the argmin, kept if closer than the row's current centroid. Block b * handles row row_order[b] when given (heaviest rows first), and each row's first-filter survivors * are added to `survivors`. + * + * The survivor list stays in the row's region of `lists`, but for its first kWarpSize entries, + * kept in shared memory; the warp reads it a window of kWarpSize entries at a time, lane l holding + * entry l, and each re-filter compacts it in place. The launch bounds leave the compiler up to 64 + * registers, for more centroid loads in flight. */ template -RAFT_KERNEL prune_kernel(const T* __restrict__ rows, - const uint32_t* __restrict__ row_order, - const T* __restrict__ centroids, - uint32_t centroid_offset, - uint32_t dim, - uint32_t partial_dim, - const float* __restrict__ ratios, - first_filter_lists lists, - uint32_t* __restrict__ labels, - float* __restrict__ dist, - uint32_t* __restrict__ survivors) +RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ rows, + const uint32_t* __restrict__ row_order, + const T* __restrict__ centroids, + uint32_t centroid_offset, + uint32_t dim, + uint32_t partial_dim, + const float* __restrict__ ratios, + first_filter_lists lists, + uint32_t* __restrict__ labels, + float* __restrict__ dist, + uint32_t* __restrict__ survivors) { - __shared__ survivor_id_t ids[kCentroidBatchSize]; - __shared__ float distances[kCentroidBatchSize]; - const uint32_t lane = threadIdx.x; - const int64_t row = row_order != nullptr ? row_order[blockIdx.x] : blockIdx.x; - const T* row_values = rows + row * dim; - uint32_t best_index = labels[row]; - float best = dist[row]; - const float bound = best; + __shared__ int64_t head[kWarpSize]; + const uint32_t lane = threadIdx.x; + const uint32_t lanes_below = (1u << lane) - 1; + const int64_t row = row_order != nullptr ? row_order[blockIdx.x] : blockIdx.x; + const T* row_values = rows + row * dim; + uint32_t best_index = labels[row]; + float best = dist[row]; + const float bound = best; - uint32_t n_left = load_survivors(lists, row, ids, distances); + int64_t* list = lists.packed + row * lists.n_col_tiles * lists.tile_n; + auto put = [&](uint32_t position, int64_t value) { + if (position < kWarpSize) { + head[position] = value; + } else { + list[position] = value; + } + }; + uint32_t n_left = compact_survivors( + list, lists.seg_count + row * lists.n_col_tiles, lists.n_col_tiles, lists.tile_n, put); if (lane == 0) { survivors[row] += n_left; } - if (n_left == 0 || (n_left == 1 && centroid_offset + ids[0] == best_index)) { return; } + if (n_left == 0 || + (n_left == 1 && + centroid_offset + static_cast(head[0] >> kSurvivorIdShift) == best_index)) { + return; + } for (uint32_t d = partial_dim; d < dim && n_left; d += kDimTileSize) { + const bool last = d + kDimTileSize >= dim; float row_tile[kDimTileSize / kWarpSize]; #pragma unroll for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { row_tile[j] = static_cast(row_values[d + lane + j * kWarpSize]); } - // Survivor tiles of kSurvivorTileSize, then halving sizes down to 1 for the ones left - uint32_t i = 0; - for_each_halving([&](auto tile) { - constexpr uint32_t kTile = decltype(tile)::value; - for (; i + kTile <= n_left; i += kTile) { - add_tile_distances( - row_tile, centroids + d, dim, ids + i, distances + i); - } - }); - __syncwarp(); // lane 0's distance updates are visible to the warp // After the last tile the distances are complete: the argmin compares them with the best - if (d + kDimTileSize < dim) { - n_left = refilter(n_left, bound * ratios[d + kDimTileSize], ids, distances); + const float threshold = last ? 0.f : bound * ratios[d + kDimTileSize]; + uint32_t kept = 0; + // The next window is loaded while the current one is computed + int64_t next = kWarpSize + lane < n_left ? list[kWarpSize + lane] : 0; + for (uint32_t base = 0; base < n_left; base += kWarpSize) { + const uint32_t w = min(kWarpSize, n_left - base); + const int64_t entry = base == 0 ? head[lane] : next; + if (base > 0 && base + kWarpSize + lane < n_left) { next = list[base + kWarpSize + lane]; } + const auto ids = static_cast(entry >> kSurvivorIdShift); + float distance = lane < w ? __int_as_float(static_cast(entry)) : 0.f; + // Survivor tiles of kSurvivorTileSize, then halving sizes down to 1 for the ones left + uint32_t i = 0; + for_each_halving([&](auto tile) { + constexpr uint32_t kTile = decltype(tile)::value; + for (; i + kTile <= w; i += kTile) { + add_tile_distances(row_tile, centroids + d, dim, ids, i, distance); + } + }); + if (!last) { + // The kept survivors go below this window's end, never into a later window + const bool keep = lane < w && distance < threshold; + const uint32_t ballot = __ballot_sync(kFullMask, keep); + if (keep) { put(kept + __popc(ballot & lanes_below), pack_survivor(ids, distance)); } + kept += __popc(ballot); + } else if (lane < w && distance < best) { + best = distance; + best_index = centroid_offset + ids; + } } + __syncwarp(); // the compacted list is visible to the warp + n_left = last ? 0 : kept; } - for (uint32_t i = lane; i < n_left; i += kWarpSize) { - if (distances[i] < best) { - best = distances[i]; - best_index = centroid_offset + ids[i]; + // No tail dims (partial_dim = dim): the first-filter distances are full, the argmin is over the + // list itself + for (uint32_t base = 0; base < n_left; base += kWarpSize) { + const uint32_t position = base + lane; + if (position < n_left) { + const int64_t entry = base == 0 ? head[lane] : list[position]; + const float distance = __int_as_float(static_cast(entry)); + if (distance < best) { + best = distance; + best_index = centroid_offset + static_cast(entry >> kSurvivorIdShift); + } } } for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index 94f6922508..7b77e0be42 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -304,7 +304,7 @@ TEST(KmeansSuperKMeans, Pruned) // dim tiles (one re-filter) constexpr int64_t n_rows = 300000; constexpr int64_t dim = 768; - constexpr int64_t n_clusters = 1100; + constexpr int64_t n_clusters = 2200; constexpr uint32_t n_iters = 8; constexpr uint64_t seed = 42; raft::handle_t handle; From 18779b18837484e5379e2956196b8c488f3aae60 Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 28 Oct 2026 00:17:55 -0700 Subject: [PATCH 08/39] Write one contiguous first-filter survivor list per row The fused filter carries each row's survivor count across centroid tiles and writes one list per row, so the pruning kernel reads it directly instead of compacting per-tile segments. Co-Authored-By: Claude Opus 5.5 --- .../cutile/export_fused_filter.py | 5 +- .../cutile/fused_filter_kernel.py | 29 ++++++----- .../cutile/fused_filter_tile.cu | 22 ++++---- .../cutile/fused_filter_tile.hpp | 14 ++--- .../distance/detail/pruned_top_1_nn/prune.cuh | 52 ++++--------------- cpp/src/distance/pruned_top_1_nn.cuh | 28 ++++------ 6 files changed, 61 insertions(+), 89 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py index 2d67c1977a..02ae4b8af7 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py @@ -57,7 +57,7 @@ def _vector(elem_dtype, alias_groups=()): ) -def _segments(): +def _counts(): return ArrayConstraint( ct.int32, ndim=2, @@ -82,11 +82,12 @@ def _signature(data_type: str, tile_m: int, tile_n: int, tile_k: int): _vector(ct.float32, read_only), # centroid_norms: ||centroid||^2 over d' _vector(ct.float32, read_only), # best: each row's current distance _vector(ct.int64), # packed survivors - _segments(), # seg_count (n_rows, n_col_tiles) + _counts(), # count (n_rows, 1) ScalarConstraint(ct.int32), # n_rows ScalarConstraint(ct.int32), # n_centroids ScalarConstraint(ct.float32), # ratio ScalarConstraint(ct.int32), # d' + ScalarConstraint(ct.int32), # row_stride ConstantConstraint(tile_m), ConstantConstraint(tile_n), ConstantConstraint(tile_k), diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py index 2c82addd0d..dde36f1c7c 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py @@ -9,16 +9,16 @@ keeps the pairs under the row's threshold best[row] * ratio, where best is the row's distance to its current centroid. -Output, for n_rows rows against n_centroids centroids (n_col_tiles = ceil(n_centroids / TN)): +Output, for n_rows rows against n_centroids centroids (row_stride >= n_centroids): - packed int64, one segment of TN slots per (row, centroid tile), at (row * n_col_tiles + tile) * TN: - slots 0 .. count-1 that tile's survivors for that row, in centroid order - slots count .. TN-1 not written - a survivor: (centroid index) << 32 | (the partial distance's float32 bits) - seg_count int32 (n_rows, n_col_tiles): count, how many slots of each segment are filled + packed int64, row r's survivors at r * row_stride + 0 .. count[r] - 1, in centroid order (the + rest not written); a survivor: (centroid index) << 32 | (the partial distance's float32 + bits) + count int32 (n_rows, 1): each row's survivor count -A segment holds a whole tile, so it cannot overflow, and a survivor's slot is just its rank among -the row's survivors in that tile: no prefix sum across rows. +A row's region holds every centroid, so it cannot overflow, and a survivor's slot is its rank among +the row's survivors: the CTA carries each row's count over the centroid tiles, so there is no +prefix sum across rows. d' is a runtime argument and must be a multiple of TK. """ @@ -51,11 +51,12 @@ def fused_filter( centroid_norms, best, packed, - seg_count, + count, n_rows, n_centroids, ratio, dp, + row_stride, tm: ConstInt, tn: ConstInt, tk: ConstInt, @@ -73,6 +74,7 @@ def fused_filter( ] * ratio ) + filled = ct.full((tm, 1), 0, dtype=ct.int32) # each row's survivors so far for centroid_tile in range(n_col_tiles): acc = ct.full((tm, tn), 0, dtype=ct.float32) for kk in range(dp // tk): @@ -96,16 +98,17 @@ def fused_filter( dist = -2.0 * acc + row_norm + centroid_norm centroid_ids = centroid_tile * tn + ct.arange(tn, dtype=ct.int32)[None, :] keep = (dist < threshold) & (centroid_ids < n_centroids) & (row_ids < n_rows) - # Slot: the start of the (row, tile) segment plus the survivor's rank in the row + # Slot: the row's region, after its survivors in the earlier tiles, plus the + # survivor's rank in this tile keep_i = keep.astype(ct.int32) - per_row = ct.sum(keep_i, axis=1, keepdims=True) - slot = (row_ids * n_col_tiles + centroid_tile) * tn + ct.cumsum(keep_i, 1) - 1 + slot = row_ids * row_stride + filled + ct.cumsum(keep_i, 1) - 1 + filled = filled + ct.sum(keep_i, axis=1, keepdims=True) # The distance's bits are masked: widening a negative int32 (a negative float, e.g. a # rounded-down 0) to int64 would sign-extend into the index index = ct.broadcast_to(centroid_ids, (tm, tn)).astype(ct.int64) dist_bits = ct.bitcast(dist, ct.int32).astype(ct.int64) & LOW_32_BITS value = ct.bitwise_lshift(index, ID_SHIFT) | dist_bits ct.scatter(packed, slot, value, mask=keep) - ct.store(seg_count, index=(row_tile, centroid_tile), tile=per_row) + ct.store(count, index=(row_tile, 0), tile=filled) return fused_filter diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu index 9b4b16fceb..479774944c 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu @@ -64,7 +64,8 @@ void launch_fused_filter(const half* rows, float ratio, int partial_dim, int64_t* packed, - int* seg_count, + int* count, + int row_stride, cudaStream_t stream) { FusedFilterTilePlanner planner; @@ -86,12 +87,13 @@ void launch_fused_filter(const half* rows, static_cast(centroid_norms), static_cast(best), static_cast(packed), - static_cast(seg_count)}) { + static_cast(count)}) { RAFT_EXPECTS(reinterpret_cast(ptr) % 16 == 0, "Pruned 1-NN fused filter: buffers must be 16-byte aligned"); } - const int n_col_tiles = (n_centroids + tile.tile_n - 1) / tile.tile_n; - const int64_t packed_size = int64_t{n_rows} * n_col_tiles * tile.tile_n; + RAFT_EXPECTS(row_stride >= n_centroids, + "Pruned 1-NN fused filter: a row's region must hold every centroid"); + const int64_t packed_size = int64_t{n_rows} * row_stride; RAFT_EXPECTS(packed_size <= std::numeric_limits::max(), "Pruned 1-NN fused filter: too many rows x centroids in one call"); @@ -125,6 +127,7 @@ void launch_fused_filter(const half* rows, int, int, float, + int, int); void* rows_ptr = const_cast(rows); void* centroids_ptr = const_cast(centroids); @@ -132,7 +135,7 @@ void launch_fused_filter(const half* rows, void* centroid_norms_ptr = const_cast(centroid_norms); void* best_ptr = const_cast(best); void* packed_ptr = packed; - void* seg_count_ptr = seg_count; + void* count_ptr = count; const int packed_len = static_cast(packed_size); const int unit = 1; @@ -165,15 +168,16 @@ void launch_fused_filter(const half* rows, packed_ptr, int{packed_len}, int{unit}, - seg_count_ptr, + count_ptr, int{n_rows}, - int{n_col_tiles}, - int{n_col_tiles}, + int{unit}, + int{unit}, int{unit}, int{n_rows}, int{n_centroids}, float{ratio}, - int{partial_dim}); + int{partial_dim}, + int{row_stride}); RAFT_CUDA_TRY(cudaGetLastError()); } diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp index b7abab3ede..eae4c594fd 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp @@ -29,10 +29,8 @@ auto fused_filter_tile_config() -> std::optional; * `partial_dim` dims, keeping the pairs below `best[row] * ratio`. FP16 only: the caller filters * float data on half copies of its first dims. * - * Survivors go to `packed`, one segment of tile_n slots per (row, centroid tile) at - * (row * n_col_tiles + tile) * tile_n, each `centroid << 32 | float bits of the partial distance`, - * in centroid order; `seg_count` [n_rows, n_col_tiles] holds how many slots of each segment are - * filled. n_col_tiles = ceil(n_centroids / tile_n). + * Row r's survivors go to `packed` at r * row_stride + 0 .. count[r] - 1, each + * `centroid << 32 | float bits of the partial distance`, in centroid order. * * @param rows [n_rows, dim] row-major, 16-byte aligned, 16-byte row pitch * @param centroids [n_centroids, dim] same layout as rows @@ -41,8 +39,9 @@ auto fused_filter_tile_config() -> std::optional; * @param best [n_rows] each row's distance to its current centroid * @param ratio the pruning bound's scale at partial_dim * @param partial_dim a multiple of tile_k, at most dim - * @param packed [n_rows * n_col_tiles * tile_n] survivors - * @param seg_count [n_rows, n_col_tiles] survivors per segment + * @param packed [n_rows * row_stride] survivors + * @param count [n_rows] survivors per row + * @param row_stride each row's region in `packed`, at least n_centroids * @param stream the stream to launch on */ void launch_fused_filter(const half* rows, @@ -56,7 +55,8 @@ void launch_fused_filter(const half* rows, float ratio, int partial_dim, int64_t* packed, - int* seg_count, + int* count, + int row_stride, cudaStream_t stream); } // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index a03a2e8607..759e9a1df8 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -142,14 +142,13 @@ void distance_to_assigned(const T* rows, /** * The first filter's survivor lists of one row batch x centroid batch (see launch_fused_filter): - * per (row, centroid tile) a segment of tile_n slots in `packed`, its count in `seg_count`. The - * pruning kernel compacts a row's survivors in place in the row's segments. + * row r's survivors at packed[r * row_stride], count[r] of them. The pruning kernel compacts them + * in place. */ struct first_filter_lists { int64_t* packed; - const int32_t* seg_count; - uint32_t n_col_tiles; - uint32_t tile_n; + const int32_t* count; + uint32_t row_stride; }; /** @@ -200,36 +199,6 @@ __device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) static_cast(__float_as_int(distance)); } -/** - * A row's first-filter survivors, from its segments (`list`, the row's region), compacted to - * positions [0, n) in centroid order and written with put(position, entry). In place: a chunk's - * entries are all read before any is written, and a write never passes the read position. - * Returns n. - */ -template -__device__ __forceinline__ uint32_t compact_survivors(const int64_t* list, - const int32_t* __restrict__ counts, - uint32_t n_col_tiles, - uint32_t tile_n, - Put&& put) -{ - const uint32_t lane = threadIdx.x % kWarpSize; - uint32_t n = 0; - for (uint32_t t = 0; t < n_col_tiles; t++) { - const auto count = static_cast(counts[t]); - for (uint32_t j0 = 0; j0 < count; j0 += kWarpSize) { - const uint32_t j = j0 + lane; - int64_t value = 0; - if (j < count) { value = list[t * tile_n + j]; } - __syncwarp(); - if (j < count && (n + j < kWarpSize || n != t * tile_n)) { put(n + j, value); } - } - n += count; - } - __syncwarp(); - return n; -} - /** * A warp per block and row, against one centroid batch: the row's first-filter survivors (partial * distances over the first partial_dim dims), the tail dims in dim tiles with a re-filter after @@ -238,9 +207,9 @@ __device__ __forceinline__ uint32_t compact_survivors(const int64_t* list, * are added to `survivors`. * * The survivor list stays in the row's region of `lists`, but for its first kWarpSize entries, - * kept in shared memory; the warp reads it a window of kWarpSize entries at a time, lane l holding - * entry l, and each re-filter compacts it in place. The launch bounds leave the compiler up to 64 - * registers, for more centroid loads in flight. + * copied to shared memory; the warp reads it a window of kWarpSize entries at a time, lane l + * holding entry l, and each re-filter compacts it in place. The launch bounds leave the compiler up + * to 64 registers, for more centroid loads in flight. */ template RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ rows, @@ -264,7 +233,7 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ float best = dist[row]; const float bound = best; - int64_t* list = lists.packed + row * lists.n_col_tiles * lists.tile_n; + int64_t* list = lists.packed + row * lists.row_stride; auto put = [&](uint32_t position, int64_t value) { if (position < kWarpSize) { head[position] = value; @@ -272,8 +241,9 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ list[position] = value; } }; - uint32_t n_left = compact_survivors( - list, lists.seg_count + row * lists.n_col_tiles, lists.n_col_tiles, lists.tile_n, put); + uint32_t n_left = static_cast(lists.count[row]); + if (lane < n_left) { head[lane] = list[lane]; } + __syncwarp(); if (lane == 0) { survivors[row] += n_left; } if (n_left == 0 || (n_left == 1 && diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 69e0ac9ddc..dad4bf807a 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -62,16 +62,13 @@ inline auto filter_tile() -> std::optional struct workspace_layout { int64_t n_row_batches; int64_t n_streams; - fused_filter_tile tile; std::size_t survivors, sorted_keys, local_ids, batch_offsets, sort_temp, sort_temp_bytes; - std::vector packed, seg_count; + std::vector packed, count; std::size_t total_bytes; workspace_layout(raft::resources const& handle, int64_t m) { - const auto t = filter_tile(); - RAFT_EXPECTS(t.has_value(), "Pruned 1-NN needs the fused filter's cuTile kernel"); - tile = *t; + RAFT_EXPECTS(filter_tile().has_value(), "Pruned 1-NN needs the fused filter's cuTile kernel"); n_row_batches = raft::ceildiv(m, kRowBatchSize); const auto pool_size = raft::resource::is_stream_pool_initialized(handle) ? static_cast(raft::resource::get_stream_pool_size(handle)) @@ -100,12 +97,12 @@ struct workspace_layout { static_cast(nullptr), static_cast(nullptr))); sort_temp = take(sort_temp_bytes); - // The first filter's survivor lists, one pair per stream, sized for a whole batch - const int64_t batch_rows = std::min(m, kRowBatchSize); - const int64_t n_col_tiles = raft::ceildiv(kCentroidBatchSize, tile.tile_n); + // The first filter's survivor lists, one pair per stream: a region of a whole centroid batch + // per row of a row batch + const int64_t batch_rows = std::min(m, kRowBatchSize); for (int64_t s = 0; s < n_streams; s++) { - packed.push_back(take(batch_rows * n_col_tiles * tile.tile_n * sizeof(int64_t))); - seg_count.push_back(take(batch_rows * n_col_tiles * sizeof(int32_t))); + packed.push_back(take(batch_rows * kCentroidBatchSize * sizeof(int64_t))); + count.push_back(take(batch_rows * sizeof(int32_t))); } total_bytes = end; } @@ -265,7 +262,7 @@ void pruned_top_1_nn(raft::resources const& handle, float* row_dist = output.nearest_dist + i; const uint32_t* order = params.row_order != nullptr ? params.row_order + i : nullptr; auto* packed = reinterpret_cast(ws + layout.packed[s]); - auto* seg_count = reinterpret_cast(ws + layout.seg_count[s]); + auto* count = reinterpret_cast(ws + layout.count[s]); // Each row's bound: its distance to its current neighbor distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream); for (int64_t j = 0; j < n; j += kCentroidBatchSize) { @@ -281,7 +278,8 @@ void pruned_top_1_nn(raft::resources const& handle, params.first_ratio, static_cast(kDimTileSize), packed, - seg_count, + count, + static_cast(kCentroidBatchSize), stream); prune(rows, n_rows, @@ -291,11 +289,7 @@ void pruned_top_1_nn(raft::resources const& handle, k, kDimTileSize, params.ratios, - first_filter_lists{ - packed, - seg_count, - static_cast(raft::ceildiv(n_centroids, layout.tile.tile_n)), - static_cast(layout.tile.tile_n)}, + first_filter_lists{packed, count, kCentroidBatchSize}, row_labels, row_dist, survivors + i, From e58ee5c1b173748fec080a5ed27324e2bd507ffa Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 28 Oct 2026 00:20:07 -0700 Subject: [PATCH 09/39] Prune only with dims past the first filter's; simplify the kernel Pruning needs dim > 256 (d' < d), so the pruning kernel always has tail dims and one argmin path. The re-filter threshold comes from the current best, and the list is read a window at a time. Co-Authored-By: Claude Opus 5.5 --- cpp/include/cuvs/cluster/kmeans.hpp | 2 +- cpp/src/cluster/detail/superkmeans.cuh | 6 +- .../distance/detail/pruned_top_1_nn/prune.cuh | 55 ++++++------------- cpp/src/distance/pruned_top_1_nn.cuh | 7 ++- cpp/tests/cluster/kmeans_superkmeans.cu | 8 +-- 5 files changed, 29 insertions(+), 49 deletions(-) diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index 52993cb9c7..b229826878 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -238,7 +238,7 @@ struct balanced_params : base_params { * row with the fused 1-NN over all dimensions; later iterations compute partial distances over the * first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the * ADSampling bound, without its random rotation), and finish the remaining dimensions only for the - * survivors. Pruning applies with more than 256 clusters and at least 256 dimensions, which must + * survivors. Pruning applies with more than 256 clusters and more than 256 dimensions, which must * then be a multiple of 256; otherwise every iteration assigns with the full 1-NN. * * The number of clusters is taken from the `centroids` argument. The start centroids are rows diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 15d7fc1317..186a024031 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -27,13 +27,13 @@ namespace cuvs::cluster::kmeans::detail::superkmeans { namespace pruned_nn = cuvs::distance::detail::pruned_nn; -// Pruning works on whole dimension tiles and only pays off with enough centroids to discard; below -// that every iteration assigns with the full 1-NN. +// Pruning works on whole dimension tiles past the first filter's, and only pays off with enough +// centroids to discard; below that every iteration assigns with the full 1-NN. constexpr int64_t kMinPrunedK = 256; inline bool prunes(int64_t n_clusters, int64_t dim) { - return dim >= pruned_nn::kDimTileSize && n_clusters > kMinPrunedK; + return dim > pruned_nn::kDimTileSize && n_clusters > kMinPrunedK; } /** Sum of squared distances of the rows to their assigned centroids. */ diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 759e9a1df8..6e6d801ba0 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -230,17 +230,10 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ const int64_t row = row_order != nullptr ? row_order[blockIdx.x] : blockIdx.x; const T* row_values = rows + row * dim; uint32_t best_index = labels[row]; - float best = dist[row]; - const float bound = best; + // The row's distance to its current centroid until the last tile's argmin + float best = dist[row]; - int64_t* list = lists.packed + row * lists.row_stride; - auto put = [&](uint32_t position, int64_t value) { - if (position < kWarpSize) { - head[position] = value; - } else { - list[position] = value; - } - }; + int64_t* list = lists.packed + row * lists.row_stride; uint32_t n_left = static_cast(lists.count[row]); if (lane < n_left) { head[lane] = list[lane]; } __syncwarp(); @@ -251,24 +244,20 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ return; } - for (uint32_t d = partial_dim; d < dim && n_left; d += kDimTileSize) { + // One pass per dim tile (partial_dim < dim); the last one ends with the argmin + for (uint32_t d = partial_dim; n_left; d += kDimTileSize) { const bool last = d + kDimTileSize >= dim; float row_tile[kDimTileSize / kWarpSize]; #pragma unroll for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { row_tile[j] = static_cast(row_values[d + lane + j * kWarpSize]); } - // After the last tile the distances are complete: the argmin compares them with the best - const float threshold = last ? 0.f : bound * ratios[d + kDimTileSize]; - uint32_t kept = 0; - // The next window is loaded while the current one is computed - int64_t next = kWarpSize + lane < n_left ? list[kWarpSize + lane] : 0; + uint32_t kept = 0; for (uint32_t base = 0; base < n_left; base += kWarpSize) { const uint32_t w = min(kWarpSize, n_left - base); - const int64_t entry = base == 0 ? head[lane] : next; - if (base > 0 && base + kWarpSize + lane < n_left) { next = list[base + kWarpSize + lane]; } - const auto ids = static_cast(entry >> kSurvivorIdShift); - float distance = lane < w ? __int_as_float(static_cast(entry)) : 0.f; + const int64_t entry = lane >= w ? 0 : base == 0 ? head[lane] : list[base + lane]; + const auto ids = static_cast(entry >> kSurvivorIdShift); + float distance = __int_as_float(static_cast(entry)); // Survivor tiles of kSurvivorTileSize, then halving sizes down to 1 for the ones left uint32_t i = 0; for_each_halving([&](auto tile) { @@ -279,11 +268,14 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ }); if (!last) { // The kept survivors go below this window's end, never into a later window - const bool keep = lane < w && distance < threshold; + const bool keep = lane < w && distance < best * ratios[d + kDimTileSize]; const uint32_t ballot = __ballot_sync(kFullMask, keep); - if (keep) { put(kept + __popc(ballot & lanes_below), pack_survivor(ids, distance)); } + if (keep) { + const uint32_t slot = kept + __popc(ballot & lanes_below); + (slot < kWarpSize ? head[slot] : list[slot]) = pack_survivor(ids, distance); + } kept += __popc(ballot); - } else if (lane < w && distance < best) { + } else if (lane < w && distance < best) { // after the last tile the distances are full best = distance; best_index = centroid_offset + ids; } @@ -292,19 +284,6 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ n_left = last ? 0 : kept; } - // No tail dims (partial_dim = dim): the first-filter distances are full, the argmin is over the - // list itself - for (uint32_t base = 0; base < n_left; base += kWarpSize) { - const uint32_t position = base + lane; - if (position < n_left) { - const int64_t entry = base == 0 ? head[lane] : list[position]; - const float distance = __int_as_float(static_cast(entry)); - if (distance < best) { - best = distance; - best_index = centroid_offset + static_cast(entry >> kSurvivorIdShift); - } - } - } for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { const uint32_t other_index = __shfl_down_sync(kFullMask, best_index, offset); const float other = __shfl_down_sync(kFullMask, best, offset); @@ -340,8 +319,8 @@ void prune(const T* rows, uint32_t* survivors, cudaStream_t stream) { - RAFT_EXPECTS(partial_dim <= dim && (dim - partial_dim) % kDimTileSize == 0, - "Pruned 1-NN needs dim - partial_dim to be a multiple of %u", + RAFT_EXPECTS(partial_dim < dim && (dim - partial_dim) % kDimTileSize == 0, + "Pruned 1-NN needs partial_dim < dim, dim - partial_dim a multiple of %u", kDimTileSize); prune_kernel<<>>(rows, row_order, diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index dad4bf807a..ac80a944ac 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -166,11 +166,12 @@ inline void order_rows_by_survivors(raft::resources const& handle, /** * Whether pruned_top_1_nn runs for k features on this build and device: it needs the fused - * filter's cuTile kernel, and k a positive multiple of detail::pruned_nn::kDimTileSize. + * filter's cuTile kernel, and k a multiple of detail::pruned_nn::kDimTileSize above it (the first + * filter's dims, with a tail left to prune on). */ inline bool is_pruned_top_1_nn_available(int64_t k) { - return k > 0 && k % detail::pruned_nn::kDimTileSize == 0 && + return k > detail::pruned_nn::kDimTileSize && k % detail::pruned_nn::kDimTileSize == 0 && detail::pruned_nn::filter_tile().has_value(); } @@ -224,7 +225,7 @@ void pruned_top_1_nn(raft::resources const& handle, using namespace detail::pruned_nn; constexpr bool kHalf = std::is_same_v; RAFT_EXPECTS(is_pruned_top_1_nn_available(k), - "Pruned 1-NN needs the fused filter's cuTile kernel and k a multiple of %u", + "Pruned 1-NN needs the fused filter's cuTile kernel and k a multiple of %u above it", kDimTileSize); RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); RAFT_EXPECTS(params.ratios != nullptr, "Pruned 1-NN needs the pruning ratios"); diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index 7b77e0be42..5bbcd2fc15 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -293,10 +293,10 @@ TEST(KmeansSuperKMeans, Fallback) } } -/* With more than 256 clusters and a multiple of 256 dims, the iterations after the first prune - * (on a build and device with the filter's kernels; otherwise they assign in full). The pruning - * bound is a heuristic without a rotation, so the result is compared with Lloyd's from the same - * start rows rather than exactly. The labels and inertia come from a full assignment. +/* With more than 256 clusters and more than 256 dims, a multiple of 256, the iterations after the + * first prune (on a build and device with the filter's kernels; otherwise they assign in full). The + * pruning bound is a heuristic without a rotation, so the result is compared with Lloyd's from the + * same start rows rather than exactly. The labels and inertia come from a full assignment. */ TEST(KmeansSuperKMeans, Pruned) { From 1edbba5256214f4ee85c9f234e672487173ea25b Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 28 Oct 2026 00:21:41 -0700 Subject: [PATCH 10/39] Return early from pruning rows with no survivors A row with no first-filter survivors now reads only its count. Co-Authored-By: Claude Opus 5.5 --- .../distance/detail/pruned_top_1_nn/prune.cuh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 6e6d801ba0..f97a751439 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -228,19 +228,21 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ const uint32_t lane = threadIdx.x; const uint32_t lanes_below = (1u << lane) - 1; const int64_t row = row_order != nullptr ? row_order[blockIdx.x] : blockIdx.x; - const T* row_values = rows + row * dim; - uint32_t best_index = labels[row]; + uint32_t n_left = static_cast(lists.count[row]); + // Nothing survived: nothing else to read or write + if (n_left == 0) { return; } + const T* row_values = rows + row * dim; + uint32_t best_index = labels[row]; // The row's distance to its current centroid until the last tile's argmin float best = dist[row]; - int64_t* list = lists.packed + row * lists.row_stride; - uint32_t n_left = static_cast(lists.count[row]); + int64_t* list = lists.packed + row * lists.row_stride; if (lane < n_left) { head[lane] = list[lane]; } __syncwarp(); if (lane == 0) { survivors[row] += n_left; } - if (n_left == 0 || - (n_left == 1 && - centroid_offset + static_cast(head[0] >> kSurvivorIdShift) == best_index)) { + // Only the current centroid survived + if (n_left == 1 && + centroid_offset + static_cast(head[0] >> kSurvivorIdShift) == best_index) { return; } From 75ea3abb897e9c4196c00e751244fe3d3ed789ed Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 28 Oct 2026 00:32:39 -0700 Subject: [PATCH 11/39] Overlap row batches on the main stream and one pool stream Row batches alternate between the main stream and, with a stream pool on the handle, the pool's first stream, so one pool stream is enough, as for kmeans' out-of-core fit. Without a pool everything runs on the main stream. Each stream needs its own first-filter survivor lists (2 GiB at 128K rows x 2K centroids). Co-Authored-By: Claude Opus 5.5 --- cpp/include/cuvs/cluster/kmeans.hpp | 4 +++ cpp/src/distance/pruned_top_1_nn.cuh | 15 ++++------ cpp/tests/cluster/kmeans_superkmeans.cu | 40 ++++++++++--------------- 3 files changed, 25 insertions(+), 34 deletions(-) diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index b229826878..c1ab6598d8 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -245,6 +245,10 @@ struct balanced_params : base_params { * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run * (there is no convergence test), and an empty cluster keeps its previous centroid. * + * With a stream pool on the handle (one stream is enough), the pruned assignment's row batches + * alternate between the main stream and the pool's, overlapping their kernels, for up to 2 GiB + * more workspace. + * * The following metrics are currently supported: * - L2Expanded */ diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index ac80a944ac..d70151b073 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -73,7 +72,7 @@ struct workspace_layout { const auto pool_size = raft::resource::is_stream_pool_initialized(handle) ? static_cast(raft::resource::get_stream_pool_size(handle)) : 0; - n_streams = std::max(1, std::min({pool_size, kMaxStreams, n_row_batches})); + n_streams = std::min({1 + pool_size, kMaxStreams, n_row_batches}); std::size_t end = 0; auto take = [&](std::size_t bytes) { @@ -190,8 +189,8 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * the current neighbor. A neighbor pruned away early can be missed. * * Rows go in batches of kRowBatchSize against batches of kCentroidBatchSize rows of y. The row - * batches round-robin over up to two streams of the handle's stream pool (the main stream without - * one), which wait on and are synchronized with the main stream. + * batches round-robin over the main stream and, with a stream pool on the handle, its first stream, + * which waits on and is synchronized with the main stream. * * @param output in: each row's current neighbor (nearest_idx); out: its nearest neighbor and * squared distance @@ -257,7 +256,7 @@ void pruned_top_1_nn(raft::resources const& handle, const int64_t n_rows = std::min(kRowBatchSize, m - i); const int64_t s = b % layout.n_streams; cudaStream_t stream = - pooled ? raft::resource::get_stream_from_stream_pool(handle, s).get() : main_stream; + s > 0 ? raft::resource::get_stream_from_stream_pool(handle, s - 1).get() : main_stream; const DataT* rows = x + i * k; uint32_t* row_labels = output.nearest_idx + i; float* row_dist = output.nearest_dist + i; @@ -297,11 +296,7 @@ void pruned_top_1_nn(raft::resources const& handle, stream); } } - if (pooled) { - std::vector used(layout.n_streams); - std::iota(used.begin(), used.end(), std::size_t{0}); - raft::resource::sync_stream_pool(handle, used); - } + if (pooled) { raft::resource::sync_stream_pool(handle, std::vector{0}); } if (params.next_row_order != nullptr) { order_rows_by_survivors(handle, layout, ws, m, n, params.next_row_order); } diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index 5bbcd2fc15..d28f3ba124 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -356,30 +356,22 @@ TEST(KmeansSuperKMeans, Pruned) params.n_iters = n_iters; params.rng_state = raft::random::RngState(seed); auto labels = raft::make_device_vector(handle, n_rows); - float inertia = 0; - cuvs::cluster::kmeans::fit_predict( - handle, params, X, centroids.view(), labels.view(), raft::make_host_scalar_view(&inertia)); - EXPECT_NEAR(inertia, lloyd_inertia, 0.01 * lloyd_inertia); - - // With a stream pool the row batches run on two streams - raft::resource::set_cuda_stream_pool(handle, std::make_shared(2)); - float pooled_inertia = 0; - cuvs::cluster::kmeans::fit_predict(handle, - params, - X, - centroids.view(), - labels.view(), - raft::make_host_scalar_view(&pooled_inertia)); - EXPECT_NEAR(pooled_inertia, lloyd_inertia, 0.01 * lloyd_inertia); - - float half_inertia = 0; - cuvs::cluster::kmeans::fit_predict(handle, - params, - raft::make_const_mdspan(X_half.view()), - centroids.view(), - labels.view(), - raft::make_host_scalar_view(&half_inertia)); - EXPECT_NEAR(half_inertia, lloyd_inertia, 0.01 * lloyd_inertia); + auto check = [&](auto data) { + float inertia = 0; + cuvs::cluster::kmeans::fit_predict( + handle, params, data, centroids.view(), labels.view(), raft::make_host_scalar_view(&inertia)); + EXPECT_NEAR(inertia, lloyd_inertia, 0.01 * lloyd_inertia); + }; + + for (bool pooled : {false, true}) { + SCOPED_TRACE(pooled ? "two streams" : "one stream"); + // With a stream pool the row batches alternate between the main stream and the pool's + if (pooled) { + raft::resource::set_cuda_stream_pool(handle, std::make_shared(1)); + } + check(X); + check(raft::make_const_mdspan(X_half.view())); + } } TEST(KmeansSuperKMeans, RejectsUnsupported) From fca97b17c3c5ca86328b01e90d85c309539e101f Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 28 Oct 2026 00:52:25 -0700 Subject: [PATCH 12/39] Fix style: SuperKMeans API reference, formatting, copyright Generate the C++ API reference for SuperKMeans, apply ruff-format and cmake-format, and use the canonical copyright notice. Co-Authored-By: Claude Opus 5.5 --- cpp/CMakeLists.txt | 9 +- cpp/src/cluster/detail/superkmeans.cuh | 2 +- .../cutile/export_fused_filter.py | 6 +- .../cutile/fused_filter_kernel.py | 28 ++- fern/pages/cpp_api/cpp-api-cluster-kmeans.md | 191 ++++++++++++++++++ 5 files changed, 221 insertions(+), 15 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 99186a0d67..e30b33ba52 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1593,7 +1593,8 @@ if(NOT BUILD_CPU_ONLY) target_sources( cuvs_objs PRIVATE ${cutile_fused_1nn_files} - src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu ${cutile_smoke_files} + src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu + ${cutile_smoke_files} ${cutile_pruned_1nn_files} src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu ) @@ -1643,8 +1644,10 @@ if(NOT BUILD_CPU_ONLY) PUBLIC "$" "$" INTERFACE "$" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" "${CMAKE_CURRENT_BINARY_DIR}/src" - "${cutile_fused_1nn_generated_dir}" "${cutile_smoke_generated_dir}" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" + "${CMAKE_CURRENT_BINARY_DIR}/src" + "${cutile_fused_1nn_generated_dir}" + "${cutile_smoke_generated_dir}" "${cutile_pruned_1nn_generated_dir}" ) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 186a024031..6fc9db3074 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py index 02ae4b8af7..9919b82e44 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/export_fused_filter.py @@ -78,9 +78,9 @@ def _signature(data_type: str, tile_m: int, tile_n: int, tile_k: int): parameters=[ _matrix(), # rows (n_rows, d) _matrix(), # centroids (n_centroids, d) - _vector(ct.float32, read_only), # row_norms: ||row||^2 over d' - _vector(ct.float32, read_only), # centroid_norms: ||centroid||^2 over d' - _vector(ct.float32, read_only), # best: each row's current distance + _vector(ct.float32, read_only), # row_norms over d' + _vector(ct.float32, read_only), # centroid_norms over d' + _vector(ct.float32, read_only), # best: current distances _vector(ct.int64), # packed survivors _counts(), # count (n_rows, 1) ScalarConstraint(ct.int32), # n_rows diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py index dde36f1c7c..3a0c036f7a 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py @@ -69,17 +69,21 @@ def fused_filter( row_norms, index=(row_tile,), shape=(tm,), padding_mode=zero_pad )[:, None] threshold = ( - ct.load(best, index=(row_tile,), shape=(tm,), padding_mode=zero_pad)[ - :, None - ] + ct.load( + best, index=(row_tile,), shape=(tm,), padding_mode=zero_pad + )[:, None] * ratio ) - filled = ct.full((tm, 1), 0, dtype=ct.int32) # each row's survivors so far + # Each row's survivors so far + filled = ct.full((tm, 1), 0, dtype=ct.int32) for centroid_tile in range(n_col_tiles): acc = ct.full((tm, tn), 0, dtype=ct.float32) for kk in range(dp // tk): a = ct.load( - rows, index=(row_tile, kk), shape=(tm, tk), padding_mode=zero_pad + rows, + index=(row_tile, kk), + shape=(tm, tk), + padding_mode=zero_pad, ) b = ct.load( centroids, @@ -96,8 +100,14 @@ def fused_filter( padding_mode=zero_pad, )[None, :] dist = -2.0 * acc + row_norm + centroid_norm - centroid_ids = centroid_tile * tn + ct.arange(tn, dtype=ct.int32)[None, :] - keep = (dist < threshold) & (centroid_ids < n_centroids) & (row_ids < n_rows) + centroid_ids = ( + centroid_tile * tn + ct.arange(tn, dtype=ct.int32)[None, :] + ) + keep = ( + (dist < threshold) + & (centroid_ids < n_centroids) + & (row_ids < n_rows) + ) # Slot: the row's region, after its survivors in the earlier tiles, plus the # survivor's rank in this tile keep_i = keep.astype(ct.int32) @@ -106,7 +116,9 @@ def fused_filter( # The distance's bits are masked: widening a negative int32 (a negative float, e.g. a # rounded-down 0) to int64 would sign-extend into the index index = ct.broadcast_to(centroid_ids, (tm, tn)).astype(ct.int64) - dist_bits = ct.bitcast(dist, ct.int32).astype(ct.int64) & LOW_32_BITS + dist_bits = ( + ct.bitcast(dist, ct.int32).astype(ct.int64) & LOW_32_BITS + ) value = ct.bitwise_lshift(index, ID_SHIFT) | dist_bits ct.scatter(packed, slot, value, mask=keep) ct.store(count, index=(row_tile, 0), tile=filled) diff --git a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md index 595293cd9f..fc0595d32a 100644 --- a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md +++ b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md @@ -85,6 +85,35 @@ enum class balanced_donor_selection { | `SizeSorted` | `0` | | `Random` | `1` | + +### cluster::kmeans::superkmeans_params + +Simple object to specify hyper-parameters to the SuperKMeans algorithm. + +SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every row with the fused 1-NN over all dimensions; later iterations compute partial distances over the first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the ADSampling bound, without its random rotation), and finish the remaining dimensions only for the survivors. Pruning applies with more than 256 clusters and more than 256 dimensions, which must then be a multiple of 256; otherwise every iteration assigns with the full 1-NN. + +The number of clusters is taken from the `centroids` argument. The start centroids are rows sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run (there is no convergence test), and an empty cluster keeps its previous centroid. + +With a stream pool on the handle (one stream is enough), the pruned assignment's row batches alternate between the main stream and the pool's, overlapping their kernels, for up to 2 GiB more workspace. + +The following metrics are currently supported: + +- L2Expanded + +```cpp +struct superkmeans_params : base_params { + uint32_t n_iters; + raft::random::RngState rng_state; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `n_iters` | `uint32_t` | Number of training iterations. | +| `rng_state` | `raft::random::RngState` | Seed to the random number generator (the start rows). | + ### cluster::kmeans::kmeans_type @@ -439,6 +468,60 @@ std::optional> inertia = std::nullopt); `void` +**Additional overload:** `cluster::kmeans::fit` + +Find clusters with the SuperKMeans algorithm. + +```cpp +void fit(const raft::resources& handle, +cuvs::cluster::kmeans::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::fit` + +Find clusters with the SuperKMeans algorithm. + +```cpp +void fit(const raft::resources& handle, +cuvs::cluster::kmeans::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +std::optional> inertia = std::nullopt); +``` + +With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the centroids, which are accumulated and returned in FP32. + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + ### cluster::kmeans::predict @@ -726,6 +809,58 @@ raft::device_vector_view labels); `void` +**Additional overload:** `cluster::kmeans::predict` + +Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + +```cpp +void predict(const raft::resources& handle, +cuvs::cluster::kmeans::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | +| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::predict` + +Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + +```cpp +void predict(const raft::resources& handle, +cuvs::cluster::kmeans::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | +| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | + +**Returns** + +`void` + ### cluster::kmeans::fit_predict @@ -907,6 +1042,62 @@ raft::device_vector_view labels); `void` +**Additional overload:** `cluster::kmeans::fit_predict` + +Compute SuperKMeans clusters and predict the cluster index of each sample. + +```cpp +void fit_predict(const raft::resources& handle, +cuvs::cluster::kmeans::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::fit_predict` + +Compute SuperKMeans clusters and predict the cluster index of each sample. + +```cpp +void fit_predict(const raft::resources& handle, +cuvs::cluster::kmeans::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + ### cluster::kmeans::transform From 2d869df7a0d7d2e2d1253ee6c51e6028168158f9 Mon Sep 17 00:00:00 2001 From: sherylll Date: Fri, 30 Oct 2026 02:44:53 -0700 Subject: [PATCH 13/39] Compute the filter's norms from its FP16 values, with the FP16 copy The fused filter's distance combines an FP16 dot product with the row and centroid norms; for float data the norms came from the FP32 values, so the two didn't cancel consistently. Take them from the same FP16 values, as half data already does, in the kernel that makes the FP16 copy. Co-Authored-By: Claude Opus 5.5 --- cpp/src/cluster/detail/superkmeans.cuh | 74 ++++++++++++++----- .../distance/detail/pruned_top_1_nn/prune.cuh | 7 +- cpp/src/distance/pruned_top_1_nn.cuh | 5 +- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 6fc9db3074..7b00525efc 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -94,15 +95,48 @@ void predict(const raft::resources& handle, cuvs::spatial::knn::detail::utils::mapping{}); } -/** The first `cols` dims of n rows (row stride `stride`) as half, packed (row stride `cols`). */ +/** + * The first `cols` dims of n rows (row stride `stride`) as half, packed (row stride `cols`), and + * the squared norms of those half values over the first kDimTileSize dims (the fused filter's, + * from the same values as its dot products); a warp per row. + */ +template +RAFT_KERNEL copy_first_columns_kernel(const S* __restrict__ in, + int64_t n, + int64_t stride, + int64_t cols, + half* __restrict__ out, + float* __restrict__ norms) +{ + const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / pruned_nn::kWarpSize; + if (row >= n) { return; } + float s = 0.f; + for (int64_t j = threadIdx.x % pruned_nn::kWarpSize; j < cols; j += pruned_nn::kWarpSize) { + const half h = static_cast(in[row * stride + j]); + out[row * cols + j] = h; + if (j < pruned_nn::kDimTileSize) { + const float f = __half2float(h); + s += f * f; + } + } + s = raft::warpReduce(s); + if (threadIdx.x % pruned_nn::kWarpSize == 0) { norms[row] = s; } +} + template -void copy_first_columns( - const raft::resources& handle, const S* in, int64_t n, int64_t stride, int64_t cols, half* out) +void copy_first_columns(const raft::resources& handle, + const S* in, + int64_t n, + int64_t stride, + int64_t cols, + half* out, + float* norms) { - raft::linalg::map_offset( - handle, raft::make_device_vector_view(out, n * cols), [=] __device__(int64_t i) { - return static_cast(in[i / cols * stride + i % cols]); - }); + constexpr uint32_t kThreads = 256; + const auto blocks = raft::ceildiv(n * pruned_nn::kWarpSize, kThreads); + auto stream = raft::resource::get_cuda_stream(handle).get(); + copy_first_columns_kernel<<>>(in, n, stride, cols, out, norms); + RAFT_CUDA_TRY(cudaPeekAtLastError()); } /** @@ -139,11 +173,17 @@ class pruned_assignment { params_.ratios = ratios_.data_handle(); params_.first_ratio = ratios[pruned_nn::kDimTileSize]; raft::copy(ratios_.data_handle(), ratios.data(), ratios.size(), stream); - pruned_nn::squared_norms( - X.data_handle(), n_rows_, dim_, pruned_nn::kDimTileSize, row_norms_.data_handle(), stream); - if constexpr (!std::is_same_v) { - copy_first_columns( - handle, X.data_handle(), n_rows_, dim_, filter_dim_, filter_rows_.data_handle()); + if constexpr (std::is_same_v) { + pruned_nn::squared_norms( + X.data_handle(), n_rows_, dim_, pruned_nn::kDimTileSize, row_norms_.data_handle(), stream); + } else { + copy_first_columns(handle, + X.data_handle(), + n_rows_, + dim_, + filter_dim_, + filter_rows_.data_handle(), + row_norms_.data_handle()); } // The host ratios are released when the copy is done raft::resource::sync_stream(handle); @@ -157,25 +197,19 @@ class pruned_assignment { raft::device_vector_view labels, bool order_next) { - auto stream = raft::resource::get_cuda_stream(handle_).get(); copy_first_columns(handle_, centroids.data_handle(), n_clusters_, dim_, filter_dim_, - filter_centroids_.data_handle()); + filter_centroids_.data_handle(), + centroid_norms_.data_handle()); const T* kernel_centroids = nullptr; if constexpr (std::is_same_v) { kernel_centroids = filter_centroids_.data_handle(); } else { kernel_centroids = centroids.data_handle(); } - pruned_nn::squared_norms(kernel_centroids, - n_clusters_, - dim_, - pruned_nn::kDimTileSize, - centroid_norms_.data_handle(), - stream); params_.next_row_order = order_next ? row_order_.data_handle() : nullptr; cuvs::distance::pruned_top_1_nn(handle_, {labels.data_handle(), dist_.data_handle()}, diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index f97a751439..6daf520fb1 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -75,8 +75,11 @@ __device__ __forceinline__ void warp_reduce_sum_n(T* values) } } -/** Squared norm of each of n rows (row stride `stride`) over their first `cols` dims; a warp per - * row. */ +/** + * Squared norm of each of n rows (row stride `stride`) over their first `cols` dims; a warp per + * row. Not raft::linalg::rowNorm: raft's row reductions take whole contiguous rows (row length = + * stride), and masking the other columns would still read them all. + */ template RAFT_KERNEL squared_norms_kernel( const T* __restrict__ rows, int64_t n, int64_t stride, int64_t cols, float* __restrict__ out) diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index d70151b073..fab6297c82 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -198,8 +198,9 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * @param y [n, k] row-major * @param x_fp16 [m, kDimTileSize]: x's first dims in FP16, for float; ignored for half * @param y_fp16 [n, kDimTileSize]: y's first dims in FP16, for float; ignored for half - * @param xn [m] squared norms of x over the first kDimTileSize dims - * @param yn [n] squared norms of y over the first kDimTileSize dims + * @param xn [m] squared norms of x over the first kDimTileSize dims, of the FP16 values + * the filter reads (x_fp16 for float) + * @param yn [n] the same for y * @param workspace pruned_top_1_nn_workspace_size(handle, m) bytes, 256-byte aligned */ template From 6a2ae57b1b6ed068b2e4929072dca26fa25927ea Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 1 Nov 2026 18:33:02 -0800 Subject: [PATCH 14/39] Compute the pruning ratios from eps0 in the kernel, not from a device array Co-Authored-By: Claude Opus 5.5 --- cpp/src/cluster/detail/superkmeans.cuh | 16 ++------ .../distance/detail/pruned_top_1_nn/prune.cuh | 38 ++++++++----------- cpp/src/distance/pruned_top_1_nn.cuh | 21 +++++----- 3 files changed, 29 insertions(+), 46 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 7b00525efc..821d57179d 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -141,9 +141,8 @@ void copy_first_columns(const raft::resources& handle, /** * The pruned assignment (cuvs::distance::pruned_top_1_nn) and what it keeps across iterations: the - * rows' norms and FP16 copy, the pruning ratios, the row order, and the workspace. The centroids - * the 1-NN reads are T: for half rows, a half copy of the float centroids, which is also the - * filter's. + * rows' norms and FP16 copy, the row order, and the workspace. The centroids the 1-NN reads are + * T: for half rows, a half copy of the float centroids, which is also the filter's. */ template class pruned_assignment { @@ -157,7 +156,6 @@ class pruned_assignment { dim_(X.extent(1)), n_clusters_(n_clusters), filter_dim_(std::is_same_v ? dim_ : pruned_nn::kDimTileSize), - ratios_(raft::make_device_vector(handle, dim_ + 1)), row_norms_(raft::make_device_vector(handle, n_rows_)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), dist_(raft::make_device_vector(handle, n_rows_)), @@ -168,11 +166,7 @@ class pruned_assignment { workspace_(raft::make_device_vector( handle, cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_))) { - auto stream = raft::resource::get_cuda_stream(handle).get(); - const auto ratios = pruned_nn::adsampling_ratios(dim_); - params_.ratios = ratios_.data_handle(); - params_.first_ratio = ratios[pruned_nn::kDimTileSize]; - raft::copy(ratios_.data_handle(), ratios.data(), ratios.size(), stream); + auto stream = raft::resource::get_cuda_stream(handle).get(); if constexpr (std::is_same_v) { pruned_nn::squared_norms( X.data_handle(), n_rows_, dim_, pruned_nn::kDimTileSize, row_norms_.data_handle(), stream); @@ -185,8 +179,6 @@ class pruned_assignment { filter_rows_.data_handle(), row_norms_.data_handle()); } - // The host ratios are released when the copy is done - raft::resource::sync_stream(handle); } /** @@ -233,7 +225,7 @@ class pruned_assignment { raft::device_matrix_view X_; int64_t n_rows_, dim_, n_clusters_, filter_dim_; cuvs::distance::PrunedTop1nnParams params_; - raft::device_vector ratios_, row_norms_, centroid_norms_, dist_; + raft::device_vector row_norms_, centroid_norms_, dist_; raft::device_vector row_order_; raft::device_matrix filter_rows_, filter_centroids_; raft::device_vector workspace_; diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 6daf520fb1..358b3c7fd5 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -14,7 +14,6 @@ #include #include #include -#include namespace cuvs::distance::detail::pruned_nn { @@ -33,24 +32,16 @@ constexpr uint32_t kWarpSize = 32; constexpr uint32_t kFullMask = 0xffffffff; /** - * ADSampling's pruning ratios: ratios[i] = (i / d) * (1 + eps0 / sqrt(i))^2 for i dims seen, - * i = 0..d (1 at both ends). A partial distance over the first i dims is compared against - * ratios[i] times the row's current best distance. The ratios assume every dim carries the same - * share of a distance, which ADSampling ensures with a random rotation of the data; without one - * they are a heuristic. + * ADSampling's pruning ratio for a partial distance over the first i of dim dims: + * (i / dim) * (1 + eps0 / sqrt(i))^2. A partial distance over i dims is compared against the ratio + * times the row's current best distance. The ratios assume every dim carries the same share of a + * distance, which ADSampling ensures with a random rotation of the data; without one they are a + * heuristic. */ -inline auto adsampling_ratios(uint32_t dim, float eps0 = 1.5f) -> std::vector +__host__ __device__ inline float adsampling_ratio(uint32_t i, uint32_t dim, float eps0) { - std::vector ratios(dim + 1); - for (uint32_t i = 0; i <= dim; i++) { - if (i == 0 || i == dim) { - ratios[i] = 1.0f; - continue; - } - const double f = 1.0 + eps0 / std::sqrt(static_cast(i)); - ratios[i] = static_cast(static_cast(i) / dim * f * f); - } - return ratios; + const float f = 1.0f + eps0 / sqrtf(static_cast(i)); + return static_cast(i) / static_cast(dim) * f * f; } /** @@ -221,7 +212,7 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ uint32_t centroid_offset, uint32_t dim, uint32_t partial_dim, - const float* __restrict__ ratios, + float eps0, first_filter_lists lists, uint32_t* __restrict__ labels, float* __restrict__ dist, @@ -252,6 +243,8 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ // One pass per dim tile (partial_dim < dim); the last one ends with the argmin for (uint32_t d = partial_dim; n_left; d += kDimTileSize) { const bool last = d + kDimTileSize >= dim; + // The re-filter's ratio + const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, eps0); float row_tile[kDimTileSize / kWarpSize]; #pragma unroll for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { @@ -273,7 +266,7 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ }); if (!last) { // The kept survivors go below this window's end, never into a later window - const bool keep = lane < w && distance < best * ratios[d + kDimTileSize]; + const bool keep = lane < w && distance < best * ratio; const uint32_t ballot = __ballot_sync(kFullMask, keep); if (keep) { const uint32_t slot = kept + __popc(ballot & lanes_below); @@ -307,7 +300,8 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ * The pruning kernel for n_rows rows against one centroid batch (its first centroid * centroid_offset), after the fused first filter wrote `lists`. `labels` and `dist` hold each * row's current centroid and distance, and get the closer survivor if there is one; `survivors` - * gets each row's first-filter survivors added. row_order (optional): the rows' order. + * gets each row's first-filter survivors added. row_order (optional): the rows' order. eps0: + * ADSampling's margin, for the re-filters' ratios. */ template void prune(const T* rows, @@ -317,7 +311,7 @@ void prune(const T* rows, uint32_t centroid_offset, int64_t dim, uint32_t partial_dim, - const float* ratios, + float eps0, first_filter_lists lists, uint32_t* labels, float* dist, @@ -333,7 +327,7 @@ void prune(const T* rows, centroid_offset, static_cast(dim), partial_dim, - ratios, + eps0, lists, labels, dist, diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index fab6297c82..0ef0693b92 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -30,11 +30,9 @@ namespace cuvs::distance { /** The pruned 1-NN's bounds, and the order its pruning kernel takes the rows in. */ struct PrunedTop1nnParams { - /** [k + 1] on device: ratios[i] scales a row's bound after i dims (see - * detail::pruned_nn::adsampling_ratios). */ - const float* ratios = nullptr; - /** ratios[detail::pruned_nn::kDimTileSize] on host: the first filter's. */ - float first_ratio = 1.0f; + /** ADSampling's margin: a row's bound after i dims is scaled by + * detail::pruned_nn::adsampling_ratio(i, k, eps0). */ + float eps0 = 1.5f; /** [m], optional: each row batch's rows in the order to prune them, heaviest first. */ const uint32_t* row_order = nullptr; /** [m], optional: the order for the next call, from this call's first-filter survivors. May be @@ -183,10 +181,10 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, /** * Updates each row of x to its nearest row of y in L2, starting from its current one. Not exact: * from the distance to the current neighbor (the bound), a fused partial GEMM over the first - * detail::pruned_nn::kDimTileSize dims (FP16) keeps the candidates under the bound times - * params.first_ratio; the pruning kernel adds the candidates' tail dims a tile at a time, keeping - * those under the bound times params.ratios[dims so far], and takes the closest left if it beats - * the current neighbor. A neighbor pruned away early can be missed. + * detail::pruned_nn::kDimTileSize dims (FP16) keeps the candidates under the bound times its + * ratio; the pruning kernel adds the candidates' tail dims a tile at a time, keeping those under + * the bound times the ratio for the dims so far (see PrunedTop1nnParams::eps0), and takes the + * closest left if it beats the current neighbor. A neighbor pruned away early can be missed. * * Rows go in batches of kRowBatchSize against batches of kCentroidBatchSize rows of y. The row * batches round-robin over the main stream and, with a stream pool on the handle, its first stream, @@ -228,7 +226,6 @@ void pruned_top_1_nn(raft::resources const& handle, "Pruned 1-NN needs the fused filter's cuTile kernel and k a multiple of %u above it", kDimTileSize); RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); - RAFT_EXPECTS(params.ratios != nullptr, "Pruned 1-NN needs the pruning ratios"); RAFT_EXPECTS(kHalf || (x_fp16 != nullptr && y_fp16 != nullptr), "Pruned 1-NN on float needs FP16 copies of x's and y's first dims"); const workspace_layout layout(handle, m); @@ -276,7 +273,7 @@ void pruned_top_1_nn(raft::resources const& handle, xn + i, yn + j, row_dist, - params.first_ratio, + adsampling_ratio(kDimTileSize, static_cast(k), params.eps0), static_cast(kDimTileSize), packed, count, @@ -289,7 +286,7 @@ void pruned_top_1_nn(raft::resources const& handle, static_cast(j), k, kDimTileSize, - params.ratios, + params.eps0, first_filter_lists{packed, count, kCentroidBatchSize}, row_labels, row_dist, From a57a157ac7181fecfd84f8894197c70801bce53e Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 1 Nov 2026 21:10:29 -0800 Subject: [PATCH 15/39] Assign in full for half rows the filter can't read in place; document superkmeans_params defaults Co-Authored-By: Claude Opus 5.5 --- cpp/include/cuvs/cluster/kmeans.hpp | 4 ++-- cpp/src/cluster/detail/superkmeans.cuh | 8 ++++++-- fern/pages/cpp_api/cpp-api-cluster-kmeans.md | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index c1ab6598d8..1bde30d72d 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -254,12 +254,12 @@ struct balanced_params : base_params { */ struct superkmeans_params : base_params { /** - * Number of training iterations. + * Number of training iterations (default 10). */ uint32_t n_iters = 10; /** - * Seed to the random number generator (the start rows). + * Seed to the random number generator (the start rows); default seed 0. */ raft::random::RngState rng_state{0}; }; diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 821d57179d..04b0e9c77d 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -257,8 +258,11 @@ void fit(const raft::resources& handle, "SuperKMeans with pruning (n_clusters > %ld) needs dim to be a multiple of %u", kMinPrunedK, pruned_nn::kDimTileSize); - // Without the filter's kernels on this build or device, every iteration assigns in full - const bool pruned = prunes(n_clusters, dim) && cuvs::distance::is_pruned_top_1_nn_available(dim); + // Without the filter's kernels on this build or device, every iteration assigns in full; so too + // for half rows the filter can't read in place (it needs them 16-byte aligned) + const bool pruned = + prunes(n_clusters, dim) && cuvs::distance::is_pruned_top_1_nn_available(dim) && + (!std::is_same_v || reinterpret_cast(X.data_handle()) % 16 == 0); auto stream = raft::resource::get_cuda_stream(handle); auto mapping_op = cuvs::spatial::knn::detail::utils::mapping{}; diff --git a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md index fc0595d32a..de0892806a 100644 --- a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md +++ b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md @@ -111,8 +111,8 @@ struct superkmeans_params : base_params { | Name | Type | Description | | --- | --- | --- | -| `n_iters` | `uint32_t` | Number of training iterations. | -| `rng_state` | `raft::random::RngState` | Seed to the random number generator (the start rows). | +| `n_iters` | `uint32_t` | Number of training iterations (default 10). | +| `rng_state` | `raft::random::RngState` | Seed to the random number generator (the start rows); default seed 0. | ### cluster::kmeans::kmeans_type From e7cb068ddc96e8108d78a4b020b1d38669bbe32a Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 1 Nov 2026 23:41:15 -0800 Subject: [PATCH 16/39] Load the pruning kernel's tiles in 16-byte loads; drop its launch bounds Update comments Co-Authored-By: Claude Opus 5.5 --- .../distance/detail/pruned_top_1_nn/prune.cuh | 59 +++++++++++++------ cpp/src/distance/pruned_top_1_nn.cuh | 26 +++++--- 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 358b3c7fd5..9916c0c6d4 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -146,7 +147,29 @@ struct first_filter_lists { }; /** - * Distances over one dim tile between the row (`row_tile`, lane l holding dims l + 32 j) and the N + * One dim tile of a vector (16-byte aligned) into the warp's registers as floats, in 16-byte loads: + * lane l holds dims (32 j + l) * V + [0, V) at tile[j * V + [0, V)], V = 16 / sizeof(T). + */ +template +__device__ __forceinline__ void load_tile(const T* __restrict__ values, + float (&tile)[Dims / kWarpSize]) +{ + constexpr uint32_t kVec = 16 / sizeof(T); + static_assert(Dims % (kWarpSize * kVec) == 0, "a dim tile is whole 16-byte loads per lane"); + const uint32_t lane = threadIdx.x % kWarpSize; +#pragma unroll + for (uint32_t j = 0; j < Dims / (kWarpSize * kVec); j++) { + raft::TxN_t v; + v.load(values, (j * kWarpSize + lane) * kVec); +#pragma unroll + for (uint32_t k = 0; k < kVec; k++) { + tile[j * kVec + k] = static_cast(v.val.data[k]); + } + } +} + +/** + * Distances over one dim tile between the row (`row_tile`, laid out by load_tile) and the N * survivors at window positions [first, first + N), their ids held one per lane in `ids`; the lane * holding each survivor adds its distance to `distance`. The N centroids' loads are issued back to * back. @@ -165,10 +188,12 @@ __device__ __forceinline__ void add_tile_distances(const float (&row_tile)[Dims for (uint32_t i = 0; i < N; i++) { const T* centroid = centroids + static_cast(__shfl_sync(kFullMask, ids, first + i)) * stride; + float centroid_tile[Dims / kWarpSize]; + load_tile(centroid, centroid_tile); partial[i] = 0; #pragma unroll for (uint32_t j = 0; j < Dims / kWarpSize; j++) { - const float diff = row_tile[j] - static_cast(centroid[lane + j * kWarpSize]); + const float diff = row_tile[j] - centroid_tile[j]; partial[i] = fmaf(diff, diff, partial[i]); } } @@ -202,21 +227,20 @@ __device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) * * The survivor list stays in the row's region of `lists`, but for its first kWarpSize entries, * copied to shared memory; the warp reads it a window of kWarpSize entries at a time, lane l - * holding entry l, and each re-filter compacts it in place. The launch bounds leave the compiler up - * to 64 registers, for more centroid loads in flight. + * holding entry l, and each re-filter compacts it in place. */ template -RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ rows, - const uint32_t* __restrict__ row_order, - const T* __restrict__ centroids, - uint32_t centroid_offset, - uint32_t dim, - uint32_t partial_dim, - float eps0, - first_filter_lists lists, - uint32_t* __restrict__ labels, - float* __restrict__ dist, - uint32_t* __restrict__ survivors) +RAFT_KERNEL prune_kernel(const T* __restrict__ rows, + const uint32_t* __restrict__ row_order, + const T* __restrict__ centroids, + uint32_t centroid_offset, + uint32_t dim, + uint32_t partial_dim, + float eps0, + first_filter_lists lists, + uint32_t* __restrict__ labels, + float* __restrict__ dist, + uint32_t* __restrict__ survivors) { __shared__ int64_t head[kWarpSize]; const uint32_t lane = threadIdx.x; @@ -246,10 +270,7 @@ RAFT_KERNEL __launch_bounds__(kWarpSize, 32) prune_kernel(const T* __restrict__ // The re-filter's ratio const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, eps0); float row_tile[kDimTileSize / kWarpSize]; -#pragma unroll - for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { - row_tile[j] = static_cast(row_values[d + lane + j * kWarpSize]); - } + load_tile(row_values + d, row_tile); uint32_t kept = 0; for (uint32_t base = 0; base < n_left; base += kWarpSize) { const uint32_t w = min(kWarpSize, n_left - base); diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 0ef0693b92..eeab0197bd 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -179,16 +179,24 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, } /** - * Updates each row of x to its nearest row of y in L2, starting from its current one. Not exact: - * from the distance to the current neighbor (the bound), a fused partial GEMM over the first - * detail::pruned_nn::kDimTileSize dims (FP16) keeps the candidates under the bound times its - * ratio; the pruning kernel adds the candidates' tail dims a tile at a time, keeping those under - * the bound times the ratio for the dims so far (see PrunedTop1nnParams::eps0), and takes the - * closest left if it beats the current neighbor. A neighbor pruned away early can be missed. + * Updates each row of x to its nearest row of y in L2, starting from its current one: the pruned + * assignment of SuperKMeans (Kuffo, Hepkema, Boncz, "A Super Fast K-means for Indexing Vector + * Embeddings", https://arxiv.org/abs/2603.20009). With b a row's squared distance to its current + * neighbor and ratio(i) = (i / k) * (1 + eps0 / sqrt(i))^2 (ADSampling's bound, see + * PrunedTop1nnParams::eps0): * - * Rows go in batches of kRowBatchSize against batches of kCentroidBatchSize rows of y. The row - * batches round-robin over the main stream and, with a stream pool on the handle, its first stream, - * which waits on and is synchronized with the main stream. + * 1. a fused FP16 GEMM over the first detail::pruned_nn::kDimTileSize dims keeps the rows of y + * whose partial distance is under b * ratio(kDimTileSize); + * 2. the pruning kernel adds the survivors' remaining dims a tile at a time, keeping those under + * b * ratio(dims so far); + * 3. the closest survivor over all k dims replaces the current neighbor, and b, if closer. + * + * Not exact: a neighbor pruned away on a partial distance is missed. + * + * Rows go in batches of kRowBatchSize against batches of kCentroidBatchSize rows of y, each + * centroid batch tightening b for the next. The row batches round-robin over the main stream and, + * with a stream pool on the handle, its first stream, which waits on and is synchronized with the + * main stream. * * @param output in: each row's current neighbor (nearest_idx); out: its nearest neighbor and * squared distance From 0ee37a6bdc5a5d07d4862e7bf1ed495dcb3dfa20 Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 2 Nov 2026 16:51:04 -0800 Subject: [PATCH 17/39] Add license --- cpp/src/cluster/detail/superkmeans.cuh | 7 +++++++ .../distance/detail/pruned_top_1_nn/prune.cuh | 7 +++++++ cpp/src/distance/pruned_top_1_nn.cuh | 7 +++++++ thirdparty/LICENSES/LICENSE.superkmeans | 21 +++++++++++++++++++ 4 files changed, 42 insertions(+) create mode 100644 thirdparty/LICENSES/LICENSE.superkmeans diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 04b0e9c77d..e22fb986a4 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -1,7 +1,14 @@ /* + * SPDX-FileCopyrightText: Copyright (c) 2025 Leonardo Xavier Kuffo Rivero * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * + * This source code is licensed under the MIT license found in the + * LICENSE file thirdparty/LICENSES/LICENSE.superkmeans + */ #pragma once #include "../../distance/pruned_top_1_nn.cuh" diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 9916c0c6d4..c43c18fc22 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -1,7 +1,14 @@ /* + * SPDX-FileCopyrightText: Copyright (c) 2025 Leonardo Xavier Kuffo Rivero * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * + * This source code is licensed under the MIT license found in the + * LICENSE file thirdparty/LICENSES/LICENSE.superkmeans + */ #pragma once #include diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index eeab0197bd..91b65a3c1a 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -1,7 +1,14 @@ /* + * SPDX-FileCopyrightText: Copyright (c) 2025 Leonardo Xavier Kuffo Rivero * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + * + * This source code is licensed under the MIT license found in the + * LICENSE file thirdparty/LICENSES/LICENSE.superkmeans + */ #pragma once #include "detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp" diff --git a/thirdparty/LICENSES/LICENSE.superkmeans b/thirdparty/LICENSES/LICENSE.superkmeans new file mode 100644 index 0000000000..b70483dcf7 --- /dev/null +++ b/thirdparty/LICENSES/LICENSE.superkmeans @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Leonardo Xavier Kuffo Rivero + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From e02f9db89c6e7d1b9c7e1a5e23250485485ab889 Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 2 Nov 2026 17:15:47 -0800 Subject: [PATCH 18/39] Replace custom warp reduce with cub::WarpReduceBatched --- .../distance/detail/pruned_top_1_nn/prune.cuh | 34 ++++--------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index c43c18fc22..40c766884d 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -19,6 +19,8 @@ #include #include +#include + #include #include #include @@ -52,28 +54,6 @@ __host__ __device__ inline float adsampling_ratio(uint32_t i, uint32_t dim, floa return static_cast(i) / static_cast(dim) * f * f; } -/** - * Sums N independent per-lane values across the warp (xor shuffles: every lane ends with the N - * sums), the N shuffles of each step issued together so they overlap. - */ -template -__device__ __forceinline__ void warp_reduce_sum_n(T* values) -{ - T shifted[N]; -#pragma unroll - for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { -#pragma unroll - for (uint32_t i = 0; i < N; i++) { - shifted[i] = __shfl_xor_sync(kFullMask, values[i], offset); - } - asm volatile("" ::: "memory"); -#pragma unroll - for (uint32_t i = 0; i < N; i++) { - values[i] += shifted[i]; - } - } -} - /** * Squared norm of each of n rows (row stride `stride`) over their first `cols` dims; a warp per * row. Not raft::linalg::rowNorm: raft's row reductions take whole contiguous rows (row length = @@ -204,11 +184,11 @@ __device__ __forceinline__ void add_tile_distances(const float (&row_tile)[Dims partial[i] = fmaf(diff, diff, partial[i]); } } - warp_reduce_sum_n(partial); -#pragma unroll - for (uint32_t i = 0; i < N; i++) { - if (lane == first + i) { distance += partial[i]; } - } + // Lane i ends with survivor first + i's sum; move it to that survivor's lane + using batched_reduce = cub::WarpReduceBatched; + typename batched_reduce::TempStorage temp_storage; + const float sum = __shfl_sync(kFullMask, batched_reduce{temp_storage}.Sum(partial), lane - first); + if (lane >= first && lane < first + N) { distance += sum; } } /** Calls f(integral_constant), then with N / 2, ..., 1. */ From 6b3a6626cc3b8298d851e5eaeafb6b82aea0236e Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 4 Nov 2026 23:08:38 -0800 Subject: [PATCH 19/39] Hardcode eps0 --- .../distance/detail/pruned_top_1_nn/prune.cuh | 19 +++++++++---------- cpp/src/distance/pruned_top_1_nn.cuh | 8 ++------ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 40c766884d..5b06376e07 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -29,6 +29,9 @@ namespace cuvs::distance::detail::pruned_nn { // Dims between the survivors' re-filters, and d' of the first filter constexpr uint32_t kDimTileSize = 256; + +/** ADSampling's margin, the eps0 of adsampling_ratio. */ +constexpr float kAdsamplingEps0 = 1.5f; // Centroids per batch: one fused filter and pruning launch per row batch and centroid batch constexpr uint32_t kCentroidBatchSize = 2048; // Rows per batch: bounds the first filter's survivor lists (row batch x centroid batch slots) @@ -43,10 +46,10 @@ constexpr uint32_t kFullMask = 0xffffffff; /** * ADSampling's pruning ratio for a partial distance over the first i of dim dims: - * (i / dim) * (1 + eps0 / sqrt(i))^2. A partial distance over i dims is compared against the ratio - * times the row's current best distance. The ratios assume every dim carries the same share of a - * distance, which ADSampling ensures with a random rotation of the data; without one they are a - * heuristic. + * (i / dim) * (1 + eps0 / sqrt(i))^2, with eps0 ADSampling's margin. A partial distance over i + * dims is compared against the ratio times the row's current best distance. The ratios assume + * every dim carries the same share of a distance, which ADSampling ensures with a random rotation + * of the data; without one they are a heuristic. */ __host__ __device__ inline float adsampling_ratio(uint32_t i, uint32_t dim, float eps0) { @@ -223,7 +226,6 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, uint32_t centroid_offset, uint32_t dim, uint32_t partial_dim, - float eps0, first_filter_lists lists, uint32_t* __restrict__ labels, float* __restrict__ dist, @@ -255,7 +257,7 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, for (uint32_t d = partial_dim; n_left; d += kDimTileSize) { const bool last = d + kDimTileSize >= dim; // The re-filter's ratio - const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, eps0); + const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); float row_tile[kDimTileSize / kWarpSize]; load_tile(row_values + d, row_tile); uint32_t kept = 0; @@ -308,8 +310,7 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, * The pruning kernel for n_rows rows against one centroid batch (its first centroid * centroid_offset), after the fused first filter wrote `lists`. `labels` and `dist` hold each * row's current centroid and distance, and get the closer survivor if there is one; `survivors` - * gets each row's first-filter survivors added. row_order (optional): the rows' order. eps0: - * ADSampling's margin, for the re-filters' ratios. + * gets each row's first-filter survivors added. row_order (optional): the rows' order. */ template void prune(const T* rows, @@ -319,7 +320,6 @@ void prune(const T* rows, uint32_t centroid_offset, int64_t dim, uint32_t partial_dim, - float eps0, first_filter_lists lists, uint32_t* labels, float* dist, @@ -335,7 +335,6 @@ void prune(const T* rows, centroid_offset, static_cast(dim), partial_dim, - eps0, lists, labels, dist, diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 91b65a3c1a..4e627a1f05 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -37,9 +37,6 @@ namespace cuvs::distance { /** The pruned 1-NN's bounds, and the order its pruning kernel takes the rows in. */ struct PrunedTop1nnParams { - /** ADSampling's margin: a row's bound after i dims is scaled by - * detail::pruned_nn::adsampling_ratio(i, k, eps0). */ - float eps0 = 1.5f; /** [m], optional: each row batch's rows in the order to prune them, heaviest first. */ const uint32_t* row_order = nullptr; /** [m], optional: the order for the next call, from this call's first-filter survivors. May be @@ -190,7 +187,7 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * assignment of SuperKMeans (Kuffo, Hepkema, Boncz, "A Super Fast K-means for Indexing Vector * Embeddings", https://arxiv.org/abs/2603.20009). With b a row's squared distance to its current * neighbor and ratio(i) = (i / k) * (1 + eps0 / sqrt(i))^2 (ADSampling's bound, see - * PrunedTop1nnParams::eps0): + * detail::pruned_nn::adsampling_ratio): * * 1. a fused FP16 GEMM over the first detail::pruned_nn::kDimTileSize dims keeps the rows of y * whose partial distance is under b * ratio(kDimTileSize); @@ -288,7 +285,7 @@ void pruned_top_1_nn(raft::resources const& handle, xn + i, yn + j, row_dist, - adsampling_ratio(kDimTileSize, static_cast(k), params.eps0), + adsampling_ratio(kDimTileSize, static_cast(k), kAdsamplingEps0), static_cast(kDimTileSize), packed, count, @@ -301,7 +298,6 @@ void pruned_top_1_nn(raft::resources const& handle, static_cast(j), k, kDimTileSize, - params.eps0, first_filter_lists{packed, count, kCentroidBatchSize}, row_labels, row_dist, From f2355d40b3fb591a461653fe6d446a9e49388f2f Mon Sep 17 00:00:00 2001 From: sherylll Date: Thu, 5 Nov 2026 16:40:33 -0800 Subject: [PATCH 20/39] Halve pruned 1-NN centroid batch to 1024 --- cpp/src/distance/detail/pruned_top_1_nn/prune.cuh | 2 +- cpp/src/distance/pruned_top_1_nn.cuh | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 5b06376e07..58241583a9 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -33,7 +33,7 @@ constexpr uint32_t kDimTileSize = 256; /** ADSampling's margin, the eps0 of adsampling_ratio. */ constexpr float kAdsamplingEps0 = 1.5f; // Centroids per batch: one fused filter and pruning launch per row batch and centroid batch -constexpr uint32_t kCentroidBatchSize = 2048; +constexpr uint32_t kCentroidBatchSize = 1024; // Rows per batch: bounds the first filter's survivor lists (row batch x centroid batch slots) constexpr int64_t kRowBatchSize = 128 * 1024; // Survivors whose loads are batched over a dim tile diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 4e627a1f05..a11449808f 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -141,10 +142,7 @@ inline void order_rows_by_survivors(raft::resources const& handle, m, raft::make_device_vector_view(local_ids, m), raft::make_device_vector_view(batch_offsets, layout.n_row_batches + 1)); - int end_bit = 1; - while (end_bit < 32 && (int64_t{1} << end_bit) <= n) { - end_bit++; - } + const int end_bit = std::clamp(std::bit_width(static_cast(n)), 1, 32); const int begin_bit = std::max(0, end_bit - 8); std::size_t temp_size = layout.sort_temp_bytes; RAFT_CUDA_TRY(cub::DeviceSegmentedRadixSort::SortPairsDescending( From 79bdbba19a8557bb291af423624e5f3a518af4c2 Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 8 Nov 2026 17:01:41 -0800 Subject: [PATCH 21/39] Prepare SuperKMeans filter rows per batch --- cpp/src/cluster/detail/superkmeans.cuh | 82 +++-------------- .../distance/detail/pruned_top_1_nn/prune.cuh | 91 ++++++++++++++----- cpp/src/distance/pruned_top_1_nn.cuh | 48 ++++++---- 3 files changed, 112 insertions(+), 109 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index e22fb986a4..8144d866ef 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -103,54 +103,11 @@ void predict(const raft::resources& handle, cuvs::spatial::knn::detail::utils::mapping{}); } -/** - * The first `cols` dims of n rows (row stride `stride`) as half, packed (row stride `cols`), and - * the squared norms of those half values over the first kDimTileSize dims (the fused filter's, - * from the same values as its dot products); a warp per row. - */ -template -RAFT_KERNEL copy_first_columns_kernel(const S* __restrict__ in, - int64_t n, - int64_t stride, - int64_t cols, - half* __restrict__ out, - float* __restrict__ norms) -{ - const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / pruned_nn::kWarpSize; - if (row >= n) { return; } - float s = 0.f; - for (int64_t j = threadIdx.x % pruned_nn::kWarpSize; j < cols; j += pruned_nn::kWarpSize) { - const half h = static_cast(in[row * stride + j]); - out[row * cols + j] = h; - if (j < pruned_nn::kDimTileSize) { - const float f = __half2float(h); - s += f * f; - } - } - s = raft::warpReduce(s); - if (threadIdx.x % pruned_nn::kWarpSize == 0) { norms[row] = s; } -} - -template -void copy_first_columns(const raft::resources& handle, - const S* in, - int64_t n, - int64_t stride, - int64_t cols, - half* out, - float* norms) -{ - constexpr uint32_t kThreads = 256; - const auto blocks = raft::ceildiv(n * pruned_nn::kWarpSize, kThreads); - auto stream = raft::resource::get_cuda_stream(handle).get(); - copy_first_columns_kernel<<>>(in, n, stride, cols, out, norms); - RAFT_CUDA_TRY(cudaPeekAtLastError()); -} - /** * The pruned assignment (cuvs::distance::pruned_top_1_nn) and what it keeps across iterations: the - * rows' norms and FP16 copy, the row order, and the workspace. The centroids the 1-NN reads are - * T: for half rows, a half copy of the float centroids, which is also the filter's. + * half rows' norms, the row order, and the workspace. Float filter rows are prepared per batch. + * The centroids the 1-NN reads are T: for half rows, a half copy of the float centroids, which is + * also the filter's. */ template class pruned_assignment { @@ -164,28 +121,19 @@ class pruned_assignment { dim_(X.extent(1)), n_clusters_(n_clusters), filter_dim_(std::is_same_v ? dim_ : pruned_nn::kDimTileSize), - row_norms_(raft::make_device_vector(handle, n_rows_)), + row_norms_( + raft::make_device_vector(handle, std::is_same_v ? n_rows_ : 0)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), dist_(raft::make_device_vector(handle, n_rows_)), row_order_(raft::make_device_vector(handle, n_rows_)), - filter_rows_(raft::make_device_matrix( - handle, std::is_same_v ? 0 : n_rows_, filter_dim_)), filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), workspace_(raft::make_device_vector( - handle, cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_))) + handle, cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_))) { auto stream = raft::resource::get_cuda_stream(handle).get(); if constexpr (std::is_same_v) { pruned_nn::squared_norms( X.data_handle(), n_rows_, dim_, pruned_nn::kDimTileSize, row_norms_.data_handle(), stream); - } else { - copy_first_columns(handle, - X.data_handle(), - n_rows_, - dim_, - filter_dim_, - filter_rows_.data_handle(), - row_norms_.data_handle()); } } @@ -197,13 +145,14 @@ class pruned_assignment { raft::device_vector_view labels, bool order_next) { - copy_first_columns(handle_, - centroids.data_handle(), - n_clusters_, - dim_, - filter_dim_, - filter_centroids_.data_handle(), - centroid_norms_.data_handle()); + pruned_nn::squared_norms(centroids.data_handle(), + n_clusters_, + dim_, + pruned_nn::kDimTileSize, + centroid_norms_.data_handle(), + raft::resource::get_cuda_stream(handle_).get(), + filter_centroids_.data_handle(), + filter_dim_); const T* kernel_centroids = nullptr; if constexpr (std::is_same_v) { kernel_centroids = filter_centroids_.data_handle(); @@ -215,7 +164,6 @@ class pruned_assignment { {labels.data_handle(), dist_.data_handle()}, X_.data_handle(), kernel_centroids, - filter_rows_.data_handle(), filter_centroids_.data_handle(), row_norms_.data_handle(), centroid_norms_.data_handle(), @@ -235,7 +183,7 @@ class pruned_assignment { cuvs::distance::PrunedTop1nnParams params_; raft::device_vector row_norms_, centroid_norms_, dist_; raft::device_vector row_order_; - raft::device_matrix filter_rows_, filter_centroids_; + raft::device_matrix filter_centroids_; raft::device_vector workspace_; }; diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 58241583a9..681e7361c5 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -21,7 +21,10 @@ #include +#include + #include +#include #include #include @@ -60,68 +63,106 @@ __host__ __device__ inline float adsampling_ratio(uint32_t i, uint32_t dim, floa /** * Squared norm of each of n rows (row stride `stride`) over their first `cols` dims; a warp per * row. Not raft::linalg::rowNorm: raft's row reductions take whole contiguous rows (row length = - * stride), and masking the other columns would still read them all. + * stride), and masking the other columns would still read them all. When copying, pack the first + * `copy_cols` dims as FP16 and compute the norm from those rounded values. */ -template -RAFT_KERNEL squared_norms_kernel( - const T* __restrict__ rows, int64_t n, int64_t stride, int64_t cols, float* __restrict__ out) +template +RAFT_KERNEL squared_norms_kernel(const T* __restrict__ rows, + int64_t n, + int64_t stride, + int64_t cols, + float* __restrict__ out, + Copy copy, + int64_t copy_cols) { const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; if (row >= n) { return; } - float s = 0.f; - for (int64_t j = threadIdx.x % kWarpSize; j < cols; j += kWarpSize) { - const float v = static_cast(rows[row * stride + j]); - s += v * v; + float s = 0.f; + constexpr bool kCopy = !std::is_same_v; + for (int64_t j = threadIdx.x % kWarpSize; j < (kCopy ? copy_cols : cols); j += kWarpSize) { + float v = static_cast(rows[row * stride + j]); + if constexpr (kCopy) { + const half h = static_cast(v); + copy[row * copy_cols + j] = h; + v = __half2float(h); + } + if (j < cols) { s += v * v; } } s = raft::warpReduce(s); if (threadIdx.x % kWarpSize == 0) { out[row] = s; } } -template -void squared_norms( - const T* rows, int64_t n, int64_t stride, int64_t cols, float* out, cudaStream_t stream) +template +void squared_norms(const T* rows, + int64_t n, + int64_t stride, + int64_t cols, + float* out, + cudaStream_t stream, + Copy copy = nullptr, + int64_t copy_cols = 0) { constexpr uint32_t kThreads = 256; const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); - squared_norms_kernel<<>>(rows, n, stride, cols, out); + squared_norms_kernel<<>>( + rows, n, stride, cols, out, copy, copy_cols); RAFT_CUDA_TRY(cudaPeekAtLastError()); } -/** Each row's squared distance to its assigned centroid over all dims; a warp per row. */ -template -RAFT_KERNEL distance_to_assigned_kernel(const T* __restrict__ rows, - int64_t n, - const C* __restrict__ centroids, - int64_t dim, - const uint32_t* __restrict__ labels, - float* __restrict__ dist) +/** Each row's squared distance to its assigned centroid; optionally pack and norm its filter dims. + */ +template +RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, + int64_t n, + const C* __restrict__ centroids, + int64_t dim, + const uint32_t* __restrict__ labels, + float* __restrict__ dist, + half* __restrict__ filter_rows, + float* __restrict__ filter_norms) { const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; if (row >= n) { return; } const T* a = rows + row * dim; const C* b = centroids + static_cast(labels[row]) * dim; float distance = 0.f; + float norm = 0.f; for (int64_t j = threadIdx.x % kWarpSize; j < dim; j += kWarpSize) { - const float diff = static_cast(a[j]) - static_cast(b[j]); + const float value = static_cast(a[j]); + const float diff = value - static_cast(b[j]); distance += diff * diff; + if constexpr (Copy) { + if (j < kDimTileSize) { + const half h = static_cast(value); + filter_rows[row * kDimTileSize + j] = h; + const float f = __half2float(h); + norm += f * f; + } + } } distance = raft::warpReduce(distance); if (threadIdx.x % kWarpSize == 0) { dist[row] = distance; } + if constexpr (Copy) { + norm = raft::warpReduce(norm); + if (threadIdx.x % kWarpSize == 0) { filter_norms[row] = norm; } + } } -template +template void distance_to_assigned(const T* rows, int64_t n, const C* centroids, int64_t dim, const uint32_t* labels, float* dist, - cudaStream_t stream) + cudaStream_t stream, + half* filter_rows = nullptr, + float* filter_norms = nullptr) { constexpr uint32_t kThreads = 256; const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); - distance_to_assigned_kernel<<>>( - rows, n, centroids, dim, labels, dist); + distance_to_assigned_and_copy_kernel<<>>( + rows, n, centroids, dim, labels, dist, filter_rows, filter_norms); RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index a11449808f..e29e144cc4 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -65,10 +65,10 @@ struct workspace_layout { int64_t n_row_batches; int64_t n_streams; std::size_t survivors, sorted_keys, local_ids, batch_offsets, sort_temp, sort_temp_bytes; - std::vector packed, count; + std::vector packed, count, filter_rows, filter_norms; std::size_t total_bytes; - workspace_layout(raft::resources const& handle, int64_t m) + workspace_layout(raft::resources const& handle, int64_t m, bool copy_rows) { RAFT_EXPECTS(filter_tile().has_value(), "Pruned 1-NN needs the fused filter's cuTile kernel"); n_row_batches = raft::ceildiv(m, kRowBatchSize); @@ -105,6 +105,10 @@ struct workspace_layout { for (int64_t s = 0; s < n_streams; s++) { packed.push_back(take(batch_rows * kCentroidBatchSize * sizeof(int64_t))); count.push_back(take(batch_rows * sizeof(int32_t))); + if (copy_rows) { + filter_rows.push_back(take(batch_rows * kDimTileSize * sizeof(half))); + filter_norms.push_back(take(batch_rows * sizeof(float))); + } } total_bytes = end; } @@ -175,9 +179,10 @@ inline bool is_pruned_top_1_nn_available(int64_t k) } /** The workspace pruned_top_1_nn needs for m rows on `handle` (the same handle as the call). */ +template inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, int64_t m) { - return detail::pruned_nn::workspace_layout(handle, m).total_bytes; + return detail::pruned_nn::workspace_layout(handle, m, !std::is_same_v).total_bytes; } /** @@ -204,19 +209,17 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * squared distance * @param x [m, k] row-major * @param y [n, k] row-major - * @param x_fp16 [m, kDimTileSize]: x's first dims in FP16, for float; ignored for half * @param y_fp16 [n, kDimTileSize]: y's first dims in FP16, for float; ignored for half - * @param xn [m] squared norms of x over the first kDimTileSize dims, of the FP16 values - * the filter reads (x_fp16 for float) + * @param xn [m] squared norms of half x over the first kDimTileSize dims; ignored for + * float, whose filter rows and norms are prepared in the workspace per batch * @param yn [n] the same for y - * @param workspace pruned_top_1_nn_workspace_size(handle, m) bytes, 256-byte aligned + * @param workspace pruned_top_1_nn_workspace_size(handle, m) bytes, 256-byte aligned */ template void pruned_top_1_nn(raft::resources const& handle, Top1nnOutput output, const DataT* x, const DataT* y, - const half* x_fp16, const half* y_fp16, const float* xn, const float* yn, @@ -236,21 +239,18 @@ void pruned_top_1_nn(raft::resources const& handle, "Pruned 1-NN needs the fused filter's cuTile kernel and k a multiple of %u above it", kDimTileSize); RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); - RAFT_EXPECTS(kHalf || (x_fp16 != nullptr && y_fp16 != nullptr), - "Pruned 1-NN on float needs FP16 copies of x's and y's first dims"); - const workspace_layout layout(handle, m); + RAFT_EXPECTS(kHalf || y_fp16 != nullptr, + "Pruned 1-NN on float needs an FP16 copy of y's first dims"); + const workspace_layout layout(handle, m, !kHalf); RAFT_EXPECTS(workspace_bytes >= layout.total_bytes, "Pruned 1-NN's workspace is too small"); auto* ws = static_cast(workspace); // The filter reads FP16 rows: half data whole, float data's FP16 copies of its first dims const int64_t filter_dim = kHalf ? k : kDimTileSize; - const half* x_filter = nullptr; const half* y_filter = nullptr; if constexpr (kHalf) { - x_filter = x; y_filter = y; } else { - x_filter = x_fp16; y_filter = y_fp16; } auto main_stream = raft::resource::get_cuda_stream(handle).get(); @@ -272,15 +272,29 @@ void pruned_top_1_nn(raft::resources const& handle, auto* packed = reinterpret_cast(ws + layout.packed[s]); auto* count = reinterpret_cast(ws + layout.count[s]); // Each row's bound: its distance to its current neighbor - distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream); + const half* x_filter; + const float* row_norms; + if constexpr (kHalf) { + x_filter = rows; + row_norms = xn + i; + distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream); + } else { + // Each stream reuses its filter inputs only after its previous row batch finishes. + auto* batch_rows = reinterpret_cast(ws + layout.filter_rows[s]); + auto* batch_norms = reinterpret_cast(ws + layout.filter_norms[s]); + distance_to_assigned( + rows, n_rows, y, k, row_labels, row_dist, stream, batch_rows, batch_norms); + x_filter = batch_rows; + row_norms = batch_norms; + } for (int64_t j = 0; j < n; j += kCentroidBatchSize) { const int64_t n_centroids = std::min(kCentroidBatchSize, n - j); - launch_fused_filter(x_filter + i * filter_dim, + launch_fused_filter(x_filter, static_cast(n_rows), y_filter + j * filter_dim, static_cast(n_centroids), static_cast(filter_dim), - xn + i, + row_norms, yn + j, row_dist, adsampling_ratio(kDimTileSize, static_cast(k), kAdsamplingEps0), From b3e94760e2716d94d7f4f5697171b66fba1f9f55 Mon Sep 17 00:00:00 2001 From: sherylll Date: Sun, 8 Nov 2026 22:44:59 -0800 Subject: [PATCH 22/39] Require supported layouts for high-dimensional SuperKMeans pruning --- cpp/include/cuvs/cluster/kmeans.hpp | 7 +- cpp/src/cluster/detail/superkmeans.cuh | 28 +++--- cpp/src/distance/pruned_top_1_nn.cuh | 4 +- cpp/tests/cluster/kmeans_superkmeans.cu | 109 +++++++++++++++++++++--- 4 files changed, 121 insertions(+), 27 deletions(-) diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index 266b5efe4c..ffc6f911e3 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -244,8 +244,11 @@ struct balanced_params : base_params { * row with the fused 1-NN over all dimensions; later iterations compute partial distances over the * first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the * ADSampling bound, without its random rotation), and finish the remaining dimensions only for the - * survivors. Pruning applies with more than 256 clusters and more than 256 dimensions, which must - * then be a multiple of 256; otherwise every iteration assigns with the full 1-NN. + * survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: + * the dimension count must be divisible by 256, dataset and centroid pointers must be 16-byte + * aligned, and the build and device must provide the pruning kernel. Inputs that violate these + * requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every + * iteration uses full nearest-centroid assignment (Lloyd's algorithm). * * The number of clusters is taken from the `centroids` argument. The start centroids are rows * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 8144d866ef..b9cd972e08 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -38,11 +38,12 @@ namespace pruned_nn = cuvs::distance::detail::pruned_nn; // Pruning works on whole dimension tiles past the first filter's, and only pays off with enough // centroids to discard; below that every iteration assigns with the full 1-NN. -constexpr int64_t kMinPrunedK = 256; +constexpr int64_t kMinPrunedK = 256; +constexpr int64_t kMinPrunedDim = 1024; inline bool prunes(int64_t n_clusters, int64_t dim) { - return dim > pruned_nn::kDimTileSize && n_clusters > kMinPrunedK; + return dim > kMinPrunedDim && n_clusters > kMinPrunedK; } /** Sum of squared distances of the rows to their assigned centroids. */ @@ -209,15 +210,20 @@ void fit(const raft::resources& handle, RAFT_EXPECTS(!labels.has_value() || labels->extent(0) == n_rows, "Number of rows in dataset and labels are different"); RAFT_EXPECTS(params.n_iters > 0, "SuperKMeans needs at least one iteration"); - RAFT_EXPECTS(!prunes(n_clusters, dim) || dim % pruned_nn::kDimTileSize == 0, - "SuperKMeans with pruning (n_clusters > %ld) needs dim to be a multiple of %u", - kMinPrunedK, - pruned_nn::kDimTileSize); - // Without the filter's kernels on this build or device, every iteration assigns in full; so too - // for half rows the filter can't read in place (it needs them 16-byte aligned) - const bool pruned = - prunes(n_clusters, dim) && cuvs::distance::is_pruned_top_1_nn_available(dim) && - (!std::is_same_v || reinterpret_cast(X.data_handle()) % 16 == 0); + const bool pruned = prunes(n_clusters, dim); + if (pruned) { + // Validate before sampling centroids or allocating scratch: eligible inputs must use pruning. + RAFT_EXPECTS(dim % pruned_nn::kDimTileSize == 0, + "SuperKMeans pruning needs dim to be a multiple of %u", + pruned_nn::kDimTileSize); + const auto aligned = [](const void* ptr) { + return reinterpret_cast(ptr) % pruned_nn::kAlignment == 0; + }; + RAFT_EXPECTS(aligned(X.data_handle()) && aligned(centroids.data_handle()), + "SuperKMeans pruning needs 16-byte-aligned dataset and centroid pointers"); + RAFT_EXPECTS(cuvs::distance::is_pruned_top_1_nn_available(dim), + "SuperKMeans pruning kernel is unavailable on this build or device"); + } auto stream = raft::resource::get_cuda_stream(handle); auto mapping_op = cuvs::spatial::knn::detail::utils::mapping{}; diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index e29e144cc4..addc369c4b 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -48,7 +48,7 @@ struct PrunedTop1nnParams { namespace detail::pruned_nn { constexpr int64_t kMaxStreams = 2; -constexpr std::size_t kAlignment = 256; +constexpr std::size_t kAlignment = 16; /** The fused first filter's tile on this device, or nullopt when the build or device has none. */ inline auto filter_tile() -> std::optional @@ -213,7 +213,7 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * @param xn [m] squared norms of half x over the first kDimTileSize dims; ignored for * float, whose filter rows and norms are prepared in the workspace per batch * @param yn [n] the same for y - * @param workspace pruned_top_1_nn_workspace_size(handle, m) bytes, 256-byte aligned + * @param workspace pruned_top_1_nn_workspace_size(handle, m) bytes, 16-byte aligned */ template void pruned_top_1_nn(raft::resources const& handle, diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index d28f3ba124..6afc497fc6 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -293,17 +294,16 @@ TEST(KmeansSuperKMeans, Fallback) } } -/* With more than 256 clusters and more than 256 dims, a multiple of 256, the iterations after the - * first prune (on a build and device with the filter's kernels; otherwise they assign in full). The +/* With more than 256 clusters and more than 1024 dims, a multiple of 256, the iterations after + * the first prune on a build and device with the filter's kernels. The * pruning bound is a heuristic without a rotation, so the result is compared with Lloyd's from the * same start rows rather than exactly. The labels and inertia come from a full assignment. */ TEST(KmeansSuperKMeans, Pruned) { - // Three row batches, the last partial; two centroid batches, the second partial; and two tail - // dim tiles (one re-filter) + // Multiple row and centroid batches, with partial final batches and several tail dimension tiles constexpr int64_t n_rows = 300000; - constexpr int64_t dim = 768; + constexpr int64_t dim = 1536; constexpr int64_t n_clusters = 2200; constexpr uint32_t n_iters = 8; constexpr uint64_t seed = 42; @@ -369,22 +369,102 @@ TEST(KmeansSuperKMeans, Pruned) if (pooled) { raft::resource::set_cuda_stream_pool(handle, std::make_shared(1)); } - check(X); - check(raft::make_const_mdspan(X_half.view())); + try { + check(X); + check(raft::make_const_mdspan(X_half.view())); + } catch (const raft::logic_error& e) { + if (std::string{e.what()}.find("SuperKMeans pruning kernel is unavailable") != + std::string::npos) { + GTEST_SKIP() << e.what(); + } + throw; + } } } -TEST(KmeansSuperKMeans, RejectsUnsupported) +TEST(KmeansSuperKMeans, AlignmentRequirements) +{ + constexpr int64_t n_rows = 1024, n_clusters = 300; + for (int64_t dim : {1024, 1280}) { + raft::handle_t handle; + auto stream = raft::resource::get_cuda_stream(handle); + auto check = [&](auto value) { + using T = decltype(value); + auto storage = raft::make_device_vector(handle, n_rows * dim + 1); + auto centers = raft::make_device_vector(handle, n_clusters * dim + 1); + auto labels = raft::make_device_vector(handle, n_rows); + std::vector host(n_rows * dim); + for (int64_t i = 0; i < n_rows * dim; ++i) { + host[i] = static_cast(std::sin(static_cast(i) * 0.13f)); + } + cuvs::cluster::kmeans::superkmeans_params params; + params.n_iters = 3; + for (auto offsets : {std::pair{1, 0}, std::pair{0, 1}, std::pair{1, 1}}) { + SCOPED_TRACE(sizeof(T)); + SCOPED_TRACE(offsets.first); + SCOPED_TRACE(offsets.second); + auto X = raft::make_device_matrix_view( + storage.data_handle() + offsets.first, n_rows, dim); + auto C = raft::make_device_matrix_view( + centers.data_handle() + offsets.second, n_clusters, dim); + raft::copy(storage.data_handle() + offsets.first, host.data(), host.size(), stream); + float inertia = 0; + if (dim > 1024) { + EXPECT_THROW( + cuvs::cluster::kmeans::fit_predict( + handle, params, X, C, labels.view(), raft::make_host_scalar_view(&inertia)), + raft::logic_error); + continue; + } + cuvs::cluster::kmeans::fit_predict( + handle, params, X, C, labels.view(), raft::make_host_scalar_view(&inertia)); + std::vector host_centers(n_clusters * dim); + std::vector host_labels(n_rows); + raft::copy(host_centers.data(), C.data_handle(), host_centers.size(), stream); + raft::copy(host_labels.data(), labels.data_handle(), n_rows, stream); + raft::resource::sync_stream(handle); + double expected = 0; + for (int64_t r = 0; r < n_rows; ++r) { + ASSERT_LT(host_labels[r], n_clusters); + for (int64_t j = 0; j < dim; ++j) { + const double diff = static_cast(host[r * dim + j]) - + host_centers[int64_t{host_labels[r]} * dim + j]; + expected += diff * diff; + } + } + EXPECT_TRUE(std::isfinite(inertia)); + EXPECT_NEAR(inertia, expected, 1e-4 * expected); + } + }; + check(float{}); + check(half{}); + } +} + +TEST(KmeansSuperKMeans, UntiledDimensionFallback) { raft::handle_t handle; cuvs::cluster::kmeans::superkmeans_params params; - // Pruning works on whole 256-dim tiles. + params.n_iters = 3; auto X_odd = raft::make_device_matrix(handle, 1000, 384); auto centroids = raft::make_device_matrix(handle, 300, 384); - EXPECT_THROW(cuvs::cluster::kmeans::fit( - handle, params, raft::make_const_mdspan(X_odd.view()), centroids.view()), - raft::logic_error); + RAFT_CUDA_TRY(cudaMemsetAsync(X_odd.data_handle(), + 0, + X_odd.size() * sizeof(float), + raft::resource::get_cuda_stream(handle).get())); + float inertia = -1; + cuvs::cluster::kmeans::fit(handle, + params, + raft::make_const_mdspan(X_odd.view()), + centroids.view(), + raft::make_host_scalar_view(&inertia)); + EXPECT_EQ(inertia, 0); +} +TEST(KmeansSuperKMeans, RejectsUnsupported) +{ + raft::handle_t handle; + cuvs::cluster::kmeans::superkmeans_params params; auto X = raft::make_device_matrix(handle, 1000, 256); auto small_centroids = raft::make_device_matrix(handle, 8, 256); params.metric = cuvs::distance::DistanceType::InnerProduct; @@ -393,6 +473,11 @@ TEST(KmeansSuperKMeans, RejectsUnsupported) raft::logic_error); params.metric = cuvs::distance::DistanceType::L2Expanded; + auto X_odd = raft::make_device_matrix(handle, 1000, 1152); + auto C_odd = raft::make_device_matrix(handle, 300, 1152); + EXPECT_THROW( + cuvs::cluster::kmeans::fit(handle, params, raft::make_const_mdspan(X_odd.view()), C_odd.view()), + raft::logic_error); auto too_many = raft::make_device_matrix(handle, 1001, 256); EXPECT_THROW( cuvs::cluster::kmeans::fit(handle, params, raft::make_const_mdspan(X.view()), too_many.view()), From a71429fe02ca358804c98bc83025c9c2f3bd4667 Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 9 Nov 2026 02:01:52 -0800 Subject: [PATCH 23/39] Size SuperKMeans row batches from workspace budget --- cpp/src/cluster/detail/superkmeans.cuh | 29 ++++++++++++- .../distance/detail/pruned_top_1_nn/prune.cuh | 2 - cpp/src/distance/pruned_top_1_nn.cuh | 42 ++++++++++++------- 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index b9cd972e08..34a60719c7 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -46,6 +46,29 @@ inline bool prunes(int64_t n_clusters, int64_t dim) return dim > kMinPrunedDim && n_clusters > kMinPrunedK; } +/** Budget the per-stream survivor lists and filter inputs together, as in balanced prediction. */ +template +auto calc_minibatch_size(const raft::resources& handle, int64_t n_rows) -> int64_t +{ + std::size_t mem_per_row = pruned_nn::kCentroidBatchSize * sizeof(int64_t) + sizeof(int32_t); + if constexpr (!std::is_same_v) { + mem_per_row += pruned_nn::kDimTileSize * sizeof(half) + sizeof(float); + } + const auto pool_size = raft::resource::is_stream_pool_initialized(handle) + ? static_cast(raft::resource::get_stream_pool_size(handle)) + : 0; + const auto n_streams = std::min({1 + pool_size, pruned_nn::kMaxStreams, n_rows}); + const auto free_ws_size = raft::resource::get_workspace_free_bytes(handle); + const auto available_ws_size = + std::min((free_ws_size / std::size_t{10}) * std::size_t{8}, std::size_t{1} << 29); + // Prefer whole 1K-row batches; under tight budgets retain float-buffer alignment. + constexpr std::size_t kAlignedRows = pruned_nn::kAlignment / sizeof(float); + auto batch_size = std::max(kAlignedRows, available_ws_size / (n_streams * mem_per_row)); + const auto row_multiple = batch_size >= 1024 ? std::size_t{1024} : kAlignedRows; + batch_size = batch_size / row_multiple * row_multiple; + return static_cast(std::min(batch_size, static_cast(n_rows))); +} + /** Sum of squared distances of the rows to their assigned centroids. */ template auto inertia(const raft::resources& handle, @@ -128,8 +151,10 @@ class pruned_assignment { dist_(raft::make_device_vector(handle, n_rows_)), row_order_(raft::make_device_vector(handle, n_rows_)), filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), + params_{.row_batch_size = calc_minibatch_size(handle, n_rows_)}, workspace_(raft::make_device_vector( - handle, cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_))) + handle, + cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, params_.row_batch_size))) { auto stream = raft::resource::get_cuda_stream(handle).get(); if constexpr (std::is_same_v) { @@ -181,10 +206,10 @@ class pruned_assignment { const raft::resources& handle_; raft::device_matrix_view X_; int64_t n_rows_, dim_, n_clusters_, filter_dim_; - cuvs::distance::PrunedTop1nnParams params_; raft::device_vector row_norms_, centroid_norms_, dist_; raft::device_vector row_order_; raft::device_matrix filter_centroids_; + cuvs::distance::PrunedTop1nnParams params_; raft::device_vector workspace_; }; diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 681e7361c5..cb69614acb 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -37,8 +37,6 @@ constexpr uint32_t kDimTileSize = 256; constexpr float kAdsamplingEps0 = 1.5f; // Centroids per batch: one fused filter and pruning launch per row batch and centroid batch constexpr uint32_t kCentroidBatchSize = 1024; -// Rows per batch: bounds the first filter's survivor lists (row batch x centroid batch slots) -constexpr int64_t kRowBatchSize = 128 * 1024; // Survivors whose loads are batched over a dim tile constexpr uint32_t kSurvivorTileSize = 16; // fused_filter_kernel.py's ID_SHIFT diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index addc369c4b..a6b7fa6ff3 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -38,6 +38,8 @@ namespace cuvs::distance { /** The pruned 1-NN's bounds, and the order its pruning kernel takes the rows in. */ struct PrunedTop1nnParams { + /** Rows per batch; keep fixed between workspace sizing and calls that reuse row_order. */ + int64_t row_batch_size = 0; /** [m], optional: each row batch's rows in the order to prune them, heaviest first. */ const uint32_t* row_order = nullptr; /** [m], optional: the order for the next call, from this call's first-filter survivors. May be @@ -64,14 +66,19 @@ inline auto filter_tile() -> std::optional struct workspace_layout { int64_t n_row_batches; int64_t n_streams; + int64_t row_batch_size; std::size_t survivors, sorted_keys, local_ids, batch_offsets, sort_temp, sort_temp_bytes; std::vector packed, count, filter_rows, filter_norms; std::size_t total_bytes; - workspace_layout(raft::resources const& handle, int64_t m, bool copy_rows) + workspace_layout(raft::resources const& handle, int64_t m, bool copy_rows, int64_t batch_size) + : row_batch_size(batch_size) { RAFT_EXPECTS(filter_tile().has_value(), "Pruned 1-NN needs the fused filter's cuTile kernel"); - n_row_batches = raft::ceildiv(m, kRowBatchSize); + RAFT_EXPECTS(row_batch_size > 0 && + (row_batch_size >= m || row_batch_size % (kAlignment / sizeof(float)) == 0), + "Pruned 1-NN row batches must preserve 16-byte alignment of per-row buffers"); + n_row_batches = raft::ceildiv(m, row_batch_size); const auto pool_size = raft::resource::is_stream_pool_initialized(handle) ? static_cast(raft::resource::get_stream_pool_size(handle)) : 0; @@ -101,7 +108,7 @@ struct workspace_layout { sort_temp = take(sort_temp_bytes); // The first filter's survivor lists, one pair per stream: a region of a whole centroid batch // per row of a row batch - const int64_t batch_rows = std::min(m, kRowBatchSize); + const int64_t batch_rows = std::min(m, row_batch_size); for (int64_t s = 0; s < n_streams; s++) { packed.push_back(take(batch_rows * kCentroidBatchSize * sizeof(int64_t))); count.push_back(take(batch_rows * sizeof(int32_t))); @@ -117,14 +124,15 @@ struct workspace_layout { /** Each row's id within its row batch, and the row batches' bounds (n_row_batches + 1). */ inline void row_batch_layout(raft::resources const& handle, int64_t m, + int64_t row_batch_size, raft::device_vector_view local_ids, raft::device_vector_view batch_offsets) { - raft::linalg::map_offset(handle, local_ids, [] __device__(int64_t i) { - return static_cast(i % kRowBatchSize); + raft::linalg::map_offset(handle, local_ids, [row_batch_size] __device__(int64_t i) { + return static_cast(i % row_batch_size); }); - raft::linalg::map_offset(handle, batch_offsets, [m] __device__(int64_t b) { - return b * kRowBatchSize < m ? b * kRowBatchSize : m; + raft::linalg::map_offset(handle, batch_offsets, [m, row_batch_size] __device__(int64_t b) { + return b * row_batch_size < m ? b * row_batch_size : m; }); } @@ -144,6 +152,7 @@ inline void order_rows_by_survivors(raft::resources const& handle, auto* batch_offsets = reinterpret_cast(workspace + layout.batch_offsets); row_batch_layout(handle, m, + layout.row_batch_size, raft::make_device_vector_view(local_ids, m), raft::make_device_vector_view(batch_offsets, layout.n_row_batches + 1)); const int end_bit = std::clamp(std::bit_width(static_cast(n)), 1, 32); @@ -180,9 +189,13 @@ inline bool is_pruned_top_1_nn_available(int64_t k) /** The workspace pruned_top_1_nn needs for m rows on `handle` (the same handle as the call). */ template -inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, int64_t m) +inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, + int64_t m, + int64_t row_batch_size) { - return detail::pruned_nn::workspace_layout(handle, m, !std::is_same_v).total_bytes; + return detail::pruned_nn::workspace_layout( + handle, m, !std::is_same_v, row_batch_size) + .total_bytes; } /** @@ -200,7 +213,7 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * * Not exact: a neighbor pruned away on a partial distance is missed. * - * Rows go in batches of kRowBatchSize against batches of kCentroidBatchSize rows of y, each + * Rows go in batches of params.row_batch_size against batches of kCentroidBatchSize rows of y, each * centroid batch tightening b for the next. The row batches round-robin over the main stream and, * with a stream pool on the handle, its first stream, which waits on and is synchronized with the * main stream. @@ -213,7 +226,8 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * @param xn [m] squared norms of half x over the first kDimTileSize dims; ignored for * float, whose filter rows and norms are prepared in the workspace per batch * @param yn [n] the same for y - * @param workspace pruned_top_1_nn_workspace_size(handle, m) bytes, 16-byte aligned + * @param workspace pruned_top_1_nn_workspace_size(handle, m, params.row_batch_size) bytes, + * 16-byte aligned */ template void pruned_top_1_nn(raft::resources const& handle, @@ -241,7 +255,7 @@ void pruned_top_1_nn(raft::resources const& handle, RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); RAFT_EXPECTS(kHalf || y_fp16 != nullptr, "Pruned 1-NN on float needs an FP16 copy of y's first dims"); - const workspace_layout layout(handle, m, !kHalf); + const workspace_layout layout(handle, m, !kHalf, params.row_batch_size); RAFT_EXPECTS(workspace_bytes >= layout.total_bytes, "Pruned 1-NN's workspace is too small"); auto* ws = static_cast(workspace); @@ -260,8 +274,8 @@ void pruned_top_1_nn(raft::resources const& handle, if (pooled) { raft::resource::wait_stream_pool_on_stream(handle); } for (int64_t b = 0; b < layout.n_row_batches; b++) { - const int64_t i = b * kRowBatchSize; - const int64_t n_rows = std::min(kRowBatchSize, m - i); + const int64_t i = b * layout.row_batch_size; + const int64_t n_rows = std::min(layout.row_batch_size, m - i); const int64_t s = b % layout.n_streams; cudaStream_t stream = s > 0 ? raft::resource::get_stream_from_stream_pool(handle, s - 1).get() : main_stream; From 3b2571b9e43747c3f581602f2d82a26946ee7f26 Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 9 Nov 2026 04:10:59 -0800 Subject: [PATCH 24/39] Tune tile sizes for different arch (coarsely) --- .../cutile/fused_filter_cutile_matrix.json | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json index 92427d264c..66c4558981 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json @@ -5,14 +5,25 @@ ], "_metric": [{"metric": "l2", "metric_abbrev": "l2"}], "_index": [{"index_type": "int32", "index_abbrev": "i32"}], - "_tile": [{"tile_m": 128, "tile_n": 128, "tile_k": 64, "occupancy": 2}], - "_export": [ - {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_80", "arch_tag": "cutile_arch_8_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, - {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_86", "arch_tag": "cutile_arch_8_6", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, - {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_90", "arch_tag": "cutile_arch_9_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, - {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_100", "arch_tag": "cutile_arch_10_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, - {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_107", "arch_tag": "cutile_arch_10_7", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"}, - {"output_format": "cubin", "register": "cubin", "gpu_code": "sm_120", "arch_tag": "cutile_arch_12_0", "artifact_basename": "@data_type@_@gpu_code@", "artifact_ext": "cubin"} + "output_format": "cubin", + "register": "cubin", + "artifact_basename": "@data_type@_@gpu_code@", + "artifact_ext": "cubin", + "_tile": [ + { + "tile_m": 64, "tile_n": 64, "tile_k": 32, "occupancy": 2, + "_architecture": [ + {"gpu_code": "sm_80", "arch_tag": "cutile_arch_8_0"}, + {"gpu_code": "sm_86", "arch_tag": "cutile_arch_8_6"}, + {"gpu_code": "sm_90", "arch_tag": "cutile_arch_9_0"}, + {"gpu_code": "sm_107", "arch_tag": "cutile_arch_10_7"}, + {"gpu_code": "sm_120", "arch_tag": "cutile_arch_12_0"} + ] + }, + { + "tile_m": 128, "tile_n": 128, "tile_k": 64, "occupancy": 2, + "gpu_code": "sm_100", "arch_tag": "cutile_arch_10_0" + } ] } ] From 4c2585a01662b74b748b5cf88dbddbd28442b956 Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 9 Nov 2026 04:11:34 -0800 Subject: [PATCH 25/39] Fix style --- fern/pages/cpp_api/cpp-api-cluster-kmeans.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md index 4ffaa5078f..3e58cc6fa4 100644 --- a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md +++ b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md @@ -90,7 +90,7 @@ enum class balanced_donor_selection { Simple object to specify hyper-parameters to the SuperKMeans algorithm. -SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every row with the fused 1-NN over all dimensions; later iterations compute partial distances over the first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the ADSampling bound, without its random rotation), and finish the remaining dimensions only for the survivors. Pruning applies with more than 256 clusters and more than 256 dimensions, which must then be a multiple of 256; otherwise every iteration assigns with the full 1-NN. +SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every row with the fused 1-NN over all dimensions; later iterations compute partial distances over the first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the ADSampling bound, without its random rotation), and finish the remaining dimensions only for the survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: the dimension count must be divisible by 256, dataset and centroid pointers must be 16-byte aligned, and the build and device must provide the pruning kernel. Inputs that violate these requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every iteration uses full nearest-centroid assignment (Lloyd's algorithm). The number of clusters is taken from the `centroids` argument. The start centroids are rows sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run (there is no convergence test), and an empty cluster keeps its previous centroid. From 26372e22996880e618d668395426dac5147c080e Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 9 Nov 2026 17:47:08 -0800 Subject: [PATCH 26/39] Add sm103 for completeness; update tile sizes based on measurements on H100 and RTX Pro 6000 --- .../cutile/fused_filter_cutile_matrix.json | 16 ++++++++++++---- .../pruned_top_1_nn/cutile/fused_filter_tile.cu | 3 +++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json index 66c4558981..7db659b5d4 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_cutile_matrix.json @@ -11,18 +11,26 @@ "artifact_ext": "cubin", "_tile": [ { - "tile_m": 64, "tile_n": 64, "tile_k": 32, "occupancy": 2, + "tile_m": 64, "tile_n": 64, "tile_k": 64, "occupancy": 2, + "_architecture": [ + {"gpu_code": "sm_90", "arch_tag": "cutile_arch_9_0"} + ] + }, + { + "tile_m": 64, "tile_n": 128, "tile_k": 64, "occupancy": 2, "_architecture": [ {"gpu_code": "sm_80", "arch_tag": "cutile_arch_8_0"}, {"gpu_code": "sm_86", "arch_tag": "cutile_arch_8_6"}, - {"gpu_code": "sm_90", "arch_tag": "cutile_arch_9_0"}, - {"gpu_code": "sm_107", "arch_tag": "cutile_arch_10_7"}, {"gpu_code": "sm_120", "arch_tag": "cutile_arch_12_0"} ] }, { "tile_m": 128, "tile_n": 128, "tile_k": 64, "occupancy": 2, - "gpu_code": "sm_100", "arch_tag": "cutile_arch_10_0" + "_architecture": [ + {"gpu_code": "sm_100", "arch_tag": "cutile_arch_10_0"}, + {"gpu_code": "sm_103", "arch_tag": "cutile_arch_10_3"}, + {"gpu_code": "sm_107", "arch_tag": "cutile_arch_10_7"} + ] } ] } diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu index 479774944c..bd3b655592 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu @@ -36,6 +36,9 @@ struct FusedFilterTilePlanner : cuvs::detail::jit_lto::TileAlgorithmPlanner { add_static_fragment>(); add_static_fragment>(); add_static_fragment>(); +#if CUVS_CUTILE_SM103_ENABLED + add_static_fragment>(); +#endif #if CUVS_CUTILE_SM107_ENABLED add_static_fragment>(); #endif From 78d844afdd2cb4319f41729576f73f0ff4fa2b6e Mon Sep 17 00:00:00 2001 From: sherylll Date: Tue, 10 Nov 2026 21:39:48 -0800 Subject: [PATCH 27/39] Remove duplicate kmeans_type definition --- cpp/include/cuvs/cluster/kmeans.hpp | 5 ----- fern/pages/cpp_api/cpp-api-cluster-kmeans.md | 19 ------------------- 2 files changed, 24 deletions(-) diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index 01b6ee2e5e..5a1a5d200d 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -338,11 +338,6 @@ struct superkmeans_params : base_params { raft::random::RngState rng_state{0}; }; -/** - * @brief Type of k-means algorithm. - */ -enum class kmeans_type { KMeans = 0, KMeansBalanced = 1 }; - /** * @} */ diff --git a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md index 39f1aeac3c..a6bc078a91 100644 --- a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md +++ b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md @@ -143,25 +143,6 @@ struct superkmeans_params : base_params { | `n_iters` | `uint32_t` | Number of training iterations (default 10). | | `rng_state` | `raft::random::RngState` | Seed to the random number generator (the start rows); default seed 0. | - -### cluster::kmeans::kmeans_type - -Type of k-means algorithm. - -```cpp -enum class kmeans_type { - KMeans = 0, - KMeansBalanced = 1 -}; -``` - -**Values** - -| Name | Value | -| --- | --- | -| `KMeans` | `0` | -| `KMeansBalanced` | `1` | - ## k-means clustering APIs From 5a4543585b3cb9281e9f19010c043babd6501f51 Mon Sep 17 00:00:00 2001 From: sherylll Date: Tue, 10 Nov 2026 23:42:06 -0800 Subject: [PATCH 28/39] Small refactor --- cpp/src/cluster/detail/superkmeans.cuh | 16 ++++---- .../distance/detail/pruned_top_1_nn/prune.cuh | 41 +++++++++---------- cpp/src/distance/pruned_top_1_nn.cuh | 28 +++++++------ 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 34a60719c7..9c1f9ea20a 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -52,7 +52,7 @@ auto calc_minibatch_size(const raft::resources& handle, int64_t n_rows) -> int64 { std::size_t mem_per_row = pruned_nn::kCentroidBatchSize * sizeof(int64_t) + sizeof(int32_t); if constexpr (!std::is_same_v) { - mem_per_row += pruned_nn::kDimTileSize * sizeof(half) + sizeof(float); + mem_per_row += pruned_nn::kPartialDim * sizeof(half) + sizeof(float); } const auto pool_size = raft::resource::is_stream_pool_initialized(handle) ? static_cast(raft::resource::get_stream_pool_size(handle)) @@ -144,7 +144,7 @@ class pruned_assignment { n_rows_(X.extent(0)), dim_(X.extent(1)), n_clusters_(n_clusters), - filter_dim_(std::is_same_v ? dim_ : pruned_nn::kDimTileSize), + filter_dim_(std::is_same_v ? dim_ : pruned_nn::kPartialDim), row_norms_( raft::make_device_vector(handle, std::is_same_v ? n_rows_ : 0)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), @@ -159,7 +159,7 @@ class pruned_assignment { auto stream = raft::resource::get_cuda_stream(handle).get(); if constexpr (std::is_same_v) { pruned_nn::squared_norms( - X.data_handle(), n_rows_, dim_, pruned_nn::kDimTileSize, row_norms_.data_handle(), stream); + X.data_handle(), n_rows_, dim_, pruned_nn::kPartialDim, row_norms_.data_handle(), stream); } } @@ -174,7 +174,7 @@ class pruned_assignment { pruned_nn::squared_norms(centroids.data_handle(), n_clusters_, dim_, - pruned_nn::kDimTileSize, + pruned_nn::kPartialDim, centroid_norms_.data_handle(), raft::resource::get_cuda_stream(handle_).get(), filter_centroids_.data_handle(), @@ -238,9 +238,11 @@ void fit(const raft::resources& handle, const bool pruned = prunes(n_clusters, dim); if (pruned) { // Validate before sampling centroids or allocating scratch: eligible inputs must use pruning. - RAFT_EXPECTS(dim % pruned_nn::kDimTileSize == 0, - "SuperKMeans pruning needs dim to be a multiple of %u", - pruned_nn::kDimTileSize); + RAFT_EXPECTS( + dim > pruned_nn::kPartialDim && (dim - pruned_nn::kPartialDim) % pruned_nn::kDimTileSize == 0, + "SuperKMeans pruning needs a positive tail after %u dims in tiles of %u", + pruned_nn::kPartialDim, + pruned_nn::kDimTileSize); const auto aligned = [](const void* ptr) { return reinterpret_cast(ptr) % pruned_nn::kAlignment == 0; }; diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index cb69614acb..5ce1d73792 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -30,7 +30,9 @@ namespace cuvs::distance::detail::pruned_nn { -// Dims between the survivors' re-filters, and d' of the first filter +// Dims used by the first filter (d') +constexpr uint32_t kPartialDim = 256; +// Dims between the survivors' re-filters constexpr uint32_t kDimTileSize = 256; /** ADSampling's margin, the eps0 of adsampling_ratio. */ @@ -130,10 +132,10 @@ RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, const float diff = value - static_cast(b[j]); distance += diff * diff; if constexpr (Copy) { - if (j < kDimTileSize) { - const half h = static_cast(value); - filter_rows[row * kDimTileSize + j] = h; - const float f = __half2float(h); + if (j < kPartialDim) { + const half h = static_cast(value); + filter_rows[row * kPartialDim + j] = h; + const float f = __half2float(h); norm += f * f; } } @@ -176,12 +178,11 @@ struct first_filter_lists { }; /** - * One dim tile of a vector (16-byte aligned) into the warp's registers as floats, in 16-byte loads: + * One dim tile of a vector (16-byte aligned) into the warp's registers, in 16-byte loads: * lane l holds dims (32 j + l) * V + [0, V) at tile[j * V + [0, V)], V = 16 / sizeof(T). */ template -__device__ __forceinline__ void load_tile(const T* __restrict__ values, - float (&tile)[Dims / kWarpSize]) +__device__ __forceinline__ void load_tile(const T* __restrict__ values, T (&tile)[Dims / kWarpSize]) { constexpr uint32_t kVec = 16 / sizeof(T); static_assert(Dims % (kWarpSize * kVec) == 0, "a dim tile is whole 16-byte loads per lane"); @@ -192,7 +193,7 @@ __device__ __forceinline__ void load_tile(const T* __restrict__ values, v.load(values, (j * kWarpSize + lane) * kVec); #pragma unroll for (uint32_t k = 0; k < kVec; k++) { - tile[j * kVec + k] = static_cast(v.val.data[k]); + tile[j * kVec + k] = v.val.data[k]; } } } @@ -204,7 +205,7 @@ __device__ __forceinline__ void load_tile(const T* __restrict__ values, * back. */ template -__device__ __forceinline__ void add_tile_distances(const float (&row_tile)[Dims / kWarpSize], +__device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / kWarpSize], const T* __restrict__ centroids, uint32_t stride, uint32_t ids, @@ -217,12 +218,12 @@ __device__ __forceinline__ void add_tile_distances(const float (&row_tile)[Dims for (uint32_t i = 0; i < N; i++) { const T* centroid = centroids + static_cast(__shfl_sync(kFullMask, ids, first + i)) * stride; - float centroid_tile[Dims / kWarpSize]; + T centroid_tile[Dims / kWarpSize]; load_tile(centroid, centroid_tile); partial[i] = 0; #pragma unroll for (uint32_t j = 0; j < Dims / kWarpSize; j++) { - const float diff = row_tile[j] - centroid_tile[j]; + const float diff = static_cast(row_tile[j]) - static_cast(centroid_tile[j]); partial[i] = fmaf(diff, diff, partial[i]); } } @@ -249,7 +250,7 @@ __device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) /** * A warp per block and row, against one centroid batch: the row's first-filter survivors (partial - * distances over the first partial_dim dims), the tail dims in dim tiles with a re-filter after + * distances over the first kPartialDim dims), the tail dims in dim tiles with a re-filter after * each but the last, then the argmin, kept if closer than the row's current centroid. Block b * handles row row_order[b] when given (heaviest rows first), and each row's first-filter survivors * are added to `survivors`. @@ -264,7 +265,6 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const T* __restrict__ centroids, uint32_t centroid_offset, uint32_t dim, - uint32_t partial_dim, first_filter_lists lists, uint32_t* __restrict__ labels, float* __restrict__ dist, @@ -292,12 +292,12 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, return; } - // One pass per dim tile (partial_dim < dim); the last one ends with the argmin - for (uint32_t d = partial_dim; n_left; d += kDimTileSize) { + // One pass per remaining dim tile; the last one ends with the argmin + for (uint32_t d = kPartialDim; n_left; d += kDimTileSize) { const bool last = d + kDimTileSize >= dim; // The re-filter's ratio const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); - float row_tile[kDimTileSize / kWarpSize]; + T row_tile[kDimTileSize / kWarpSize]; load_tile(row_values + d, row_tile); uint32_t kept = 0; for (uint32_t base = 0; base < n_left; base += kWarpSize) { @@ -358,22 +358,21 @@ void prune(const T* rows, const T* centroids, uint32_t centroid_offset, int64_t dim, - uint32_t partial_dim, first_filter_lists lists, uint32_t* labels, float* dist, uint32_t* survivors, cudaStream_t stream) { - RAFT_EXPECTS(partial_dim < dim && (dim - partial_dim) % kDimTileSize == 0, - "Pruned 1-NN needs partial_dim < dim, dim - partial_dim a multiple of %u", + RAFT_EXPECTS(dim > kPartialDim && (dim - kPartialDim) % kDimTileSize == 0, + "Pruned 1-NN needs a positive tail after %u dims in tiles of %u", + kPartialDim, kDimTileSize); prune_kernel<<>>(rows, row_order, centroids, centroid_offset, static_cast(dim), - partial_dim, lists, labels, dist, diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index a6b7fa6ff3..6edf6c87f3 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -113,7 +113,7 @@ struct workspace_layout { packed.push_back(take(batch_rows * kCentroidBatchSize * sizeof(int64_t))); count.push_back(take(batch_rows * sizeof(int32_t))); if (copy_rows) { - filter_rows.push_back(take(batch_rows * kDimTileSize * sizeof(half))); + filter_rows.push_back(take(batch_rows * kPartialDim * sizeof(half))); filter_norms.push_back(take(batch_rows * sizeof(float))); } } @@ -178,12 +178,13 @@ inline void order_rows_by_survivors(raft::resources const& handle, /** * Whether pruned_top_1_nn runs for k features on this build and device: it needs the fused - * filter's cuTile kernel, and k a multiple of detail::pruned_nn::kDimTileSize above it (the first - * filter's dims, with a tail left to prune on). + * filter's cuTile kernel, the first detail::pruned_nn::kPartialDim dims for filtering, and a + * positive tail in whole detail::pruned_nn::kDimTileSize tiles to prune on. */ inline bool is_pruned_top_1_nn_available(int64_t k) { - return k > detail::pruned_nn::kDimTileSize && k % detail::pruned_nn::kDimTileSize == 0 && + return k > detail::pruned_nn::kPartialDim && + (k - detail::pruned_nn::kPartialDim) % detail::pruned_nn::kDimTileSize == 0 && detail::pruned_nn::filter_tile().has_value(); } @@ -205,8 +206,8 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * neighbor and ratio(i) = (i / k) * (1 + eps0 / sqrt(i))^2 (ADSampling's bound, see * detail::pruned_nn::adsampling_ratio): * - * 1. a fused FP16 GEMM over the first detail::pruned_nn::kDimTileSize dims keeps the rows of y - * whose partial distance is under b * ratio(kDimTileSize); + * 1. a fused FP16 GEMM over the first detail::pruned_nn::kPartialDim dims keeps the rows of y + * whose partial distance is under b * ratio(kPartialDim); * 2. the pruning kernel adds the survivors' remaining dims a tile at a time, keeping those under * b * ratio(dims so far); * 3. the closest survivor over all k dims replaces the current neighbor, and b, if closer. @@ -222,8 +223,8 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * squared distance * @param x [m, k] row-major * @param y [n, k] row-major - * @param y_fp16 [n, kDimTileSize]: y's first dims in FP16, for float; ignored for half - * @param xn [m] squared norms of half x over the first kDimTileSize dims; ignored for + * @param y_fp16 [n, kPartialDim]: y's first dims in FP16, for float; ignored for half + * @param xn [m] squared norms of half x over the first kPartialDim dims; ignored for * float, whose filter rows and norms are prepared in the workspace per batch * @param yn [n] the same for y * @param workspace pruned_top_1_nn_workspace_size(handle, m, params.row_batch_size) bytes, @@ -250,7 +251,9 @@ void pruned_top_1_nn(raft::resources const& handle, using namespace detail::pruned_nn; constexpr bool kHalf = std::is_same_v; RAFT_EXPECTS(is_pruned_top_1_nn_available(k), - "Pruned 1-NN needs the fused filter's cuTile kernel and k a multiple of %u above it", + "Pruned 1-NN needs the fused filter's cuTile kernel and a positive tail after %u " + "dims in tiles of %u", + kPartialDim, kDimTileSize); RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); RAFT_EXPECTS(kHalf || y_fp16 != nullptr, @@ -260,7 +263,7 @@ void pruned_top_1_nn(raft::resources const& handle, auto* ws = static_cast(workspace); // The filter reads FP16 rows: half data whole, float data's FP16 copies of its first dims - const int64_t filter_dim = kHalf ? k : kDimTileSize; + const int64_t filter_dim = kHalf ? k : kPartialDim; const half* y_filter = nullptr; if constexpr (kHalf) { y_filter = y; @@ -311,8 +314,8 @@ void pruned_top_1_nn(raft::resources const& handle, row_norms, yn + j, row_dist, - adsampling_ratio(kDimTileSize, static_cast(k), kAdsamplingEps0), - static_cast(kDimTileSize), + adsampling_ratio(kPartialDim, static_cast(k), kAdsamplingEps0), + static_cast(kPartialDim), packed, count, static_cast(kCentroidBatchSize), @@ -323,7 +326,6 @@ void pruned_top_1_nn(raft::resources const& handle, y + j * k, static_cast(j), k, - kDimTileSize, first_filter_lists{packed, count, kCentroidBatchSize}, row_labels, row_dist, From 976b3722dd64d452a889da5c6e927a570231fefc Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 11 Nov 2026 00:24:01 -0800 Subject: [PATCH 29/39] More refactoring: in F32 mode, use F32 data to calculate norm, even though fused filter is F16 --- cpp/src/cluster/detail/superkmeans.cuh | 16 +++++------ .../distance/detail/pruned_top_1_nn/prune.cuh | 24 +++++------------ cpp/src/distance/pruned_top_1_nn.cuh | 27 +++++++------------ 3 files changed, 22 insertions(+), 45 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 9c1f9ea20a..eba792f8e5 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -51,9 +51,7 @@ template auto calc_minibatch_size(const raft::resources& handle, int64_t n_rows) -> int64_t { std::size_t mem_per_row = pruned_nn::kCentroidBatchSize * sizeof(int64_t) + sizeof(int32_t); - if constexpr (!std::is_same_v) { - mem_per_row += pruned_nn::kPartialDim * sizeof(half) + sizeof(float); - } + if constexpr (!std::is_same_v) { mem_per_row += pruned_nn::kPartialDim * sizeof(half); } const auto pool_size = raft::resource::is_stream_pool_initialized(handle) ? static_cast(raft::resource::get_stream_pool_size(handle)) : 0; @@ -129,7 +127,7 @@ void predict(const raft::resources& handle, /** * The pruned assignment (cuvs::distance::pruned_top_1_nn) and what it keeps across iterations: the - * half rows' norms, the row order, and the workspace. Float filter rows are prepared per batch. + * row norms, the row order, and the workspace. Float filter rows are prepared per batch. * The centroids the 1-NN reads are T: for half rows, a half copy of the float centroids, which is * also the filter's. */ @@ -145,8 +143,7 @@ class pruned_assignment { dim_(X.extent(1)), n_clusters_(n_clusters), filter_dim_(std::is_same_v ? dim_ : pruned_nn::kPartialDim), - row_norms_( - raft::make_device_vector(handle, std::is_same_v ? n_rows_ : 0)), + row_norms_(raft::make_device_vector(handle, n_rows_)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), dist_(raft::make_device_vector(handle, n_rows_)), row_order_(raft::make_device_vector(handle, n_rows_)), @@ -157,10 +154,9 @@ class pruned_assignment { cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, params_.row_batch_size))) { auto stream = raft::resource::get_cuda_stream(handle).get(); - if constexpr (std::is_same_v) { - pruned_nn::squared_norms( - X.data_handle(), n_rows_, dim_, pruned_nn::kPartialDim, row_norms_.data_handle(), stream); - } + // The input rows do not change between iterations, so cache their filter norms once. + pruned_nn::squared_norms( + X.data_handle(), n_rows_, dim_, pruned_nn::kPartialDim, row_norms_.data_handle(), stream); } /** diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 5ce1d73792..97b0eec334 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -109,7 +109,7 @@ void squared_norms(const T* rows, RAFT_CUDA_TRY(cudaPeekAtLastError()); } -/** Each row's squared distance to its assigned centroid; optionally pack and norm its filter dims. +/** Each row's squared distance to its assigned centroid; optionally pack its filter dims as FP16. */ template RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, @@ -118,34 +118,23 @@ RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, int64_t dim, const uint32_t* __restrict__ labels, float* __restrict__ dist, - half* __restrict__ filter_rows, - float* __restrict__ filter_norms) + half* __restrict__ filter_rows) { const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; if (row >= n) { return; } const T* a = rows + row * dim; const C* b = centroids + static_cast(labels[row]) * dim; float distance = 0.f; - float norm = 0.f; for (int64_t j = threadIdx.x % kWarpSize; j < dim; j += kWarpSize) { const float value = static_cast(a[j]); const float diff = value - static_cast(b[j]); distance += diff * diff; if constexpr (Copy) { - if (j < kPartialDim) { - const half h = static_cast(value); - filter_rows[row * kPartialDim + j] = h; - const float f = __half2float(h); - norm += f * f; - } + if (j < kPartialDim) { filter_rows[row * kPartialDim + j] = static_cast(value); } } } distance = raft::warpReduce(distance); if (threadIdx.x % kWarpSize == 0) { dist[row] = distance; } - if constexpr (Copy) { - norm = raft::warpReduce(norm); - if (threadIdx.x % kWarpSize == 0) { filter_norms[row] = norm; } - } } template @@ -156,13 +145,12 @@ void distance_to_assigned(const T* rows, const uint32_t* labels, float* dist, cudaStream_t stream, - half* filter_rows = nullptr, - float* filter_norms = nullptr) + half* filter_rows = nullptr) { constexpr uint32_t kThreads = 256; const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); - distance_to_assigned_and_copy_kernel<<>>( - rows, n, centroids, dim, labels, dist, filter_rows, filter_norms); + distance_to_assigned_and_copy_kernel + <<>>(rows, n, centroids, dim, labels, dist, filter_rows); RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 6edf6c87f3..da3660f8b1 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -68,7 +68,7 @@ struct workspace_layout { int64_t n_streams; int64_t row_batch_size; std::size_t survivors, sorted_keys, local_ids, batch_offsets, sort_temp, sort_temp_bytes; - std::vector packed, count, filter_rows, filter_norms; + std::vector packed, count, filter_rows; std::size_t total_bytes; workspace_layout(raft::resources const& handle, int64_t m, bool copy_rows, int64_t batch_size) @@ -112,10 +112,7 @@ struct workspace_layout { for (int64_t s = 0; s < n_streams; s++) { packed.push_back(take(batch_rows * kCentroidBatchSize * sizeof(int64_t))); count.push_back(take(batch_rows * sizeof(int32_t))); - if (copy_rows) { - filter_rows.push_back(take(batch_rows * kPartialDim * sizeof(half))); - filter_norms.push_back(take(batch_rows * sizeof(float))); - } + if (copy_rows) { filter_rows.push_back(take(batch_rows * kPartialDim * sizeof(half))); } } total_bytes = end; } @@ -224,9 +221,8 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * @param x [m, k] row-major * @param y [n, k] row-major * @param y_fp16 [n, kPartialDim]: y's first dims in FP16, for float; ignored for half - * @param xn [m] squared norms of half x over the first kPartialDim dims; ignored for - * float, whose filter rows and norms are prepared in the workspace per batch - * @param yn [n] the same for y + * @param xn [m] squared norms of the original x over the first kPartialDim dims + * @param yn [n] squared norms of the FP16 filter centroids over the first kPartialDim dims * @param workspace pruned_top_1_nn_workspace_size(handle, m, params.row_batch_size) bytes, * 16-byte aligned */ @@ -258,6 +254,7 @@ void pruned_top_1_nn(raft::resources const& handle, RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); RAFT_EXPECTS(kHalf || y_fp16 != nullptr, "Pruned 1-NN on float needs an FP16 copy of y's first dims"); + RAFT_EXPECTS(xn != nullptr && yn != nullptr, "Pruned 1-NN needs row and centroid norms"); const workspace_layout layout(handle, m, !kHalf, params.row_batch_size); RAFT_EXPECTS(workspace_bytes >= layout.total_bytes, "Pruned 1-NN's workspace is too small"); auto* ws = static_cast(workspace); @@ -290,19 +287,15 @@ void pruned_top_1_nn(raft::resources const& handle, auto* count = reinterpret_cast(ws + layout.count[s]); // Each row's bound: its distance to its current neighbor const half* x_filter; - const float* row_norms; + const float* row_norms = xn + i; if constexpr (kHalf) { - x_filter = rows; - row_norms = xn + i; + x_filter = rows; distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream); } else { // Each stream reuses its filter inputs only after its previous row batch finishes. - auto* batch_rows = reinterpret_cast(ws + layout.filter_rows[s]); - auto* batch_norms = reinterpret_cast(ws + layout.filter_norms[s]); - distance_to_assigned( - rows, n_rows, y, k, row_labels, row_dist, stream, batch_rows, batch_norms); - x_filter = batch_rows; - row_norms = batch_norms; + auto* batch_rows = reinterpret_cast(ws + layout.filter_rows[s]); + distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream, batch_rows); + x_filter = batch_rows; } for (int64_t j = 0; j < n; j += kCentroidBatchSize) { const int64_t n_centroids = std::min(kCentroidBatchSize, n - j); From f92bdc499e62559e0e784874c1ebe9ccb9bac985 Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 11 Nov 2026 17:09:13 -0800 Subject: [PATCH 30/39] Use raft launch_kernel --- cpp/src/cluster/detail/superkmeans.cuh | 13 ++-- .../distance/detail/pruned_top_1_nn/prune.cuh | 76 ++++++++++++------- cpp/src/distance/pruned_top_1_nn.cuh | 39 ++++++---- 3 files changed, 77 insertions(+), 51 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index eba792f8e5..c0c4d6d9af 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -76,13 +76,13 @@ auto inertia(const raft::resources& handle, { auto dist = raft::make_device_vector(handle, X.extent(0)); cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); - pruned_nn::distance_to_assigned(X.data_handle(), + pruned_nn::distance_to_assigned(handle, + X.data_handle(), X.extent(0), centroids.data_handle(), X.extent(1), labels.data_handle(), - dist.data_handle(), - stream); + dist.data_handle()); auto sum = raft::make_device_scalar(handle, 0.0f); raft::linalg::mapThenSumReduce( sum.data_handle(), dist.size(), raft::identity_op{}, stream, dist.data_handle()); @@ -153,10 +153,9 @@ class pruned_assignment { handle, cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, params_.row_batch_size))) { - auto stream = raft::resource::get_cuda_stream(handle).get(); // The input rows do not change between iterations, so cache their filter norms once. pruned_nn::squared_norms( - X.data_handle(), n_rows_, dim_, pruned_nn::kPartialDim, row_norms_.data_handle(), stream); + handle, X.data_handle(), n_rows_, dim_, pruned_nn::kPartialDim, row_norms_.data_handle()); } /** @@ -167,12 +166,12 @@ class pruned_assignment { raft::device_vector_view labels, bool order_next) { - pruned_nn::squared_norms(centroids.data_handle(), + pruned_nn::squared_norms(handle_, + centroids.data_handle(), n_clusters_, dim_, pruned_nn::kPartialDim, centroid_norms_.data_handle(), - raft::resource::get_cuda_stream(handle_).get(), filter_centroids_.data_handle(), filter_dim_); const T* kernel_centroids = nullptr; diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 97b0eec334..d23b2cd750 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -93,20 +94,28 @@ RAFT_KERNEL squared_norms_kernel(const T* __restrict__ rows, } template -void squared_norms(const T* rows, +void squared_norms(const raft::resources& handle, + const T* rows, int64_t n, int64_t stride, int64_t cols, float* out, - cudaStream_t stream, Copy copy = nullptr, int64_t copy_cols = 0) { constexpr uint32_t kThreads = 256; const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); - squared_norms_kernel<<>>( - rows, n, stride, cols, out, copy, copy_cols); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(handle, + blocks, + kThreads, + squared_norms_kernel, + rows, + n, + stride, + cols, + out, + copy, + copy_cols); } /** Each row's squared distance to its assigned centroid; optionally pack its filter dims as FP16. @@ -122,12 +131,12 @@ RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, { const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; if (row >= n) { return; } - const T* a = rows + row * dim; - const C* b = centroids + static_cast(labels[row]) * dim; - float distance = 0.f; + const T* row_data = rows + row * dim; + const C* assigned_centroid = centroids + static_cast(labels[row]) * dim; + float distance = 0.f; for (int64_t j = threadIdx.x % kWarpSize; j < dim; j += kWarpSize) { - const float value = static_cast(a[j]); - const float diff = value - static_cast(b[j]); + const float value = static_cast(row_data[j]); + const float diff = value - static_cast(assigned_centroid[j]); distance += diff * diff; if constexpr (Copy) { if (j < kPartialDim) { filter_rows[row * kPartialDim + j] = static_cast(value); } @@ -138,20 +147,28 @@ RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, } template -void distance_to_assigned(const T* rows, +void distance_to_assigned(const raft::resources& handle, + const T* rows, int64_t n, const C* centroids, int64_t dim, const uint32_t* labels, float* dist, - cudaStream_t stream, half* filter_rows = nullptr) { constexpr uint32_t kThreads = 256; const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); - distance_to_assigned_and_copy_kernel - <<>>(rows, n, centroids, dim, labels, dist, filter_rows); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(handle, + blocks, + kThreads, + distance_to_assigned_and_copy_kernel, + rows, + n, + centroids, + dim, + labels, + dist, + filter_rows); } /** @@ -340,7 +357,8 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, * gets each row's first-filter survivors added. row_order (optional): the rows' order. */ template -void prune(const T* rows, +void prune(const raft::resources& handle, + const T* rows, int64_t n_rows, const uint32_t* row_order, const T* centroids, @@ -349,23 +367,25 @@ void prune(const T* rows, first_filter_lists lists, uint32_t* labels, float* dist, - uint32_t* survivors, - cudaStream_t stream) + uint32_t* survivors) { RAFT_EXPECTS(dim > kPartialDim && (dim - kPartialDim) % kDimTileSize == 0, "Pruned 1-NN needs a positive tail after %u dims in tiles of %u", kPartialDim, kDimTileSize); - prune_kernel<<>>(rows, - row_order, - centroids, - centroid_offset, - static_cast(dim), - lists, - labels, - dist, - survivors); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + raft::launch_kernel(handle, + n_rows, + kWarpSize, + prune_kernel, + rows, + row_order, + centroids, + centroid_offset, + static_cast(dim), + lists, + labels, + dist, + survivors); } } // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index da3660f8b1..86428bf95b 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -273,28 +273,35 @@ void pruned_top_1_nn(raft::resources const& handle, const bool pooled = layout.n_streams > 1; if (pooled) { raft::resource::wait_stream_pool_on_stream(handle); } + std::vector stream_handles(layout.n_streams, handle); + for (int64_t s = 1; s < layout.n_streams; s++) { + raft::resource::set_cuda_stream(stream_handles[s], + raft::resource::get_stream_from_stream_pool(handle, s - 1)); + } + for (int64_t b = 0; b < layout.n_row_batches; b++) { - const int64_t i = b * layout.row_batch_size; - const int64_t n_rows = std::min(layout.row_batch_size, m - i); - const int64_t s = b % layout.n_streams; - cudaStream_t stream = - s > 0 ? raft::resource::get_stream_from_stream_pool(handle, s - 1).get() : main_stream; - const DataT* rows = x + i * k; - uint32_t* row_labels = output.nearest_idx + i; - float* row_dist = output.nearest_dist + i; - const uint32_t* order = params.row_order != nullptr ? params.row_order + i : nullptr; - auto* packed = reinterpret_cast(ws + layout.packed[s]); - auto* count = reinterpret_cast(ws + layout.count[s]); + const int64_t i = b * layout.row_batch_size; + const int64_t n_rows = std::min(layout.row_batch_size, m - i); + const int64_t s = b % layout.n_streams; + const auto& batch_handle = stream_handles[s]; + auto stream = raft::resource::get_cuda_stream(batch_handle).get(); + const DataT* rows = x + i * k; + uint32_t* row_labels = output.nearest_idx + i; + float* row_dist = output.nearest_dist + i; + const uint32_t* order = params.row_order != nullptr ? params.row_order + i : nullptr; + auto* packed = reinterpret_cast(ws + layout.packed[s]); + auto* count = reinterpret_cast(ws + layout.count[s]); // Each row's bound: its distance to its current neighbor const half* x_filter; const float* row_norms = xn + i; if constexpr (kHalf) { x_filter = rows; - distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream); + distance_to_assigned(batch_handle, rows, n_rows, y, k, row_labels, row_dist); } else { // Each stream reuses its filter inputs only after its previous row batch finishes. auto* batch_rows = reinterpret_cast(ws + layout.filter_rows[s]); - distance_to_assigned(rows, n_rows, y, k, row_labels, row_dist, stream, batch_rows); + distance_to_assigned( + batch_handle, rows, n_rows, y, k, row_labels, row_dist, batch_rows); x_filter = batch_rows; } for (int64_t j = 0; j < n; j += kCentroidBatchSize) { @@ -313,7 +320,8 @@ void pruned_top_1_nn(raft::resources const& handle, count, static_cast(kCentroidBatchSize), stream); - prune(rows, + prune(batch_handle, + rows, n_rows, order, y + j * k, @@ -322,8 +330,7 @@ void pruned_top_1_nn(raft::resources const& handle, first_filter_lists{packed, count, kCentroidBatchSize}, row_labels, row_dist, - survivors + i, - stream); + survivors + i); } } if (pooled) { raft::resource::sync_stream_pool(handle, std::vector{0}); } From 6f9fb0aea980b7f72e56669e389adfdc8ef2c3dd Mon Sep 17 00:00:00 2001 From: sherylll Date: Thu, 12 Nov 2026 19:53:56 -0800 Subject: [PATCH 31/39] Peel single-survivor tail from pruned nearest-neighbor search --- .../distance/detail/pruned_top_1_nn/prune.cuh | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index d23b2cd750..12eb31b4a9 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -291,14 +291,10 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, if (lane < n_left) { head[lane] = list[lane]; } __syncwarp(); if (lane == 0) { survivors[row] += n_left; } - // Only the current centroid survived - if (n_left == 1 && - centroid_offset + static_cast(head[0] >> kSurvivorIdShift) == best_index) { - return; - } // One pass per remaining dim tile; the last one ends with the argmin - for (uint32_t d = kPartialDim; n_left; d += kDimTileSize) { + uint32_t d = kPartialDim; + for (; n_left > 1; d += kDimTileSize) { const bool last = d + kDimTileSize >= dim; // The re-filter's ratio const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); @@ -332,8 +328,43 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, best_index = centroid_offset + ids; } } + // The final tile leaves lane-local full distances for the warp argmin. + if (last) { break; } __syncwarp(); // the compacted list is visible to the warp - n_left = last ? 0 : kept; + n_left = kept; + } + if (n_left == 0) { return; } + + // A single survivor needs no windows or list compaction, but still gets re-filtered per tile. + if (n_left == 1) { + const int64_t entry = head[0]; + const auto centroid_id = static_cast(entry >> kSurvivorIdShift); + // The assigned centroid's full distance is already known. + if (centroid_offset + centroid_id == best_index) { return; } + float distance = __int_as_float(static_cast(entry)); + const T* centroid = centroids + static_cast(centroid_id) * dim; + for (; d < dim; d += kDimTileSize) { + const bool last = d + kDimTileSize >= dim; + const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); + T row_tile[kDimTileSize / kWarpSize]; + T centroid_tile[kDimTileSize / kWarpSize]; + load_tile(row_values + d, row_tile); + load_tile(centroid + d, centroid_tile); + float partial = 0.f; +#pragma unroll + for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { + const float diff = static_cast(row_tile[j]) - static_cast(centroid_tile[j]); + partial = fmaf(diff, diff, partial); + } + distance += raft::warpReduce(partial); + if (!last) { + if (!(distance < best * ratio)) { break; } + } else if (lane == 0 && distance < best) { + labels[row] = centroid_offset + centroid_id; + dist[row] = distance; + } + } + return; } for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { From 91ee26aad294d198feac94c5c708aa504bfb0fca Mon Sep 17 00:00:00 2001 From: sherylll Date: Thu, 12 Nov 2026 22:08:47 -0800 Subject: [PATCH 32/39] Order pruning rows from each fused-filter tile --- cpp/src/cluster/detail/superkmeans.cuh | 27 +-- .../distance/detail/pruned_top_1_nn/prune.cuh | 25 +-- cpp/src/distance/pruned_top_1_nn.cuh | 177 +++++++++--------- 3 files changed, 105 insertions(+), 124 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index c0c4d6d9af..3d0fa661a2 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -46,11 +46,12 @@ inline bool prunes(int64_t n_clusters, int64_t dim) return dim > kMinPrunedDim && n_clusters > kMinPrunedK; } -/** Budget the per-stream survivor lists and filter inputs together, as in balanced prediction. */ +/** Budget the per-stream survivor lists, counts, row order, and optional FP16 filter rows. */ template auto calc_minibatch_size(const raft::resources& handle, int64_t n_rows) -> int64_t { - std::size_t mem_per_row = pruned_nn::kCentroidBatchSize * sizeof(int64_t) + sizeof(int32_t); + std::size_t mem_per_row = + pruned_nn::kCentroidBatchSize * sizeof(int64_t) + sizeof(int32_t) + sizeof(uint32_t); if constexpr (!std::is_same_v) { mem_per_row += pruned_nn::kPartialDim * sizeof(half); } const auto pool_size = raft::resource::is_stream_pool_initialized(handle) ? static_cast(raft::resource::get_stream_pool_size(handle)) @@ -127,7 +128,7 @@ void predict(const raft::resources& handle, /** * The pruned assignment (cuvs::distance::pruned_top_1_nn) and what it keeps across iterations: the - * row norms, the row order, and the workspace. Float filter rows are prepared per batch. + * row norms and the workspace. Float filter rows are prepared per batch. * The centroids the 1-NN reads are T: for half rows, a half copy of the float centroids, which is * also the filter's. */ @@ -146,12 +147,11 @@ class pruned_assignment { row_norms_(raft::make_device_vector(handle, n_rows_)), centroid_norms_(raft::make_device_vector(handle, n_clusters)), dist_(raft::make_device_vector(handle, n_rows_)), - row_order_(raft::make_device_vector(handle, n_rows_)), filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), - params_{.row_batch_size = calc_minibatch_size(handle, n_rows_)}, + row_batch_size_(calc_minibatch_size(handle, n_rows_)), workspace_(raft::make_device_vector( handle, - cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, params_.row_batch_size))) + cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, row_batch_size_))) { // The input rows do not change between iterations, so cache their filter norms once. pruned_nn::squared_norms( @@ -160,11 +160,9 @@ class pruned_assignment { /** * Updates `labels`, each row's centroid from the previous assignment, for `centroids`. - * `order_next`: another call follows, so order the rows for it. */ void assign(raft::device_matrix_view centroids, - raft::device_vector_view labels, - bool order_next) + raft::device_vector_view labels) { pruned_nn::squared_norms(handle_, centroids.data_handle(), @@ -180,7 +178,6 @@ class pruned_assignment { } else { kernel_centroids = centroids.data_handle(); } - params_.next_row_order = order_next ? row_order_.data_handle() : nullptr; cuvs::distance::pruned_top_1_nn(handle_, {labels.data_handle(), dist_.data_handle()}, X_.data_handle(), @@ -191,10 +188,9 @@ class pruned_assignment { n_rows_, n_clusters_, dim_, - params_, + row_batch_size_, workspace_.data_handle(), workspace_.size()); - if (order_next) { params_.row_order = row_order_.data_handle(); } } private: @@ -202,9 +198,8 @@ class pruned_assignment { raft::device_matrix_view X_; int64_t n_rows_, dim_, n_clusters_, filter_dim_; raft::device_vector row_norms_, centroid_norms_, dist_; - raft::device_vector row_order_; raft::device_matrix filter_centroids_; - cuvs::distance::PrunedTop1nnParams params_; + int64_t row_batch_size_; raft::device_vector workspace_; }; @@ -272,9 +267,7 @@ void fit(const raft::resources& handle, for (uint32_t iter = 0; iter < params.n_iters; iter++) { if (pruned_assign.has_value() && iter > 0) { - // The last pruned call orders no rows: the final labels come from a full assignment - pruned_assign->assign( - raft::make_const_mdspan(centroids), assigned.view(), iter + 1 < params.n_iters); + pruned_assign->assign(raft::make_const_mdspan(centroids), assigned.view()); } else { assign(); } diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 12eb31b4a9..13a5161ee6 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -257,8 +257,7 @@ __device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) * A warp per block and row, against one centroid batch: the row's first-filter survivors (partial * distances over the first kPartialDim dims), the tail dims in dim tiles with a re-filter after * each but the last, then the argmin, kept if closer than the row's current centroid. Block b - * handles row row_order[b] when given (heaviest rows first), and each row's first-filter survivors - * are added to `survivors`. + * handles row row_order[b], ordered using this tile's first-filter survivor counts. * * The survivor list stays in the row's region of `lists`, but for its first kWarpSize entries, * copied to shared memory; the warp reads it a window of kWarpSize entries at a time, lane l @@ -272,25 +271,23 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, uint32_t dim, first_filter_lists lists, uint32_t* __restrict__ labels, - float* __restrict__ dist, - uint32_t* __restrict__ survivors) + float* __restrict__ dist) { __shared__ int64_t head[kWarpSize]; const uint32_t lane = threadIdx.x; const uint32_t lanes_below = (1u << lane) - 1; - const int64_t row = row_order != nullptr ? row_order[blockIdx.x] : blockIdx.x; + const int64_t row = row_order[blockIdx.x]; uint32_t n_left = static_cast(lists.count[row]); // Nothing survived: nothing else to read or write if (n_left == 0) { return; } const T* row_values = rows + row * dim; - uint32_t best_index = labels[row]; + uint32_t best_index = UINT32_MAX; // No improving candidate yet. // The row's distance to its current centroid until the last tile's argmin float best = dist[row]; int64_t* list = lists.packed + row * lists.row_stride; if (lane < n_left) { head[lane] = list[lane]; } __syncwarp(); - if (lane == 0) { survivors[row] += n_left; } // One pass per remaining dim tile; the last one ends with the argmin uint32_t d = kPartialDim; @@ -340,7 +337,7 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const int64_t entry = head[0]; const auto centroid_id = static_cast(entry >> kSurvivorIdShift); // The assigned centroid's full distance is already known. - if (centroid_offset + centroid_id == best_index) { return; } + if (centroid_offset + centroid_id == labels[row]) { return; } float distance = __int_as_float(static_cast(entry)); const T* centroid = centroids + static_cast(centroid_id) * dim; for (; d < dim; d += kDimTileSize) { @@ -375,7 +372,7 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, best_index = other_index; } } - if (lane == 0) { + if (lane == 0 && best_index != UINT32_MAX) { labels[row] = best_index; dist[row] = best; } @@ -384,8 +381,8 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, /** * The pruning kernel for n_rows rows against one centroid batch (its first centroid * centroid_offset), after the fused first filter wrote `lists`. `labels` and `dist` hold each - * row's current centroid and distance, and get the closer survivor if there is one; `survivors` - * gets each row's first-filter survivors added. row_order (optional): the rows' order. + * row's current centroid and distance, and get the closer survivor if there is one. + * row_order: the rows' order. */ template void prune(const raft::resources& handle, @@ -397,8 +394,7 @@ void prune(const raft::resources& handle, int64_t dim, first_filter_lists lists, uint32_t* labels, - float* dist, - uint32_t* survivors) + float* dist) { RAFT_EXPECTS(dim > kPartialDim && (dim - kPartialDim) % kDimTileSize == 0, "Pruned 1-NN needs a positive tail after %u dims in tiles of %u", @@ -415,8 +411,7 @@ void prune(const raft::resources& handle, static_cast(dim), lists, labels, - dist, - survivors); + dist); } } // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 86428bf95b..4a83479292 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -18,12 +18,12 @@ #include #include #include +#include #include -#include #include #include -#include +#include #include @@ -36,17 +36,6 @@ namespace cuvs::distance { -/** The pruned 1-NN's bounds, and the order its pruning kernel takes the rows in. */ -struct PrunedTop1nnParams { - /** Rows per batch; keep fixed between workspace sizing and calls that reuse row_order. */ - int64_t row_batch_size = 0; - /** [m], optional: each row batch's rows in the order to prune them, heaviest first. */ - const uint32_t* row_order = nullptr; - /** [m], optional: the order for the next call, from this call's first-filter survivors. May be - * row_order. */ - uint32_t* next_row_order = nullptr; -}; - namespace detail::pruned_nn { constexpr int64_t kMaxStreams = 2; @@ -67,13 +56,14 @@ struct workspace_layout { int64_t n_row_batches; int64_t n_streams; int64_t row_batch_size; - std::size_t survivors, sorted_keys, local_ids, batch_offsets, sort_temp, sort_temp_bytes; - std::vector packed, count, filter_rows; + int max_blocks_per_sm; + std::vector packed, count, filter_rows, row_order; std::size_t total_bytes; workspace_layout(raft::resources const& handle, int64_t m, bool copy_rows, int64_t batch_size) : row_batch_size(batch_size) { + max_blocks_per_sm = raft::resource::get_device_properties(handle).maxBlocksPerMultiProcessor; RAFT_EXPECTS(filter_tile().has_value(), "Pruned 1-NN needs the fused filter's cuTile kernel"); RAFT_EXPECTS(row_batch_size > 0 && (row_batch_size >= m || row_batch_size % (kAlignment / sizeof(float)) == 0), @@ -90,85 +80,89 @@ struct workspace_layout { end = offset + bytes; return offset; }; - survivors = take(m * sizeof(uint32_t)); - sorted_keys = take(m * sizeof(uint32_t)); - local_ids = take(m * sizeof(uint32_t)); - batch_offsets = take((n_row_batches + 1) * sizeof(int64_t)); - RAFT_CUDA_TRY( - cub::DeviceSegmentedRadixSort::SortPairsDescending(nullptr, - sort_temp_bytes, - static_cast(nullptr), - static_cast(nullptr), - static_cast(nullptr), - static_cast(nullptr), - m, - n_row_batches, - static_cast(nullptr), - static_cast(nullptr))); - sort_temp = take(sort_temp_bytes); - // The first filter's survivor lists, one pair per stream: a region of a whole centroid batch - // per row of a row batch const int64_t batch_rows = std::min(m, row_batch_size); + // Streams reuse their buffers only after the previous row batch finishes. for (int64_t s = 0; s < n_streams; s++) { packed.push_back(take(batch_rows * kCentroidBatchSize * sizeof(int64_t))); count.push_back(take(batch_rows * sizeof(int32_t))); + row_order.push_back(take(batch_rows * sizeof(uint32_t))); if (copy_rows) { filter_rows.push_back(take(batch_rows * kPartialDim * sizeof(half))); } } total_bytes = end; } }; -/** Each row's id within its row batch, and the row batches' bounds (n_row_batches + 1). */ -inline void row_batch_layout(raft::resources const& handle, - int64_t m, - int64_t row_batch_size, - raft::device_vector_view local_ids, - raft::device_vector_view batch_offsets) +/** + * Order the rows of one row batch for pruning against the current centroid batch. count[r] is + * the number of centroids that survived the fused first filter for row r. Each sorting block + * sorts (count, row id) pairs within one balanced, contiguous segment, largest counts first. + * Only row ids are written; the counts and input data stay in their original positions. + * + * Interleave the sorted segments by rank to submit heavier rows from every segment early: + * + * sorted segment 0: a(12), b(10), c(8) // row ids, with counts in parentheses + * sorted segment 1: d(7), e(4), f(1) + * row_order: a, d, b, e, c, f + * + * This approximates longest-processing-time-first (LPT) scheduling without a device-wide sort. + * LPT applies across pruning blocks; similar-cost rows should share a block when multiple + * row-processing warps are grouped together. Prune currently launches one warp (one row) per + * block, with block b processing row_order[b], so there is no within-block row grouping. + */ +template +RAFT_KERNEL order_rows_by_survivors_kernel(const int32_t* __restrict__ count, + uint32_t* __restrict__ row_order, + int64_t n_rows) { - raft::linalg::map_offset(handle, local_ids, [row_batch_size] __device__(int64_t i) { - return static_cast(i % row_batch_size); - }); - raft::linalg::map_offset(handle, batch_offsets, [m, row_batch_size] __device__(int64_t b) { - return b * row_batch_size < m ? b * row_batch_size : m; - }); + using Sort = cub::BlockRadixSort; + __shared__ typename Sort::TempStorage scratch; + const int64_t n_segments = gridDim.x; + const int64_t rows_per_segment = n_rows / n_segments; + const int64_t extra_rows = n_rows % n_segments; + const int64_t base = blockIdx.x * rows_per_segment + min(int64_t{blockIdx.x}, extra_rows); + const int64_t segment_rows = rows_per_segment + (blockIdx.x < extra_rows); + uint32_t keys[ItemsPerThread]; + uint32_t ids[ItemsPerThread]; +#pragma unroll + for (int j = 0; j < ItemsPerThread; j++) { + const int64_t rank = threadIdx.x * ItemsPerThread + j; + const int64_t row = base + rank; + ids[j] = static_cast(row); + // Zero is reserved for padding, so it follows valid rows with no survivors. + keys[j] = rank < segment_rows ? static_cast(count[row]) + 1 : 0; + } + constexpr int kCountBits = std::bit_width(kCentroidBatchSize + 1); + Sort(scratch).SortDescendingBlockedToStriped(keys, ids, 0, kCountBits); +#pragma unroll + for (int j = 0; j < ItemsPerThread; j++) { + const int64_t rank = threadIdx.x + j * BlockSize; + // Longer segments come first, so the final partial rank leaves no holes. + if (rank < segment_rows) { row_order[rank * n_segments + blockIdx.x] = ids[j]; } + } } -/** - * Each row batch's local row ids ordered by survivor count, descending, into `row_order`. - * Approximate: one radix pass over the counts' top 8 significant bits (the counts are at most n), - * rows of a bucket keeping their order. - */ inline void order_rows_by_survivors(raft::resources const& handle, - const workspace_layout& layout, - char* workspace, - int64_t m, - int64_t n, - uint32_t* row_order) + const int32_t* count, + uint32_t* row_order, + int64_t n_rows, + int max_blocks_per_sm) { - auto* local_ids = reinterpret_cast(workspace + layout.local_ids); - auto* batch_offsets = reinterpret_cast(workspace + layout.batch_offsets); - row_batch_layout(handle, - m, - layout.row_batch_size, - raft::make_device_vector_view(local_ids, m), - raft::make_device_vector_view(batch_offsets, layout.n_row_batches + 1)); - const int end_bit = std::clamp(std::bit_width(static_cast(n)), 1, 32); - const int begin_bit = std::max(0, end_bit - 8); - std::size_t temp_size = layout.sort_temp_bytes; - RAFT_CUDA_TRY(cub::DeviceSegmentedRadixSort::SortPairsDescending( - workspace + layout.sort_temp, - temp_size, - reinterpret_cast(workspace + layout.survivors), - reinterpret_cast(workspace + layout.sorted_keys), - local_ids, - row_order, - m, - layout.n_row_batches, - batch_offsets, - batch_offsets + 1, - begin_bit, - end_bit, - raft::resource::get_cuda_stream(handle).get())); + constexpr int64_t kMaxSegmentRows = 256 * 32; + // Use the resident-block limit as the interleaved rank width, adding segments only when + // needed to keep each segment within one sorting block's capacity. + const int64_t n_segments = + std::min(n_rows, std::max(int64_t{max_blocks_per_sm}, raft::ceildiv(n_rows, kMaxSegmentRows))); + const int64_t segment_rows = raft::ceildiv(n_rows, n_segments); + // CUB requires both sizes at compile time. Keep a compact 2048-entry variant to avoid + // excessive padding on short segments; the 8192-entry variant uses more threads to limit + // per-thread storage for larger segments. + if (segment_rows <= 128 * 16) { + raft::launch_kernel( + handle, n_segments, 128, order_rows_by_survivors_kernel<128, 16>, count, row_order, n_rows); + } else { + raft::launch_kernel( + handle, n_segments, 256, order_rows_by_survivors_kernel<256, 32>, count, row_order, n_rows); + } } } // namespace detail::pruned_nn @@ -211,7 +205,11 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * * Not exact: a neighbor pruned away on a partial distance is missed. * - * Rows go in batches of params.row_batch_size against batches of kCentroidBatchSize rows of y, each + * Each first-filter tile orders its rows by the newly produced survivor counts before pruning. + * Counts are sorted within segments and their ranks interleaved, approximating heavy-first order. + * The row order and survivor lists are scratch buffers private to each stream. + * + * Rows go in batches of row_batch_size against batches of kCentroidBatchSize rows of y, each * centroid batch tightening b for the next. The row batches round-robin over the main stream and, * with a stream pool on the handle, its first stream, which waits on and is synchronized with the * main stream. @@ -223,7 +221,8 @@ inline std::size_t pruned_top_1_nn_workspace_size(raft::resources const& handle, * @param y_fp16 [n, kPartialDim]: y's first dims in FP16, for float; ignored for half * @param xn [m] squared norms of the original x over the first kPartialDim dims * @param yn [n] squared norms of the FP16 filter centroids over the first kPartialDim dims - * @param workspace pruned_top_1_nn_workspace_size(handle, m, params.row_batch_size) bytes, + * @param row_batch_size rows per batch, matching workspace sizing + * @param workspace pruned_top_1_nn_workspace_size(handle, m, row_batch_size) bytes, * 16-byte aligned */ template @@ -237,7 +236,7 @@ void pruned_top_1_nn(raft::resources const& handle, int64_t m, int64_t n, int64_t k, - const PrunedTop1nnParams& params, + int64_t row_batch_size, void* workspace, std::size_t workspace_bytes) { @@ -255,7 +254,7 @@ void pruned_top_1_nn(raft::resources const& handle, RAFT_EXPECTS(kHalf || y_fp16 != nullptr, "Pruned 1-NN on float needs an FP16 copy of y's first dims"); RAFT_EXPECTS(xn != nullptr && yn != nullptr, "Pruned 1-NN needs row and centroid norms"); - const workspace_layout layout(handle, m, !kHalf, params.row_batch_size); + const workspace_layout layout(handle, m, !kHalf, row_batch_size); RAFT_EXPECTS(workspace_bytes >= layout.total_bytes, "Pruned 1-NN's workspace is too small"); auto* ws = static_cast(workspace); @@ -267,9 +266,6 @@ void pruned_top_1_nn(raft::resources const& handle, } else { y_filter = y_fp16; } - auto main_stream = raft::resource::get_cuda_stream(handle).get(); - auto* survivors = reinterpret_cast(ws + layout.survivors); - RAFT_CUDA_TRY(cudaMemsetAsync(survivors, 0, m * sizeof(uint32_t), main_stream)); const bool pooled = layout.n_streams > 1; if (pooled) { raft::resource::wait_stream_pool_on_stream(handle); } @@ -288,7 +284,7 @@ void pruned_top_1_nn(raft::resources const& handle, const DataT* rows = x + i * k; uint32_t* row_labels = output.nearest_idx + i; float* row_dist = output.nearest_dist + i; - const uint32_t* order = params.row_order != nullptr ? params.row_order + i : nullptr; + auto* order = reinterpret_cast(ws + layout.row_order[s]); auto* packed = reinterpret_cast(ws + layout.packed[s]); auto* count = reinterpret_cast(ws + layout.count[s]); // Each row's bound: its distance to its current neighbor @@ -320,6 +316,7 @@ void pruned_top_1_nn(raft::resources const& handle, count, static_cast(kCentroidBatchSize), stream); + order_rows_by_survivors(batch_handle, count, order, n_rows, layout.max_blocks_per_sm); prune(batch_handle, rows, n_rows, @@ -329,14 +326,10 @@ void pruned_top_1_nn(raft::resources const& handle, k, first_filter_lists{packed, count, kCentroidBatchSize}, row_labels, - row_dist, - survivors + i); + row_dist); } } if (pooled) { raft::resource::sync_stream_pool(handle, std::vector{0}); } - if (params.next_row_order != nullptr) { - order_rows_by_survivors(handle, layout, ws, m, n, params.next_row_order); - } #else RAFT_FAIL("Pruned 1-NN needs a build with cuTile"); #endif From 7054a3fc771a66a3a9bd5901cf7966c5afb3470a Mon Sep 17 00:00:00 2001 From: sherylll Date: Mon, 16 Nov 2026 00:07:19 -0800 Subject: [PATCH 33/39] Small cleanup --- cpp/include/cuvs/cluster/kmeans.hpp | 59 ++----------------- .../cutile/fused_filter_kernel.py | 10 +--- fern/pages/cpp_api/cpp-api-cluster-kmeans.md | 4 +- 3 files changed, 12 insertions(+), 61 deletions(-) diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index 5a1a5d200d..681bc9e542 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -319,9 +319,12 @@ struct balanced_params : base_params { * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run * (there is no convergence test), and an empty cluster keeps its previous centroid. * - * With a stream pool on the handle (one stream is enough), the pruned assignment's row batches - * alternate between the main stream and the pool's, overlapping their kernels, for up to 2 GiB - * more workspace. + * Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 + * tensor cores for both float and half input; surviving candidates are refined in FP32 over + * subsequent 256-dimension tiles, with further pruning between tiles. + * + * Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional + * inertia are computed with a full assignment against the final centroids. * * The following metrics are currently supported: * - L2Expanded @@ -877,22 +880,6 @@ void fit(const raft::resources& handle, /** * @brief Find clusters with the SuperKMeans algorithm. * - * @code{.cpp} - * #include - * #include - * using namespace cuvs::cluster; - * ... - * raft::resources handle; - * cuvs::cluster::kmeans::superkmeans_params params; - * int64_t n_features = 1024, n_clusters = 4096; - * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); - * - * kmeans::fit(handle, - * params, - * X, - * centroids.view()); - * @endcode - * * @param[in] handle The raft handle. * @param[in] params Parameters for SuperKMeans. * @param[in] X Training instances to cluster. The data must @@ -1445,22 +1432,6 @@ void predict(const raft::resources& handle, /** * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. * - * @code{.cpp} - * #include - * #include - * using namespace cuvs::cluster; - * ... - * raft::resources handle; - * cuvs::cluster::kmeans::superkmeans_params params; - * auto labels = raft::make_device_vector(handle, X.extent(0)); - * - * kmeans::predict(handle, - * params, - * X, - * centroids, - * labels.view()); - * @endcode - * * @param[in] handle The raft handle. * @param[in] params Parameters for SuperKMeans. * @param[in] X New data to predict. @@ -1805,24 +1776,6 @@ void fit_predict(const raft::resources& handle, /** * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. * - * @code{.cpp} - * #include - * #include - * using namespace cuvs::cluster; - * ... - * raft::resources handle; - * cuvs::cluster::kmeans::superkmeans_params params; - * int64_t n_features = 1024, n_clusters = 4096; - * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); - * auto labels = raft::make_device_vector(handle, X.extent(0)); - * - * kmeans::fit_predict(handle, - * params, - * X, - * centroids.view(), - * labels.view()); - * @endcode - * * @param[in] handle The raft handle. * @param[in] params Parameters for SuperKMeans. * @param[in] X Training instances to cluster. The data must be diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py index 3a0c036f7a..5bdcf17a18 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_kernel.py @@ -31,7 +31,6 @@ # float32 bits in the low 32. One scatter per survivor is the costly part of the kernel; separate # 32-bit scatters per field are much slower. ID_SHIFT = 32 -LOW_32_BITS = (1 << 32) - 1 def kernel_symbol(data_type: str) -> str: @@ -113,12 +112,9 @@ def fused_filter( keep_i = keep.astype(ct.int32) slot = row_ids * row_stride + filled + ct.cumsum(keep_i, 1) - 1 filled = filled + ct.sum(keep_i, axis=1, keepdims=True) - # The distance's bits are masked: widening a negative int32 (a negative float, e.g. a - # rounded-down 0) to int64 would sign-extend into the index - index = ct.broadcast_to(centroid_ids, (tm, tn)).astype(ct.int64) - dist_bits = ( - ct.bitcast(dist, ct.int32).astype(ct.int64) & LOW_32_BITS - ) + # Unsigned widening keeps the distance's sign bit out of the centroid index. + index = centroid_ids.astype(ct.int64) + dist_bits = ct.bitcast(dist, ct.uint32).astype(ct.int64) value = ct.bitwise_lshift(index, ID_SHIFT) | dist_bits ct.scatter(packed, slot, value, mask=keep) ct.store(count, index=(row_tile, 0), tile=filled) diff --git a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md index a6bc078a91..ff5f851852 100644 --- a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md +++ b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md @@ -123,7 +123,9 @@ SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteratio The number of clusters is taken from the `centroids` argument. The start centroids are rows sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run (there is no convergence test), and an empty cluster keeps its previous centroid. -With a stream pool on the handle (one stream is enough), the pruned assignment's row batches alternate between the main stream and the pool's, overlapping their kernels, for up to 2 GiB more workspace. +Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 tensor cores for both float and half input; surviving candidates are refined in FP32 over subsequent 256-dimension tiles, with further pruning between tiles. + +Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional inertia are computed with a full assignment against the final centroids. The following metrics are currently supported: From e3a25500066dd8a55883fa87055267439b2c666a Mon Sep 17 00:00:00 2001 From: sherylll Date: Tue, 17 Nov 2026 18:21:26 -0800 Subject: [PATCH 34/39] Mark SuperKMeans API experimental --- cpp/include/cuvs/cluster/kmeans.hpp | 333 ++++++++-------- cpp/src/cluster/detail/superkmeans.cuh | 6 +- cpp/src/cluster/kmeans_superkmeans.cu | 16 +- cpp/tests/cluster/kmeans_superkmeans.cu | 75 ++-- fern/pages/cpp_api/cpp-api-cluster-kmeans.md | 391 ++++++++++--------- 5 files changed, 420 insertions(+), 401 deletions(-) diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp index 681bc9e542..bfb21c1901 100644 --- a/cpp/include/cuvs/cluster/kmeans.hpp +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -302,45 +302,6 @@ struct balanced_params : base_params { bool is_packed_binary = false; }; -/** - * Simple object to specify hyper-parameters to the SuperKMeans algorithm. - * - * SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every - * row with the fused 1-NN over all dimensions; later iterations compute partial distances over the - * first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the - * ADSampling bound, without its random rotation), and finish the remaining dimensions only for the - * survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: - * the dimension count must be divisible by 256, dataset and centroid pointers must be 16-byte - * aligned, and the build and device must provide the pruning kernel. Inputs that violate these - * requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every - * iteration uses full nearest-centroid assignment (Lloyd's algorithm). - * - * The number of clusters is taken from the `centroids` argument. The start centroids are rows - * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run - * (there is no convergence test), and an empty cluster keeps its previous centroid. - * - * Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 - * tensor cores for both float and half input; surviving candidates are refined in FP32 over - * subsequent 256-dimension tiles, with further pruning between tiles. - * - * Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional - * inertia are computed with a full assignment against the final centroids. - * - * The following metrics are currently supported: - * - L2Expanded - */ -struct superkmeans_params : base_params { - /** - * Number of training iterations (default 10). - */ - uint32_t n_iters = 10; - - /** - * Seed to the random number generator (the start rows); default seed 0. - */ - raft::random::RngState rng_state{0}; -}; - /** * @} */ @@ -877,47 +838,6 @@ void fit(const raft::resources& handle, raft::device_matrix_view centroids, std::optional> inertia = std::nullopt); -/** - * @brief Find clusters with the SuperKMeans algorithm. - * - * @param[in] handle The raft handle. - * @param[in] params Parameters for SuperKMeans. - * @param[in] X Training instances to cluster. The data must - * be in row-major format. - * [dim = n_samples x n_features] - * @param[out] centroids The generated centroids. - * [dim = n_clusters x n_features] - * @param[out] inertia Sum of squared distances of samples to their - * closest cluster center. - */ -void fit(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - std::optional> inertia = std::nullopt); - -/** - * @brief Find clusters with the SuperKMeans algorithm. - * - * With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the - * centroids, which are accumulated and returned in FP32. - * - * @param[in] handle The raft handle. - * @param[in] params Parameters for SuperKMeans. - * @param[in] X Training instances to cluster. The data must - * be in row-major format. - * [dim = n_samples x n_features] - * @param[out] centroids The generated centroids. - * [dim = n_clusters x n_features] - * @param[out] inertia Sum of squared distances of samples to their - * closest cluster center. - */ -void fit(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - std::optional> inertia = std::nullopt); - /** * @brief Predict the closest cluster each sample in X belongs to. * @@ -1429,42 +1349,6 @@ void predict(const raft::resources& handle, raft::device_matrix_view centroids, raft::device_vector_view labels); -/** - * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. - * - * @param[in] handle The raft handle. - * @param[in] params Parameters for SuperKMeans. - * @param[in] X New data to predict. - * [dim = n_samples x n_features] - * @param[in] centroids Cluster centroids. The data must be in row-major format. - * [dim = n_clusters x n_features] - * @param[out] labels Index of the closest centroid for each sample. - * [len = n_samples] - */ -void predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - raft::device_vector_view labels); - -/** - * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. - * - * @param[in] handle The raft handle. - * @param[in] params Parameters for SuperKMeans. - * @param[in] X New data to predict. - * [dim = n_samples x n_features] - * @param[in] centroids Cluster centroids. The data must be in row-major format. - * [dim = n_clusters x n_features] - * @param[out] labels Index of the closest centroid for each sample. - * [len = n_samples] - */ -void predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - raft::device_vector_view labels); - /** * @brief Compute k-means clustering and predicts cluster index for each sample * in the input. @@ -1773,50 +1657,6 @@ void fit_predict(const raft::resources& handle, raft::device_matrix_view centroids, raft::device_vector_view labels); -/** - * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. - * - * @param[in] handle The raft handle. - * @param[in] params Parameters for SuperKMeans. - * @param[in] X Training instances to cluster. The data must be - * in row-major format. - * [dim = n_samples x n_features] - * @param[out] centroids The generated centroids. - * [dim = n_clusters x n_features] - * @param[out] labels Index of the closest centroid for each sample. - * [len = n_samples] - * @param[out] inertia Sum of squared distances of samples to their - * closest cluster center. - */ -void fit_predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - raft::device_vector_view labels, - std::optional> inertia = std::nullopt); - -/** - * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. - * - * @param[in] handle The raft handle. - * @param[in] params Parameters for SuperKMeans. - * @param[in] X Training instances to cluster. The data must be - * in row-major format. - * [dim = n_samples x n_features] - * @param[out] centroids The generated centroids. - * [dim = n_clusters x n_features] - * @param[out] labels Index of the closest centroid for each sample. - * [len = n_samples] - * @param[out] inertia Sum of squared distances of samples to their - * closest cluster center. - */ -void fit_predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, - raft::device_matrix_view X, - raft::device_matrix_view centroids, - raft::device_vector_view labels, - std::optional> inertia = std::nullopt); - /** * @brief Transform X to a cluster-distance space. * @@ -2147,6 +1987,179 @@ void find_k(raft::resources const& handle, */ } // namespace helpers +namespace experimental { + +/** + * @defgroup superkmeans Experimental SuperKMeans APIs + * @{ + */ + +/** + * Simple object to specify hyper-parameters to the SuperKMeans algorithm. + * + * SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every + * row with the fused 1-NN over all dimensions; later iterations compute partial distances over the + * first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the + * ADSampling bound, without its random rotation), and finish the remaining dimensions only for the + * survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: + * row strides must be divisible by 16 bytes, dataset and centroid pointers must be 16-byte + * aligned, and the build and device must provide the pruning kernel. Inputs that violate these + * requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every + * iteration uses full nearest-centroid assignment (Lloyd's algorithm). + * + * The number of clusters is taken from the `centroids` argument. The start centroids are rows + * sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run + * (there is no convergence test), and an empty cluster keeps its previous centroid. + * + * Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 + * tensor cores for both float and half input; surviving candidates are refined in FP32 over + * subsequent tiles of up to 256 dimensions, with further pruning between tiles. + * + * Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional + * inertia are computed with a full assignment against the final centroids. + * + * The following metrics are currently supported: + * - L2Expanded + */ +struct superkmeans_params : base_params { + /** + * Number of training iterations (default 10). + */ + uint32_t n_iters = 10; + + /** + * Seed to the random number generator (the start rows); default seed 0. + */ + raft::random::RngState rng_state{0}; +}; + +/** + * @brief Find clusters with the SuperKMeans algorithm. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia = std::nullopt); + +/** + * @brief Find clusters with the SuperKMeans algorithm. + * + * With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the + * centroids, which are accumulated and returned in FP32. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + std::optional> inertia = std::nullopt); + +/** + * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia = std::nullopt); + +/** + * @brief Compute SuperKMeans clusters and predict the cluster index of each sample. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for SuperKMeans. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids The generated centroids. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the closest centroid for each sample. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + std::optional> inertia = std::nullopt); + +/** + * @} + */ + +} // namespace experimental + } // namespace kmeans } // namespace cluster } // namespace CUVS_EXPORT cuvs diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 3d0fa661a2..daa2b7a86f 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -94,7 +94,7 @@ auto inertia(const raft::resources& handle, } template -void validate(const cuvs::cluster::kmeans::superkmeans_params& params, +void validate(const cuvs::cluster::kmeans::experimental::superkmeans_params& params, raft::device_matrix_view X, raft::device_matrix_view centroids) { @@ -108,7 +108,7 @@ void validate(const cuvs::cluster::kmeans::superkmeans_params& params, /** Each row's closest centroid over all dims. */ template void predict(const raft::resources& handle, - const cuvs::cluster::kmeans::superkmeans_params& params, + const cuvs::cluster::kmeans::experimental::superkmeans_params& params, raft::device_matrix_view X, raft::device_matrix_view centroids, raft::device_vector_view labels) @@ -211,7 +211,7 @@ class pruned_assignment { */ template void fit(const raft::resources& handle, - const cuvs::cluster::kmeans::superkmeans_params& params, + const cuvs::cluster::kmeans::experimental::superkmeans_params& params, raft::device_matrix_view X, raft::device_matrix_view centroids, std::optional> labels, diff --git a/cpp/src/cluster/kmeans_superkmeans.cu b/cpp/src/cluster/kmeans_superkmeans.cu index 00690dd6e6..2a2353ddd5 100644 --- a/cpp/src/cluster/kmeans_superkmeans.cu +++ b/cpp/src/cluster/kmeans_superkmeans.cu @@ -7,10 +7,10 @@ #include -namespace cuvs::cluster::kmeans { +namespace cuvs::cluster::kmeans::experimental { void fit(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, raft::device_matrix_view X, raft::device_matrix_view centroids, std::optional> inertia) @@ -19,7 +19,7 @@ void fit(const raft::resources& handle, } void fit(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, raft::device_matrix_view X, raft::device_matrix_view centroids, std::optional> inertia) @@ -28,7 +28,7 @@ void fit(const raft::resources& handle, } void predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, raft::device_matrix_view X, raft::device_matrix_view centroids, raft::device_vector_view labels) @@ -37,7 +37,7 @@ void predict(const raft::resources& handle, } void predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, raft::device_matrix_view X, raft::device_matrix_view centroids, raft::device_vector_view labels) @@ -46,7 +46,7 @@ void predict(const raft::resources& handle, } void fit_predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, raft::device_matrix_view X, raft::device_matrix_view centroids, raft::device_vector_view labels, @@ -56,7 +56,7 @@ void fit_predict(const raft::resources& handle, } void fit_predict(const raft::resources& handle, - cuvs::cluster::kmeans::superkmeans_params const& params, + cuvs::cluster::kmeans::experimental::superkmeans_params const& params, raft::device_matrix_view X, raft::device_matrix_view centroids, raft::device_vector_view labels, @@ -65,4 +65,4 @@ void fit_predict(const raft::resources& handle, detail::superkmeans::fit(handle, params, X, centroids, labels, inertia); } -} // namespace cuvs::cluster::kmeans +} // namespace cuvs::cluster::kmeans::experimental diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index 6afc497fc6..abb2171e95 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "../test_utils.h" +#include "../test_utils.cuh" #include @@ -169,11 +169,11 @@ void fit_predict(raft::handle_t& handle, float& inertia) { cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); - cuvs::cluster::kmeans::superkmeans_params params; + cuvs::cluster::kmeans::experimental::superkmeans_params params; params.n_iters = n_iters; auto d_centroids = raft::make_device_matrix(handle, kClusters, kDim); auto d_labels = raft::make_device_vector(handle, kRows); - cuvs::cluster::kmeans::fit_predict( + cuvs::cluster::kmeans::experimental::fit_predict( handle, params, X, d_centroids.view(), d_labels.view(), raft::make_host_scalar_view(&inertia)); centroids.resize(kClusters * kDim); labels.resize(kRows); @@ -208,13 +208,13 @@ TEST(KmeansSuperKMeans, Fallback) kDim, kClusters, 1, - start_centroids(handle, h_X, cuvs::cluster::kmeans::superkmeans_params{}.rng_state.seed), + start_centroids( + handle, h_X, cuvs::cluster::kmeans::experimental::superkmeans_params{}.rng_state.seed), ref_centroids, ref_labels, ref_inertia); - for (size_t i = 0; i < centroids.size(); i++) { - ASSERT_NEAR(centroids[i], ref_centroids[i], 1e-5f) << "centroid " << i / kDim; - } + ASSERT_TRUE( + cuvs::hostVecMatch(ref_centroids, centroids, cuvs::CompareApproxNoScaling(1e-5f))); int64_t same_label = 0; for (int64_t r = 0; r < kRows; r++) { same_label += labels[r] == ref_labels[r]; @@ -270,26 +270,27 @@ TEST(KmeansSuperKMeans, Fallback) auto d_labels = raft::make_device_vector(handle, kRows); cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); raft::copy(d_centroids_fp.data_handle(), centroids.data(), centroids.size(), stream); - cuvs::cluster::kmeans::predict(handle, - cuvs::cluster::kmeans::superkmeans_params{}, - raft::make_const_mdspan(X.view()), - raft::make_const_mdspan(d_centroids_fp.view()), - d_labels.view()); + cuvs::cluster::kmeans::experimental::predict( + handle, + cuvs::cluster::kmeans::experimental::superkmeans_params{}, + raft::make_const_mdspan(X.view()), + raft::make_const_mdspan(d_centroids_fp.view()), + d_labels.view()); std::vector predicted(kRows); raft::copy(predicted.data(), d_labels.data_handle(), kRows, stream); raft::resource::sync_stream(handle); EXPECT_EQ(predicted, labels); // fit runs the same iterations; its centroids may differ in the last bits (atomic sums). - cuvs::cluster::kmeans::superkmeans_params params; + cuvs::cluster::kmeans::experimental::superkmeans_params params; params.n_iters = n_iters; auto d_centroids = raft::make_device_matrix(handle, kClusters, kDim); float fit_inertia = 0; - cuvs::cluster::kmeans::fit(handle, - params, - raft::make_const_mdspan(X.view()), - d_centroids.view(), - raft::make_host_scalar_view(&fit_inertia)); + cuvs::cluster::kmeans::experimental::fit(handle, + params, + raft::make_const_mdspan(X.view()), + d_centroids.view(), + raft::make_host_scalar_view(&fit_inertia)); EXPECT_NEAR(fit_inertia, inertia, 1e-3 * inertia); } } @@ -352,13 +353,13 @@ TEST(KmeansSuperKMeans, Pruned) raft::make_host_scalar_view(&lloyd_inertia), raft::make_host_scalar_view(&lloyd_iters)); - cuvs::cluster::kmeans::superkmeans_params params; + cuvs::cluster::kmeans::experimental::superkmeans_params params; params.n_iters = n_iters; params.rng_state = raft::random::RngState(seed); auto labels = raft::make_device_vector(handle, n_rows); auto check = [&](auto data) { float inertia = 0; - cuvs::cluster::kmeans::fit_predict( + cuvs::cluster::kmeans::experimental::fit_predict( handle, params, data, centroids.view(), labels.view(), raft::make_host_scalar_view(&inertia)); EXPECT_NEAR(inertia, lloyd_inertia, 0.01 * lloyd_inertia); }; @@ -397,7 +398,7 @@ TEST(KmeansSuperKMeans, AlignmentRequirements) for (int64_t i = 0; i < n_rows * dim; ++i) { host[i] = static_cast(std::sin(static_cast(i) * 0.13f)); } - cuvs::cluster::kmeans::superkmeans_params params; + cuvs::cluster::kmeans::experimental::superkmeans_params params; params.n_iters = 3; for (auto offsets : {std::pair{1, 0}, std::pair{0, 1}, std::pair{1, 1}}) { SCOPED_TRACE(sizeof(T)); @@ -411,12 +412,12 @@ TEST(KmeansSuperKMeans, AlignmentRequirements) float inertia = 0; if (dim > 1024) { EXPECT_THROW( - cuvs::cluster::kmeans::fit_predict( + cuvs::cluster::kmeans::experimental::fit_predict( handle, params, X, C, labels.view(), raft::make_host_scalar_view(&inertia)), raft::logic_error); continue; } - cuvs::cluster::kmeans::fit_predict( + cuvs::cluster::kmeans::experimental::fit_predict( handle, params, X, C, labels.view(), raft::make_host_scalar_view(&inertia)); std::vector host_centers(n_clusters * dim); std::vector host_labels(n_rows); @@ -444,7 +445,7 @@ TEST(KmeansSuperKMeans, AlignmentRequirements) TEST(KmeansSuperKMeans, UntiledDimensionFallback) { raft::handle_t handle; - cuvs::cluster::kmeans::superkmeans_params params; + cuvs::cluster::kmeans::experimental::superkmeans_params params; params.n_iters = 3; auto X_odd = raft::make_device_matrix(handle, 1000, 384); auto centroids = raft::make_device_matrix(handle, 300, 384); @@ -453,35 +454,35 @@ TEST(KmeansSuperKMeans, UntiledDimensionFallback) X_odd.size() * sizeof(float), raft::resource::get_cuda_stream(handle).get())); float inertia = -1; - cuvs::cluster::kmeans::fit(handle, - params, - raft::make_const_mdspan(X_odd.view()), - centroids.view(), - raft::make_host_scalar_view(&inertia)); + cuvs::cluster::kmeans::experimental::fit(handle, + params, + raft::make_const_mdspan(X_odd.view()), + centroids.view(), + raft::make_host_scalar_view(&inertia)); EXPECT_EQ(inertia, 0); } TEST(KmeansSuperKMeans, RejectsUnsupported) { raft::handle_t handle; - cuvs::cluster::kmeans::superkmeans_params params; + cuvs::cluster::kmeans::experimental::superkmeans_params params; auto X = raft::make_device_matrix(handle, 1000, 256); auto small_centroids = raft::make_device_matrix(handle, 8, 256); params.metric = cuvs::distance::DistanceType::InnerProduct; - EXPECT_THROW(cuvs::cluster::kmeans::fit( + EXPECT_THROW(cuvs::cluster::kmeans::experimental::fit( handle, params, raft::make_const_mdspan(X.view()), small_centroids.view()), raft::logic_error); params.metric = cuvs::distance::DistanceType::L2Expanded; auto X_odd = raft::make_device_matrix(handle, 1000, 1152); auto C_odd = raft::make_device_matrix(handle, 300, 1152); - EXPECT_THROW( - cuvs::cluster::kmeans::fit(handle, params, raft::make_const_mdspan(X_odd.view()), C_odd.view()), - raft::logic_error); + EXPECT_THROW(cuvs::cluster::kmeans::experimental::fit( + handle, params, raft::make_const_mdspan(X_odd.view()), C_odd.view()), + raft::logic_error); auto too_many = raft::make_device_matrix(handle, 1001, 256); - EXPECT_THROW( - cuvs::cluster::kmeans::fit(handle, params, raft::make_const_mdspan(X.view()), too_many.view()), - raft::logic_error); + EXPECT_THROW(cuvs::cluster::kmeans::experimental::fit( + handle, params, raft::make_const_mdspan(X.view()), too_many.view()), + raft::logic_error); } } // namespace cuvs diff --git a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md index ff5f851852..1dc0df18ef 100644 --- a/fern/pages/cpp_api/cpp-api-cluster-kmeans.md +++ b/fern/pages/cpp_api/cpp-api-cluster-kmeans.md @@ -114,37 +114,6 @@ enum class balanced_donor_selection { | `SizeSorted` | `0` | | `Random` | `1` | - -### cluster::kmeans::superkmeans_params - -Simple object to specify hyper-parameters to the SuperKMeans algorithm. - -SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every row with the fused 1-NN over all dimensions; later iterations compute partial distances over the first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the ADSampling bound, without its random rotation), and finish the remaining dimensions only for the survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: the dimension count must be divisible by 256, dataset and centroid pointers must be 16-byte aligned, and the build and device must provide the pruning kernel. Inputs that violate these requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every iteration uses full nearest-centroid assignment (Lloyd's algorithm). - -The number of clusters is taken from the `centroids` argument. The start centroids are rows sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run (there is no convergence test), and an empty cluster keeps its previous centroid. - -Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 tensor cores for both float and half input; surviving candidates are refined in FP32 over subsequent 256-dimension tiles, with further pruning between tiles. - -Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional inertia are computed with a full assignment against the final centroids. - -The following metrics are currently supported: - -- L2Expanded - -```cpp -struct superkmeans_params : base_params { - uint32_t n_iters; - raft::random::RngState rng_state; -}; -``` - -**Fields** - -| Name | Type | Description | -| --- | --- | --- | -| `n_iters` | `uint32_t` | Number of training iterations (default 10). | -| `rng_state` | `raft::random::RngState` | Seed to the random number generator (the start rows); default seed 0. | - ## k-means clustering APIs @@ -480,60 +449,6 @@ std::optional> inertia = std::nullopt); `void` -**Additional overload:** `cluster::kmeans::fit` - -Find clusters with the SuperKMeans algorithm. - -```cpp -void fit(const raft::resources& handle, -cuvs::cluster::kmeans::superkmeans_params const& params, -raft::device_matrix_view X, -raft::device_matrix_view centroids, -std::optional> inertia = std::nullopt); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `handle` | in | `const raft::resources&` | The raft handle. | -| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | -| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | -| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | -| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | - -**Returns** - -`void` - -**Additional overload:** `cluster::kmeans::fit` - -Find clusters with the SuperKMeans algorithm. - -```cpp -void fit(const raft::resources& handle, -cuvs::cluster::kmeans::superkmeans_params const& params, -raft::device_matrix_view X, -raft::device_matrix_view centroids, -std::optional> inertia = std::nullopt); -``` - -With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the centroids, which are accumulated and returned in FP32. - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `handle` | in | `const raft::resources&` | The raft handle. | -| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | -| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | -| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | -| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | - -**Returns** - -`void` - ### cluster::kmeans::predict @@ -821,58 +736,6 @@ raft::device_vector_view labels); `void` -**Additional overload:** `cluster::kmeans::predict` - -Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. - -```cpp -void predict(const raft::resources& handle, -cuvs::cluster::kmeans::superkmeans_params const& params, -raft::device_matrix_view X, -raft::device_matrix_view centroids, -raft::device_vector_view labels); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `handle` | in | `const raft::resources&` | The raft handle. | -| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | -| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | -| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | -| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | - -**Returns** - -`void` - -**Additional overload:** `cluster::kmeans::predict` - -Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. - -```cpp -void predict(const raft::resources& handle, -cuvs::cluster::kmeans::superkmeans_params const& params, -raft::device_matrix_view X, -raft::device_matrix_view centroids, -raft::device_vector_view labels); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `handle` | in | `const raft::resources&` | The raft handle. | -| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | -| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | -| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | -| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | - -**Returns** - -`void` - ### cluster::kmeans::fit_predict @@ -1054,62 +917,6 @@ raft::device_vector_view labels); `void` -**Additional overload:** `cluster::kmeans::fit_predict` - -Compute SuperKMeans clusters and predict the cluster index of each sample. - -```cpp -void fit_predict(const raft::resources& handle, -cuvs::cluster::kmeans::superkmeans_params const& params, -raft::device_matrix_view X, -raft::device_matrix_view centroids, -raft::device_vector_view labels, -std::optional> inertia = std::nullopt); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `handle` | in | `const raft::resources&` | The raft handle. | -| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | -| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | -| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | -| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | -| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | - -**Returns** - -`void` - -**Additional overload:** `cluster::kmeans::fit_predict` - -Compute SuperKMeans clusters and predict the cluster index of each sample. - -```cpp -void fit_predict(const raft::resources& handle, -cuvs::cluster::kmeans::superkmeans_params const& params, -raft::device_matrix_view X, -raft::device_matrix_view centroids, -raft::device_vector_view labels, -std::optional> inertia = std::nullopt); -``` - -**Parameters** - -| Name | Direction | Type | Description | -| --- | --- | --- | --- | -| `handle` | in | `const raft::resources&` | The raft handle. | -| `params` | in | [`cuvs::cluster::kmeans::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-superkmeans-params) | Parameters for SuperKMeans. | -| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | -| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | -| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | -| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | - -**Returns** - -`void` - ### cluster::kmeans::transform @@ -1503,3 +1310,201 @@ float tol = 1e-3); **Returns** `void` + +## Experimental SuperKMeans APIs + + +### cluster::kmeans::experimental::superkmeans_params + +Simple object to specify hyper-parameters to the SuperKMeans algorithm. + +SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every row with the fused 1-NN over all dimensions; later iterations compute partial distances over the first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the ADSampling bound, without its random rotation), and finish the remaining dimensions only for the survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required: row strides must be divisible by 16 bytes, dataset and centroid pointers must be 16-byte aligned, and the build and device must provide the pruning kernel. Inputs that violate these requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every iteration uses full nearest-centroid assignment (Lloyd's algorithm). + +The number of clusters is taken from the `centroids` argument. The start centroids are rows sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run (there is no convergence test), and an empty cluster keeps its previous centroid. + +Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16 tensor cores for both float and half input; surviving candidates are refined in FP32 over subsequent tiles of up to 256 dimensions, with further pruning between tiles. + +Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional inertia are computed with a full assignment against the final centroids. + +The following metrics are currently supported: + +- L2Expanded + +```cpp +struct superkmeans_params : base_params { + uint32_t n_iters; + raft::random::RngState rng_state; +}; +``` + +**Fields** + +| Name | Type | Description | +| --- | --- | --- | +| `n_iters` | `uint32_t` | Number of training iterations (default 10). | +| `rng_state` | `raft::random::RngState` | Seed to the random number generator (the start rows); default seed 0. | + + +### cluster::kmeans::experimental::fit + +Find clusters with the SuperKMeans algorithm. + +```cpp +void fit(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::experimental::fit` + +Find clusters with the SuperKMeans algorithm. + +```cpp +void fit(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +std::optional> inertia = std::nullopt); +``` + +With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the centroids, which are accumulated and returned in FP32. + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + + +### cluster::kmeans::experimental::predict + +Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + +```cpp +void predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | +| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::experimental::predict` + +Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids. + +```cpp +void predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | New data to predict. [dim = n_samples x n_features] | +| `centroids` | in | `raft::device_matrix_view` | Cluster centroids. The data must be in row-major format. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | + +**Returns** + +`void` + + +### cluster::kmeans::experimental::fit_predict + +Compute SuperKMeans clusters and predict the cluster index of each sample. + +```cpp +void fit_predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` + +**Additional overload:** `cluster::kmeans::experimental::fit_predict` + +Compute SuperKMeans clusters and predict the cluster index of each sample. + +```cpp +void fit_predict(const raft::resources& handle, +cuvs::cluster::kmeans::experimental::superkmeans_params const& params, +raft::device_matrix_view X, +raft::device_matrix_view centroids, +raft::device_vector_view labels, +std::optional> inertia = std::nullopt); +``` + +**Parameters** + +| Name | Direction | Type | Description | +| --- | --- | --- | --- | +| `handle` | in | `const raft::resources&` | The raft handle. | +| `params` | in | [`cuvs::cluster::kmeans::experimental::superkmeans_params const&`](/api-reference/cpp-api-cluster-kmeans#cluster-kmeans-experimental-superkmeans-params) | Parameters for SuperKMeans. | +| `X` | in | `raft::device_matrix_view` | Training instances to cluster. The data must be in row-major format. [dim = n_samples x n_features] | +| `centroids` | out | `raft::device_matrix_view` | The generated centroids. [dim = n_clusters x n_features] | +| `labels` | out | `raft::device_vector_view` | Index of the closest centroid for each sample. [len = n_samples] | +| `inertia` | out | `std::optional>` | Sum of squared distances of samples to their closest cluster center.
Default: `std::nullopt`. | + +**Returns** + +`void` From 0a70f8cf3c3420b5af7209e2c2d8238adcdaeeff Mon Sep 17 00:00:00 2001 From: sherylll Date: Tue, 17 Nov 2026 18:29:54 -0800 Subject: [PATCH 35/39] Use RAFT warp primitives in pruned assignment --- .../distance/detail/pruned_top_1_nn/prune.cuh | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 13a5161ee6..0ac6a5d9a0 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -12,6 +12,7 @@ #pragma once #include +#include #include #include #include @@ -221,8 +222,7 @@ __device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / kW float partial[N]; #pragma unroll for (uint32_t i = 0; i < N; i++) { - const T* centroid = - centroids + static_cast(__shfl_sync(kFullMask, ids, first + i)) * stride; + const T* centroid = centroids + static_cast(raft::shfl(ids, first + i)) * stride; T centroid_tile[Dims / kWarpSize]; load_tile(centroid, centroid_tile); partial[i] = 0; @@ -235,7 +235,7 @@ __device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / kW // Lane i ends with survivor first + i's sum; move it to that survivor's lane using batched_reduce = cub::WarpReduceBatched; typename batched_reduce::TempStorage temp_storage; - const float sum = __shfl_sync(kFullMask, batched_reduce{temp_storage}.Sum(partial), lane - first); + const float sum = raft::shfl(batched_reduce{temp_storage}.Sum(partial), lane - first); if (lane >= first && lane < first + N) { distance += sum; } } @@ -364,17 +364,11 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, return; } - for (int offset = kWarpSize / 2; offset > 0; offset >>= 1) { - const uint32_t other_index = __shfl_down_sync(kFullMask, best_index, offset); - const float other = __shfl_down_sync(kFullMask, best, offset); - if (other < best) { - best = other; - best_index = other_index; - } - } - if (lane == 0 && best_index != UINT32_MAX) { - labels[row] = best_index; - dist[row] = best; + const auto nearest = + raft::warpReduce(raft::KeyValuePair{best_index, best}, raft::argmin_op{}); + if (lane == 0 && nearest.key != UINT32_MAX) { + labels[row] = nearest.key; + dist[row] = nearest.value; } } From 572253e5983c469bac2b917996f67fa40819430d Mon Sep 17 00:00:00 2001 From: sherylll Date: Tue, 17 Nov 2026 19:35:46 -0800 Subject: [PATCH 36/39] Support partial dimension tiles in pruned assignment --- cpp/src/cluster/detail/superkmeans.cuh | 11 ++-- .../distance/detail/pruned_top_1_nn/prune.cuh | 54 ++++++++++++------- cpp/src/distance/pruned_top_1_nn.cuh | 14 ++--- cpp/tests/cluster/kmeans_superkmeans.cu | 34 +++++++----- 4 files changed, 66 insertions(+), 47 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index daa2b7a86f..1290b76d01 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -36,8 +36,8 @@ namespace cuvs::cluster::kmeans::detail::superkmeans { namespace pruned_nn = cuvs::distance::detail::pruned_nn; -// Pruning works on whole dimension tiles past the first filter's, and only pays off with enough -// centroids to discard; below that every iteration assigns with the full 1-NN. +// Pruning only pays off with enough dimensions and centroids to discard; below these thresholds +// every iteration assigns with the full 1-NN. constexpr int64_t kMinPrunedK = 256; constexpr int64_t kMinPrunedDim = 1024; @@ -228,11 +228,8 @@ void fit(const raft::resources& handle, const bool pruned = prunes(n_clusters, dim); if (pruned) { // Validate before sampling centroids or allocating scratch: eligible inputs must use pruning. - RAFT_EXPECTS( - dim > pruned_nn::kPartialDim && (dim - pruned_nn::kPartialDim) % pruned_nn::kDimTileSize == 0, - "SuperKMeans pruning needs a positive tail after %u dims in tiles of %u", - pruned_nn::kPartialDim, - pruned_nn::kDimTileSize); + RAFT_EXPECTS(dim % (pruned_nn::kAlignment / sizeof(T)) == 0, + "SuperKMeans pruning needs row strides divisible by 16 bytes"); const auto aligned = [](const void* ptr) { return reinterpret_cast(ptr) % pruned_nn::kAlignment == 0; }; diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 0ac6a5d9a0..a0420f5206 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -46,8 +46,9 @@ constexpr uint32_t kSurvivorTileSize = 16; // fused_filter_kernel.py's ID_SHIFT constexpr int kSurvivorIdShift = 32; -constexpr uint32_t kWarpSize = 32; -constexpr uint32_t kFullMask = 0xffffffff; +constexpr std::size_t kAlignment = 16; +constexpr uint32_t kWarpSize = 32; +constexpr uint32_t kFullMask = 0xffffffff; /** * ADSampling's pruning ratio for a partial distance over the first i of dim dims: @@ -184,19 +185,31 @@ struct first_filter_lists { }; /** - * One dim tile of a vector (16-byte aligned) into the warp's registers, in 16-byte loads: - * lane l holds dims (32 j + l) * V + [0, V) at tile[j * V + [0, V)], V = 16 / sizeof(T). + * One dim tile in 16-byte loads. Non-256-aligned dimensions still have a whole-vector tail; + * zero-fill vectors past its end. Aligned instantiations omit the bounds checks. */ -template -__device__ __forceinline__ void load_tile(const T* __restrict__ values, T (&tile)[Dims / kWarpSize]) +template +__device__ __forceinline__ void load_tile(const T* __restrict__ values, + uint32_t valid_dims, + T (&tile)[Dims / kWarpSize]) { constexpr uint32_t kVec = 16 / sizeof(T); static_assert(Dims % (kWarpSize * kVec) == 0, "a dim tile is whole 16-byte loads per lane"); const uint32_t lane = threadIdx.x % kWarpSize; #pragma unroll for (uint32_t j = 0; j < Dims / (kWarpSize * kVec); j++) { + const uint32_t offset = (j * kWarpSize + lane) * kVec; + if constexpr (!Aligned) { + if (offset >= valid_dims) { +#pragma unroll + for (uint32_t k = 0; k < kVec; k++) { + tile[j * kVec + k] = T{0}; + } + continue; + } + } raft::TxN_t v; - v.load(values, (j * kWarpSize + lane) * kVec); + v.load(values, offset); #pragma unroll for (uint32_t k = 0; k < kVec; k++) { tile[j * kVec + k] = v.val.data[k]; @@ -210,10 +223,11 @@ __device__ __forceinline__ void load_tile(const T* __restrict__ values, T (&tile * holding each survivor adds its distance to `distance`. The N centroids' loads are issued back to * back. */ -template +template __device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / kWarpSize], const T* __restrict__ centroids, uint32_t stride, + uint32_t valid_dims, uint32_t ids, uint32_t first, float& distance) @@ -224,7 +238,7 @@ __device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / kW for (uint32_t i = 0; i < N; i++) { const T* centroid = centroids + static_cast(raft::shfl(ids, first + i)) * stride; T centroid_tile[Dims / kWarpSize]; - load_tile(centroid, centroid_tile); + load_tile(centroid, valid_dims, centroid_tile); partial[i] = 0; #pragma unroll for (uint32_t j = 0; j < Dims / kWarpSize; j++) { @@ -263,7 +277,7 @@ __device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) * copied to shared memory; the warp reads it a window of kWarpSize entries at a time, lane l * holding entry l, and each re-filter compacts it in place. */ -template +template RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const uint32_t* __restrict__ row_order, const T* __restrict__ centroids, @@ -296,7 +310,7 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, // The re-filter's ratio const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); T row_tile[kDimTileSize / kWarpSize]; - load_tile(row_values + d, row_tile); + load_tile(row_values + d, dim - d, row_tile); uint32_t kept = 0; for (uint32_t base = 0; base < n_left; base += kWarpSize) { const uint32_t w = min(kWarpSize, n_left - base); @@ -308,7 +322,8 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, for_each_halving([&](auto tile) { constexpr uint32_t kTile = decltype(tile)::value; for (; i + kTile <= w; i += kTile) { - add_tile_distances(row_tile, centroids + d, dim, ids, i, distance); + add_tile_distances( + row_tile, centroids + d, dim, dim - d, ids, i, distance); } }); if (!last) { @@ -345,8 +360,8 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); T row_tile[kDimTileSize / kWarpSize]; T centroid_tile[kDimTileSize / kWarpSize]; - load_tile(row_values + d, row_tile); - load_tile(centroid + d, centroid_tile); + load_tile(row_values + d, dim - d, row_tile); + load_tile(centroid + d, dim - d, centroid_tile); float partial = 0.f; #pragma unroll for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { @@ -390,14 +405,15 @@ void prune(const raft::resources& handle, uint32_t* labels, float* dist) { - RAFT_EXPECTS(dim > kPartialDim && (dim - kPartialDim) % kDimTileSize == 0, - "Pruned 1-NN needs a positive tail after %u dims in tiles of %u", - kPartialDim, - kDimTileSize); + RAFT_EXPECTS(dim > kPartialDim, "Pruned 1-NN needs a positive tail after %u dims", kPartialDim); + RAFT_EXPECTS(dim % (kAlignment / sizeof(T)) == 0, + "Pruned 1-NN needs row strides divisible by 16 bytes"); + // Whole dimension tiles keep the vectorized kernel free of tail checks. + const auto kernel = dim % kDimTileSize == 0 ? prune_kernel : prune_kernel; raft::launch_kernel(handle, n_rows, kWarpSize, - prune_kernel, + kernel, rows, row_order, centroids, diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 4a83479292..61c8339587 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -38,8 +38,7 @@ namespace cuvs::distance { namespace detail::pruned_nn { -constexpr int64_t kMaxStreams = 2; -constexpr std::size_t kAlignment = 16; +constexpr int64_t kMaxStreams = 2; /** The fused first filter's tile on this device, or nullopt when the build or device has none. */ inline auto filter_tile() -> std::optional @@ -170,13 +169,11 @@ inline void order_rows_by_survivors(raft::resources const& handle, /** * Whether pruned_top_1_nn runs for k features on this build and device: it needs the fused * filter's cuTile kernel, the first detail::pruned_nn::kPartialDim dims for filtering, and a - * positive tail in whole detail::pruned_nn::kDimTileSize tiles to prune on. + * positive tail to prune on. */ inline bool is_pruned_top_1_nn_available(int64_t k) { - return k > detail::pruned_nn::kPartialDim && - (k - detail::pruned_nn::kPartialDim) % detail::pruned_nn::kDimTileSize == 0 && - detail::pruned_nn::filter_tile().has_value(); + return k > detail::pruned_nn::kPartialDim && detail::pruned_nn::filter_tile().has_value(); } /** The workspace pruned_top_1_nn needs for m rows on `handle` (the same handle as the call). */ @@ -247,9 +244,8 @@ void pruned_top_1_nn(raft::resources const& handle, constexpr bool kHalf = std::is_same_v; RAFT_EXPECTS(is_pruned_top_1_nn_available(k), "Pruned 1-NN needs the fused filter's cuTile kernel and a positive tail after %u " - "dims in tiles of %u", - kPartialDim, - kDimTileSize); + "dims", + kPartialDim); RAFT_EXPECTS(m > 0 && n > 0, "Pruned 1-NN needs rows in x and y"); RAFT_EXPECTS(kHalf || y_fp16 != nullptr, "Pruned 1-NN on float needs an FP16 copy of y's first dims"); diff --git a/cpp/tests/cluster/kmeans_superkmeans.cu b/cpp/tests/cluster/kmeans_superkmeans.cu index abb2171e95..f16637df86 100644 --- a/cpp/tests/cluster/kmeans_superkmeans.cu +++ b/cpp/tests/cluster/kmeans_superkmeans.cu @@ -24,6 +24,7 @@ #include +#include #include #include #include @@ -35,9 +36,11 @@ namespace cuvs { namespace { -constexpr int64_t kRows = 4000; -constexpr int64_t kDim = 200; // below one dimension tile: no pruning -constexpr int64_t kClusters = 300; +constexpr int64_t kRows = 4000; +constexpr int64_t kDim = 200; // below one dimension tile: no pruning +constexpr int64_t kClusters = 300; +constexpr int64_t kPrunedDim = 1536; +constexpr int64_t kRowAlignmentBytes = 16; double sq_dist(const float* a, const float* b, int64_t dim) { @@ -295,16 +298,17 @@ TEST(KmeansSuperKMeans, Fallback) } } -/* With more than 256 clusters and more than 1024 dims, a multiple of 256, the iterations after - * the first prune on a build and device with the filter's kernels. The - * pruning bound is a heuristic without a rotation, so the result is compared with Lloyd's from the - * same start rows rather than exactly. The labels and inertia come from a full assignment. +/* Exercise pruned assignment with whole dimension tiles and a partial tail. The pruning bound + * is a heuristic without a rotation, so compare with Lloyd's from the same start rows rather than + * exactly. The labels and inertia come from a full assignment. */ -TEST(KmeansSuperKMeans, Pruned) +class KmeansSuperKMeansPruned : public ::testing::TestWithParam {}; + +TEST_P(KmeansSuperKMeansPruned, Pruned) { // Multiple row and centroid batches, with partial final batches and several tail dimension tiles constexpr int64_t n_rows = 300000; - constexpr int64_t dim = 1536; + const int64_t dim = GetParam(); constexpr int64_t n_clusters = 2200; constexpr uint32_t n_iters = 8; constexpr uint64_t seed = 42; @@ -383,6 +387,11 @@ TEST(KmeansSuperKMeans, Pruned) } } +INSTANTIATE_TEST_SUITE_P(DimensionTails, + KmeansSuperKMeansPruned, + ::testing::Values(kPrunedDim, + kPrunedDim + kRowAlignmentBytes / sizeof(half))); + TEST(KmeansSuperKMeans, AlignmentRequirements) { constexpr int64_t n_rows = 1024, n_clusters = 300; @@ -473,9 +482,10 @@ TEST(KmeansSuperKMeans, RejectsUnsupported) handle, params, raft::make_const_mdspan(X.view()), small_centroids.view()), raft::logic_error); - params.metric = cuvs::distance::DistanceType::L2Expanded; - auto X_odd = raft::make_device_matrix(handle, 1000, 1152); - auto C_odd = raft::make_device_matrix(handle, 300, 1152); + params.metric = cuvs::distance::DistanceType::L2Expanded; + constexpr int64_t invalid_stride_dim = kPrunedDim + 1; + auto X_odd = raft::make_device_matrix(handle, 1000, invalid_stride_dim); + auto C_odd = raft::make_device_matrix(handle, 300, invalid_stride_dim); EXPECT_THROW(cuvs::cluster::kmeans::experimental::fit( handle, params, raft::make_const_mdspan(X_odd.view()), C_odd.view()), raft::logic_error); From d2713a5dcb88a7e3b6f930e6559732ce05ef71ba Mon Sep 17 00:00:00 2001 From: sherylll Date: Tue, 17 Nov 2026 19:45:48 -0800 Subject: [PATCH 37/39] Remove redundant initial survivor check in prune --- cpp/src/distance/detail/pruned_top_1_nn/prune.cuh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index a0420f5206..77af8ea7f8 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -292,10 +292,8 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const uint32_t lanes_below = (1u << lane) - 1; const int64_t row = row_order[blockIdx.x]; uint32_t n_left = static_cast(lists.count[row]); - // Nothing survived: nothing else to read or write - if (n_left == 0) { return; } - const T* row_values = rows + row * dim; - uint32_t best_index = UINT32_MAX; // No improving candidate yet. + const T* row_values = rows + row * dim; + uint32_t best_index = UINT32_MAX; // No improving candidate yet. // The row's distance to its current centroid until the last tile's argmin float best = dist[row]; From 8cb98b7c330083c8e522f10979772d3ee4a00fdc Mon Sep 17 00:00:00 2001 From: sherylll Date: Tue, 17 Nov 2026 22:27:11 -0800 Subject: [PATCH 38/39] Simplify prune survivor arguments and use RAFT warp helpers --- .../distance/detail/pruned_top_1_nn/prune.cuh | 99 +++++++++---------- cpp/src/distance/pruned_top_1_nn.cuh | 3 +- 2 files changed, 47 insertions(+), 55 deletions(-) diff --git a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh index 77af8ea7f8..38e1e9e944 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh +++ b/cpp/src/distance/detail/pruned_top_1_nn/prune.cuh @@ -47,8 +47,6 @@ constexpr uint32_t kSurvivorTileSize = 16; constexpr int kSurvivorIdShift = 32; constexpr std::size_t kAlignment = 16; -constexpr uint32_t kWarpSize = 32; -constexpr uint32_t kFullMask = 0xffffffff; /** * ADSampling's pruning ratio for a partial distance over the first i of dim dims: @@ -78,11 +76,12 @@ RAFT_KERNEL squared_norms_kernel(const T* __restrict__ rows, Copy copy, int64_t copy_cols) { - const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; + const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / raft::warp_size(); if (row >= n) { return; } float s = 0.f; constexpr bool kCopy = !std::is_same_v; - for (int64_t j = threadIdx.x % kWarpSize; j < (kCopy ? copy_cols : cols); j += kWarpSize) { + for (int64_t j = threadIdx.x % raft::warp_size(); j < (kCopy ? copy_cols : cols); + j += raft::warp_size()) { float v = static_cast(rows[row * stride + j]); if constexpr (kCopy) { const half h = static_cast(v); @@ -92,7 +91,7 @@ RAFT_KERNEL squared_norms_kernel(const T* __restrict__ rows, if (j < cols) { s += v * v; } } s = raft::warpReduce(s); - if (threadIdx.x % kWarpSize == 0) { out[row] = s; } + if (threadIdx.x % raft::warp_size() == 0) { out[row] = s; } } template @@ -106,7 +105,7 @@ void squared_norms(const raft::resources& handle, int64_t copy_cols = 0) { constexpr uint32_t kThreads = 256; - const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); + const auto blocks = raft::ceildiv(n * raft::warp_size(), kThreads); raft::launch_kernel(handle, blocks, kThreads, @@ -131,12 +130,12 @@ RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, float* __restrict__ dist, half* __restrict__ filter_rows) { - const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / kWarpSize; + const int64_t row = (blockIdx.x * int64_t{blockDim.x} + threadIdx.x) / raft::warp_size(); if (row >= n) { return; } const T* row_data = rows + row * dim; const C* assigned_centroid = centroids + static_cast(labels[row]) * dim; float distance = 0.f; - for (int64_t j = threadIdx.x % kWarpSize; j < dim; j += kWarpSize) { + for (int64_t j = threadIdx.x % raft::warp_size(); j < dim; j += raft::warp_size()) { const float value = static_cast(row_data[j]); const float diff = value - static_cast(assigned_centroid[j]); distance += diff * diff; @@ -145,7 +144,7 @@ RAFT_KERNEL distance_to_assigned_and_copy_kernel(const T* __restrict__ rows, } } distance = raft::warpReduce(distance); - if (threadIdx.x % kWarpSize == 0) { dist[row] = distance; } + if (threadIdx.x % raft::warp_size() == 0) { dist[row] = distance; } } template @@ -159,7 +158,7 @@ void distance_to_assigned(const raft::resources& handle, half* filter_rows = nullptr) { constexpr uint32_t kThreads = 256; - const auto blocks = raft::ceildiv(n * kWarpSize, kThreads); + const auto blocks = raft::ceildiv(n * raft::warp_size(), kThreads); raft::launch_kernel(handle, blocks, kThreads, @@ -173,17 +172,6 @@ void distance_to_assigned(const raft::resources& handle, filter_rows); } -/** - * The first filter's survivor lists of one row batch x centroid batch (see launch_fused_filter): - * row r's survivors at packed[r * row_stride], count[r] of them. The pruning kernel compacts them - * in place. - */ -struct first_filter_lists { - int64_t* packed; - const int32_t* count; - uint32_t row_stride; -}; - /** * One dim tile in 16-byte loads. Non-256-aligned dimensions still have a whole-vector tail; * zero-fill vectors past its end. Aligned instantiations omit the bounds checks. @@ -191,14 +179,15 @@ struct first_filter_lists { template __device__ __forceinline__ void load_tile(const T* __restrict__ values, uint32_t valid_dims, - T (&tile)[Dims / kWarpSize]) + T (&tile)[Dims / raft::warp_size()]) { constexpr uint32_t kVec = 16 / sizeof(T); - static_assert(Dims % (kWarpSize * kVec) == 0, "a dim tile is whole 16-byte loads per lane"); - const uint32_t lane = threadIdx.x % kWarpSize; + static_assert(Dims % (raft::warp_size() * kVec) == 0, + "a dim tile is whole 16-byte loads per lane"); + const uint32_t lane = threadIdx.x % raft::warp_size(); #pragma unroll - for (uint32_t j = 0; j < Dims / (kWarpSize * kVec); j++) { - const uint32_t offset = (j * kWarpSize + lane) * kVec; + for (uint32_t j = 0; j < Dims / (raft::warp_size() * kVec); j++) { + const uint32_t offset = (j * raft::warp_size() + lane) * kVec; if constexpr (!Aligned) { if (offset >= valid_dims) { #pragma unroll @@ -224,7 +213,7 @@ __device__ __forceinline__ void load_tile(const T* __restrict__ values, * back. */ template -__device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / kWarpSize], +__device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / raft::warp_size()], const T* __restrict__ centroids, uint32_t stride, uint32_t valid_dims, @@ -232,16 +221,16 @@ __device__ __forceinline__ void add_tile_distances(const T (&row_tile)[Dims / kW uint32_t first, float& distance) { - const uint32_t lane = threadIdx.x % kWarpSize; + const uint32_t lane = threadIdx.x % raft::warp_size(); float partial[N]; #pragma unroll for (uint32_t i = 0; i < N; i++) { const T* centroid = centroids + static_cast(raft::shfl(ids, first + i)) * stride; - T centroid_tile[Dims / kWarpSize]; + T centroid_tile[Dims / raft::warp_size()]; load_tile(centroid, valid_dims, centroid_tile); partial[i] = 0; #pragma unroll - for (uint32_t j = 0; j < Dims / kWarpSize; j++) { + for (uint32_t j = 0; j < Dims / raft::warp_size(); j++) { const float diff = static_cast(row_tile[j]) - static_cast(centroid_tile[j]); partial[i] = fmaf(diff, diff, partial[i]); } @@ -273,9 +262,9 @@ __device__ __forceinline__ int64_t pack_survivor(uint32_t id, float distance) * each but the last, then the argmin, kept if closer than the row's current centroid. Block b * handles row row_order[b], ordered using this tile's first-filter survivor counts. * - * The survivor list stays in the row's region of `lists`, but for its first kWarpSize entries, - * copied to shared memory; the warp reads it a window of kWarpSize entries at a time, lane l - * holding entry l, and each re-filter compacts it in place. + * Row r has count[r] survivors at packed[r * kCentroidBatchSize]. The first warp's entries are + * copied to shared memory; the warp reads one window of entries at a time, lane l holding entry l, + * and each re-filter compacts the list in place. */ template RAFT_KERNEL prune_kernel(const T* __restrict__ rows, @@ -283,21 +272,22 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const T* __restrict__ centroids, uint32_t centroid_offset, uint32_t dim, - first_filter_lists lists, + int64_t* __restrict__ packed, + const int32_t* __restrict__ count, uint32_t* __restrict__ labels, float* __restrict__ dist) { - __shared__ int64_t head[kWarpSize]; + __shared__ int64_t head[raft::warp_size()]; const uint32_t lane = threadIdx.x; const uint32_t lanes_below = (1u << lane) - 1; const int64_t row = row_order[blockIdx.x]; - uint32_t n_left = static_cast(lists.count[row]); + uint32_t n_left = static_cast(count[row]); const T* row_values = rows + row * dim; uint32_t best_index = UINT32_MAX; // No improving candidate yet. // The row's distance to its current centroid until the last tile's argmin float best = dist[row]; - int64_t* list = lists.packed + row * lists.row_stride; + int64_t* list = packed + row * kCentroidBatchSize; if (lane < n_left) { head[lane] = list[lane]; } __syncwarp(); @@ -307,11 +297,11 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, const bool last = d + kDimTileSize >= dim; // The re-filter's ratio const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); - T row_tile[kDimTileSize / kWarpSize]; + T row_tile[kDimTileSize / raft::warp_size()]; load_tile(row_values + d, dim - d, row_tile); uint32_t kept = 0; - for (uint32_t base = 0; base < n_left; base += kWarpSize) { - const uint32_t w = min(kWarpSize, n_left - base); + for (uint32_t base = 0; base < n_left; base += raft::warp_size()) { + const uint32_t w = min(uint32_t{raft::warp_size()}, n_left - base); const int64_t entry = lane >= w ? 0 : base == 0 ? head[lane] : list[base + lane]; const auto ids = static_cast(entry >> kSurvivorIdShift); float distance = __int_as_float(static_cast(entry)); @@ -327,10 +317,10 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, if (!last) { // The kept survivors go below this window's end, never into a later window const bool keep = lane < w && distance < best * ratio; - const uint32_t ballot = __ballot_sync(kFullMask, keep); + const uint32_t ballot = __ballot_sync(raft::warp_full_mask(), keep); if (keep) { - const uint32_t slot = kept + __popc(ballot & lanes_below); - (slot < kWarpSize ? head[slot] : list[slot]) = pack_survivor(ids, distance); + const uint32_t slot = kept + __popc(ballot & lanes_below); + (slot < raft::warp_size() ? head[slot] : list[slot]) = pack_survivor(ids, distance); } kept += __popc(ballot); } else if (lane < w && distance < best) { // after the last tile the distances are full @@ -354,20 +344,19 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, float distance = __int_as_float(static_cast(entry)); const T* centroid = centroids + static_cast(centroid_id) * dim; for (; d < dim; d += kDimTileSize) { - const bool last = d + kDimTileSize >= dim; - const float ratio = last ? 0.0f : adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); - T row_tile[kDimTileSize / kWarpSize]; - T centroid_tile[kDimTileSize / kWarpSize]; + T row_tile[kDimTileSize / raft::warp_size()]; + T centroid_tile[kDimTileSize / raft::warp_size()]; load_tile(row_values + d, dim - d, row_tile); load_tile(centroid + d, dim - d, centroid_tile); float partial = 0.f; #pragma unroll - for (uint32_t j = 0; j < kDimTileSize / kWarpSize; j++) { + for (uint32_t j = 0; j < kDimTileSize / raft::warp_size(); j++) { const float diff = static_cast(row_tile[j]) - static_cast(centroid_tile[j]); partial = fmaf(diff, diff, partial); } distance += raft::warpReduce(partial); - if (!last) { + if (d + kDimTileSize < dim) { + const float ratio = adsampling_ratio(d + kDimTileSize, dim, kAdsamplingEps0); if (!(distance < best * ratio)) { break; } } else if (lane == 0 && distance < best) { labels[row] = centroid_offset + centroid_id; @@ -387,8 +376,8 @@ RAFT_KERNEL prune_kernel(const T* __restrict__ rows, /** * The pruning kernel for n_rows rows against one centroid batch (its first centroid - * centroid_offset), after the fused first filter wrote `lists`. `labels` and `dist` hold each - * row's current centroid and distance, and get the closer survivor if there is one. + * centroid_offset), after the fused first filter wrote `packed` and `count`. `labels` and `dist` + * hold each row's current centroid and distance, and get the closer survivor if there is one. * row_order: the rows' order. */ template @@ -399,7 +388,8 @@ void prune(const raft::resources& handle, const T* centroids, uint32_t centroid_offset, int64_t dim, - first_filter_lists lists, + int64_t* packed, + const int32_t* count, uint32_t* labels, float* dist) { @@ -410,14 +400,15 @@ void prune(const raft::resources& handle, const auto kernel = dim % kDimTileSize == 0 ? prune_kernel : prune_kernel; raft::launch_kernel(handle, n_rows, - kWarpSize, + raft::warp_size(), kernel, rows, row_order, centroids, centroid_offset, static_cast(dim), - lists, + packed, + count, labels, dist); } diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index 61c8339587..f746bef5e1 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -320,7 +320,8 @@ void pruned_top_1_nn(raft::resources const& handle, y + j * k, static_cast(j), k, - first_filter_lists{packed, count, kCentroidBatchSize}, + packed, + count, row_labels, row_dist); } From 29970b19334d0124338450d8a39466de91eb8d0d Mon Sep 17 00:00:00 2001 From: sherylll Date: Wed, 18 Nov 2026 16:49:38 -0800 Subject: [PATCH 39/39] Use workspace memory and dry-run sizing in SuperKMeans --- cpp/src/cluster/detail/superkmeans.cuh | 123 +++++++++++++----- .../cutile/fused_filter_tile.cu | 80 ++++++------ .../cutile/fused_filter_tile.hpp | 10 +- cpp/src/distance/pruned_top_1_nn.cuh | 7 +- 4 files changed, 136 insertions(+), 84 deletions(-) diff --git a/cpp/src/cluster/detail/superkmeans.cuh b/cpp/src/cluster/detail/superkmeans.cuh index 1290b76d01..a14699fb9c 100644 --- a/cpp/src/cluster/detail/superkmeans.cuh +++ b/cpp/src/cluster/detail/superkmeans.cuh @@ -19,16 +19,23 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include #include +#include + #include #include +#include +#include #include #include @@ -46,28 +53,6 @@ inline bool prunes(int64_t n_clusters, int64_t dim) return dim > kMinPrunedDim && n_clusters > kMinPrunedK; } -/** Budget the per-stream survivor lists, counts, row order, and optional FP16 filter rows. */ -template -auto calc_minibatch_size(const raft::resources& handle, int64_t n_rows) -> int64_t -{ - std::size_t mem_per_row = - pruned_nn::kCentroidBatchSize * sizeof(int64_t) + sizeof(int32_t) + sizeof(uint32_t); - if constexpr (!std::is_same_v) { mem_per_row += pruned_nn::kPartialDim * sizeof(half); } - const auto pool_size = raft::resource::is_stream_pool_initialized(handle) - ? static_cast(raft::resource::get_stream_pool_size(handle)) - : 0; - const auto n_streams = std::min({1 + pool_size, pruned_nn::kMaxStreams, n_rows}); - const auto free_ws_size = raft::resource::get_workspace_free_bytes(handle); - const auto available_ws_size = - std::min((free_ws_size / std::size_t{10}) * std::size_t{8}, std::size_t{1} << 29); - // Prefer whole 1K-row batches; under tight budgets retain float-buffer alignment. - constexpr std::size_t kAlignedRows = pruned_nn::kAlignment / sizeof(float); - auto batch_size = std::max(kAlignedRows, available_ws_size / (n_streams * mem_per_row)); - const auto row_multiple = batch_size >= 1024 ? std::size_t{1024} : kAlignedRows; - batch_size = batch_size / row_multiple * row_multiple; - return static_cast(std::min(batch_size, static_cast(n_rows))); -} - /** Sum of squared distances of the rows to their assigned centroids. */ template auto inertia(const raft::resources& handle, @@ -75,7 +60,9 @@ auto inertia(const raft::resources& handle, raft::device_matrix_view centroids, raft::device_vector_view labels) -> float { - auto dist = raft::make_device_vector(handle, X.extent(0)); + auto mr = raft::resource::get_workspace_resource_ref(handle); + auto dist = + raft::make_device_mdarray(handle, mr, raft::make_extents(X.extent(0))); cudaStream_t stream = raft::resource::get_cuda_stream(handle).get(); pruned_nn::distance_to_assigned(handle, X.data_handle(), @@ -84,7 +71,7 @@ auto inertia(const raft::resources& handle, X.extent(1), labels.data_handle(), dist.data_handle()); - auto sum = raft::make_device_scalar(handle, 0.0f); + auto sum = raft::make_device_mdarray(handle, mr, raft::make_extents()); raft::linalg::mapThenSumReduce( sum.data_handle(), dist.size(), raft::identity_op{}, stream, dist.data_handle()); float out = 0; @@ -138,20 +125,42 @@ class pruned_assignment { pruned_assignment(const raft::resources& handle, raft::device_matrix_view X, int64_t n_clusters) + : pruned_assignment(handle, X, n_clusters, select_batch_size(handle, X, n_clusters)) + { + } + + pruned_assignment(const raft::resources& handle, + raft::device_matrix_view X, + int64_t n_clusters, + int64_t row_batch_size) : handle_(handle), X_(X), n_rows_(X.extent(0)), dim_(X.extent(1)), n_clusters_(n_clusters), filter_dim_(std::is_same_v ? dim_ : pruned_nn::kPartialDim), - row_norms_(raft::make_device_vector(handle, n_rows_)), - centroid_norms_(raft::make_device_vector(handle, n_clusters)), - dist_(raft::make_device_vector(handle, n_rows_)), - filter_centroids_(raft::make_device_matrix(handle, n_clusters, filter_dim_)), - row_batch_size_(calc_minibatch_size(handle, n_rows_)), - workspace_(raft::make_device_vector( + row_norms_(raft::make_device_mdarray( handle, - cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, row_batch_size_))) + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_rows_))), + centroid_norms_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_clusters))), + dist_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_rows_))), + filter_centroids_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents(n_clusters, filter_dim_))), + row_batch_size_(row_batch_size), + workspace_(raft::make_device_mdarray( + handle, + raft::resource::get_workspace_resource_ref(handle), + raft::make_extents( + cuvs::distance::pruned_top_1_nn_workspace_size(handle, n_rows_, row_batch_size_)))) { // The input rows do not change between iterations, so cache their filter norms once. pruned_nn::squared_norms( @@ -194,6 +203,43 @@ class pruned_assignment { } private: + static auto select_batch_size(const raft::resources& handle, + raft::device_matrix_view X, + int64_t n_clusters) -> int64_t + { + const auto n_rows = X.extent(0); + const auto budget = std::min( + (raft::resource::get_workspace_free_bytes(handle) / 10) * 8, std::size_t{1} << 29); + const auto fits = [&](int64_t batch_size) { + try { + // Only workspace is used here; keep the probe local rather than swapping global resources. + raft::mr::dry_run_resource probe{ + raft::resource::get_workspace_resource_ref(handle)}; + raft::resources dry_handle(handle); + dry_handle.add_resource_factory( + std::make_shared(true)); + raft::resource::set_workspace_resource( + dry_handle, probe, std::numeric_limits::max()); + pruned_assignment candidate(dry_handle, X, n_clusters, batch_size); + return raft::resource::get_workspace_used_bytes(dry_handle) <= budget; + } catch (const rmm::bad_alloc&) { + return false; + } + }; + + constexpr int64_t alignment = pruned_nn::kAlignment / sizeof(float); + const auto min_batch_size = std::min(n_rows, alignment); + auto batch_size = n_rows; + while (!fits(batch_size)) { + RAFT_EXPECTS(batch_size > min_batch_size, + "Insufficient workspace for SuperKMeans pruned assignment"); + const auto half_batch = batch_size / 2; + const int64_t row_multiple = half_batch >= 1024 ? 1024 : alignment; + batch_size = std::max(min_batch_size, half_batch / row_multiple * row_multiple); + } + return batch_size; + } + const raft::resources& handle_; raft::device_matrix_view X_; int64_t n_rows_, dim_, n_clusters_, filter_dim_; @@ -239,6 +285,7 @@ void fit(const raft::resources& handle, "SuperKMeans pruning kernel is unavailable on this build or device"); } + auto mr = raft::resource::get_workspace_resource_ref(handle); auto stream = raft::resource::get_cuda_stream(handle); auto mapping_op = cuvs::spatial::knn::detail::utils::mapping{}; @@ -247,15 +294,19 @@ void fit(const raft::resources& handle, if constexpr (std::is_same_v) { raft::matrix::sample_rows(handle, random_state, X, centroids); } else { - auto start = raft::make_device_matrix(handle, n_clusters, dim); + auto start = raft::make_device_mdarray( + handle, mr, raft::make_extents(n_clusters, dim)); raft::matrix::sample_rows(handle, random_state, X, start.view()); raft::linalg::map(handle, centroids, mapping_op, raft::make_const_mdspan(start.view())); } - auto assigned = raft::make_device_vector(handle, n_rows); - auto sizes = raft::make_device_vector(handle, n_clusters); - auto means = raft::make_device_matrix(handle, n_clusters, dim); - auto assign = [&]() { + auto assigned = + raft::make_device_mdarray(handle, mr, raft::make_extents(n_rows)); + auto sizes = raft::make_device_mdarray( + handle, mr, raft::make_extents(n_clusters)); + auto means = raft::make_device_mdarray( + handle, mr, raft::make_extents(n_clusters, dim)); + auto assign = [&]() { predict(handle, params, X, raft::make_const_mdspan(centroids), assigned.view()); }; diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu index bd3b655592..61abcaa9fd 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu @@ -12,6 +12,7 @@ #include #include +#include #include @@ -56,7 +57,8 @@ auto fused_filter_tile_config() -> std::optional return fused_filter_tile{cfg.tile_m, cfg.tile_n, cfg.tile_k}; } -void launch_fused_filter(const half* rows, +void launch_fused_filter(const raft::resources& handle, + const half* rows, int n_rows, const half* centroids, int n_centroids, @@ -68,8 +70,7 @@ void launch_fused_filter(const half* rows, int partial_dim, int64_t* packed, int* count, - int row_stride, - cudaStream_t stream) + int row_stride) { FusedFilterTilePlanner planner; auto launcher = planner.try_get_launcher(); @@ -145,43 +146,42 @@ void launch_fused_filter(const half* rows, dim3 grid((n_rows + tile.tile_m - 1) / tile.tile_m, 1, 1); dim3 block(1, 1, 1); // Each array as (pointer, shape..., stride...), then the scalars; the tile sizes are compiled in. - launcher->template dispatch(stream, - grid, - block, - 0, - rows_ptr, - int{n_rows}, - int{dim}, - int{dim}, - int{unit}, - centroids_ptr, - int{n_centroids}, - int{dim}, - int{dim}, - int{unit}, - row_norms_ptr, - int{n_rows}, - int{unit}, - centroid_norms_ptr, - int{n_centroids}, - int{unit}, - best_ptr, - int{n_rows}, - int{unit}, - packed_ptr, - int{packed_len}, - int{unit}, - count_ptr, - int{n_rows}, - int{unit}, - int{unit}, - int{unit}, - int{n_rows}, - int{n_centroids}, - float{ratio}, - int{partial_dim}, - int{row_stride}); - RAFT_CUDA_TRY(cudaGetLastError()); + raft::launch_kernel(handle, + grid, + block, + raft::kernel_ref{launcher->get_kernel()}, + rows_ptr, + int{n_rows}, + int{dim}, + int{dim}, + int{unit}, + centroids_ptr, + int{n_centroids}, + int{dim}, + int{dim}, + int{unit}, + row_norms_ptr, + int{n_rows}, + int{unit}, + centroid_norms_ptr, + int{n_centroids}, + int{unit}, + best_ptr, + int{n_rows}, + int{unit}, + packed_ptr, + int{packed_len}, + int{unit}, + count_ptr, + int{n_rows}, + int{unit}, + int{unit}, + int{unit}, + int{n_rows}, + int{n_centroids}, + float{ratio}, + int{partial_dim}, + int{row_stride}); } } // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp index eae4c594fd..3fa26e7720 100644 --- a/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp +++ b/cpp/src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.hpp @@ -8,6 +8,8 @@ #include #include +#include + #include #include @@ -32,6 +34,7 @@ auto fused_filter_tile_config() -> std::optional; * Row r's survivors go to `packed` at r * row_stride + 0 .. count[r] - 1, each * `centroid << 32 | float bits of the partial distance`, in centroid order. * + * @param handle RAFT resources providing the stream and dry-run state * @param rows [n_rows, dim] row-major, 16-byte aligned, 16-byte row pitch * @param centroids [n_centroids, dim] same layout as rows * @param row_norms [n_rows] squared norms over the first partial_dim dims @@ -42,9 +45,9 @@ auto fused_filter_tile_config() -> std::optional; * @param packed [n_rows * row_stride] survivors * @param count [n_rows] survivors per row * @param row_stride each row's region in `packed`, at least n_centroids - * @param stream the stream to launch on */ -void launch_fused_filter(const half* rows, +void launch_fused_filter(const raft::resources& handle, + const half* rows, int n_rows, const half* centroids, int n_centroids, @@ -56,7 +59,6 @@ void launch_fused_filter(const half* rows, int partial_dim, int64_t* packed, int* count, - int row_stride, - cudaStream_t stream); + int row_stride); } // namespace cuvs::distance::detail::pruned_nn diff --git a/cpp/src/distance/pruned_top_1_nn.cuh b/cpp/src/distance/pruned_top_1_nn.cuh index f746bef5e1..ef6edb7d3f 100644 --- a/cpp/src/distance/pruned_top_1_nn.cuh +++ b/cpp/src/distance/pruned_top_1_nn.cuh @@ -276,7 +276,6 @@ void pruned_top_1_nn(raft::resources const& handle, const int64_t n_rows = std::min(layout.row_batch_size, m - i); const int64_t s = b % layout.n_streams; const auto& batch_handle = stream_handles[s]; - auto stream = raft::resource::get_cuda_stream(batch_handle).get(); const DataT* rows = x + i * k; uint32_t* row_labels = output.nearest_idx + i; float* row_dist = output.nearest_dist + i; @@ -298,7 +297,8 @@ void pruned_top_1_nn(raft::resources const& handle, } for (int64_t j = 0; j < n; j += kCentroidBatchSize) { const int64_t n_centroids = std::min(kCentroidBatchSize, n - j); - launch_fused_filter(x_filter, + launch_fused_filter(batch_handle, + x_filter, static_cast(n_rows), y_filter + j * filter_dim, static_cast(n_centroids), @@ -310,8 +310,7 @@ void pruned_top_1_nn(raft::resources const& handle, static_cast(kPartialDim), packed, count, - static_cast(kCentroidBatchSize), - stream); + static_cast(kCentroidBatchSize)); order_rows_by_survivors(batch_handle, count, order, n_rows, layout.max_blocks_per_sm); prune(batch_handle, rows,