Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
8b35602
Add SuperKMeans k-means API with an unpruned fallback
sherylll Oct 26, 2026
e83a28f
Add SuperKMeans' fused first-filter cuTile kernel
sherylll Oct 26, 2026
19b3fb4
Add SuperKMeans' pruned assignment
sherylll Oct 26, 2026
443bb05
Order SuperKMeans' pruning by survivor count (LPT)
sherylll Oct 26, 2026
669fcc1
Run SuperKMeans' row batches on the handle's stream pool
sherylll Oct 26, 2026
844f2f0
Move SuperKMeans' pruned assignment into a pruned 1-NN distance
sherylll Oct 26, 2026
4bf9ccc
Prune survivor lists in place, out of shared memory
sherylll Oct 28, 2026
18779b1
Write one contiguous first-filter survivor list per row
sherylll Oct 28, 2026
e58ee5c
Prune only with dims past the first filter's; simplify the kernel
sherylll Oct 28, 2026
1edbba5
Return early from pruning rows with no survivors
sherylll Oct 28, 2026
75ea3ab
Overlap row batches on the main stream and one pool stream
sherylll Oct 28, 2026
fca97b1
Fix style: SuperKMeans API reference, formatting, copyright
sherylll Oct 28, 2026
2d869df
Compute the filter's norms from its FP16 values, with the FP16 copy
sherylll Oct 30, 2026
6a2ae57
Compute the pruning ratios from eps0 in the kernel, not from a device…
sherylll Nov 2, 2026
a57a157
Assign in full for half rows the filter can't read in place; document…
sherylll Nov 2, 2026
e7cb068
Load the pruning kernel's tiles in 16-byte loads; drop its launch bounds
sherylll Nov 2, 2026
0ee37a6
Add license
sherylll Nov 3, 2026
e02f9db
Replace custom warp reduce with cub::WarpReduceBatched
sherylll Nov 3, 2026
31b9674
Merge branch 'main' into superkmeans
sherylll Nov 4, 2026
6b3a662
Hardcode eps0
sherylll Nov 5, 2026
f2355d4
Halve pruned 1-NN centroid batch to 1024
sherylll Nov 6, 2026
9565eb9
Merge branch 'main' into superkmeans
sherylll Nov 6, 2026
4760f3b
Merge branch 'main' into superkmeans
sherylll Nov 8, 2026
79bdbba
Prepare SuperKMeans filter rows per batch
sherylll Nov 9, 2026
b3e9476
Require supported layouts for high-dimensional SuperKMeans pruning
sherylll Nov 9, 2026
a71429f
Size SuperKMeans row batches from workspace budget
sherylll Nov 9, 2026
3b2571b
Tune tile sizes for different arch (coarsely)
sherylll Nov 9, 2026
4c2585a
Fix style
sherylll Nov 9, 2026
26372e2
Add sm103 for completeness; update tile sizes based on measurements on
sherylll Nov 10, 2026
0bbf70c
Merge upstream/main into superkmeans
sherylll Nov 11, 2026
78d844a
Remove duplicate kmeans_type definition
sherylll Nov 11, 2026
5a45435
Small refactor
sherylll Nov 11, 2026
976b372
More refactoring: in F32 mode, use F32 data to calculate norm, even t…
sherylll Nov 11, 2026
f92bdc4
Use raft launch_kernel
sherylll Nov 12, 2026
6f9fb0a
Peel single-survivor tail from pruned nearest-neighbor search
sherylll Nov 13, 2026
91ee26a
Order pruning rows from each fused-filter tile
sherylll Nov 13, 2026
7054a3f
Small cleanup
sherylll Nov 16, 2026
3e2964b
Merge branch 'release/26.12' into superkmeans
sherylll Nov 16, 2026
e3a2550
Mark SuperKMeans API experimental
sherylll Nov 18, 2026
0a70f8c
Use RAFT warp primitives in pruned assignment
sherylll Nov 18, 2026
572253e
Support partial dimension tiles in pruned assignment
sherylll Nov 18, 2026
d2713a5
Remove redundant initial survivor check in prune
sherylll Nov 18, 2026
7fa7a8b
Merge branch 'release/26.12' into superkmeans
sherylll Nov 18, 2026
8cb98b7
Simplify prune survivor arguments and use RAFT warp helpers
sherylll Nov 18, 2026
29970b1
Use workspace memory and dry-run sizing in SuperKMeans
sherylll Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 32 additions & 3 deletions cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1241,6 +1241,28 @@ if(NOT BUILD_CPU_ONLY)
"${cutile_fused_1nn_tiles}" @ONLY
)
list(APPEND cutile_fused_1nn_files "${cutile_fused_1nn_tiles}")

set(pruned_1nn_cutile_dir
"${CMAKE_CURRENT_SOURCE_DIR}/src/distance/detail/pruned_top_1_nn/cutile"
)
set(cutile_pruned_1nn_generated_dir
"${CMAKE_CURRENT_BINARY_DIR}/generated_kernels/distance/pruned_1nn/cutile"
)
generate_cutile_kernels(
cutile_pruned_1nn_files
KERNEL_DIR "${pruned_1nn_cutile_dir}"
KERNEL_BASENAME "pruned_1nn_fused_filter"
KERNEL_PYTHON "fused_filter_kernel.py"
EXPORT_SCRIPT "export_fused_filter.py"
OUTPUT_DIRECTORY "${cutile_pruned_1nn_generated_dir}"
MATRIX_JSON_FILE "${pruned_1nn_cutile_dir}/fused_filter_cutile_matrix.json"
PYTHON "${cutile_python}" BIN2C "${cutile_bin2c}"
FRAGMENT_TAG_FORMAT_CUBIN
"cuvs::distance::detail::pruned_nn::fragment_tag_fused_filter_cubin<cuvs::neighbors::detail::tag_@data_abbrev@, cuvs::detail::jit_lto::@arch_tag@>"
FRAGMENT_TAG_HEADER_FILES
"<cuvs/detail/jit_lto/pruned_top_1_nn/fused_filter_fragments.hpp>"
"<cuvs/detail/jit_lto/cutile_arch_tags.hpp>" "<cuvs/detail/jit_lto/common_fragments.hpp>"
)
endif()
# Note that this matrix contains an `arch_includes` placeholder, since we don't currently have a
# way to do an item-wise transform on a list after computing the matrix product and before
Expand Down Expand Up @@ -1463,6 +1485,7 @@ if(NOT BUILD_CPU_ONLY)
src/cluster/kmeans_balanced_fit_predict_int8.cu
src/cluster/kmeans_balanced_predict_int8.cu
src/cluster/kmeans_balanced_predict_uint8.cu
src/cluster/kmeans_superkmeans.cu
src/cluster/kmeans_transform_double.cu
src/cluster/kmeans_transform_float.cu
src/cluster/single_linkage_float.cu
Expand Down Expand Up @@ -1577,7 +1600,10 @@ if(NOT BUILD_CPU_ONLY)
target_sources(
cuvs_objs
PRIVATE ${cutile_fused_1nn_files}
src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu ${cutile_smoke_files}
src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu
${cutile_smoke_files}
${cutile_pruned_1nn_files}
src/distance/detail/pruned_top_1_nn/cutile/fused_filter_tile.cu
)
endif()

Expand Down Expand Up @@ -1626,8 +1652,11 @@ if(NOT BUILD_CPU_ONLY)
PUBLIC "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>"
"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../c/include>"
INTERFACE "$<INSTALL_INTERFACE:include>"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" "${CMAKE_CURRENT_BINARY_DIR}/src"
"${cutile_fused_1nn_generated_dir}" "${cutile_smoke_generated_dir}"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src"
"${CMAKE_CURRENT_BINARY_DIR}/src"
"${cutile_fused_1nn_generated_dir}"
"${cutile_smoke_generated_dir}"
"${cutile_pruned_1nn_generated_dir}"
)

# Endian detection
Expand Down
173 changes: 173 additions & 0 deletions cpp/include/cuvs/cluster/kmeans.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -1987,6 +1987,179 @@ void find_k(raft::resources const& handle,
*/
} // namespace helpers

namespace experimental {

/**
* @defgroup superkmeans Experimental SuperKMeans APIs
* @{
*/

/**
* Simple object to specify hyper-parameters to the SuperKMeans algorithm.
*
* SuperKMeans is Lloyd's k-means with a pruned assignment step: the first iteration assigns every
* row with the fused 1-NN over all dimensions; later iterations compute partial distances over the
* first 256 dimensions, discard centroids that are unlikely to beat a row's current one (the
* ADSampling bound, without its random rotation), and finish the remaining dimensions only for the
* survivors. With more than 256 clusters and more than 1024 dimensions, pruning is required:
* row strides must be divisible by 16 bytes, dataset and centroid pointers must be 16-byte
* aligned, and the build and device must provide the pruning kernel. Inputs that violate these
* requirements raise an error. With at most 256 clusters or at most 1024 dimensions, every
* iteration uses full nearest-centroid assignment (Lloyd's algorithm).
*
* The number of clusters is taken from the `centroids` argument. The start centroids are rows
* sampled at random, as with kmeans::params::InitMethod::Random; all `n_iters` iterations run
* (there is no convergence test), and an empty cluster keeps its previous centroid.
*
* Pruning is approximate and can discard the true nearest centroid. The first filter uses FP16
* tensor cores for both float and half input; surviving candidates are refined in FP32 over
* subsequent tiles of up to 256 dimensions, with further pruning between tiles.
*
* Prediction uses full nearest-centroid assignment without pruning. Returned labels and optional
* inertia are computed with a full assignment against the final centroids.
*
* The following metrics are currently supported:
* - L2Expanded
*/
struct superkmeans_params : base_params {
/**
* Number of training iterations (default 10).
*/
uint32_t n_iters = 10;

/**
* Seed to the random number generator (the start rows); default seed 0.
*/
raft::random::RngState rng_state{0};
};

/**
* @brief Find clusters with the SuperKMeans algorithm.
*
* @param[in] handle The raft handle.
* @param[in] params Parameters for SuperKMeans.
* @param[in] X Training instances to cluster. The data must
* be in row-major format.
* [dim = n_samples x n_features]
* @param[out] centroids The generated centroids.
* [dim = n_clusters x n_features]
* @param[out] inertia Sum of squared distances of samples to their
* closest cluster center.
*/
void fit(const raft::resources& handle,
cuvs::cluster::kmeans::experimental::superkmeans_params const& params,
raft::device_matrix_view<const float, int64_t> X,
raft::device_matrix_view<float, int64_t> centroids,
std::optional<raft::host_scalar_view<float>> inertia = std::nullopt);

/**
* @brief Find clusters with the SuperKMeans algorithm.
*
* With half input the distance GEMMs run on FP16 tensor cores against an FP16 copy of the
* centroids, which are accumulated and returned in FP32.
*
* @param[in] handle The raft handle.
* @param[in] params Parameters for SuperKMeans.
* @param[in] X Training instances to cluster. The data must
* be in row-major format.
* [dim = n_samples x n_features]
* @param[out] centroids The generated centroids.
* [dim = n_clusters x n_features]
* @param[out] inertia Sum of squared distances of samples to their
* closest cluster center.
*/
void fit(const raft::resources& handle,
cuvs::cluster::kmeans::experimental::superkmeans_params const& params,
raft::device_matrix_view<const half, int64_t> X,
raft::device_matrix_view<float, int64_t> centroids,
std::optional<raft::host_scalar_view<float>> inertia = std::nullopt);

/**
* @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids.
*
* @param[in] handle The raft handle.
* @param[in] params Parameters for SuperKMeans.
* @param[in] X New data to predict.
* [dim = n_samples x n_features]
* @param[in] centroids Cluster centroids. The data must be in row-major format.
* [dim = n_clusters x n_features]
* @param[out] labels Index of the closest centroid for each sample.
* [len = n_samples]
*/
void predict(const raft::resources& handle,
cuvs::cluster::kmeans::experimental::superkmeans_params const& params,
raft::device_matrix_view<const float, int64_t> X,
raft::device_matrix_view<const float, int64_t> centroids,
raft::device_vector_view<uint32_t, int64_t> labels);

/**
* @brief Predict the closest cluster each sample in X belongs to, for SuperKMeans centroids.
*
* @param[in] handle The raft handle.
* @param[in] params Parameters for SuperKMeans.
* @param[in] X New data to predict.
* [dim = n_samples x n_features]
* @param[in] centroids Cluster centroids. The data must be in row-major format.
* [dim = n_clusters x n_features]
* @param[out] labels Index of the closest centroid for each sample.
* [len = n_samples]
*/
void predict(const raft::resources& handle,
cuvs::cluster::kmeans::experimental::superkmeans_params const& params,
raft::device_matrix_view<const half, int64_t> X,
raft::device_matrix_view<const float, int64_t> centroids,
raft::device_vector_view<uint32_t, int64_t> labels);

/**
* @brief Compute SuperKMeans clusters and predict the cluster index of each sample.
*
* @param[in] handle The raft handle.
* @param[in] params Parameters for SuperKMeans.
* @param[in] X Training instances to cluster. The data must be
* in row-major format.
* [dim = n_samples x n_features]
* @param[out] centroids The generated centroids.
* [dim = n_clusters x n_features]
* @param[out] labels Index of the closest centroid for each sample.
* [len = n_samples]
* @param[out] inertia Sum of squared distances of samples to their
* closest cluster center.
*/
void fit_predict(const raft::resources& handle,
cuvs::cluster::kmeans::experimental::superkmeans_params const& params,
raft::device_matrix_view<const float, int64_t> X,
raft::device_matrix_view<float, int64_t> centroids,
raft::device_vector_view<uint32_t, int64_t> labels,
std::optional<raft::host_scalar_view<float>> inertia = std::nullopt);

/**
* @brief Compute SuperKMeans clusters and predict the cluster index of each sample.
*
* @param[in] handle The raft handle.
* @param[in] params Parameters for SuperKMeans.
* @param[in] X Training instances to cluster. The data must be
* in row-major format.
* [dim = n_samples x n_features]
* @param[out] centroids The generated centroids.
* [dim = n_clusters x n_features]
* @param[out] labels Index of the closest centroid for each sample.
* [len = n_samples]
* @param[out] inertia Sum of squared distances of samples to their
* closest cluster center.
*/
void fit_predict(const raft::resources& handle,
cuvs::cluster::kmeans::experimental::superkmeans_params const& params,
raft::device_matrix_view<const half, int64_t> X,
raft::device_matrix_view<float, int64_t> centroids,
raft::device_vector_view<uint32_t, int64_t> labels,
std::optional<raft::host_scalar_view<float>> inertia = std::nullopt);

/**
* @}
*/

} // namespace experimental

} // namespace kmeans
} // namespace cluster
} // namespace CUVS_EXPORT cuvs
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

#pragma once

namespace cuvs::distance::detail::pruned_nn {

template <typename DataTag, typename ArchTag>
struct fragment_tag_fused_filter_cubin {
static constexpr int cc_major = ArchTag::cc_major;
static constexpr int cc_minor = ArchTag::cc_minor;
};

} // namespace cuvs::distance::detail::pruned_nn
Loading
Loading