Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 31 additions & 58 deletions cpp/src/cluster/detail/kmeans.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ void cluster_cost(
rmm::device_uvector<char>& workspace,
int batch_samples,
int batch_centroids,
std::optional<raft::device_vector_view<const DataT, IndexT>> sample_weight = std::nullopt)
std::optional<raft::device_vector_view<const DataT, IndexT>> sample_weight = std::nullopt,
std::optional<raft::device_vector_view<const DataT, IndexT>> tf32_norms = std::nullopt)
{
auto n_samples = static_cast<IndexT>(X.extent(0));
RAFT_EXPECTS(norms.extent(0) >= n_samples, "Cached norms must cover the input rows");
Expand All @@ -90,7 +91,7 @@ void cluster_cost(
batch_samples,
batch_centroids,
workspace,
std::nullopt,
tf32_norms,
/*accurate_distances=*/true);

if (sample_weight.has_value()) {
Expand Down Expand Up @@ -1151,13 +1152,26 @@ void kmeans_fit(

auto batch_data_view = raft::make_device_matrix_view<const DataT, IndexT>(
data_batch.data(), cur_batch_size, n_features);
const bool batch_uses_tf32 =
data_on_device ? device_uses_tf32
: assignment_uses_tf32(
data_batch.data(), cur_batch_size, centroids_const.data_handle());
if constexpr (!data_on_device) {
if (need_compute_norms) { compute_batch_norms(data_batch.data(), cur_batch_size, false); }
if (need_compute_norms) {
compute_batch_norms(data_batch.data(), cur_batch_size, batch_uses_tf32);
}
}
const auto l2_norm_offset =
data_on_device ? static_cast<IndexT>(data_batch.offset()) : IndexT{0};
auto l2_const_view = raft::make_device_vector_view<const DataT, IndexT>(
L2NormBatch.data_handle() + l2_norm_offset, cur_batch_size);
std::optional<raft::device_vector_view<const DataT, IndexT>> tf32_const_view = std::nullopt;
if constexpr (kmeans_may_require_tf32_norms_v<DataT>) {
if (batch_uses_tf32) {
tf32_const_view = raft::make_device_vector_view<const DataT, IndexT>(
Tf32NormBatch.data_handle() + l2_norm_offset, cur_batch_size);
}
}
std::optional<raft::device_vector_view<const DataT, IndexT>> batch_sw = std::nullopt;
if (weight_ptr != nullptr) {
batch_sw =
Expand All @@ -1178,7 +1192,8 @@ void kmeans_fit(
ws,
iter_params.batch_samples,
iter_params.batch_centroids,
batch_sw);
batch_sw,
tf32_const_view);
raft::linalg::add(clustering_cost.data_handle(),
clustering_cost.data_handle(),
batch_cost.data_handle(),
Expand Down Expand Up @@ -1280,28 +1295,25 @@ void kmeans_predict(raft::resources const& handle,
"invalid parameter (centroids.extent(1) != n_features)");

raft::default_logger().set_level(pams.verbosity);
auto metric = pams.metric;

// Allocate memory
// Device-accessible allocation of expandable storage used as temporary buffers
rmm::device_uvector<char> workspace(0, stream);
auto weight = raft::make_device_vector<DataT, IndexT>(handle, n_samples);
if (sample_weight.has_value())
raft::copy(handle, weight.view(), sample_weight.value());
else
raft::matrix::fill(handle, weight.view(), DataT(1));

if (normalize_weight && sample_weight.has_value()) {
const DataT* weight_ptr = sample_weight.has_value() ? sample_weight->data_handle() : nullptr;
auto normalized_weight =
raft::make_device_vector<DataT, IndexT>(handle, normalize_weight && weight_ptr ? n_samples : 0);
if (normalize_weight && weight_ptr != nullptr) {
auto d_wt_sum = raft::make_device_scalar<DataT>(handle, DataT{0});
weightSum(handle, raft::make_const_mdspan(weight.view()), d_wt_sum.view(), true);
weightSum(handle, sample_weight.value(), d_wt_sum.view(), true);
const DataT* d_wt_sum_ptr = d_wt_sum.data_handle();
raft::linalg::map(
handle,
weight.view(),
normalized_weight.view(),
[n_samples, d_wt_sum_ptr] __device__(DataT w) {
return w * static_cast<DataT>(n_samples) / *d_wt_sum_ptr;
},
raft::make_const_mdspan(weight.view()));
sample_weight.value());
weight_ptr = normalized_weight.data_handle();
}

// Optional CAGRA-based assignment: the `Top1nnBackend::Cagra` backend over an index of the
Expand All @@ -1316,58 +1328,18 @@ void kmeans_predict(raft::resources const& handle,
rmm::device_uvector<char> assignment_output(0, stream);
rmm::device_uvector<DataT> L2NormBuf_OR_DistBuf(0, stream);

auto L2NormX = raft::make_device_vector<DataT, IndexT>(handle, n_samples);
auto Tf32NormX = raft::make_device_vector<DataT, IndexT>(
handle, kmeans_may_require_tf32_norms_v<DataT> ? n_samples : IndexT{0});
std::optional<raft::device_vector_view<const DataT, IndexT>> tf32_normx_view = std::nullopt;
const bool needs_norms = !use_ann && (metric == cuvs::distance::DistanceType::L2Expanded ||
metric == cuvs::distance::DistanceType::L2SqrtExpanded ||
metric == cuvs::distance::DistanceType::CosineExpanded);
if (needs_norms) {
bool use_tf32_norms = false;
if constexpr (kmeans_may_require_tf32_norms_v<DataT>) {
const auto plan = probe_kmeans_top_1_nn(handle,
X.data_handle(),
centroids.data_handle(),
n_samples,
static_cast<IndexT>(centroids.extent(0)),
n_features,
metric,
pams.batch_samples,
pams.batch_centroids,
cuvs::distance::detail::Top1nnBackend::Auto);
use_tf32_norms =
plan.available && plan.norm_policy == cuvs::distance::detail::Top1nnNormPolicy::Tf32;
if (use_tf32_norms) {
compute_fp32_tf32_norms(handle,
X,
L2NormX.view(),
Tf32NormX.view(),
metric == cuvs::distance::DistanceType::CosineExpanded);
tf32_normx_view = raft::make_const_mdspan(Tf32NormX.view());
} else {
raft::linalg::norm<raft::linalg::L2Norm, raft::Apply::ALONG_ROWS>(
handle, X, L2NormX.view());
}
} else {
raft::linalg::norm<raft::linalg::L2Norm, raft::Apply::ALONG_ROWS>(handle, X, L2NormX.view());
}
}

auto l2normx_view =
raft::make_device_vector_view<const DataT, IndexT>(L2NormX.data_handle(), n_samples);
const auto result = cuvs::cluster::kmeans::detail::minClusterAndDistanceCompute<DataT, IndexT>(
handle,
X,
centroids,
assignment_output,
l2normx_view,
raft::make_device_vector_view<const DataT, IndexT>(nullptr, IndexT{0}), // norms computed there
L2NormBuf_OR_DistBuf,
pams.metric,
pams.batch_samples,
pams.batch_centroids,
workspace,
tf32_normx_view,
std::nullopt,
use_ann ? cuvs::distance::detail::Top1nnBackend::Cagra
: cuvs::distance::detail::Top1nnBackend::Auto,
use_ann ? &ann_index : nullptr);
Expand All @@ -1378,7 +1350,8 @@ void kmeans_predict(raft::resources const& handle,
cuvs::cluster::kmeans::detail::weightAndComputeClusterCost(
handle,
result,
raft::make_const_mdspan(weight.view()),
raft::make_device_vector_view<const DataT, IndexT>(weight_ptr,
weight_ptr == nullptr ? 0 : n_samples),
pams.metric,
workspace,
raft::make_device_scalar_view(clusterCostD.data()));
Expand Down
85 changes: 40 additions & 45 deletions cpp/src/cluster/detail/kmeans_balanced.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,19 @@ auto make_bitwise_expanded_iterator(const uint8_t* packed_data)
return thrust::make_transform_iterator(counting_iter, decoder);
}

/** Buffers for predict_core, reused across calls so each minibatch does not reallocate them. */
template <typename MathT>
struct predict_core_workspace {
predict_core_workspace(cuda::stream_ref stream, rmm::device_async_resource_ref mr)
: norms_or_distances(0, stream, mr), workspace(0, stream, mr), output(0, stream, mr)
{
}

rmm::device_uvector<MathT> norms_or_distances;
rmm::device_uvector<char> workspace;
rmm::device_uvector<char> output;
};

/**
* @brief Predict labels for the dataset; floating-point types only.
*
Expand All @@ -117,6 +130,7 @@ auto make_bitwise_expanded_iterator(const uint8_t* packed_data)
* @param[in] n_rows Number samples in the `dataset`
* @param[out] labels Output predictions [n_rows]
* @param[inout] mr (optional) Memory resource to use for temporary allocations
* @param[inout] scratch Buffers reused across calls
*/
template <typename MathT, typename IdxT, typename LabelT>
inline std::enable_if_t<std::is_floating_point_v<MathT>> predict_core(
Expand All @@ -129,34 +143,35 @@ inline std::enable_if_t<std::is_floating_point_v<MathT>> predict_core(
const MathT* dataset_norm,
IdxT n_rows,
LabelT* labels,
rmm::device_async_resource_ref mr)
rmm::device_async_resource_ref mr,
predict_core_workspace<MathT>& scratch)
{
auto stream = raft::resource::get_cuda_stream(handle);
switch (params.metric) {
case cuvs::distance::DistanceType::L2Expanded:
case cuvs::distance::DistanceType::L2SqrtExpanded:
case cuvs::distance::DistanceType::CosineExpanded: {
rmm::device_uvector<MathT> L2NormBuf_OR_DistBuf(0, stream, mr);
rmm::device_uvector<char> workspace(0, stream, mr);

auto X_view = raft::make_device_matrix_view<const MathT, IdxT>(dataset, n_rows, dim);
auto centroids_view =
raft::make_device_matrix_view<const MathT, IdxT>(centers, n_clusters, dim);
auto X_norm_view = raft::make_device_vector_view<const MathT, IdxT>(dataset_norm, n_rows);

rmm::device_uvector<char> assignment_output(0, stream, mr);
auto X_norm_view = raft::make_device_vector_view<const MathT, IdxT>(
dataset_norm, dataset_norm == nullptr ? IdxT{0} : n_rows);

const auto result = cuvs::cluster::kmeans::detail::minClusterAndDistanceCompute<MathT, IdxT>(
handle,
X_view,
centroids_view,
assignment_output,
scratch.output,
X_norm_view,
L2NormBuf_OR_DistBuf,
scratch.norms_or_distances,
params.metric,
0, // default top_1_nn row tuning
0, // default top_1_nn candidate tuning
workspace);
scratch.workspace,
std::nullopt,
cuvs::distance::detail::Top1nnBackend::Auto,
nullptr,
/*need_compute_distance=*/false);

cuvs::cluster::kmeans::detail::copyClusterLabels(handle, result, labels);
break;
Expand Down Expand Up @@ -696,14 +711,14 @@ void predict(const raft::resources& handle,
rmm::device_uvector<MathT> cur_dataset(
std::is_same_v<T, MathT> ? 0 : max_minibatch_size * transformed_dim, stream, mem_res);
constexpr bool native_half = std::is_same_v<T, half> && std::is_same_v<MathT, float>;
bool need_norm =
dataset_norm == nullptr && (params.metric == cuvs::distance::DistanceType::L2Expanded ||
params.metric == cuvs::distance::DistanceType::L2SqrtExpanded ||
params.metric == cuvs::distance::DistanceType::CosineExpanded);
bool need_compute_norm = need_norm && !params.is_packed_binary;
// Packed binary rows decode to +/-1, so every row norm is the decoded dimension: filled once.
const bool packed_norms = dataset_norm == nullptr && params.is_packed_binary &&
(params.metric == cuvs::distance::DistanceType::L2Expanded ||
params.metric == cuvs::distance::DistanceType::L2SqrtExpanded ||
params.metric == cuvs::distance::DistanceType::CosineExpanded);
rmm::device_uvector<MathT> cur_dataset_norm(
need_norm || native_half ? max_minibatch_size : 0, stream, mem_res);
if (need_norm && params.is_packed_binary) {
packed_norms || native_half ? max_minibatch_size : 0, stream, mem_res);
if (packed_norms) {
raft::matrix::fill(
handle,
raft::make_device_matrix_view<MathT, IdxT>(cur_dataset_norm.data(), max_minibatch_size, 1),
Expand All @@ -716,9 +731,10 @@ void predict(const raft::resources& handle,
native_half_scratch.emplace(
native_centers_size, n_clusters, max_minibatch_size, stream, mem_res);
}
const MathT* dataset_norm_ptr =
need_norm && params.is_packed_binary ? cur_dataset_norm.data() : nullptr;
auto cur_dataset_ptr = cur_dataset.data();

// Buffers reused by every minibatch, instead of allocated per predict_core call.
predict_core_workspace<MathT> core_scratch(stream, mem_res);
for (IdxT offset = 0; offset < n_rows; offset += max_minibatch_size) {
IdxT minibatch_size = std::min<IdxT>(max_minibatch_size, n_rows - offset);

Expand Down Expand Up @@ -756,31 +772,9 @@ void predict(const raft::resources& handle,
mapping_op);
}

// Compute the norm now if it hasn't been pre-computed.
if (need_compute_norm) {
if (params.metric == cuvs::distance::DistanceType::CosineExpanded)
compute_norm(handle,
cur_dataset_norm.data(),
cur_dataset_ptr,
dim,
minibatch_size,
mapping_op,
raft::sqrt_op{},
mr);
else
compute_norm(handle,
cur_dataset_norm.data(),
cur_dataset_ptr,
dim,
minibatch_size,
mapping_op,
raft::identity_op{},
mr);
dataset_norm_ptr = cur_dataset_norm.data();
} else if (dataset_norm != nullptr) {
dataset_norm_ptr = dataset_norm + offset;
}

const MathT* dataset_norm_ptr = dataset_norm != nullptr ? dataset_norm + offset
: packed_norms ? cur_dataset_norm.data()
: nullptr;
predict_core(handle,
params,
centers,
Expand All @@ -790,7 +784,8 @@ void predict(const raft::resources& handle,
dataset_norm_ptr,
minibatch_size,
labels + offset,
mem_res);
mem_res,
core_scratch);
}
}

Expand Down
12 changes: 7 additions & 5 deletions cpp/src/cluster/detail/kmeans_common.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -110,13 +110,13 @@ struct CopyClusterLabelOp {
template <bool SquareDistance, typename DataT, typename IndexT, typename DistanceIterator>
struct WeightedDistanceOp {
DistanceIterator distances;
const DataT* weights;
const DataT* weights; // nullptr: every weight is effectively 1

__device__ DataT operator()(IndexT offset) const
{
auto distance = static_cast<DataT>(distances[offset]);
if constexpr (SquareDistance) { distance *= distance; }
return distance * weights[offset];
return weights == nullptr ? distance : distance * weights[offset];
}
};

Expand Down Expand Up @@ -489,7 +489,7 @@ MinClusterAndDistanceResult<DataT, IndexT> minClusterAndDistanceCompute(
raft::device_matrix_view<const DataT, IndexT> X,
raft::device_matrix_view<const DataT, IndexT> centroids,
rmm::device_uvector<char>& output_storage,
raft::device_vector_view<const DataT, IndexT> L2NormX,
raft::device_vector_view<const DataT, IndexT> L2NormX, // if empty, compute here
rmm::device_uvector<DataT>& L2NormBuf_OR_DistBuf,
cuvs::distance::DistanceType metric,
int batch_samples,
Expand All @@ -498,7 +498,8 @@ MinClusterAndDistanceResult<DataT, IndexT> minClusterAndDistanceCompute(
std::optional<raft::device_vector_view<const DataT, IndexT>> Tf32NormX = std::nullopt,
cuvs::distance::detail::Top1nnBackend backend = cuvs::distance::detail::Top1nnBackend::Auto,
// Required (and only used) with Top1nnBackend::Cagra: index built over `centroids`.
const cuvs::distance::detail::top_1_nn_cagra_index* ann_index = nullptr);
const cuvs::distance::detail::top_1_nn_cagra_index* ann_index = nullptr,
bool need_compute_distance = true);

#define EXTERN_TEMPLATE_MIN_CLUSTER_AND_DISTANCE(DataT, IndexT) \
extern template MinClusterAndDistanceResult<DataT, IndexT> \
Expand All @@ -515,7 +516,8 @@ MinClusterAndDistanceResult<DataT, IndexT> minClusterAndDistanceCompute(
rmm::device_uvector<char>&, \
std::optional<raft::device_vector_view<const DataT, IndexT>>, \
cuvs::distance::detail::Top1nnBackend, \
const cuvs::distance::detail::top_1_nn_cagra_index*);
const cuvs::distance::detail::top_1_nn_cagra_index*, \
bool);

EXTERN_TEMPLATE_MIN_CLUSTER_AND_DISTANCE(float, int64_t)
EXTERN_TEMPLATE_MIN_CLUSTER_AND_DISTANCE(float, int)
Expand Down
Loading
Loading