From e94bbe854243eb780b0fefff0f50a8337fafbe13 Mon Sep 17 00:00:00 2001 From: huuanhhuyn Date: Fri, 13 Nov 2026 17:03:06 +0100 Subject: [PATCH] K-means - allocations, norms compute, row batching (#3047) Improve K-means performance towards r27.02 ## Optimizations - Scratch space to avoid re-allocations of the same space across calls / minibatches - Row batching to improve L2 hit rate for cutile 1-NN - Remove unnecessary weight compute when not given instead of filling with ones - Compute just the norms needed by the backend (TF32, FP32, or None) ## Performance evaluation ### H200 image ## B100 image ### A100 image ### RTX Pro 6000 image Authors: - https://github.com/huuanhhuyn Approvers: - Divye Gala (https://github.com/divyegala) URL: https://github.com/NVIDIA/cuvs/pull/3047 (cherry picked from commit 471e6d71b49342472a6a1c03ed2e00c17ffdec55) --- cpp/src/cluster/detail/kmeans.cuh | 89 ++--- cpp/src/cluster/detail/kmeans_balanced.cuh | 85 +++-- cpp/src/cluster/detail/kmeans_common.cuh | 12 +- .../detail/minClusterDistanceCompute.cu | 318 ++++++++++-------- cpp/src/distance/detail/fused_distance_nn.cuh | 8 +- .../cutile/fused_1nn_tile.cu | 112 +++--- cpp/src/distance/fused_distance_nn-inl.cuh | 9 +- 7 files changed, 310 insertions(+), 323 deletions(-) diff --git a/cpp/src/cluster/detail/kmeans.cuh b/cpp/src/cluster/detail/kmeans.cuh index 5a0b189097..b8afcad5ed 100644 --- a/cpp/src/cluster/detail/kmeans.cuh +++ b/cpp/src/cluster/detail/kmeans.cuh @@ -71,7 +71,8 @@ void cluster_cost( rmm::device_uvector& workspace, int batch_samples, int batch_centroids, - std::optional> sample_weight = std::nullopt) + std::optional> sample_weight = std::nullopt, + std::optional> tf32_norms = std::nullopt) { auto n_samples = static_cast(X.extent(0)); RAFT_EXPECTS(norms.extent(0) >= n_samples, "Cached norms must cover the input rows"); @@ -90,7 +91,7 @@ void cluster_cost( batch_samples, batch_centroids, workspace, - std::nullopt, + tf32_norms, /*accurate_distances=*/true); if (sample_weight.has_value()) { @@ -1151,13 +1152,26 @@ void kmeans_fit( auto batch_data_view = raft::make_device_matrix_view( data_batch.data(), cur_batch_size, n_features); + const bool batch_uses_tf32 = + data_on_device ? device_uses_tf32 + : assignment_uses_tf32( + data_batch.data(), cur_batch_size, centroids_const.data_handle()); if constexpr (!data_on_device) { - if (need_compute_norms) { compute_batch_norms(data_batch.data(), cur_batch_size, false); } + if (need_compute_norms) { + compute_batch_norms(data_batch.data(), cur_batch_size, batch_uses_tf32); + } } const auto l2_norm_offset = data_on_device ? static_cast(data_batch.offset()) : IndexT{0}; auto l2_const_view = raft::make_device_vector_view( L2NormBatch.data_handle() + l2_norm_offset, cur_batch_size); + std::optional> tf32_const_view = std::nullopt; + if constexpr (kmeans_may_require_tf32_norms_v) { + if (batch_uses_tf32) { + tf32_const_view = raft::make_device_vector_view( + Tf32NormBatch.data_handle() + l2_norm_offset, cur_batch_size); + } + } std::optional> batch_sw = std::nullopt; if (weight_ptr != nullptr) { batch_sw = @@ -1178,7 +1192,8 @@ void kmeans_fit( ws, iter_params.batch_samples, iter_params.batch_centroids, - batch_sw); + batch_sw, + tf32_const_view); raft::linalg::add(clustering_cost.data_handle(), clustering_cost.data_handle(), batch_cost.data_handle(), @@ -1280,28 +1295,25 @@ void kmeans_predict(raft::resources const& handle, "invalid parameter (centroids.extent(1) != n_features)"); raft::default_logger().set_level(pams.verbosity); - auto metric = pams.metric; // Allocate memory // Device-accessible allocation of expandable storage used as temporary buffers rmm::device_uvector workspace(0, stream); - auto weight = raft::make_device_vector(handle, n_samples); - if (sample_weight.has_value()) - raft::copy(handle, weight.view(), sample_weight.value()); - else - raft::matrix::fill(handle, weight.view(), DataT(1)); - - if (normalize_weight && sample_weight.has_value()) { + const DataT* weight_ptr = sample_weight.has_value() ? sample_weight->data_handle() : nullptr; + auto normalized_weight = + raft::make_device_vector(handle, normalize_weight && weight_ptr ? n_samples : 0); + if (normalize_weight && weight_ptr != nullptr) { auto d_wt_sum = raft::make_device_scalar(handle, DataT{0}); - weightSum(handle, raft::make_const_mdspan(weight.view()), d_wt_sum.view(), true); + weightSum(handle, sample_weight.value(), d_wt_sum.view(), true); const DataT* d_wt_sum_ptr = d_wt_sum.data_handle(); raft::linalg::map( handle, - weight.view(), + normalized_weight.view(), [n_samples, d_wt_sum_ptr] __device__(DataT w) { return w * static_cast(n_samples) / *d_wt_sum_ptr; }, - raft::make_const_mdspan(weight.view())); + sample_weight.value()); + weight_ptr = normalized_weight.data_handle(); } // Optional CAGRA-based assignment: the `Top1nnBackend::Cagra` backend over an index of the @@ -1316,58 +1328,18 @@ void kmeans_predict(raft::resources const& handle, rmm::device_uvector assignment_output(0, stream); rmm::device_uvector L2NormBuf_OR_DistBuf(0, stream); - auto L2NormX = raft::make_device_vector(handle, n_samples); - auto Tf32NormX = raft::make_device_vector( - handle, kmeans_may_require_tf32_norms_v ? n_samples : IndexT{0}); - std::optional> tf32_normx_view = std::nullopt; - const bool needs_norms = !use_ann && (metric == cuvs::distance::DistanceType::L2Expanded || - metric == cuvs::distance::DistanceType::L2SqrtExpanded || - metric == cuvs::distance::DistanceType::CosineExpanded); - if (needs_norms) { - bool use_tf32_norms = false; - if constexpr (kmeans_may_require_tf32_norms_v) { - const auto plan = probe_kmeans_top_1_nn(handle, - X.data_handle(), - centroids.data_handle(), - n_samples, - static_cast(centroids.extent(0)), - n_features, - metric, - pams.batch_samples, - pams.batch_centroids, - cuvs::distance::detail::Top1nnBackend::Auto); - use_tf32_norms = - plan.available && plan.norm_policy == cuvs::distance::detail::Top1nnNormPolicy::Tf32; - if (use_tf32_norms) { - compute_fp32_tf32_norms(handle, - X, - L2NormX.view(), - Tf32NormX.view(), - metric == cuvs::distance::DistanceType::CosineExpanded); - tf32_normx_view = raft::make_const_mdspan(Tf32NormX.view()); - } else { - raft::linalg::norm( - handle, X, L2NormX.view()); - } - } else { - raft::linalg::norm(handle, X, L2NormX.view()); - } - } - - auto l2normx_view = - raft::make_device_vector_view(L2NormX.data_handle(), n_samples); const auto result = cuvs::cluster::kmeans::detail::minClusterAndDistanceCompute( handle, X, centroids, assignment_output, - l2normx_view, + raft::make_device_vector_view(nullptr, IndexT{0}), // norms computed there L2NormBuf_OR_DistBuf, pams.metric, pams.batch_samples, pams.batch_centroids, workspace, - tf32_normx_view, + std::nullopt, use_ann ? cuvs::distance::detail::Top1nnBackend::Cagra : cuvs::distance::detail::Top1nnBackend::Auto, use_ann ? &ann_index : nullptr); @@ -1378,7 +1350,8 @@ void kmeans_predict(raft::resources const& handle, cuvs::cluster::kmeans::detail::weightAndComputeClusterCost( handle, result, - raft::make_const_mdspan(weight.view()), + raft::make_device_vector_view(weight_ptr, + weight_ptr == nullptr ? 0 : n_samples), pams.metric, workspace, raft::make_device_scalar_view(clusterCostD.data())); diff --git a/cpp/src/cluster/detail/kmeans_balanced.cuh b/cpp/src/cluster/detail/kmeans_balanced.cuh index 7b05bef3cc..cf0f81b775 100644 --- a/cpp/src/cluster/detail/kmeans_balanced.cuh +++ b/cpp/src/cluster/detail/kmeans_balanced.cuh @@ -97,6 +97,19 @@ auto make_bitwise_expanded_iterator(const uint8_t* packed_data) return thrust::make_transform_iterator(counting_iter, decoder); } +/** Buffers for predict_core, reused across calls so each minibatch does not reallocate them. */ +template +struct predict_core_workspace { + predict_core_workspace(cuda::stream_ref stream, rmm::device_async_resource_ref mr) + : norms_or_distances(0, stream, mr), workspace(0, stream, mr), output(0, stream, mr) + { + } + + rmm::device_uvector norms_or_distances; + rmm::device_uvector workspace; + rmm::device_uvector output; +}; + /** * @brief Predict labels for the dataset; floating-point types only. * @@ -117,6 +130,7 @@ auto make_bitwise_expanded_iterator(const uint8_t* packed_data) * @param[in] n_rows Number samples in the `dataset` * @param[out] labels Output predictions [n_rows] * @param[inout] mr (optional) Memory resource to use for temporary allocations + * @param[inout] scratch Buffers reused across calls */ template inline std::enable_if_t> predict_core( @@ -129,34 +143,35 @@ inline std::enable_if_t> predict_core( const MathT* dataset_norm, IdxT n_rows, LabelT* labels, - rmm::device_async_resource_ref mr) + rmm::device_async_resource_ref mr, + predict_core_workspace& scratch) { auto stream = raft::resource::get_cuda_stream(handle); switch (params.metric) { case cuvs::distance::DistanceType::L2Expanded: case cuvs::distance::DistanceType::L2SqrtExpanded: case cuvs::distance::DistanceType::CosineExpanded: { - rmm::device_uvector L2NormBuf_OR_DistBuf(0, stream, mr); - rmm::device_uvector workspace(0, stream, mr); - auto X_view = raft::make_device_matrix_view(dataset, n_rows, dim); auto centroids_view = raft::make_device_matrix_view(centers, n_clusters, dim); - auto X_norm_view = raft::make_device_vector_view(dataset_norm, n_rows); - - rmm::device_uvector assignment_output(0, stream, mr); + auto X_norm_view = raft::make_device_vector_view( + dataset_norm, dataset_norm == nullptr ? IdxT{0} : n_rows); const auto result = cuvs::cluster::kmeans::detail::minClusterAndDistanceCompute( handle, X_view, centroids_view, - assignment_output, + scratch.output, X_norm_view, - L2NormBuf_OR_DistBuf, + scratch.norms_or_distances, params.metric, 0, // default top_1_nn row tuning 0, // default top_1_nn candidate tuning - workspace); + scratch.workspace, + std::nullopt, + cuvs::distance::detail::Top1nnBackend::Auto, + nullptr, + /*need_compute_distance=*/false); cuvs::cluster::kmeans::detail::copyClusterLabels(handle, result, labels); break; @@ -696,14 +711,14 @@ void predict(const raft::resources& handle, rmm::device_uvector cur_dataset( std::is_same_v ? 0 : max_minibatch_size * transformed_dim, stream, mem_res); constexpr bool native_half = std::is_same_v && std::is_same_v; - bool need_norm = - dataset_norm == nullptr && (params.metric == cuvs::distance::DistanceType::L2Expanded || - params.metric == cuvs::distance::DistanceType::L2SqrtExpanded || - params.metric == cuvs::distance::DistanceType::CosineExpanded); - bool need_compute_norm = need_norm && !params.is_packed_binary; + // Packed binary rows decode to +/-1, so every row norm is the decoded dimension: filled once. + const bool packed_norms = dataset_norm == nullptr && params.is_packed_binary && + (params.metric == cuvs::distance::DistanceType::L2Expanded || + params.metric == cuvs::distance::DistanceType::L2SqrtExpanded || + params.metric == cuvs::distance::DistanceType::CosineExpanded); rmm::device_uvector cur_dataset_norm( - need_norm || native_half ? max_minibatch_size : 0, stream, mem_res); - if (need_norm && params.is_packed_binary) { + packed_norms || native_half ? max_minibatch_size : 0, stream, mem_res); + if (packed_norms) { raft::matrix::fill( handle, raft::make_device_matrix_view(cur_dataset_norm.data(), max_minibatch_size, 1), @@ -716,9 +731,10 @@ void predict(const raft::resources& handle, native_half_scratch.emplace( native_centers_size, n_clusters, max_minibatch_size, stream, mem_res); } - const MathT* dataset_norm_ptr = - need_norm && params.is_packed_binary ? cur_dataset_norm.data() : nullptr; auto cur_dataset_ptr = cur_dataset.data(); + + // Buffers reused by every minibatch, instead of allocated per predict_core call. + predict_core_workspace core_scratch(stream, mem_res); for (IdxT offset = 0; offset < n_rows; offset += max_minibatch_size) { IdxT minibatch_size = std::min(max_minibatch_size, n_rows - offset); @@ -756,31 +772,9 @@ void predict(const raft::resources& handle, mapping_op); } - // Compute the norm now if it hasn't been pre-computed. - if (need_compute_norm) { - if (params.metric == cuvs::distance::DistanceType::CosineExpanded) - compute_norm(handle, - cur_dataset_norm.data(), - cur_dataset_ptr, - dim, - minibatch_size, - mapping_op, - raft::sqrt_op{}, - mr); - else - compute_norm(handle, - cur_dataset_norm.data(), - cur_dataset_ptr, - dim, - minibatch_size, - mapping_op, - raft::identity_op{}, - mr); - dataset_norm_ptr = cur_dataset_norm.data(); - } else if (dataset_norm != nullptr) { - dataset_norm_ptr = dataset_norm + offset; - } - + const MathT* dataset_norm_ptr = dataset_norm != nullptr ? dataset_norm + offset + : packed_norms ? cur_dataset_norm.data() + : nullptr; predict_core(handle, params, centers, @@ -790,7 +784,8 @@ void predict(const raft::resources& handle, dataset_norm_ptr, minibatch_size, labels + offset, - mem_res); + mem_res, + core_scratch); } } diff --git a/cpp/src/cluster/detail/kmeans_common.cuh b/cpp/src/cluster/detail/kmeans_common.cuh index 6f42d20aad..ce94e92a5e 100644 --- a/cpp/src/cluster/detail/kmeans_common.cuh +++ b/cpp/src/cluster/detail/kmeans_common.cuh @@ -110,13 +110,13 @@ struct CopyClusterLabelOp { template struct WeightedDistanceOp { DistanceIterator distances; - const DataT* weights; + const DataT* weights; // nullptr: every weight is effectively 1 __device__ DataT operator()(IndexT offset) const { auto distance = static_cast(distances[offset]); if constexpr (SquareDistance) { distance *= distance; } - return distance * weights[offset]; + return weights == nullptr ? distance : distance * weights[offset]; } }; @@ -489,7 +489,7 @@ MinClusterAndDistanceResult minClusterAndDistanceCompute( raft::device_matrix_view X, raft::device_matrix_view centroids, rmm::device_uvector& output_storage, - raft::device_vector_view L2NormX, + raft::device_vector_view L2NormX, // if empty, compute here rmm::device_uvector& L2NormBuf_OR_DistBuf, cuvs::distance::DistanceType metric, int batch_samples, @@ -498,7 +498,8 @@ MinClusterAndDistanceResult minClusterAndDistanceCompute( std::optional> Tf32NormX = std::nullopt, cuvs::distance::detail::Top1nnBackend backend = cuvs::distance::detail::Top1nnBackend::Auto, // Required (and only used) with Top1nnBackend::Cagra: index built over `centroids`. - const cuvs::distance::detail::top_1_nn_cagra_index* ann_index = nullptr); + const cuvs::distance::detail::top_1_nn_cagra_index* ann_index = nullptr, + bool need_compute_distance = true); #define EXTERN_TEMPLATE_MIN_CLUSTER_AND_DISTANCE(DataT, IndexT) \ extern template MinClusterAndDistanceResult \ @@ -515,7 +516,8 @@ MinClusterAndDistanceResult minClusterAndDistanceCompute( rmm::device_uvector&, \ std::optional>, \ cuvs::distance::detail::Top1nnBackend, \ - const cuvs::distance::detail::top_1_nn_cagra_index*); + const cuvs::distance::detail::top_1_nn_cagra_index*, \ + bool); EXTERN_TEMPLATE_MIN_CLUSTER_AND_DISTANCE(float, int64_t) EXTERN_TEMPLATE_MIN_CLUSTER_AND_DISTANCE(float, int) diff --git a/cpp/src/cluster/detail/minClusterDistanceCompute.cu b/cpp/src/cluster/detail/minClusterDistanceCompute.cu index b10c6c8896..01555a7e95 100644 --- a/cpp/src/cluster/detail/minClusterDistanceCompute.cu +++ b/cpp/src/cluster/detail/minClusterDistanceCompute.cu @@ -6,8 +6,9 @@ #include "../../distance/top_1_nn.cuh" #include "kmeans_common.cuh" -#include #include +#include +#include #include @@ -38,49 +39,13 @@ void* reserve_aligned_workspace(rmm::device_uvector& workspace, return reinterpret_cast(aligned); } -struct tf32_square_op { - template - __device__ float operator()(float value, IndexT) const - { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 - value = nvcuda::wmma::__float_to_tf32(value); -#endif - return value * value; - } -}; - -struct compensated_sum { - float sum; - float correction; - - __device__ void add(float value) - { - const float next = sum + value; - correction += fabsf(sum) >= fabsf(value) ? (sum - next) + value : (value - next) + sum; - sum = next; - } - - __device__ float value() const { return sum + correction; } -}; - -struct dual_norm_accumulator { - compensated_sum fp32; - compensated_sum tf32; -}; - -struct add_dual_norm_accumulators { - __device__ dual_norm_accumulator operator()(dual_norm_accumulator lhs, - dual_norm_accumulator rhs) const - { - lhs.fp32.add(rhs.fp32.sum); - lhs.fp32.add(rhs.fp32.correction); - lhs.tf32.add(rhs.tf32.sum); - lhs.tf32.add(rhs.tf32.correction); - return lhs; - } -}; - -template +/** + * TF32-rounded (and with `Fp32`, also FP32) squared norms of each row in one pass over the input. + * + * `LanesPerRow` lanes reduce one row, so short rows do not leave most of a warp idle, and each + * lane group handles kRows rows at a time to keep more loads in flight. + */ +template __global__ void fp32_tf32_norms_kernel(const float* input, float* fp32_norms, float* tf32_norms, @@ -88,47 +53,93 @@ __global__ void fp32_tf32_norms_kernel(const float* input, IndexT cols, bool take_sqrt) { - constexpr int warp_size = 32; - constexpr int warps_per_block = ThreadsPerBlock / warp_size; - const int lane = threadIdx.x % warp_size; - const int warp = threadIdx.x / warp_size; - const auto row_stride = static_cast(gridDim.x) * static_cast(warps_per_block); - - for (IndexT row = static_cast(blockIdx.x) * static_cast(warps_per_block) + - static_cast(warp); - row < rows; - row += row_stride) { - dual_norm_accumulator thread_sum{}; - const auto row_offset = static_cast(row) * static_cast(cols); - for (IndexT col = static_cast(lane); col < cols; - col += static_cast(warp_size)) { - const float value = input[row_offset + static_cast(col)]; + constexpr int kRows = 2; + constexpr int groups_per_warp = raft::WarpSize / LanesPerRow; + const int lane = threadIdx.x % raft::WarpSize; + const int sub = lane % LanesPerRow; + // Rows are counted in 64 bits: with a 32-bit IndexT, base + stride can pass INT_MAX. + const auto warp = + (static_cast(blockIdx.x) * blockDim.x + threadIdx.x) / raft::WarpSize; + const auto stride = + static_cast(gridDim.x) * (blockDim.x / raft::WarpSize) * groups_per_warp * kRows; + + for (std::int64_t base = warp * groups_per_warp * kRows; base < rows; base += stride) { + float fp32_sum[kRows]{}; + float tf32_sum[kRows]{}; + const std::int64_t first_row = base + lane / LanesPerRow; +#pragma unroll + for (int r = 0; r < kRows; r++) { + const std::int64_t row = first_row + r * groups_per_warp; + if (row >= rows) { continue; } + auto add = [&](float value) { + if constexpr (Fp32) { fp32_sum[r] += value * value; } #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 - const float tf32_value = nvcuda::wmma::__float_to_tf32(value); -#else - const float tf32_value = value; + value = nvcuda::wmma::__float_to_tf32(value); #endif - thread_sum.fp32.add(value * value); - thread_sum.tf32.add(tf32_value * tf32_value); - } - - for (int offset = warp_size / 2; offset > 0; offset /= 2) { - dual_norm_accumulator other{ - {__shfl_down_sync(0xffffffff, thread_sum.fp32.sum, offset), - __shfl_down_sync(0xffffffff, thread_sum.fp32.correction, offset)}, - {__shfl_down_sync(0xffffffff, thread_sum.tf32.sum, offset), - __shfl_down_sync(0xffffffff, thread_sum.tf32.correction, offset)}}; - thread_sum = add_dual_norm_accumulators{}(thread_sum, other); + tf32_sum[r] += value * value; + }; + const float* row_ptr = input + row * static_cast(cols); + for (IndexT col = sub; col < cols; col += LanesPerRow) { + add(row_ptr[col]); + } } - if (lane == 0) { - const float fp32_norm = thread_sum.fp32.value(); - const float tf32_norm = thread_sum.tf32.value(); - fp32_norms[row] = take_sqrt ? sqrtf(fp32_norm) : fp32_norm; - tf32_norms[row] = take_sqrt ? sqrtf(tf32_norm) : tf32_norm; +#pragma unroll + for (int r = 0; r < kRows; r++) { + for (int offset = LanesPerRow / 2; offset > 0; offset /= 2) { + if constexpr (Fp32) { fp32_sum[r] += __shfl_xor_sync(0xffffffff, fp32_sum[r], offset); } + tf32_sum[r] += __shfl_xor_sync(0xffffffff, tf32_sum[r], offset); + } + const std::int64_t row = first_row + r * groups_per_warp; + if (sub == 0 && row < rows) { + if constexpr (Fp32) { fp32_norms[row] = take_sqrt ? sqrtf(fp32_sum[r]) : fp32_sum[r]; } + tf32_norms[row] = take_sqrt ? sqrtf(tf32_sum[r]) : tf32_sum[r]; + } } } } +/** Launches fp32_tf32_norms_kernel; `fp32_norms` is written only when `Fp32`. */ +template +void launch_row_norms(raft::resources const& handle, + const float* x, + float* fp32_norms, + float* tf32_norms, + IndexT rows, + IndexT cols, + bool take_sqrt) +{ + // About 8 elements of a row per lane: the largest power-of-two lane group up to a warp. + int lanes = 1; + while (lanes < raft::WarpSize && lanes * 2 * 8 <= cols) { + lanes *= 2; + } + constexpr int threads_per_block = 256; + auto launch = [&](auto lanes_per_row) { + constexpr int kLanes = decltype(lanes_per_row)::value; + const auto rows_per_block = std::size_t{threads_per_block / kLanes * 2}; + const auto blocks = static_cast(std::min( + raft::ceildiv(static_cast(rows), rows_per_block), 65535)); + raft::launch_kernel(handle, + blocks, + threads_per_block, + fp32_tf32_norms_kernel, + x, + fp32_norms, + tf32_norms, + rows, + cols, + take_sqrt); + }; + switch (lanes) { + case 1: launch(std::integral_constant{}); break; + case 2: launch(std::integral_constant{}); break; + case 4: launch(std::integral_constant{}); break; + case 8: launch(std::integral_constant{}); break; + case 16: launch(std::integral_constant{}); break; + default: launch(std::integral_constant{}); break; + } +} + template void compute_tf32_norms(raft::resources const& handle, const float* matrix, @@ -137,15 +148,7 @@ void compute_tf32_norms(raft::resources const& handle, IndexT cols, bool take_sqrt) { - auto input = raft::make_device_matrix_view(matrix, rows, cols); - auto output = raft::make_device_vector_view(norms, rows); - if (take_sqrt) { - raft::linalg::coalesced_reduction( - handle, input, output, 0.0f, false, tf32_square_op{}, raft::add_op{}, raft::sqrt_op{}); - } else { - raft::linalg::coalesced_reduction( - handle, input, output, 0.0f, false, tf32_square_op{}, raft::add_op{}, raft::identity_op{}); - } + launch_row_norms(handle, matrix, nullptr, norms, rows, cols, take_sqrt); } template @@ -174,19 +177,13 @@ void compute_fp32_tf32_norms(raft::resources const& handle, "TF32 norm output must have one element per input row"); if (input.extent(0) == 0) { return; } - constexpr int threads_per_block = 256; - constexpr int warps_per_block = threads_per_block / 32; - const auto blocks = static_cast(std::min( - raft::ceildiv(static_cast(input.extent(0)), warps_per_block), 65535)); - fp32_tf32_norms_kernel - <<>>( - input.data_handle(), - fp32_norms.data_handle(), - tf32_norms.data_handle(), - input.extent(0), - input.extent(1), - take_sqrt); - RAFT_CUDA_TRY(cudaPeekAtLastError()); + launch_row_norms(handle, + input.data_handle(), + fp32_norms.data_handle(), + tf32_norms.data_handle(), + input.extent(0), + input.extent(1), + take_sqrt); } template void compute_fp32_tf32_norms(raft::resources const&, @@ -215,7 +212,8 @@ MinClusterAndDistanceResult minClusterAndDistanceCompute( rmm::device_uvector& workspace, std::optional> Tf32NormX, cuvs::distance::detail::Top1nnBackend backend, - const cuvs::distance::detail::top_1_nn_cagra_index* ann_index) + const cuvs::distance::detail::top_1_nn_cagra_index* ann_index, + bool need_compute_distance) { auto stream = raft::resource::get_cuda_stream(handle); auto n_samples = X.extent(0); @@ -245,55 +243,80 @@ MinClusterAndDistanceResult minClusterAndDistanceCompute( backend); RAFT_EXPECTS(plan.available, "Requested top_1_nn backend is unavailable for KMeans assignment"); - const DataT* x_norm = L2NormX.data_handle(); - const DataT* y_norm = nullptr; - const bool take_sqrt = metric == cuvs::distance::DistanceType::CosineExpanded; - if constexpr (std::is_same_v) { - if (plan.norm_policy == cuvs::distance::detail::Top1nnNormPolicy::Tf32) { - if (Tf32NormX.has_value()) { - RAFT_EXPECTS(Tf32NormX->extent(0) == n_samples, - "Cached TF32 norms must have one element per input row"); - x_norm = Tf32NormX->data_handle(); - L2NormBuf_OR_DistBuf.resize(n_clusters, stream); - auto* y_norm_output = L2NormBuf_OR_DistBuf.data(); - y_norm = y_norm_output; - compute_tf32_norms( - handle, centroids.data_handle(), y_norm_output, n_clusters, n_features, take_sqrt); - } else { - const auto x_norm_bytes = sizeof(DataT) * static_cast(n_samples); - const auto y_norm_bytes_offset = - (x_norm_bytes + plan.norm_alignment - 1) / plan.norm_alignment * plan.norm_alignment; - const auto y_norm_offset = y_norm_bytes_offset / sizeof(DataT); - L2NormBuf_OR_DistBuf.resize(y_norm_offset + static_cast(n_clusters), stream); - x_norm = L2NormBuf_OR_DistBuf.data(); - y_norm = x_norm + y_norm_offset; + // The plan's norm policy decides the norms of x and the centroids: TF32 (x from the cached + // `Tf32NormX` or computed here), Native (x from the caller's `L2NormX`, or computed here when + // it is empty) or None. One allocation holds the result and everything the 1-NN needs, so a + // call allocates once: [output | backend workspace | x norms | centroid norms]. + using cuvs::distance::detail::Top1nnNormPolicy; + RAFT_EXPECTS(L2NormX.extent(0) == 0 || L2NormX.extent(0) == n_samples, + "FP32 norms must be empty or have one element per input row"); + const auto norm_policy = plan.norm_policy; + const bool take_sqrt = metric == cuvs::distance::DistanceType::CosineExpanded; + const bool need_compute_x_norms = norm_policy == Top1nnNormPolicy::Tf32 ? !Tf32NormX.has_value() + : norm_policy == Top1nnNormPolicy::Native + ? L2NormX.extent(0) == 0 + : false; + const bool need_y_norms = norm_policy != Top1nnNormPolicy::None; + const auto alignment = std::max( + {std::size_t{256}, plan.output_alignment, plan.workspace_alignment, plan.norm_alignment}); + const auto workspace_offset = raft::round_up_safe(plan.output_bytes, alignment); + const auto x_norm_offset = + raft::round_up_safe(workspace_offset + plan.workspace_bytes, alignment); + const auto y_norm_offset = raft::round_up_safe( + x_norm_offset + + (need_compute_x_norms ? sizeof(DataT) * static_cast(n_samples) : 0), + alignment); + const auto storage_bytes = + y_norm_offset + (need_y_norms ? sizeof(DataT) * static_cast(n_clusters) : 0); + auto* storage = static_cast( + reserve_aligned_workspace(output_storage, 0, storage_bytes, alignment, stream)); + void* backend_workspace = plan.workspace_bytes == 0 ? nullptr : storage + workspace_offset; + + auto* computed_x_norm = reinterpret_cast(storage + x_norm_offset); + const DataT* x_norm = need_compute_x_norms ? computed_x_norm : L2NormX.data_handle(); + DataT* y_norm = need_y_norms ? reinterpret_cast(storage + y_norm_offset) : nullptr; + switch (norm_policy) { + case Top1nnNormPolicy::Tf32: + if constexpr (std::is_same_v) { + if (need_compute_x_norms && !need_compute_distance) { + raft::matrix::fill( + handle, + raft::make_device_vector_view(computed_x_norm, n_samples), + DataT{0}); + } else if (need_compute_x_norms) { + compute_tf32_norms( + handle, X.data_handle(), computed_x_norm, n_samples, n_features, take_sqrt); + } else { + RAFT_EXPECTS(Tf32NormX->extent(0) == n_samples, + "Cached TF32 norms must have one element per input row"); + x_norm = Tf32NormX->data_handle(); + } compute_tf32_norms( - handle, X.data_handle(), L2NormBuf_OR_DistBuf.data(), n_samples, n_features, take_sqrt); - compute_tf32_norms(handle, - centroids.data_handle(), - L2NormBuf_OR_DistBuf.data() + y_norm_offset, - n_clusters, - n_features, - take_sqrt); + handle, centroids.data_handle(), y_norm, n_clusters, n_features, take_sqrt); } + break; + case Top1nnNormPolicy::Native: { + auto native_norms = [&](raft::device_matrix_view matrix, DataT* out) { + auto rows = raft::make_device_matrix_view( + matrix.data_handle(), matrix.extent(0), matrix.extent(1)); + auto out_view = raft::make_device_vector_view(out, rows.extent(0)); + if (take_sqrt) { + raft::linalg::norm( + handle, rows, out_view, raft::sqrt_op{}); + } else { + raft::linalg::norm( + handle, rows, out_view); + } + }; + if (need_compute_x_norms) { native_norms(X, computed_x_norm); } + native_norms(centroids, y_norm); + break; } - } - // The CAGRA (ANN) backend does not use norms. - if (y_norm == nullptr && plan.backend != cuvs::distance::detail::Top1nnBackend::Cagra) { - L2NormBuf_OR_DistBuf.resize(n_clusters, stream); - auto centroids_norm = - raft::make_device_vector_view(L2NormBuf_OR_DistBuf.data(), n_clusters); - if (take_sqrt) { - raft::linalg::norm( - handle, centroids, centroids_norm, raft::sqrt_op{}); - } else { - raft::linalg::norm( - handle, centroids, centroids_norm); - } - y_norm = L2NormBuf_OR_DistBuf.data(); + case Top1nnNormPolicy::None: break; } - auto result = make_native_result(plan, n_samples, output_storage, stream); + auto result = + cuvs::distance::bind_top_1_nn_result_view(plan, n_samples, storage, plan.output_bytes); bool init_out_buffer = true; if (plan.output_layout == cuvs::distance::detail::Top1nnOutputLayout::KeyValuePair) { @@ -304,8 +327,6 @@ MinClusterAndDistanceResult minClusterAndDistanceCompute( init_out_buffer = false; } - auto* backend_workspace = reserve_aligned_workspace( - workspace, 0, plan.workspace_bytes, plan.workspace_alignment, stream); auto launch = [&](auto output) { cuvs::distance::top_1_nn(handle, output, @@ -406,7 +427,8 @@ MinClusterAndDistanceResult minClusterAndDistanceCompute( rmm::device_uvector&, \ std::optional>, \ cuvs::distance::detail::Top1nnBackend, \ - const cuvs::distance::detail::top_1_nn_cagra_index*); + const cuvs::distance::detail::top_1_nn_cagra_index*, \ + bool); INSTANTIATE_MIN_CLUSTER_AND_DISTANCE(float, int64_t) INSTANTIATE_MIN_CLUSTER_AND_DISTANCE(double, int64_t) diff --git a/cpp/src/distance/detail/fused_distance_nn.cuh b/cpp/src/distance/detail/fused_distance_nn.cuh index 2e00ff7764..82f8bfcf6c 100644 --- a/cpp/src/distance/detail/fused_distance_nn.cuh +++ b/cpp/src/distance/detail/fused_distance_nn.cuh @@ -73,8 +73,12 @@ struct Top1nnTuning { /** Native result representation required by the resolved top-1 NN backend. */ enum class Top1nnOutputLayout : std::uint8_t { KeyValuePair, Separate }; -/** Arithmetic policy callers must use when preparing row norms. */ -enum class Top1nnNormPolicy : std::uint8_t { Native, Tf32 }; +/** Row norms the resolved top-1 NN backend reads */ +enum class Top1nnNormPolicy : std::uint8_t { + Native, // norms in the data type's precision (legacy backends) + Tf32, // norms of TF32-rounded values, matching a TF32 GEMM (float cuTile) + None, // no norms are read (CAGRA) +}; /** Reusable result of probing one exact top-1 NN invocation. */ template diff --git a/cpp/src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu b/cpp/src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu index c444265b72..7bfbebb70c 100644 --- a/cpp/src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu +++ b/cpp/src/distance/detail/fused_distance_nn/cutile/fused_1nn_tile.cu @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -26,6 +27,12 @@ namespace detail { namespace { +/** Rows per cuTile launch when the candidates do not fit in L2. + * cuTile requires each new batch to be 16-byte aligned. + */ +constexpr int64_t kFused1nnCutileL2RowBatch = 32768; +static_assert(kFused1nnCutileL2RowBatch % 16 == 0); + /** * Returns true when the problem dimensions exceed the per-arch high-dim crossover thresholds. * @@ -352,75 +359,58 @@ void launch_fused_1nn_tile(raft::resources const& handle, use_strict_abi && should_use_high_dim_tile(n, k, capabilities.cc_major, capabilities.cc_minor); - if constexpr (std::is_same_v) { + // Row batching for L2-optimization + const auto candidate_bytes = + static_cast(n) * static_cast(k) * static_cast(sizeof(DataT)); + const int64_t row_batch = + candidate_bytes > raft::resource::get_device_properties(handle).l2CacheSize + ? kFused1nnCutileL2RowBatch + : fused_1nn_cutile_max_batch_m; + + const auto stream = raft::resource::get_cuda_stream(handle); + for (int64_t offset = 0; offset < static_cast(m);) { + const int64_t batch_m64 = std::min(row_batch, static_cast(m) - offset); + const int batch_m = static_cast(batch_m64); + // int64 indices go through an int32 index workspace and are widened per batch. + int* batch_idx = nullptr; + if (nearest_idx != nullptr) { + if constexpr (std::is_same_v) { + batch_idx = nearest_idx + offset; + } else { + batch_idx = static_cast(index_workspace); + } + } + + auto launch = [&](auto abi) { + launch_fused_1nn_tile_impl( + handle, + batch_idx, + nearest_dist + offset, + x + static_cast(offset) * static_cast(k), + y, + xn == nullptr ? nullptr : xn + offset, + yn, + batch_m, + static_cast(n), + static_cast(k), + metric, + is_sqrt); + }; if (use_high_dim_abi) { - launch_fused_1nn_tile_impl( - handle, nearest_idx, nearest_dist, x, y, xn, yn, m, n, k, metric, is_sqrt); + launch(cutile_abi_high_dim{}); } else if (use_strict_abi) { - launch_fused_1nn_tile_impl( - handle, nearest_idx, nearest_dist, x, y, xn, yn, m, n, k, metric, is_sqrt); + launch(cutile_abi_strict{}); } else { - launch_fused_1nn_tile_impl( - handle, nearest_idx, nearest_dist, x, y, xn, yn, m, n, k, metric, is_sqrt); + launch(cutile_abi_relaxed{}); } - } else { - const auto stream = raft::resource::get_cuda_stream(handle); - constexpr int64_t max_batch_m = fused_1nn_cutile_max_batch_m; - auto* tmp_idx = nearest_idx == nullptr ? nullptr : static_cast(index_workspace); - for (int64_t offset = 0; offset < m;) { - const int64_t batch_m64 = std::min(max_batch_m, m - offset); - const int batch_m = static_cast(batch_m64); - const auto* batch_x = x + static_cast(offset) * static_cast(k); - const auto* batch_xn = xn == nullptr ? nullptr : xn + offset; - auto* batch_dist = nearest_dist + offset; - - if (use_high_dim_abi) { - launch_fused_1nn_tile_impl(handle, - tmp_idx, - batch_dist, - batch_x, - y, - batch_xn, - yn, - batch_m, - static_cast(n), - static_cast(k), - metric, - is_sqrt); - } else if (use_strict_abi) { - launch_fused_1nn_tile_impl(handle, - tmp_idx, - batch_dist, - batch_x, - y, - batch_xn, - yn, - batch_m, - static_cast(n), - static_cast(k), - metric, - is_sqrt); - } else { - launch_fused_1nn_tile_impl(handle, - tmp_idx, - batch_dist, - batch_x, - y, - batch_xn, - yn, - batch_m, - static_cast(n), - static_cast(k), - metric, - is_sqrt); - } - if (nearest_idx != nullptr) { + if constexpr (!std::is_same_v) { + if (batch_idx != nullptr) { raft::linalg::unaryOp( - nearest_idx + offset, tmp_idx, batch_m, raft::cast_op{}, stream.get()); + nearest_idx + offset, batch_idx, batch_m, raft::cast_op{}, stream.get()); } - offset += batch_m64; } + offset += batch_m64; } } diff --git a/cpp/src/distance/fused_distance_nn-inl.cuh b/cpp/src/distance/fused_distance_nn-inl.cuh index 8a99b32c14..51fbfc2eea 100644 --- a/cpp/src/distance/fused_distance_nn-inl.cuh +++ b/cpp/src/distance/fused_distance_nn-inl.cuh @@ -869,9 +869,9 @@ detail::Top1nnPlan probe_top_1_nn(raft::resources const& handle, plan.output_alignment = alignment; plan.output_layout = detail::Top1nnOutputLayout::Separate; plan.norm_alignment = alignment; - plan.norm_policy = (resolved == detail::Top1nnBackend::Cutile && std::is_same_v) - ? detail::Top1nnNormPolicy::Tf32 - : detail::Top1nnNormPolicy::Native; + plan.norm_policy = resolved == detail::Top1nnBackend::Cagra ? detail::Top1nnNormPolicy::None + : std::is_same_v ? detail::Top1nnNormPolicy::Tf32 + : detail::Top1nnNormPolicy::Native; } else { plan.output_bytes = detail::checked_top_1_nn_workspace_multiply( detail::checked_top_1_nn_extent(m), sizeof(raft::KeyValuePair)); @@ -913,7 +913,8 @@ void top_1_nn(raft::resources const& handle, "Selected top_1_nn backend does not support the requested metric"); RAFT_EXPECTS(x != nullptr && y != nullptr, "top_1_nn requires non-null input buffers"); RAFT_EXPECTS(metric == cuvs::distance::DistanceType::InnerProduct || - plan.backend == detail::Top1nnBackend::Cagra || (xn != nullptr && yn != nullptr), + plan.norm_policy == detail::Top1nnNormPolicy::None || + (xn != nullptr && yn != nullptr), "top_1_nn requires non-null norm buffers for the requested metric"); using output_type = std::remove_cvref_t; constexpr bool is_separate_output =