Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions include/cutlass/arch/warp_reduce.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/***************************************************************************************************
* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once

#include "cutlass/cutlass.h"
#include "cutlass/arch/arch.h"
#include "cutlass/arch/config.h"
#include <cuda_runtime.h>

#if defined(CUTLASS_ARCH_MMA_SM100A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM100F_ENABLED) ||\
defined(CUTLASS_ARCH_MMA_SM103A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM103F_ENABLED)
# define CUTLASS_ARCH_CREDUX_ENABLED
#endif

namespace cutlass {
namespace arch {

/////////////////////////////////////////////////////////////////////////////////////////////////

/// Reduce independent groups of Width consecutive lanes, broadcasting each group's result.
/// All 32 lanes of the warp must participate, even when Width is smaller than 32.
/// The operator must be associative; the shuffle tree order matches a down-shuffle reduction.
template <int Width, class ReductionOp>
CUTLASS_DEVICE
float warp_reduce(float value, ReductionOp op) {
static_assert(Width > 0 && Width <= 32 && (Width & (Width - 1)) == 0,
"Warp reduction width must be a power of two between 1 and 32");
#if defined(__CUDA_ARCH__)
CUTLASS_PRAGMA_UNROLL
for (int delta = Width / 2; delta > 0; delta /= 2) {
float other = __shfl_down_sync(0xffffffff, value, delta, Width);
value = op(value, other);
}
return __shfl_sync(0xffffffff, value, 0, Width);
#else
CUTLASS_UNUSED(value);
CUTLASS_UNUSED(op);
CUTLASS_NOT_IMPLEMENTED();
return 0;
#endif
}

/// Native absolute maximum for independent contiguous lane groups on supported architectures.
/// All 32 lanes must participate. LaneId, rather than threadIdx.x, also supports 2D/3D blocks.
template <int Width = 32>
CUTLASS_DEVICE
float redux_abs_max_nan_sync(float value) {
static_assert(Width > 0 && Width <= 32 && (Width & (Width - 1)) == 0,
"Warp reduction width must be a power of two between 1 and 32");
#if defined(CUTLASS_ARCH_CREDUX_ENABLED)
unsigned int group_mask = (0xffffffffu >> (32 - Width)) << ((LaneId() / Width) * Width);
float result;
asm volatile("redux.sync.max.abs.NaN.f32 %0, %1, %2;\n"
: "=f"(result) : "f"(value), "r"(group_mask));
return result;
#else
CUTLASS_UNUSED(value);
CUTLASS_NOT_IMPLEMENTED();
return 0;
#endif
}

/////////////////////////////////////////////////////////////////////////////////////////////////

} // namespace arch
} // namespace cutlass
Original file line number Diff line number Diff line change
Expand Up @@ -488,15 +488,12 @@ struct Sm100BlockScaleFactorColStore {
// For VS32, t0~t31 should generate an amax.
// For VS64, t0~t63 should generate an amax. We would first do the reduciton within a warp,
// and then use smem to do inter-warp reduction.
if constexpr (SFVecSize == 32) {
return cutlass::redux_abs_max_nan_propagation_sync_warp<ElementCompute>{}(max);
}
else if constexpr (SFVecSize == 16) {
return cutlass::redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31<ElementCompute>{}(max);
if constexpr (SFVecSize == 16 || SFVecSize == 32) {
return cutlass::redux_abs_max_nan_propagation_sync<ElementCompute, SFVecSize>{}(max);
}
else if constexpr (SFVecSize == 64) {
// Get abs_max per warp
auto abs_max = cutlass::redux_abs_max_nan_propagation_sync_warp<ElementCompute>{}(max);
auto abs_max = cutlass::redux_abs_max_nan_propagation_sync<ElementCompute, 32>{}(max);

// Switch the amax of adjacent warps
const bool leading_thread = (threadIdx.x % NumThreadsPerWarp) == 0;
Expand Down
81 changes: 26 additions & 55 deletions include/cutlass/functional.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#pragma once

#include "cutlass/cutlass.h"
#include "cutlass/arch/warp_reduce.h"
#include "cutlass/numeric_types.h"
#include "cutlass/platform/platform.h"
#include "cutlass/detail/dependent_false.hpp"
Expand All @@ -56,11 +57,6 @@
#include <intrin.h>
#endif // _MSC_VER

#if defined(CUTLASS_ARCH_MMA_SM100A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM100F_ENABLED) ||\
defined(CUTLASS_ARCH_MMA_SM103A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM103F_ENABLED)
# define CUTLASS_ARCH_CREDUX_ENABLED
#endif

namespace cutlass {

/////////////////////////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -1046,29 +1042,25 @@ struct is_atomic<atomic_maximum<T>> : platform::true_type {};

//////////////////////////////////////////////////////////////////////////////////////////////////
/// Parallel Synchronization and Communication Instructions
template <typename T>
struct redux_abs_max_nan_propagation_sync_warp;
/// Absolute maximum in each contiguous Width-lane group, broadcast to every group member.
/// Width must be a power of two in [1, 32]. All 32 lanes must execute the call.
/// A NaN propagates within its group only; signed zeros produce positive zero.
template <typename T, int Width>
struct redux_abs_max_nan_propagation_sync;

template <int Width>
struct redux_abs_max_nan_propagation_sync<float, Width> {
static_assert(Width > 0 && Width <= 32 && (Width & (Width - 1)) == 0,
"Warp reduction width must be a power of two between 1 and 32");

template <>
struct redux_abs_max_nan_propagation_sync_warp <float>{
CUTLASS_DEVICE
float operator()(float const &lhs) const {
#if defined(CUTLASS_ARCH_CREDUX_ENABLED)
float result;
asm volatile("redux.sync.max.abs.NaN.f32 %0, %1, 0xffffffff;\n" : "=f"(result) : "f"(lhs));
return result;
return arch::redux_abs_max_nan_sync<Width>(lhs);
#elif defined(__CUDA_ARCH__)
cutlass::maximum<float, /*PropagateNaN*/true> max_op;
int shuffle_width = 32;
float abs_max = cutlass::absolute_value_op<float>{}(lhs);
CUTLASS_PRAGMA_UNROLL
for(int offset = shuffle_width / 2; offset > 0; offset /= 2) {
float value = __shfl_down_sync(0xffffffff, abs_max, offset, shuffle_width);
abs_max = max_op(abs_max,value);
}
// Broadcast the maximum to all threads participating in the reduction.
abs_max = __shfl_sync(0xffffffff, abs_max, 0, shuffle_width);
return abs_max;
return arch::warp_reduce<Width>(
cutlass::absolute_value_op<float>{}(lhs),
cutlass::maximum<float, /*PropagateNaN*/true>{});
#else
CUTLASS_UNUSED(lhs);
CUTLASS_NOT_IMPLEMENTED();
Expand All @@ -1077,42 +1069,21 @@ struct redux_abs_max_nan_propagation_sync_warp <float>{
}
};

/// Backward-compatible full-warp spelling.
template <typename T>
struct redux_abs_max_nan_propagation_sync_warp;

template <>
struct redux_abs_max_nan_propagation_sync_warp<float>
: redux_abs_max_nan_propagation_sync<float, 32> {};

/// Backward-compatible spelling for the two independent half-warps.
template <typename T>
struct redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31;

template <>
struct redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31<float>{
CUTLASS_DEVICE
float operator()(float const &max) const {
#if defined(CUTLASS_ARCH_CREDUX_ENABLED)
int half_warp_idx = threadIdx.x / (NumThreadsPerWarp / 2);
bool first_half_threads = (half_warp_idx % 2) == 0;
float value0 = first_half_threads ? max : 0;
float v0 = cutlass::redux_abs_max_nan_propagation_sync_warp<float>{}(value0);

float value1 = !first_half_threads ? max : 0;
float v1 = cutlass::redux_abs_max_nan_propagation_sync_warp<float>{}(value1);
return first_half_threads ? v0: v1;

#elif defined(__CUDA_ARCH__)
float abs_max = cutlass::absolute_value_op<float>{}(max);
cutlass::maximum<float, /*PropagateNaN*/true> max_op;
constexpr int shuffle_width = 16;
CUTLASS_PRAGMA_UNROLL
for(int offset = shuffle_width/2; offset > 0; offset /= 2) {
float value = __shfl_down_sync(0xffffffff, abs_max, offset, shuffle_width);
abs_max = max_op(abs_max,value);
}
// Broadcast the maximum to all threads participating in the reduction.
abs_max = __shfl_sync(0xffffffff, abs_max, 0, shuffle_width);
return abs_max;
#else
CUTLASS_UNUSED(max);
CUTLASS_NOT_IMPLEMENTED();
return 0;
#endif
}
};
struct redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31<float>
: redux_abs_max_nan_propagation_sync<float, 16> {};


/////////////////////////////////////////////////////////////////////////////////////////////////
Expand Down
48 changes: 48 additions & 0 deletions media/docs/cpp/fundamental_types.md
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,54 @@ multiply_add<Array<half_t, kN>> mad_op;
d = mad_op(a, b, c); // efficient multiply-add for Array of half-precision elements
```

### Warp and subgroup absolute maximum

`redux_abs_max_nan_propagation_sync<float, Width>` computes the absolute maximum
of each contiguous group of `Width` lanes and broadcasts it to every lane in
that group. It is a device-only function object in `cutlass/functional.h`.
Supported widths are 1, 2, 4, 8, 16, and 32. Invalid widths fail at compile time.

All 32 lanes in each participating warp must execute the call. Width is a
logical partition of a full warp, **not** an active-thread count: it does not
make an incomplete warp or a divergent call safe. Groups follow CUDA lane
ordering, including when a warp spans rows or planes of a multidimensional
thread block. A block should contain a multiple of 32 threads.

```c++
#include "cutlass/functional.h"

__global__ void group_abs_max(float const *input, float *output, int count) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
// Do not return early or predicate the collective on idx < count.
// Zero is the identity for absolute maximum, so tail lanes can contribute zero.
float value = idx < count ? input[idx] : 0.0f;
float amax = cutlass::redux_abs_max_nan_propagation_sync<float, 8>{}(value);
if (idx < count) {
output[idx] = amax;
}
}
// For count > 0, launch group_abs_max<<<(count + 127) / 128, 128>>>(...).
```

NaNs propagate only within their group; a NaN in lanes 0–7 does not affect
lanes 8–15 for width 8. Negative inputs contribute their absolute values,
negative infinity produces positive infinity, and either sign of zero produces
positive zero. Subnormal inputs are preserved when compiling without
flush-to-zero options. NaN payload preservation is not part of the contract.

On architectures with native floating-point redux support, the implementation
uses `redux.sync.max.abs.NaN.f32` with a lane-group member mask. Other CUDA
targets use a down-shuffle tree followed by a broadcast. The group mask is
derived from the hardware lane ID, not `threadIdx.x`, so the native path also
handles 2D and 3D blocks. This API does not synchronize memory accesses.

The existing `redux_abs_max_nan_propagation_sync_warp<float>` and
`redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31<float>` names remain
available as the width-32 and width-16 forms. The SM100 block-scale column-store
epilogue uses widths 16 and 32 directly. Its scale-vector-64 path still performs
a separate shared-memory reduction between adjacent warps; this primitive does
not communicate across warp boundaries.

## Numeric Conversion

Operators are define to convert between numeric types in `numeric_conversion.h`. Conversion operators are defined in
Expand Down
Loading
Loading