From 6a9e782a8e7d907c5ef4b5bc3db844d057c2c4a7 Mon Sep 17 00:00:00 2001 From: ivanmilevtues Date: Mon, 9 Nov 2026 18:18:11 +0100 Subject: [PATCH 1/2] Extract architecture warp-reduction primitives with GPU coverage Keep the public absolute-maximum functors and shuffle tree unchanged while sharing the lane reduction and isolating native redux PTX in cutlass::arch. Cover every peak and NaN lane, independent half-warps, multiple warps and blocks, infinity, and signed zero. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c14c-fd64-772c-bcc8-cf34fb3d71f3 Co-authored-by: Amp --- include/cutlass/arch/warp_reduce.h | 88 ++++++++++++++++++++++++++++ include/cutlass/functional.h | 38 +++--------- test/unit/core/functional.cu | 93 ++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 30 deletions(-) create mode 100644 include/cutlass/arch/warp_reduce.h diff --git a/include/cutlass/arch/warp_reduce.h b/include/cutlass/arch/warp_reduce.h new file mode 100644 index 0000000000..8b358e70ed --- /dev/null +++ b/include/cutlass/arch/warp_reduce.h @@ -0,0 +1,88 @@ +/*************************************************************************************************** + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +#pragma once + +#include "cutlass/cutlass.h" +#include "cutlass/arch/config.h" +#include + +#if defined(CUTLASS_ARCH_MMA_SM100A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM100F_ENABLED) ||\ + defined(CUTLASS_ARCH_MMA_SM103A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM103F_ENABLED) +# define CUTLASS_ARCH_CREDUX_ENABLED +#endif + +namespace cutlass { +namespace arch { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +/// Reduce independent groups of Width consecutive lanes, broadcasting each group's result. +/// All 32 lanes of the warp must participate, even when Width is smaller than 32. +/// The operator must be associative; the shuffle tree order matches a down-shuffle reduction. +template +CUTLASS_DEVICE +float warp_reduce(float value, ReductionOp op) { + static_assert(Width > 0 && Width <= 32 && (Width & (Width - 1)) == 0, + "Warp reduction width must be a power of two between 1 and 32"); +#if defined(__CUDA_ARCH__) + CUTLASS_PRAGMA_UNROLL + for (int delta = Width / 2; delta > 0; delta /= 2) { + float other = __shfl_down_sync(0xffffffff, value, delta, Width); + value = op(value, other); + } + return __shfl_sync(0xffffffff, value, 0, Width); +#else + CUTLASS_UNUSED(value); + CUTLASS_UNUSED(op); + CUTLASS_NOT_IMPLEMENTED(); + return 0; +#endif +} + +/// Native warp-wide absolute maximum, with NaN propagation, on supported architectures. +/// All 32 lanes must participate. Callers select the shuffle fallback on other architectures. +CUTLASS_DEVICE +float redux_abs_max_nan_sync(float value) { +#if defined(CUTLASS_ARCH_CREDUX_ENABLED) + float result; + asm volatile("redux.sync.max.abs.NaN.f32 %0, %1, 0xffffffff;\n" : "=f"(result) : "f"(value)); + return result; +#else + CUTLASS_UNUSED(value); + CUTLASS_NOT_IMPLEMENTED(); + return 0; +#endif +} + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace arch +} // namespace cutlass diff --git a/include/cutlass/functional.h b/include/cutlass/functional.h index eb9d20b984..01a5c50f2b 100644 --- a/include/cutlass/functional.h +++ b/include/cutlass/functional.h @@ -36,6 +36,7 @@ #pragma once #include "cutlass/cutlass.h" +#include "cutlass/arch/warp_reduce.h" #include "cutlass/numeric_types.h" #include "cutlass/platform/platform.h" #include "cutlass/detail/dependent_false.hpp" @@ -56,11 +57,6 @@ #include #endif // _MSC_VER -#if defined(CUTLASS_ARCH_MMA_SM100A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM100F_ENABLED) ||\ - defined(CUTLASS_ARCH_MMA_SM103A_ENABLED) || defined(CUTLASS_ARCH_MMA_SM103F_ENABLED) -# define CUTLASS_ARCH_CREDUX_ENABLED -#endif - namespace cutlass { ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -1054,21 +1050,11 @@ struct redux_abs_max_nan_propagation_sync_warp { CUTLASS_DEVICE float operator()(float const &lhs) const { #if defined(CUTLASS_ARCH_CREDUX_ENABLED) - float result; - asm volatile("redux.sync.max.abs.NaN.f32 %0, %1, 0xffffffff;\n" : "=f"(result) : "f"(lhs)); - return result; + return arch::redux_abs_max_nan_sync(lhs); #elif defined(__CUDA_ARCH__) - cutlass::maximum max_op; - int shuffle_width = 32; - float abs_max = cutlass::absolute_value_op{}(lhs); - CUTLASS_PRAGMA_UNROLL - for(int offset = shuffle_width / 2; offset > 0; offset /= 2) { - float value = __shfl_down_sync(0xffffffff, abs_max, offset, shuffle_width); - abs_max = max_op(abs_max,value); - } - // Broadcast the maximum to all threads participating in the reduction. - abs_max = __shfl_sync(0xffffffff, abs_max, 0, shuffle_width); - return abs_max; + return arch::warp_reduce<32>( + cutlass::absolute_value_op{}(lhs), + cutlass::maximum{}); #else CUTLASS_UNUSED(lhs); CUTLASS_NOT_IMPLEMENTED(); @@ -1095,17 +1081,9 @@ struct redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31{ return first_half_threads ? v0: v1; #elif defined(__CUDA_ARCH__) - float abs_max = cutlass::absolute_value_op{}(max); - cutlass::maximum max_op; - constexpr int shuffle_width = 16; - CUTLASS_PRAGMA_UNROLL - for(int offset = shuffle_width/2; offset > 0; offset /= 2) { - float value = __shfl_down_sync(0xffffffff, abs_max, offset, shuffle_width); - abs_max = max_op(abs_max,value); - } - // Broadcast the maximum to all threads participating in the reduction. - abs_max = __shfl_sync(0xffffffff, abs_max, 0, shuffle_width); - return abs_max; + return arch::warp_reduce<16>( + cutlass::absolute_value_op{}(max), + cutlass::maximum{}); #else CUTLASS_UNUSED(max); CUTLASS_NOT_IMPLEMENTED(); diff --git a/test/unit/core/functional.cu b/test/unit/core/functional.cu index 607c1e36a8..32ffa5b44e 100644 --- a/test/unit/core/functional.cu +++ b/test/unit/core/functional.cu @@ -34,6 +34,9 @@ #include "../common/cutlass_unit_test.h" +#include +#include + #include "cutlass/functional.h" #include "cutlass/core_io.h" @@ -566,3 +569,93 @@ TEST(FUnction, maximum_half_device) { } ///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace test { +namespace core { +namespace kernel { + +template +__global__ void warp_abs_max(float *output, float const *input) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + output[idx] = Operator{}(input[idx]); +} + +} // namespace kernel + +/// Check every lane, not only lane zero: the public functors promise a broadcast result. +template +void check_warp_abs_max() { + using Tensor = cutlass::HostTensor; + constexpr int count = 128; + Tensor input({1, count}); + Tensor output({1, count}); + + // Distinct magnitudes and signs distinguish neighboring half-warps, warps, and blocks. + // Move the maximum through every position to catch a missing shuffle stage. + for (int peak = 0; peak < Width; ++peak) { + for (int i = 0; i < count; ++i) { + input.host_data()[i] = float((i % 7) - 3); + if (i % Width == peak) { + input.host_data()[i] = -float(100 + i / Width); + } + } + input.sync_device(); + kernel::warp_abs_max<<<2, 64>>>(output.device_data(), input.device_data()); + ASSERT_EQ(cudaGetLastError(), cudaSuccess); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.sync_host(); + for (int i = 0; i < count; ++i) { + EXPECT_EQ(output.host_data()[i], float(100 + i / Width)) << "lane " << i; + } + } + + // A NaN in either half must propagate only within its reduction group. + for (int nan_lane = 0; nan_lane < 32; ++nan_lane) { + for (int i = 0; i < count; ++i) { + input.host_data()[i] = -float(i + 1); + } + input.host_data()[nan_lane] = std::numeric_limits::quiet_NaN(); + input.sync_device(); + kernel::warp_abs_max<<<2, 64>>>(output.device_data(), input.device_data()); + ASSERT_EQ(cudaGetLastError(), cudaSuccess); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.sync_host(); + for (int i = 0; i < count; ++i) { + if (i / Width == nan_lane / Width) { + EXPECT_TRUE(std::isnan(output.host_data()[i])) << "lane " << i; + } + else { + EXPECT_EQ(output.host_data()[i], float((i / Width + 1) * Width)) << "lane " << i; + } + } + } + + // Alternate negative infinity and signed zeros in adjacent groups. + for (int i = 0; i < count; ++i) { + input.host_data()[i] = i % 2 ? -0.0f : 0.0f; + if (i % (2 * Width) == Width - 1) { + input.host_data()[i] = -std::numeric_limits::infinity(); + } + } + input.sync_device(); + kernel::warp_abs_max<<<2, 64>>>(output.device_data(), input.device_data()); + ASSERT_EQ(cudaGetLastError(), cudaSuccess); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.sync_host(); + for (int i = 0; i < count; ++i) { + float expected = (i / Width) % 2 ? 0.0f : std::numeric_limits::infinity(); + EXPECT_EQ(output.host_data()[i], expected) << "lane " << i; + EXPECT_FALSE(std::signbit(output.host_data()[i])) << "lane " << i; + } +} + +} // namespace core +} // namespace test + +TEST(Functional, warp_abs_max_nan) { + test::core::check_warp_abs_max<32, cutlass::redux_abs_max_nan_propagation_sync_warp>(); +} + +TEST(Functional, half_warp_abs_max_nan) { + test::core::check_warp_abs_max<16, cutlass::redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31>(); +} From 2bd5dcfe046e0110a252f052cfc4cd6a66b8f1ca Mon Sep 17 00:00:00 2001 From: ivanmilevtues Date: Mon, 9 Nov 2026 18:24:51 +0100 Subject: [PATCH 2/2] Add subgroup absolute-max reductions and integrate block-scale epilogues Expose power-of-two lane groups with NaN propagation and native member masks derived from hardware lane IDs. Retain legacy full/half-warp functors, integrate SF16/32/64 epilogue callers, and document participation and numerical semantics. Verify all widths and 1D/2D/3D blocks on SM86, and compile native SM100a paths. Inspired by the project's scale-vector epilogue design and upstream warp API discussion #2754. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c14c-fd64-772c-bcc8-cf34fb3d71f3 Co-authored-by: Amp --- include/cutlass/arch/warp_reduce.h | 12 +++- ...m100_visitor_store_tma_warpspecialized.hpp | 9 +-- include/cutlass/functional.h | 53 ++++++++--------- media/docs/cpp/fundamental_types.md | 48 ++++++++++++++++ test/unit/core/functional.cu | 57 +++++++++++++++++-- 5 files changed, 135 insertions(+), 44 deletions(-) diff --git a/include/cutlass/arch/warp_reduce.h b/include/cutlass/arch/warp_reduce.h index 8b358e70ed..c1b858ef80 100644 --- a/include/cutlass/arch/warp_reduce.h +++ b/include/cutlass/arch/warp_reduce.h @@ -31,6 +31,7 @@ #pragma once #include "cutlass/cutlass.h" +#include "cutlass/arch/arch.h" #include "cutlass/arch/config.h" #include @@ -67,13 +68,18 @@ float warp_reduce(float value, ReductionOp op) { #endif } -/// Native warp-wide absolute maximum, with NaN propagation, on supported architectures. -/// All 32 lanes must participate. Callers select the shuffle fallback on other architectures. +/// Native absolute maximum for independent contiguous lane groups on supported architectures. +/// All 32 lanes must participate. LaneId, rather than threadIdx.x, also supports 2D/3D blocks. +template CUTLASS_DEVICE float redux_abs_max_nan_sync(float value) { + static_assert(Width > 0 && Width <= 32 && (Width & (Width - 1)) == 0, + "Warp reduction width must be a power of two between 1 and 32"); #if defined(CUTLASS_ARCH_CREDUX_ENABLED) + unsigned int group_mask = (0xffffffffu >> (32 - Width)) << ((LaneId() / Width) * Width); float result; - asm volatile("redux.sync.max.abs.NaN.f32 %0, %1, 0xffffffff;\n" : "=f"(result) : "f"(value)); + asm volatile("redux.sync.max.abs.NaN.f32 %0, %1, %2;\n" + : "=f"(result) : "f"(value), "r"(group_mask)); return result; #else CUTLASS_UNUSED(value); diff --git a/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp b/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp index 915fac9bdb..d2214a354a 100644 --- a/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp +++ b/include/cutlass/epilogue/fusion/sm100_visitor_store_tma_warpspecialized.hpp @@ -488,15 +488,12 @@ struct Sm100BlockScaleFactorColStore { // For VS32, t0~t31 should generate an amax. // For VS64, t0~t63 should generate an amax. We would first do the reduciton within a warp, // and then use smem to do inter-warp reduction. - if constexpr (SFVecSize == 32) { - return cutlass::redux_abs_max_nan_propagation_sync_warp{}(max); - } - else if constexpr (SFVecSize == 16) { - return cutlass::redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31{}(max); + if constexpr (SFVecSize == 16 || SFVecSize == 32) { + return cutlass::redux_abs_max_nan_propagation_sync{}(max); } else if constexpr (SFVecSize == 64) { // Get abs_max per warp - auto abs_max = cutlass::redux_abs_max_nan_propagation_sync_warp{}(max); + auto abs_max = cutlass::redux_abs_max_nan_propagation_sync{}(max); // Switch the amax of adjacent warps const bool leading_thread = (threadIdx.x % NumThreadsPerWarp) == 0; diff --git a/include/cutlass/functional.h b/include/cutlass/functional.h index 01a5c50f2b..0df3032a46 100644 --- a/include/cutlass/functional.h +++ b/include/cutlass/functional.h @@ -1042,17 +1042,23 @@ struct is_atomic> : platform::true_type {}; ////////////////////////////////////////////////////////////////////////////////////////////////// /// Parallel Synchronization and Communication Instructions -template -struct redux_abs_max_nan_propagation_sync_warp; +/// Absolute maximum in each contiguous Width-lane group, broadcast to every group member. +/// Width must be a power of two in [1, 32]. All 32 lanes must execute the call. +/// A NaN propagates within its group only; signed zeros produce positive zero. +template +struct redux_abs_max_nan_propagation_sync; + +template +struct redux_abs_max_nan_propagation_sync { + static_assert(Width > 0 && Width <= 32 && (Width & (Width - 1)) == 0, + "Warp reduction width must be a power of two between 1 and 32"); -template <> -struct redux_abs_max_nan_propagation_sync_warp { CUTLASS_DEVICE float operator()(float const &lhs) const { #if defined(CUTLASS_ARCH_CREDUX_ENABLED) - return arch::redux_abs_max_nan_sync(lhs); + return arch::redux_abs_max_nan_sync(lhs); #elif defined(__CUDA_ARCH__) - return arch::warp_reduce<32>( + return arch::warp_reduce( cutlass::absolute_value_op{}(lhs), cutlass::maximum{}); #else @@ -1063,34 +1069,21 @@ struct redux_abs_max_nan_propagation_sync_warp { } }; +/// Backward-compatible full-warp spelling. +template +struct redux_abs_max_nan_propagation_sync_warp; + +template <> +struct redux_abs_max_nan_propagation_sync_warp + : redux_abs_max_nan_propagation_sync {}; + +/// Backward-compatible spelling for the two independent half-warps. template struct redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31; template <> -struct redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31{ - CUTLASS_DEVICE - float operator()(float const &max) const { -#if defined(CUTLASS_ARCH_CREDUX_ENABLED) - int half_warp_idx = threadIdx.x / (NumThreadsPerWarp / 2); - bool first_half_threads = (half_warp_idx % 2) == 0; - float value0 = first_half_threads ? max : 0; - float v0 = cutlass::redux_abs_max_nan_propagation_sync_warp{}(value0); - - float value1 = !first_half_threads ? max : 0; - float v1 = cutlass::redux_abs_max_nan_propagation_sync_warp{}(value1); - return first_half_threads ? v0: v1; - -#elif defined(__CUDA_ARCH__) - return arch::warp_reduce<16>( - cutlass::absolute_value_op{}(max), - cutlass::maximum{}); -#else - CUTLASS_UNUSED(max); - CUTLASS_NOT_IMPLEMENTED(); - return 0; -#endif - } -}; +struct redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31 + : redux_abs_max_nan_propagation_sync {}; ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/media/docs/cpp/fundamental_types.md b/media/docs/cpp/fundamental_types.md index 03b3fbaccf..62132221db 100644 --- a/media/docs/cpp/fundamental_types.md +++ b/media/docs/cpp/fundamental_types.md @@ -344,6 +344,54 @@ multiply_add> mad_op; d = mad_op(a, b, c); // efficient multiply-add for Array of half-precision elements ``` +### Warp and subgroup absolute maximum + +`redux_abs_max_nan_propagation_sync` computes the absolute maximum +of each contiguous group of `Width` lanes and broadcasts it to every lane in +that group. It is a device-only function object in `cutlass/functional.h`. +Supported widths are 1, 2, 4, 8, 16, and 32. Invalid widths fail at compile time. + +All 32 lanes in each participating warp must execute the call. Width is a +logical partition of a full warp, **not** an active-thread count: it does not +make an incomplete warp or a divergent call safe. Groups follow CUDA lane +ordering, including when a warp spans rows or planes of a multidimensional +thread block. A block should contain a multiple of 32 threads. + +```c++ +#include "cutlass/functional.h" + +__global__ void group_abs_max(float const *input, float *output, int count) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + // Do not return early or predicate the collective on idx < count. + // Zero is the identity for absolute maximum, so tail lanes can contribute zero. + float value = idx < count ? input[idx] : 0.0f; + float amax = cutlass::redux_abs_max_nan_propagation_sync{}(value); + if (idx < count) { + output[idx] = amax; + } +} +// For count > 0, launch group_abs_max<<<(count + 127) / 128, 128>>>(...). +``` + +NaNs propagate only within their group; a NaN in lanes 0–7 does not affect +lanes 8–15 for width 8. Negative inputs contribute their absolute values, +negative infinity produces positive infinity, and either sign of zero produces +positive zero. Subnormal inputs are preserved when compiling without +flush-to-zero options. NaN payload preservation is not part of the contract. + +On architectures with native floating-point redux support, the implementation +uses `redux.sync.max.abs.NaN.f32` with a lane-group member mask. Other CUDA +targets use a down-shuffle tree followed by a broadcast. The group mask is +derived from the hardware lane ID, not `threadIdx.x`, so the native path also +handles 2D and 3D blocks. This API does not synchronize memory accesses. + +The existing `redux_abs_max_nan_propagation_sync_warp` and +`redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31` names remain +available as the width-32 and width-16 forms. The SM100 block-scale column-store +epilogue uses widths 16 and 32 directly. Its scale-vector-64 path still performs +a separate shared-memory reduction between adjacent warps; this primitive does +not communicate across warp boundaries. + ## Numeric Conversion Operators are define to convert between numeric types in `numeric_conversion.h`. Conversion operators are defined in diff --git a/test/unit/core/functional.cu b/test/unit/core/functional.cu index 32ffa5b44e..5c080ad75e 100644 --- a/test/unit/core/functional.cu +++ b/test/unit/core/functional.cu @@ -576,7 +576,8 @@ namespace kernel { template __global__ void warp_abs_max(float *output, float const *input) { - int idx = blockIdx.x * blockDim.x + threadIdx.x; + int thread = threadIdx.x + blockDim.x * (threadIdx.y + blockDim.y * threadIdx.z); + int idx = blockIdx.x * (blockDim.x * blockDim.y * blockDim.z) + thread; output[idx] = Operator{}(input[idx]); } @@ -584,7 +585,7 @@ __global__ void warp_abs_max(float *output, float const *input) { /// Check every lane, not only lane zero: the public functors promise a broadcast result. template -void check_warp_abs_max() { +void check_warp_abs_max(dim3 block = dim3(64)) { using Tensor = cutlass::HostTensor; constexpr int count = 128; Tensor input({1, count}); @@ -600,7 +601,7 @@ void check_warp_abs_max() { } } input.sync_device(); - kernel::warp_abs_max<<<2, 64>>>(output.device_data(), input.device_data()); + kernel::warp_abs_max<<<2, block>>>(output.device_data(), input.device_data()); ASSERT_EQ(cudaGetLastError(), cudaSuccess); ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); output.sync_host(); @@ -616,7 +617,7 @@ void check_warp_abs_max() { } input.host_data()[nan_lane] = std::numeric_limits::quiet_NaN(); input.sync_device(); - kernel::warp_abs_max<<<2, 64>>>(output.device_data(), input.device_data()); + kernel::warp_abs_max<<<2, block>>>(output.device_data(), input.device_data()); ASSERT_EQ(cudaGetLastError(), cudaSuccess); ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); output.sync_host(); @@ -638,7 +639,7 @@ void check_warp_abs_max() { } } input.sync_device(); - kernel::warp_abs_max<<<2, 64>>>(output.device_data(), input.device_data()); + kernel::warp_abs_max<<<2, block>>>(output.device_data(), input.device_data()); ASSERT_EQ(cudaGetLastError(), cudaSuccess); ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); output.sync_host(); @@ -647,6 +648,28 @@ void check_warp_abs_max() { EXPECT_EQ(output.host_data()[i], expected) << "lane " << i; EXPECT_FALSE(std::signbit(output.host_data()[i])) << "lane " << i; } + + // Preserve subnormals; an absolute-max reduction must not silently flush them to zero. + for (int i = 0; i < count; ++i) { + input.host_data()[i] = -std::numeric_limits::denorm_min() * float(i + 1); + } + input.sync_device(); + kernel::warp_abs_max<<<2, block>>>(output.device_data(), input.device_data()); + ASSERT_EQ(cudaGetLastError(), cudaSuccess); + ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess); + output.sync_host(); + for (int i = 0; i < count; ++i) { + float expected = std::numeric_limits::denorm_min() * float((i / Width + 1) * Width); + EXPECT_EQ(output.host_data()[i], expected) << "lane " << i; + } +} + +template +void check_warp_abs_max_block_shapes() { + using Operator = cutlass::redux_abs_max_nan_propagation_sync; + check_warp_abs_max(dim3(64)); + check_warp_abs_max(dim3(8, 8)); + check_warp_abs_max(dim3(4, 4, 4)); } } // namespace core @@ -659,3 +682,27 @@ TEST(Functional, warp_abs_max_nan) { TEST(Functional, half_warp_abs_max_nan) { test::core::check_warp_abs_max<16, cutlass::redux_abs_max_nan_propagation_sync_warp_t0t15_t16t31>(); } + +TEST(Functional, subgroup_abs_max_nan_width1) { + test::core::check_warp_abs_max_block_shapes<1>(); +} + +TEST(Functional, subgroup_abs_max_nan_width2) { + test::core::check_warp_abs_max_block_shapes<2>(); +} + +TEST(Functional, subgroup_abs_max_nan_width4) { + test::core::check_warp_abs_max_block_shapes<4>(); +} + +TEST(Functional, subgroup_abs_max_nan_width8) { + test::core::check_warp_abs_max_block_shapes<8>(); +} + +TEST(Functional, subgroup_abs_max_nan_width16) { + test::core::check_warp_abs_max_block_shapes<16>(); +} + +TEST(Functional, subgroup_abs_max_nan_width32) { + test::core::check_warp_abs_max_block_shapes<32>(); +}