diff --git a/c/parallel/src/util/tuning.cpp b/c/parallel/src/util/tuning.cpp index d89608fd24ea..e4d487543584 100644 --- a/c/parallel/src/util/tuning.cpp +++ b/c/parallel/src/util/tuning.cpp @@ -10,9 +10,9 @@ #include "tuning.h" -#include +#include int nominal_4b_items_to_items(int nominal_4b_items_per_thread, int key_size) { - return std::clamp(nominal_4b_items_per_thread * 4 / key_size, 1, nominal_4b_items_per_thread); + return cub::detail::nominal_4B_items_to_items(nominal_4b_items_per_thread, key_size); } diff --git a/libcudacxx/include/cuda/std/__bit/bit_reverse.h b/libcudacxx/include/cuda/std/__bit/bit_reverse.h index b0da7e2d107d..67bbd28b0165 100644 --- a/libcudacxx/include/cuda/std/__bit/bit_reverse.h +++ b/libcudacxx/include/cuda/std/__bit/bit_reverse.h @@ -30,6 +30,10 @@ #include +#if _CCCL_HAS_BUILTIN(__builtin_bitreverseg) +# define _CCCL_BUILTIN_BITREVERSEG(...) __builtin_bitreverseg(__VA_ARGS__) +#endif // has __builtin_bitreverseg + #if _CCCL_HAS_BUILTIN(__builtin_bitreverse8) # define _CCCL_BUILTIN_BITREVERSE8(...) __builtin_bitreverse8(__VA_ARGS__) #endif // has __builtin_bitreverse8 @@ -113,55 +117,59 @@ template template [[nodiscard]] _CCCL_API constexpr _Tp __bit_reverse_impl_host(_Tp __v) noexcept { +# if defined(_CCCL_BUILTIN_BITREVERSEG) + return _CCCL_BUILTIN_BITREVERSEG(__v); +# else // ^^^ _CCCL_BUILTIN_BITREVERSEG ^^^ / vvv !_CCCL_BUILTIN_BITREVERSEG vvv if constexpr (sizeof(_Tp) == sizeof(uint8_t)) { -# if defined(_CCCL_BUILTIN_BITREVERSE8) +# if defined(_CCCL_BUILTIN_BITREVERSE8) return _CCCL_BUILTIN_BITREVERSE8(__v); -# else // ^^^ _CCCL_BUILTIN_BITREVERSE8 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE8 vvv +# else // ^^^ _CCCL_BUILTIN_BITREVERSE8 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE8 vvv return ::cuda::std::__bit_reverse_impl_generic(__v); -# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE8 ^^^ +# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE8 ^^^ } else if constexpr (sizeof(_Tp) == sizeof(uint16_t)) { -# if defined(_CCCL_BUILTIN_BITREVERSE16) +# if defined(_CCCL_BUILTIN_BITREVERSE16) return _CCCL_BUILTIN_BITREVERSE16(__v); -# else // ^^^ _CCCL_BUILTIN_BITREVERSE16 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE16 vvv +# else // ^^^ _CCCL_BUILTIN_BITREVERSE16 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE16 vvv return ::cuda::std::__bit_reverse_impl_generic(__v); -# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE16 ^^^ +# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE16 ^^^ } else if constexpr (sizeof(_Tp) == sizeof(uint32_t)) { -# if defined(_CCCL_BUILTIN_BITREVERSE32) +# if defined(_CCCL_BUILTIN_BITREVERSE32) return _CCCL_BUILTIN_BITREVERSE32(__v); -# else // ^^^ _CCCL_BUILTIN_BITREVERSE32 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE32 vvv +# else // ^^^ _CCCL_BUILTIN_BITREVERSE32 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE32 vvv return ::cuda::std::__bit_reverse_impl_generic(__v); -# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE32 ^^^ +# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE32 ^^^ } else if constexpr (sizeof(_Tp) == sizeof(uint64_t)) { -# if defined(_CCCL_BUILTIN_BITREVERSE64) +# if defined(_CCCL_BUILTIN_BITREVERSE64) return _CCCL_BUILTIN_BITREVERSE64(__v); -# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv +# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv return ::cuda::std::__bit_reverse_impl_generic(__v); -# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^ +# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^ } -# if _CCCL_HAS_INT128() +# if _CCCL_HAS_INT128() else if constexpr (sizeof(_Tp) == sizeof(__uint128_t)) { -# if defined(_CCCL_BUILTIN_BITREVERSE128) +# if defined(_CCCL_BUILTIN_BITREVERSE128) return _CCCL_BUILTIN_BITREVERSE128(__v); -# elif defined(_CCCL_BUILTIN_BITREVERSE64) +# elif defined(_CCCL_BUILTIN_BITREVERSE64) return (__uint128_t{_CCCL_BUILTIN_BITREVERSE64(static_cast(__v))} << 64) | _CCCL_BUILTIN_BITREVERSE64(static_cast(__v >> 64)); -# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv +# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv return ::cuda::std::__bit_reverse_impl_generic(__v); -# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^ +# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^ } -# endif // _CCCL_HAS_INT128() +# endif // _CCCL_HAS_INT128() else { return ::cuda::std::__bit_reverse_impl_generic(__v); } +# endif // !_CCCL_BUILTIN_BITREVERSEG } #endif // _CCCL_HOST_COMPILATION() diff --git a/libcudacxx/include/cuda/std/__bit/countl.h b/libcudacxx/include/cuda/std/__bit/countl.h index 12c6c5c362ff..76a461dd4eda 100644 --- a/libcudacxx/include/cuda/std/__bit/countl.h +++ b/libcudacxx/include/cuda/std/__bit/countl.h @@ -21,6 +21,7 @@ # pragma system_header #endif // no system header +#include #include #include #include // IWYU pragma: keep @@ -73,17 +74,13 @@ template # if defined(_CCCL_BUILTIN_CLZ) return _CCCL_BUILTIN_CLZ(__v); # else // ^^^ _CCCL_BUILTIN_CLZ ^^^ // vvv !_CCCL_BUILTIN_CLZ vvv - uint32_t __res = 0; - for (uint32_t __i = __digits / 2; __i >= 1; __i /= 2) - { - const auto __mark = (~uint32_t{0} >> (__digits - __i)) << __i; - if (__v & __mark) - { - __v >>= __i; - __res |= __i; - } - } - return __digits - 1 - __res; + // Fill the bits below the highest set bit; the remaining zeros are leading. + __v |= __v >> 1; + __v |= __v >> 2; + __v |= __v >> 4; + __v |= __v >> 8; + __v |= __v >> 16; + return ::cuda::std::popcount(static_cast<_Tp>(~__v)); # endif // ^^^ !_CCCL_BUILTIN_CLZ ^^^ } else diff --git a/libcudacxx/include/cuda/std/__bit/countr.h b/libcudacxx/include/cuda/std/__bit/countr.h index a02d208b7390..2e91fb6c254d 100644 --- a/libcudacxx/include/cuda/std/__bit/countr.h +++ b/libcudacxx/include/cuda/std/__bit/countr.h @@ -21,6 +21,7 @@ # pragma system_header #endif // no system header +#include #include #include #include // IWYU pragma: keep @@ -55,38 +56,9 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD template [[nodiscard]] _CCCL_API constexpr int __cccl_countr_zero_impl_constexpr(_Tp __v) noexcept { - constexpr auto __digits = numeric_limits::digits; - - if (__v == 0) - { - return __digits; - } - - if constexpr (sizeof(_Tp) == sizeof(uint32_t)) - { - if (__v & 1) - { - return 0; - } - int __pos = 1; - for (int __i = __digits / 2; __i >= 2; __i /= 2) - { - const auto __mark = ~uint32_t{0} >> (__digits - __i); - if ((__v & __mark) == 0) - { - __v >>= __i; - __pos += __i; - } - } - return (__pos - (__v & 1)); - } - else - { - const auto __hi = static_cast(__v >> 32); - const auto __lo = static_cast(__v); - return (__lo != 0) ? ::cuda::std::__cccl_countr_zero_impl_constexpr(__lo) - : numeric_limits::digits + ::cuda::std::__cccl_countr_zero_impl_constexpr(__hi); - } + // Subtracting one sets exactly the bits below the lowest set bit. For zero, + // unsigned wraparound makes the mask all ones, yielding the full bit width. + return ::cuda::std::popcount(static_cast<_Tp>((__v - _Tp{1}) & ~__v)); } # if !_CCCL_COMPILER(NVRTC) diff --git a/libcudacxx/test/libcudacxx/std/numerics/bit/bit.count/bit_reverse.pass.cpp b/libcudacxx/test/libcudacxx/std/numerics/bit/bit.count/bit_reverse.pass.cpp index 24b5a0ed7395..34692338e43c 100644 --- a/libcudacxx/test/libcudacxx/std/numerics/bit/bit.count/bit_reverse.pass.cpp +++ b/libcudacxx/test/libcudacxx/std/numerics/bit/bit.count/bit_reverse.pass.cpp @@ -72,6 +72,12 @@ TEST_FUNC constexpr void test() { test_bit_reverse(value); } + + for (int bit = 0; bit < digits; ++bit) + { + test_bit_reverse(static_cast(T{1} << bit)); + test_bit_reverse(static_cast(all_ones ^ (T{1} << bit))); + } } TEST_FUNC constexpr bool test() diff --git a/thrust/examples/bit_reverse_copy.cu b/thrust/examples/bit_reverse_copy.cu new file mode 100644 index 000000000000..7f82f99f039d --- /dev/null +++ b/thrust/examples/bit_reverse_copy.cu @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include + +#include +#include + +// Radix-2 FFTs commonly reorder samples by reversing the bits of their indices. +// For eight samples, index 001 becomes 100, 011 becomes 110, and so on. +// This example performs only that permutation, not the FFT butterflies. +int main() +{ + const thrust::host_vector samples{10, 11, 12, 13, 14, 15, 16, 17}; + const thrust::device_vector input = samples; + thrust::device_vector reordered(input.size()); + + thrust::bit_reverse_copy(thrust::device, input.begin(), input.end(), reordered.begin()); + + const thrust::host_vector actual = reordered; + const thrust::host_vector expected{10, 14, 12, 16, 11, 15, 13, 17}; + assert(actual == expected); + std::cout << "Bit-reversed samples:"; + for (const auto sample : actual) + { + std::cout << ' ' << sample; + } + std::cout << '\n'; + + // The permutation is its own inverse. Use a separate destination: overlapping + // input and output are not supported by bit_reverse_copy. + thrust::device_vector restored(input.size()); + thrust::bit_reverse_copy(reordered.begin(), reordered.end(), restored.begin()); + const thrust::host_vector round_trip = restored; + assert(round_trip == samples); + std::cout << "Original order restored\n"; +} diff --git a/thrust/testing/bit_reverse.cu b/thrust/testing/bit_reverse.cu new file mode 100644 index 000000000000..34b5eeb963b2 --- /dev/null +++ b/thrust/testing/bit_reverse.cu @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include + +#include + +#include +#include + +#include + +template +void test_bit_reverse_copy_subrange() +{ + const Vector input{99, 10, 23, 35, 47, 52, 61, 76, 88, 98}; + Vector output(10, 97); + const auto end = thrust::bit_reverse_copy(input.begin() + 1, input.end() - 1, output.begin() + 1); + const Vector expected{97, 10, 52, 35, 76, 23, 61, 47, 88, 97}; + const Vector unchanged{99, 10, 23, 35, 47, 52, 61, 76, 88, 98}; + REQUIRE(output == expected); + REQUIRE(end == output.end() - 1); + REQUIRE(input == unchanged); + + Vector restored(8); + thrust::bit_reverse_copy(output.begin() + 1, output.end() - 1, restored.begin()); + const Vector original(input.begin() + 1, input.end() - 1); + REQUIRE(restored == original); +} +DECLARE_VECTOR_UNITTEST(test_bit_reverse_copy_subrange); + +TEST_CASE("Bit reversal handles power-of-two lengths on host and device", "[bit_reverse]") +{ + const int count = GENERATE(0, 1, 2, 4, 16, 256, 1024); + thrust::host_vector input(count); + thrust::host_vector expected(count); + for (int i = 0; i < count; ++i) + { + input[i] = 3 * i + 7; + } + for (int i = 0; i < count; ++i) + { + // Reverse one bit at a time, independently of the implementation's + // full-width reversal and leading-zero calculation. + int index = i; + int reversed = 0; + for (int remaining = count; remaining > 1; remaining /= 2) + { + reversed = 2 * reversed + index % 2; + index /= 2; + } + expected[i] = input[reversed]; + } + + thrust::host_vector host_output(count + 2, -1); + const auto host_end = thrust::bit_reverse_copy(thrust::host, input.begin(), input.end(), host_output.begin() + 1); + REQUIRE(host_end == host_output.end() - 1); + REQUIRE(host_output.front() == -1); + REQUIRE(host_output.back() == -1); + const thrust::host_vector host_values(host_output.begin() + 1, host_output.end() - 1); + REQUIRE(host_values == expected); + + const thrust::device_vector device_input = input; + thrust::device_vector device_output(count + 2, -1); + const auto device_end = + thrust::bit_reverse_copy(thrust::device, device_input.begin(), device_input.end(), device_output.begin() + 1); + REQUIRE(device_end == device_output.end() - 1); + const thrust::host_vector device_values = device_output; + REQUIRE(device_values.front() == -1); + REQUIRE(device_values.back() == -1); + const thrust::host_vector device_middle(device_values.begin() + 1, device_values.end() - 1); + REQUIRE(device_middle == expected); +} + +TEST_CASE("Bit reversal supports generated input and output-only iterators", "[bit_reverse]") +{ + const thrust::counting_iterator first(100); + std::vector output; + thrust::bit_reverse_copy(thrust::seq, first, first + 8, std::back_inserter(output)); + const std::vector expected{100, 104, 102, 106, 101, 105, 103, 107}; + REQUIRE(output == expected); + + const auto discarded = thrust::bit_reverse_copy(thrust::seq, first, first + 8, thrust::make_discard_iterator()); + REQUIRE(discarded == thrust::make_discard_iterator(8)); + + // Exercise a 32-bit difference type as well as the default pointer-sized type. + using Iterator = thrust::counting_iterator; + thrust::device_vector device_output(8); + const auto end = thrust::bit_reverse_copy(thrust::device, Iterator(100), Iterator(108), device_output.begin()); + REQUIRE(end == device_output.end()); + const thrust::host_vector actual = device_output; + const thrust::host_vector expected32{100, 104, 102, 106, 101, 105, 103, 107}; + REQUIRE(actual == expected32); +} diff --git a/thrust/thrust/bit_reverse.h b/thrust/thrust/bit_reverse.h new file mode 100644 index 000000000000..a71d1cb95ed6 --- /dev/null +++ b/thrust/thrust/bit_reverse.h @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +//! @file +//! @brief Copy a power-of-two range in bit-reversed index order. + +#pragma once + +#include + +#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) +# pragma GCC system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) +# pragma clang system_header +#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) +# pragma system_header +#endif // no system header + +#include + +THRUST_NAMESPACE_BEGIN + +//! @brief Copy elements in bit-reversed index order using an execution policy. +//! +//! For a range of size N = 2^k, output position i receives the input element +//! whose index is obtained by reversing the low k bits of i. This is the +//! permutation used by radix-2 FFT algorithms; the bits of the values themselves +//! are not changed. An empty range is also supported. +//! +//! For example, eight input elements {10, 11, 12, 13, 14, 15, 16, 17} produce +//! {10, 14, 12, 16, 11, 15, 13, 17}. Applying the permutation twice restores +//! the original order. The index map is computed lazily, without allocating +//! an N-element index array. The selected backend may use temporary storage. +//! +//! @param[in] exec The execution policy controlling parallelization. +//! @param[in] first Beginning of the input range. +//! @param[in] last End of the input range. +//! @param[out] result Beginning of the destination range. +//! @return The output iterator advanced by last - first. +//! @pre The input iterators support random access and last - first is nonnegative. +//! @pre The input length is zero or a power of two. +//! @pre The destination has space for the input length and does not overlap the input. +//! @note This is an out-of-place reordering, not an in-place FFT implementation. +template +_CCCL_HOST_DEVICE OutputIterator bit_reverse_copy( + const THRUST_NS_QUALIFIER::detail::execution_policy_base& exec, + RandomAccessIterator first, + RandomAccessIterator last, + OutputIterator result); + +//! @brief Copy elements in bit-reversed index order, selecting the iterator system. +//! +//! This overload has the same ordering and preconditions as the policy overload. +//! The input length must be zero or a power of two, and the input and output +//! ranges must not overlap. Empty and one-element ranges retain their order. +//! +//! @param[in] first Beginning of the random-access input range. +//! @param[in] last End of the input range. +//! @param[out] result Beginning of the destination range. +//! @return The output iterator advanced by last - first. +template +OutputIterator bit_reverse_copy(RandomAccessIterator first, RandomAccessIterator last, OutputIterator result); + +THRUST_NAMESPACE_END + +#include diff --git a/thrust/thrust/detail/bit_reverse.inl b/thrust/thrust/detail/bit_reverse.inl new file mode 100644 index 000000000000..f49d10114cc7 --- /dev/null +++ b/thrust/thrust/detail/bit_reverse.inl @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include + +#include +#include +#include +#include + +THRUST_NAMESPACE_BEGIN + +_CCCL_EXEC_CHECK_DISABLE +template +_CCCL_HOST_DEVICE OutputIterator bit_reverse_copy( + const THRUST_NS_QUALIFIER::detail::execution_policy_base& exec, + RandomAccessIterator first, + RandomAccessIterator last, + OutputIterator result) +{ + _CCCL_NVTX_RANGE_SCOPE("thrust::bit_reverse_copy"); + using THRUST_NS_QUALIFIER::system::detail::generic::bit_reverse_copy; + return bit_reverse_copy( + THRUST_NS_QUALIFIER::detail::derived_cast(THRUST_NS_QUALIFIER::detail::strip_const(exec)), first, last, result); +} + +template +OutputIterator bit_reverse_copy(RandomAccessIterator first, RandomAccessIterator last, OutputIterator result) +{ + _CCCL_NVTX_RANGE_SCOPE("thrust::bit_reverse_copy"); + using THRUST_NS_QUALIFIER::system::detail::generic::select_system; + using InputSystem = typename THRUST_NS_QUALIFIER::iterator_system::type; + using OutputSystem = typename THRUST_NS_QUALIFIER::iterator_system::type; + InputSystem input_system; + OutputSystem output_system; + return THRUST_NS_QUALIFIER::bit_reverse_copy(select_system(input_system, output_system), first, last, result); +} + +THRUST_NAMESPACE_END diff --git a/thrust/thrust/system/cuda/detail/terminate.h b/thrust/thrust/system/cuda/detail/terminate.h index 0a4066a144a3..84e2074f8467 100644 --- a/thrust/thrust/system/cuda/detail/terminate.h +++ b/thrust/thrust/system/cuda/detail/terminate.h @@ -20,10 +20,16 @@ THRUST_NAMESPACE_BEGIN namespace system::cuda::detail { -inline _CCCL_HOST_DEVICE void terminate_with_message(const char* message) +template +[[noreturn]] inline _CCCL_HOST_DEVICE void terminate_with_format(const char* format, Args... args) { - printf("%s\n", message); + printf(format, args...); ::cuda::std::terminate(); } + +inline _CCCL_HOST_DEVICE void terminate_with_message(const char* message) +{ + THRUST_NS_QUALIFIER::system::cuda::detail::terminate_with_format("%s\n", message); +} } // namespace system::cuda::detail THRUST_NAMESPACE_END diff --git a/thrust/thrust/system/cuda/detail/util.h b/thrust/thrust/system/cuda/detail/util.h index 214fdb987f44..c5ebbf4789c4 100644 --- a/thrust/thrust/system/cuda/detail/util.h +++ b/thrust/thrust/system/cuda/detail/util.h @@ -18,6 +18,8 @@ #include +#include + #include #include @@ -178,22 +180,22 @@ _CCCL_HOST_DEVICE inline void throw_on_error(cudaError_t status) // instructions out of the target logic. #ifdef THRUST_RDC_ENABLED -# define THRUST_TEMP_DEVICE_CODE \ - printf("Thrust CUDA backend error: %s: %s\n", cudaGetErrorName(status), cudaGetErrorString(status)) +# define THRUST_TEMP_DEVICE_CODE \ + THRUST_NS_QUALIFIER::system::cuda::detail::terminate_with_format( \ + "Thrust CUDA backend error: %s: %s\n", cudaGetErrorName(status), cudaGetErrorString(status)) #else -# define THRUST_TEMP_DEVICE_CODE printf("Thrust CUDA backend error: %d\n", static_cast(status)) +# define THRUST_TEMP_DEVICE_CODE \ + THRUST_NS_QUALIFIER::system::cuda::detail::terminate_with_format( \ + "Thrust CUDA backend error: %d\n", static_cast(status)) #endif #if _CCCL_HOSTED() - NV_IF_TARGET(NV_IS_HOST, - (throw thrust::system_error(status, thrust::cuda_category());), - (THRUST_TEMP_DEVICE_CODE; ::cuda::std::terminate();)); + NV_IF_TARGET(NV_IS_HOST, (throw thrust::system_error(status, thrust::cuda_category());), (THRUST_TEMP_DEVICE_CODE;)); #else // ^^^ _CCCL_HOSTED() ^^^ / vvv _CCCL_FREESTANDING() vvv THRUST_TEMP_DEVICE_CODE; - ::cuda::std::terminate(); #endif // _CCCL_FREESTANDING() #undef THRUST_TEMP_DEVICE_CODE @@ -216,22 +218,23 @@ _CCCL_HOST_DEVICE inline void throw_on_error(cudaError_t status, char const* msg // instructions out of the target logic. #ifdef THRUST_RDC_ENABLED -# define THRUST_TEMP_DEVICE_CODE \ - printf("Thrust CUDA backend error: %s: %s: %s\n", cudaGetErrorName(status), cudaGetErrorString(status), msg) +# define THRUST_TEMP_DEVICE_CODE \ + THRUST_NS_QUALIFIER::system::cuda::detail::terminate_with_format( \ + "Thrust CUDA backend error: %s: %s: %s\n", cudaGetErrorName(status), cudaGetErrorString(status), msg) #else -# define THRUST_TEMP_DEVICE_CODE printf("Thrust CUDA backend error: %d: %s\n", static_cast(status), msg) +# define THRUST_TEMP_DEVICE_CODE \ + THRUST_NS_QUALIFIER::system::cuda::detail::terminate_with_format( \ + "Thrust CUDA backend error: %d: %s\n", static_cast(status), msg) #endif #if _CCCL_HOSTED() - NV_IF_TARGET(NV_IS_HOST, - (throw thrust::system_error(status, thrust::cuda_category(), msg);), - (THRUST_TEMP_DEVICE_CODE; ::cuda::std::terminate();)); + NV_IF_TARGET( + NV_IS_HOST, (throw thrust::system_error(status, thrust::cuda_category(), msg);), (THRUST_TEMP_DEVICE_CODE;)); #else // ^^^ _CCCL_HOSTED() ^^^ / vvv _CCCL_FREESTANDING() vvv THRUST_TEMP_DEVICE_CODE; - ::cuda::std::terminate(); #endif // _CCCL_FREESTANDING() #undef THRUST_TEMP_DEVICE_CODE diff --git a/thrust/thrust/system/detail/generic/bit_reverse.h b/thrust/thrust/system/detail/generic/bit_reverse.h new file mode 100644 index 000000000000..c8740fb91b8f --- /dev/null +++ b/thrust/thrust/system/detail/generic/bit_reverse.h @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include + +#include + +THRUST_NAMESPACE_BEGIN +namespace system::detail::generic +{ +template +_CCCL_HOST_DEVICE OutputIterator bit_reverse_copy( + THRUST_NS_QUALIFIER::execution_policy& exec, + RandomAccessIterator first, + RandomAccessIterator last, + OutputIterator result); +} // namespace system::detail::generic +THRUST_NAMESPACE_END + +#include diff --git a/thrust/thrust/system/detail/generic/bit_reverse.inl b/thrust/thrust/system/detail/generic/bit_reverse.inl new file mode 100644 index 000000000000..eb3b1dba6e2a --- /dev/null +++ b/thrust/thrust/system/detail/generic/bit_reverse.inl @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +THRUST_NAMESPACE_BEGIN +namespace system::detail::generic +{ +template +struct bit_reverse_index +{ + int shift; + + [[nodiscard]] _CCCL_HOST_DEVICE constexpr Index operator()(Index index) const noexcept + { + using Unsigned = ::cuda::std::make_unsigned_t; + return static_cast(::cuda::std::bit_reverse(static_cast(index)) >> shift); + } +}; + +template +_CCCL_HOST_DEVICE OutputIterator bit_reverse_copy( + THRUST_NS_QUALIFIER::execution_policy& exec, + RandomAccessIterator first, + RandomAccessIterator last, + OutputIterator result) +{ + using Index = typename ::cuda::std::iterator_traits::difference_type; + using Unsigned = ::cuda::std::make_unsigned_t; + const Index count = last - first; + _CCCL_ASSERT(count >= 0, "bit_reverse_copy requires a nonnegative range length"); + if (count == 0) + { + return result; + } + _CCCL_ASSERT(::cuda::std::has_single_bit(static_cast(count)), + "bit_reverse_copy requires a power-of-two range length"); + + // A singleton has a zero-bit index. Use a zero shift rather than shifting + // by the full word width; its only index is zero either way. + const int shift = count == 1 ? 0 : ::cuda::std::countl_zero(static_cast(count)) + 1; + const auto map = THRUST_NS_QUALIFIER::make_transform_iterator( + THRUST_NS_QUALIFIER::counting_iterator(0), bit_reverse_index{shift}); + return THRUST_NS_QUALIFIER::copy( + exec, + THRUST_NS_QUALIFIER::make_permutation_iterator(first, map), + THRUST_NS_QUALIFIER::make_permutation_iterator(first, map + count), + result); +} +} // namespace system::detail::generic +THRUST_NAMESPACE_END