Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions c/parallel/src/util/tuning.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@

#include "tuning.h"

#include <algorithm>
#include <cub/util_math.cuh>

int nominal_4b_items_to_items(int nominal_4b_items_per_thread, int key_size)
{
return std::clamp(nominal_4b_items_per_thread * 4 / key_size, 1, nominal_4b_items_per_thread);
return cub::detail::nominal_4B_items_to_items(nominal_4b_items_per_thread, key_size);
}
44 changes: 26 additions & 18 deletions libcudacxx/include/cuda/std/__bit/bit_reverse.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@

#include <cuda/std/__cccl/prologue.h>

#if _CCCL_HAS_BUILTIN(__builtin_bitreverseg)
# define _CCCL_BUILTIN_BITREVERSEG(...) __builtin_bitreverseg(__VA_ARGS__)
#endif // has __builtin_bitreverseg

#if _CCCL_HAS_BUILTIN(__builtin_bitreverse8)
# define _CCCL_BUILTIN_BITREVERSE8(...) __builtin_bitreverse8(__VA_ARGS__)
#endif // has __builtin_bitreverse8
Expand Down Expand Up @@ -113,55 +117,59 @@ template <class _Tp>
template <class _Tp>
[[nodiscard]] _CCCL_API constexpr _Tp __bit_reverse_impl_host(_Tp __v) noexcept
{
# if defined(_CCCL_BUILTIN_BITREVERSEG)
return _CCCL_BUILTIN_BITREVERSEG(__v);
# else // ^^^ _CCCL_BUILTIN_BITREVERSEG ^^^ / vvv !_CCCL_BUILTIN_BITREVERSEG vvv
if constexpr (sizeof(_Tp) == sizeof(uint8_t))
{
# if defined(_CCCL_BUILTIN_BITREVERSE8)
# if defined(_CCCL_BUILTIN_BITREVERSE8)
return _CCCL_BUILTIN_BITREVERSE8(__v);
# else // ^^^ _CCCL_BUILTIN_BITREVERSE8 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE8 vvv
# else // ^^^ _CCCL_BUILTIN_BITREVERSE8 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE8 vvv
return ::cuda::std::__bit_reverse_impl_generic(__v);
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE8 ^^^
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE8 ^^^
}
else if constexpr (sizeof(_Tp) == sizeof(uint16_t))
{
# if defined(_CCCL_BUILTIN_BITREVERSE16)
# if defined(_CCCL_BUILTIN_BITREVERSE16)
return _CCCL_BUILTIN_BITREVERSE16(__v);
# else // ^^^ _CCCL_BUILTIN_BITREVERSE16 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE16 vvv
# else // ^^^ _CCCL_BUILTIN_BITREVERSE16 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE16 vvv
return ::cuda::std::__bit_reverse_impl_generic(__v);
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE16 ^^^
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE16 ^^^
}
else if constexpr (sizeof(_Tp) == sizeof(uint32_t))
{
# if defined(_CCCL_BUILTIN_BITREVERSE32)
# if defined(_CCCL_BUILTIN_BITREVERSE32)
return _CCCL_BUILTIN_BITREVERSE32(__v);
# else // ^^^ _CCCL_BUILTIN_BITREVERSE32 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE32 vvv
# else // ^^^ _CCCL_BUILTIN_BITREVERSE32 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE32 vvv
return ::cuda::std::__bit_reverse_impl_generic(__v);
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE32 ^^^
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE32 ^^^
}
else if constexpr (sizeof(_Tp) == sizeof(uint64_t))
{
# if defined(_CCCL_BUILTIN_BITREVERSE64)
# if defined(_CCCL_BUILTIN_BITREVERSE64)
return _CCCL_BUILTIN_BITREVERSE64(__v);
# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv
# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv
return ::cuda::std::__bit_reverse_impl_generic(__v);
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^
}
# if _CCCL_HAS_INT128()
# if _CCCL_HAS_INT128()
else if constexpr (sizeof(_Tp) == sizeof(__uint128_t))
{
# if defined(_CCCL_BUILTIN_BITREVERSE128)
# if defined(_CCCL_BUILTIN_BITREVERSE128)
return _CCCL_BUILTIN_BITREVERSE128(__v);
# elif defined(_CCCL_BUILTIN_BITREVERSE64)
# elif defined(_CCCL_BUILTIN_BITREVERSE64)
return (__uint128_t{_CCCL_BUILTIN_BITREVERSE64(static_cast<uint64_t>(__v))} << 64)
| _CCCL_BUILTIN_BITREVERSE64(static_cast<uint64_t>(__v >> 64));
# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv
# else // ^^^ _CCCL_BUILTIN_BITREVERSE64 ^^^ / vvv !_CCCL_BUILTIN_BITREVERSE64 vvv
return ::cuda::std::__bit_reverse_impl_generic(__v);
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^
# endif // ^^^ !_CCCL_BUILTIN_BITREVERSE64 ^^^
}
# endif // _CCCL_HAS_INT128()
# endif // _CCCL_HAS_INT128()
else
{
return ::cuda::std::__bit_reverse_impl_generic(__v);
}
# endif // !_CCCL_BUILTIN_BITREVERSEG
}
#endif // _CCCL_HOST_COMPILATION()

Expand Down
19 changes: 8 additions & 11 deletions libcudacxx/include/cuda/std/__bit/countl.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
# pragma system_header
#endif // no system header

#include <cuda/std/__bit/popcount.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/conditional.h> // IWYU pragma: keep
Expand Down Expand Up @@ -73,17 +74,13 @@ template <typename _Tp>
# if defined(_CCCL_BUILTIN_CLZ)
return _CCCL_BUILTIN_CLZ(__v);
# else // ^^^ _CCCL_BUILTIN_CLZ ^^^ // vvv !_CCCL_BUILTIN_CLZ vvv
uint32_t __res = 0;
for (uint32_t __i = __digits / 2; __i >= 1; __i /= 2)
{
const auto __mark = (~uint32_t{0} >> (__digits - __i)) << __i;
if (__v & __mark)
{
__v >>= __i;
__res |= __i;
}
}
return __digits - 1 - __res;
// Fill the bits below the highest set bit; the remaining zeros are leading.
__v |= __v >> 1;
__v |= __v >> 2;
__v |= __v >> 4;
__v |= __v >> 8;
__v |= __v >> 16;
return ::cuda::std::popcount(static_cast<_Tp>(~__v));
# endif // ^^^ !_CCCL_BUILTIN_CLZ ^^^
}
else
Expand Down
36 changes: 4 additions & 32 deletions libcudacxx/include/cuda/std/__bit/countr.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
# pragma system_header
#endif // no system header

#include <cuda/std/__bit/popcount.h>
#include <cuda/std/__concepts/concept_macros.h>
#include <cuda/std/__limits/numeric_limits.h>
#include <cuda/std/__type_traits/conditional.h> // IWYU pragma: keep
Expand Down Expand Up @@ -55,38 +56,9 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD
template <typename _Tp>
[[nodiscard]] _CCCL_API constexpr int __cccl_countr_zero_impl_constexpr(_Tp __v) noexcept
{
constexpr auto __digits = numeric_limits<uint32_t>::digits;

if (__v == 0)
{
return __digits;
}

if constexpr (sizeof(_Tp) == sizeof(uint32_t))
{
if (__v & 1)
{
return 0;
}
int __pos = 1;
for (int __i = __digits / 2; __i >= 2; __i /= 2)
{
const auto __mark = ~uint32_t{0} >> (__digits - __i);
if ((__v & __mark) == 0)
{
__v >>= __i;
__pos += __i;
}
}
return (__pos - (__v & 1));
}
else
{
const auto __hi = static_cast<uint32_t>(__v >> 32);
const auto __lo = static_cast<uint32_t>(__v);
return (__lo != 0) ? ::cuda::std::__cccl_countr_zero_impl_constexpr(__lo)
: numeric_limits<uint32_t>::digits + ::cuda::std::__cccl_countr_zero_impl_constexpr(__hi);
}
// Subtracting one sets exactly the bits below the lowest set bit. For zero,
// unsigned wraparound makes the mask all ones, yielding the full bit width.
return ::cuda::std::popcount(static_cast<_Tp>((__v - _Tp{1}) & ~__v));
}

# if !_CCCL_COMPILER(NVRTC)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,12 @@ TEST_FUNC constexpr void test()
{
test_bit_reverse(value);
}

for (int bit = 0; bit < digits; ++bit)
{
test_bit_reverse(static_cast<T>(T{1} << bit));
test_bit_reverse(static_cast<T>(all_ones ^ (T{1} << bit)));
}
}

TEST_FUNC constexpr bool test()
Expand Down
40 changes: 40 additions & 0 deletions thrust/examples/bit_reverse_copy.cu
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include <thrust/bit_reverse.h>
#include <thrust/device_vector.h>
#include <thrust/execution_policy.h>
#include <thrust/host_vector.h>

#include <cassert>
#include <iostream>

// Radix-2 FFTs commonly reorder samples by reversing the bits of their indices.
// For eight samples, index 001 becomes 100, 011 becomes 110, and so on.
// This example performs only that permutation, not the FFT butterflies.
int main()
{
const thrust::host_vector<int> samples{10, 11, 12, 13, 14, 15, 16, 17};
const thrust::device_vector<int> input = samples;
thrust::device_vector<int> reordered(input.size());

thrust::bit_reverse_copy(thrust::device, input.begin(), input.end(), reordered.begin());

const thrust::host_vector<int> actual = reordered;
const thrust::host_vector<int> expected{10, 14, 12, 16, 11, 15, 13, 17};
assert(actual == expected);
std::cout << "Bit-reversed samples:";
for (const auto sample : actual)
{
std::cout << ' ' << sample;
}
std::cout << '\n';

// The permutation is its own inverse. Use a separate destination: overlapping
// input and output are not supported by bit_reverse_copy.
thrust::device_vector<int> restored(input.size());
thrust::bit_reverse_copy(reordered.begin(), reordered.end(), restored.begin());
const thrust::host_vector<int> round_trip = restored;
assert(round_trip == samples);
std::cout << "Original order restored\n";
}
97 changes: 97 additions & 0 deletions thrust/testing/bit_reverse.cu
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#include <thrust/bit_reverse.h>
#include <thrust/execution_policy.h>
#include <thrust/iterator/counting_iterator.h>
#include <thrust/iterator/discard_iterator.h>

#include <cuda/std/cstdint>

#include <iterator>
#include <vector>

#include <unittest/unittest.h>

template <class Vector>
void test_bit_reverse_copy_subrange()
{
const Vector input{99, 10, 23, 35, 47, 52, 61, 76, 88, 98};
Vector output(10, 97);
const auto end = thrust::bit_reverse_copy(input.begin() + 1, input.end() - 1, output.begin() + 1);
const Vector expected{97, 10, 52, 35, 76, 23, 61, 47, 88, 97};
const Vector unchanged{99, 10, 23, 35, 47, 52, 61, 76, 88, 98};
REQUIRE(output == expected);
REQUIRE(end == output.end() - 1);
REQUIRE(input == unchanged);

Vector restored(8);
thrust::bit_reverse_copy(output.begin() + 1, output.end() - 1, restored.begin());
const Vector original(input.begin() + 1, input.end() - 1);
REQUIRE(restored == original);
}
DECLARE_VECTOR_UNITTEST(test_bit_reverse_copy_subrange);

TEST_CASE("Bit reversal handles power-of-two lengths on host and device", "[bit_reverse]")
{
const int count = GENERATE(0, 1, 2, 4, 16, 256, 1024);
thrust::host_vector<int> input(count);
thrust::host_vector<int> expected(count);
for (int i = 0; i < count; ++i)
{
input[i] = 3 * i + 7;
}
for (int i = 0; i < count; ++i)
{
// Reverse one bit at a time, independently of the implementation's
// full-width reversal and leading-zero calculation.
int index = i;
int reversed = 0;
for (int remaining = count; remaining > 1; remaining /= 2)
{
reversed = 2 * reversed + index % 2;
index /= 2;
}
expected[i] = input[reversed];
}

thrust::host_vector<int> host_output(count + 2, -1);
const auto host_end = thrust::bit_reverse_copy(thrust::host, input.begin(), input.end(), host_output.begin() + 1);
REQUIRE(host_end == host_output.end() - 1);
REQUIRE(host_output.front() == -1);
REQUIRE(host_output.back() == -1);
const thrust::host_vector<int> host_values(host_output.begin() + 1, host_output.end() - 1);
REQUIRE(host_values == expected);

const thrust::device_vector<int> device_input = input;
thrust::device_vector<int> device_output(count + 2, -1);
const auto device_end =
thrust::bit_reverse_copy(thrust::device, device_input.begin(), device_input.end(), device_output.begin() + 1);
REQUIRE(device_end == device_output.end() - 1);
const thrust::host_vector<int> device_values = device_output;
REQUIRE(device_values.front() == -1);
REQUIRE(device_values.back() == -1);
const thrust::host_vector<int> device_middle(device_values.begin() + 1, device_values.end() - 1);
REQUIRE(device_middle == expected);
}

TEST_CASE("Bit reversal supports generated input and output-only iterators", "[bit_reverse]")
{
const thrust::counting_iterator<cuda::std::int64_t> first(100);
std::vector<cuda::std::int64_t> output;
thrust::bit_reverse_copy(thrust::seq, first, first + 8, std::back_inserter(output));
const std::vector<cuda::std::int64_t> expected{100, 104, 102, 106, 101, 105, 103, 107};
REQUIRE(output == expected);

const auto discarded = thrust::bit_reverse_copy(thrust::seq, first, first + 8, thrust::make_discard_iterator());
REQUIRE(discarded == thrust::make_discard_iterator(8));

// Exercise a 32-bit difference type as well as the default pointer-sized type.
using Iterator = thrust::counting_iterator<int, thrust::use_default, thrust::use_default, cuda::std::int32_t>;
thrust::device_vector<int> device_output(8);
const auto end = thrust::bit_reverse_copy(thrust::device, Iterator(100), Iterator(108), device_output.begin());
REQUIRE(end == device_output.end());
const thrust::host_vector<int> actual = device_output;
const thrust::host_vector<int> expected32{100, 104, 102, 106, 101, 105, 103, 107};
REQUIRE(actual == expected32);
}
Loading
Loading