diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e45cd87f3..8905ffbf7 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -500,6 +500,15 @@ if (BUILD_BENCHMARK) ${PROJECT_SRC_DIR}) target_link_libraries(sprintz_batch_decode_benchmark PRIVATE tsfile Threads::Threads) + + if (ENABLE_SIMD) + add_executable(alp_avx2_benchmark + bench_mark/alp_avx2_benchmark.cc) + target_include_directories(alp_avx2_benchmark PRIVATE + ${PROJECT_SRC_DIR}) + target_link_libraries(alp_avx2_benchmark PRIVATE + tsfile Threads::Threads) + endif () endif () if (BUILD_TOOLS) add_subdirectory(tools) diff --git a/cpp/bench_mark/README.md b/cpp/bench_mark/README.md index e7851641f..fd0f54e22 100644 --- a/cpp/bench_mark/README.md +++ b/cpp/bench_mark/README.md @@ -55,6 +55,21 @@ file sizes, compiler flags, and whether the OS page cache was warm. The checksum makes accidental short reads, metadata-load failures, or optimizer removal visible; it is not a TsFile-content checksum. +`alp_avx2_benchmark` measures the ALP SIMD kernels: the SIMDe portable +implementation against the runtime-dispatched native AVX2 copies for +`EncodeValues`, `DecodeValues`, and bit-unpacking. It is meant to be built with +the default flags (no `-mavx2`), which is what a portable build ships: + +```bash +cmake -S cpp -B cpp/build/Release \ + -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARK=ON -DBUILD_TEST=OFF +cmake --build cpp/build/Release --target alp_avx2_benchmark +./cpp/build/Release/alp_avx2_benchmark +``` + +On non-x86-64 targets, and on x86-64 CPUs without AVX2, only the SIMDe column +is measured. The `ok` column checks that both kernels produce identical values. + `gorilla_v1_batch_decode_benchmark` compares the per-value GORILLA_V1 decode loop with the raw-pointer batch decoder on the same encoded stream for FLOAT and DOUBLE columns. It reports elapsed time, values per second and the speedup diff --git a/cpp/bench_mark/alp_avx2_benchmark.cc b/cpp/bench_mark/alp_avx2_benchmark.cc new file mode 100644 index 000000000..c39dbb048 --- /dev/null +++ b/cpp/bench_mark/alp_avx2_benchmark.cc @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Micro-benchmark for the ALP SIMD kernels: SIMDe (portable fallback) versus +// the runtime-dispatched native AVX2 copies. Build with the default flags +// (no -mavx2) so the SIMDe side is what a portable build ships: +// +// cmake -S cpp -B cpp/build/Release \ +// -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARK=ON -DBUILD_TEST=OFF +// cmake --build cpp/build/Release --target alp_avx2_benchmark +// ./cpp/build/Release/alp_avx2_benchmark +// +// When ALP AVX2 dispatch is not compiled (non-x86-64 targets) or the CPU has no +// AVX2, only the SIMDe column is measured. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "encoding/alp/alp_scalar.h" + +#ifdef ENABLE_SIMD +#include "encoding/alp/alp_simde.h" +#ifdef TSFILE_X86_AVX2_DISPATCH +#include "encoding/alp/alp_avx2.h" +#endif +#endif + +namespace { + +using Clock = std::chrono::steady_clock; +using storage::alp::AlpChooseFactorExponent; +using storage::alp::AlpPackBits; +using storage::alp::AlpSimdeKernel; +using storage::alp::AlpSimdeUnpackKernel; +using storage::alp::AlpTypeTraits; +#ifdef ALP_AVX2_DISPATCH +using storage::alp::avx2_detail::AlpAvx2Kernel; +using storage::alp::avx2_detail::AlpAvx2UnpackKernel; +#endif + +const double kTargetBatchMs = 150.0; + +// Keeps the timed kernels from being optimized away when the AVX2 column is +// not compiled in: the kernels write these buffers, and main consumes the +// checksum. +volatile uint64_t g_checksum_sink = 0; + +template +void Consume(const std::vector& values) { + uint64_t checksum = 0; + for (size_t i = 0; i < values.size(); ++i) { + uint64_t bits = 0; + std::memcpy(&bits, &values[i], sizeof(T)); + checksum = checksum * 1099511628211ULL + bits; + } + g_checksum_sink ^= checksum; +} + +template +double MeasureMs(Fn fn, int repetitions) { + const Clock::time_point start = Clock::now(); + for (int i = 0; i < repetitions; ++i) { + fn(); + } + const Clock::time_point end = Clock::now(); + return std::chrono::duration_cast< + std::chrono::duration>(end - start) + .count() / + repetitions; +} + +// Runs one timing sample, calibrating the repetition count so a sample takes +// roughly kTargetBatchMs. Returns nanoseconds per value. +template +double TimeNsPerValue(Fn fn, uint32_t count) { + if (count == 0) { + return 0.0; + } + fn(); // warm up + const double single_ms = MeasureMs(fn, 1); + int repetitions = 1; + if (single_ms > 0.0) { + repetitions = static_cast(kTargetBatchMs / single_ms); + } + repetitions = std::max(1, std::min(repetitions, 100000)); + const double ms = MeasureMs(fn, repetitions); + return ms * 1.0e6 / static_cast(count); +} + +template +std::vector MakeDecimalData(uint64_t count, double scale, double offset) { + std::vector values(static_cast(count)); + for (uint64_t i = 0; i < count; ++i) { + const double raw = static_cast(i % 100000) * scale + offset; + values[static_cast(i)] = static_cast(raw); + } + return values; +} + +template +std::vector MakeRandomData(uint64_t count) { + std::vector values(static_cast(count)); + uint64_t state = 0x9e3779b97f4a7c15ULL; + for (uint64_t i = 0; i < count; ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + const uint64_t bits = state ^ (state >> 29); + if (sizeof(T) == sizeof(float)) { + uint32_t fbits = static_cast(bits); + float v = 0.0f; + std::memcpy(&v, &fbits, sizeof(v)); + if (!std::isfinite(v)) { + v = static_cast(i % 1024) * 0.001f; + } + values[static_cast(i)] = v; + } else { + double v = 0.0; + std::memcpy(&v, &bits, sizeof(v)); + if (!std::isfinite(v)) { + v = static_cast(i % 1024) * 0.001; + } + values[static_cast(i)] = v; + } + } + return values; +} + +template +std::vector MakeConstantData(uint64_t count) { + return std::vector(static_cast(count), + static_cast(3.141592653589793)); +} + +void PrintHeader(const char* title) { + std::cout << std::endl << "==== " << title << " ====" << std::endl; + std::cout << std::left << std::setw(10) << "pattern" << std::setw(8) + << "pair" << std::right << std::setw(16) << "simde(M/s)" + << std::setw(16) << "avx2(M/s)" << std::setw(12) << "speedup" + << std::setw(10) << "ok" << std::endl; +} + +void PrintRow(const std::string& pattern, const std::string& pair, + double simde_mps, double avx2_mps, bool ok) { + std::cout << std::left << std::setw(10) << pattern << std::setw(8) << pair + << std::right << std::setw(16) << std::fixed + << std::setprecision(2) << simde_mps; + if (avx2_mps > 0.0) { + std::cout << std::setw(16) << avx2_mps; + } else { + std::cout << std::setw(16) << "-"; + } + if (avx2_mps > 0.0 && simde_mps > 0.0) { + std::cout << std::setw(12) << std::setprecision(2) + << (avx2_mps / simde_mps); + } else { + std::cout << std::setw(12) << "-"; + } + std::cout << std::setw(10) << (ok ? "yes" : "no") << std::endl; +} + +#ifdef ENABLE_SIMD + +template +void BenchEncode(const std::string& pattern, const std::vector& values) { + typedef AlpTypeTraits Traits; + typedef typename Traits::Encoded Encoded; + const uint32_t count = static_cast(values.size()); + uint8_t factor = 0; + uint8_t exponent = 0; + if (!AlpChooseFactorExponent(&values[0], count, factor, exponent)) { + return; + } + + std::vector simde_encoded(count, 0); + std::vector avx2_encoded(count, 0); + std::vector simde_bitmap((count + 7) / 8, 0); + std::vector avx2_bitmap((count + 7) / 8, 0); + std::vector simde_exceptions(count, T()); + std::vector avx2_exceptions(count, T()); + uint32_t simde_exception_count = 0; + uint32_t avx2_exception_count = 0; +#ifndef ALP_AVX2_DISPATCH + (void)avx2_exception_count; +#endif + const double simde_ns = TimeNsPerValue( + [&]() { + simde_exception_count = 0; + std::fill(simde_bitmap.begin(), simde_bitmap.end(), 0); + AlpSimdeKernel::EncodeValues( + &values[0], count, factor, exponent, &simde_encoded[0], + &simde_bitmap[0], &simde_exceptions[0], &simde_exception_count); + }, + count); + + double avx2_ns = 0.0; + bool ok = true; +#ifdef ALP_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + avx2_ns = TimeNsPerValue( + [&]() { + avx2_exception_count = 0; + std::fill(avx2_bitmap.begin(), avx2_bitmap.end(), 0); + AlpAvx2Kernel::EncodeValues( + &values[0], count, factor, exponent, &avx2_encoded[0], + &avx2_bitmap[0], &avx2_exceptions[0], + &avx2_exception_count); + }, + count); + ok = simde_exception_count == avx2_exception_count && + simde_bitmap == avx2_bitmap && simde_encoded == avx2_encoded; + for (uint32_t i = 0; ok && i < simde_exception_count; ++i) { + ok = Traits::BitwiseEqual(simde_exceptions[i], avx2_exceptions[i]); + } + } +#endif + + Consume(simde_encoded); + Consume(simde_bitmap); + Consume(simde_exceptions); + Consume(avx2_encoded); + Consume(avx2_bitmap); + Consume(avx2_exceptions); + + std::string pair = std::to_string(static_cast(factor)) + "/" + + std::to_string(static_cast(exponent)); + const double simde_mps = simde_ns > 0.0 ? 1.0e3 / simde_ns : 0.0; + const double avx2_mps = avx2_ns > 0.0 ? 1.0e3 / avx2_ns : 0.0; + PrintRow(pattern, pair, simde_mps, avx2_mps, ok); +} + +template +void BenchDecode(const std::string& pattern, const std::vector& values) { + typedef AlpTypeTraits Traits; + typedef typename Traits::Encoded Encoded; + const uint32_t count = static_cast(values.size()); + uint8_t factor = 0; + uint8_t exponent = 0; + if (!AlpChooseFactorExponent(&values[0], count, factor, exponent)) { + return; + } + + std::vector encoded(count, 0); + std::vector bitmap((count + 7) / 8, 0); + std::vector exceptions(count, T()); + uint32_t exception_count = 0; + AlpSimdeKernel::EncodeValues(&values[0], count, factor, exponent, + &encoded[0], &bitmap[0], &exceptions[0], + &exception_count); + + std::vector simde_out(count, T()); + std::vector avx2_out(count, T()); + + const double simde_ns = TimeNsPerValue( + [&]() { + AlpSimdeKernel::DecodeValues( + &encoded[0], count, factor, exponent, &bitmap[0], + exception_count == 0 ? nullptr : &exceptions[0], + exception_count, &simde_out[0]); + }, + count); + + double avx2_ns = 0.0; + bool ok = true; +#ifdef ALP_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + avx2_ns = TimeNsPerValue( + [&]() { + AlpAvx2Kernel::DecodeValues( + &encoded[0], count, factor, exponent, &bitmap[0], + exception_count == 0 ? nullptr : &exceptions[0], + exception_count, &avx2_out[0]); + }, + count); + for (uint32_t i = 0; ok && i < count; ++i) { + ok = Traits::BitwiseEqual(simde_out[i], avx2_out[i]); + } + } +#endif + + Consume(simde_out); + Consume(avx2_out); + + std::string pair = std::to_string(static_cast(factor)) + "/" + + std::to_string(static_cast(exponent)); + const double simde_mps = simde_ns > 0.0 ? 1.0e3 / simde_ns : 0.0; + const double avx2_mps = avx2_ns > 0.0 ? 1.0e3 / avx2_ns : 0.0; + PrintRow(pattern, pair, simde_mps, avx2_mps, ok); +} + +template +void BenchUnpack(uint8_t bit_width, uint32_t count) { + const Unsigned mask = + bit_width >= sizeof(Unsigned) * 8 + ? static_cast(~static_cast(0)) + : static_cast((static_cast(1) << bit_width) - + 1); + std::vector values(count, 0); + uint64_t state = 0x243f6a8885a308d3ULL; + for (uint32_t i = 0; i < count; ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + values[i] = static_cast(state) & mask; + } + std::vector body; + AlpPackBits(&values[0], count, bit_width, body); + + std::vector simde_out(count, 0); + std::vector avx2_out(count, 0); + + const double simde_ns = TimeNsPerValue( + [&]() { + AlpSimdeUnpackKernel::Unpack(body.data(), count, + bit_width, &simde_out[0]); + }, + count); + + double avx2_ns = 0.0; + bool ok = true; +#ifdef ALP_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + avx2_ns = TimeNsPerValue( + [&]() { + AlpAvx2UnpackKernel::Unpack(body.data(), count, + bit_width, &avx2_out[0]); + }, + count); + ok = simde_out == avx2_out; + } +#endif + + Consume(simde_out); + Consume(avx2_out); + + std::string pair = "w=" + std::to_string(static_cast(bit_width)); + const double simde_mps = simde_ns > 0.0 ? 1.0e3 / simde_ns : 0.0; + const double avx2_mps = avx2_ns > 0.0 ? 1.0e3 / avx2_ns : 0.0; + PrintRow(sizeof(Unsigned) == 4 ? "u32" : "u64", pair, simde_mps, avx2_mps, + ok); +} + +template +void RunType(const std::string& type_name) { + const uint64_t count = storage::alp::ALP_BLOCK_SIZE; + const std::vector decimal = MakeDecimalData(count, 0.01, -1.0); + const std::vector random = MakeRandomData(count); + const std::vector constant = MakeConstantData(count); + + PrintHeader((type_name + " EncodeValues").c_str()); + BenchEncode("decimal", decimal); + BenchEncode("random", random); + BenchEncode("constant", constant); + + PrintHeader((type_name + " DecodeValues").c_str()); + BenchDecode("decimal", decimal); + BenchDecode("random", random); + BenchDecode("constant", constant); +} + +#endif // ENABLE_SIMD + +} // namespace + +int main() { +#ifndef ENABLE_SIMD + std::cout << "ENABLE_SIMD is OFF; nothing to benchmark." << std::endl; + return 0; +#else +#ifdef ALP_AVX2_DISPATCH + if (!common::cpu_has_avx2()) { + std::cout << "CPU has no AVX2; only the SIMDe column is measured." + << std::endl; + } +#else + std::cout << "ALP AVX2 dispatch is not compiled on this target; only the " + "SIMDe column is measured." + << std::endl; +#endif + + std::cout << "ALP SIMD kernel micro-benchmark, " + << storage::alp::ALP_BLOCK_SIZE << " values per call" + << std::endl; + RunType("FLOAT"); + RunType("DOUBLE"); + + PrintHeader("Unpack (uint32_t)"); + const uint8_t u32_widths[] = {1, 2, 3, 4, 5, 8, 12, 16, 20, 24, 28, 32}; + for (size_t i = 0; i < sizeof(u32_widths); ++i) { + BenchUnpack(u32_widths[i], storage::alp::ALP_BLOCK_SIZE); + } + PrintHeader("Unpack (uint64_t)"); + const uint8_t u64_widths[] = {1, 2, 4, 8, 12, 16, 20, + 24, 32, 40, 48, 56, 57}; + for (size_t i = 0; i < sizeof(u64_widths); ++i) { + BenchUnpack(u64_widths[i], storage::alp::ALP_BLOCK_SIZE); + } + std::cout << "sink " << g_checksum_sink << std::endl; + return 0; +#endif +} diff --git a/cpp/docs/alp-codec-design.md b/cpp/docs/alp-codec-design.md index ea0f5d633..3026a1b6f 100644 --- a/cpp/docs/alp-codec-design.md +++ b/cpp/docs/alp-codec-design.md @@ -152,18 +152,23 @@ be bit-exactly reproducible by the decoder. SIMDe is already a TsFile C++ dependency. The ALP kernels will use the existing `ENABLE_SIMD` switch and `TsFile::SIMDe` target. -- Write the kernels with `simde__m256i` and `simde_mm256_*`, matching the - existing encoding code style. -- The kernels are header-only and compile into baseline TsFile translation - units with `ENABLE_SIMD`; there is no separate `-mavx2` translation unit and - no runtime CPU dispatch yet. On GCC/Clang, SIMDe annotates the AVX2 helpers - with target attributes, so native AVX2 instructions may be emitted inside - outlined helper calls; on MSVC without `/arch:AVX2` SIMDe emulates them. - A dedicated AVX2 object library plus runtime dispatch is a follow-up. -- On AArch64 the same SIMDe source maps to NEON without the AVX2 flag. -- Add a scalar fallback used when SIMD is disabled or unavailable. -- Select the kernel at compile time through `ENABLE_SIMD`; there is no runtime - dispatch or function table yet. +- Write the portable kernels with `simde__m256i` and `simde_mm256_*` in + `alp_simde.h`, matching the existing encoding code style. +- The portable kernels are header-only and compile into baseline TsFile + translation units with `ENABLE_SIMD`. On AArch64 the same SIMDe source maps + to NEON without the AVX2 flag; on MSVC, 32-bit x86 and CPUs without AVX2 + they are the fallback. +- On x86-64 with GCC/Clang, `alp_avx2.h` carries native AVX2 copies of the + `EncodeValues`, `DecodeValues` and bit-unpack kernels, compiled with + `TSFILE_TARGET_AVX2` (`__attribute__((target("avx2")))`). ALP AVX2 is + compiled only for x86-64 (`ALP_AVX2_DISPATCH`); 32-bit x86 keeps the SIMDe + path. The dispatchers `AlpSimdKernel` / `AlpSimdUnpackKernel` in + `alp_simde.h` select the AVX2 copies once per batch call through + `common::cpu_has_avx2()` from `common/simd_dispatch.h`. + The library is not built with `-mavx2`, so the binary's ISA floor does not + change: a CPU without AVX2 runs the SIMDe path, and both paths must encode + to identical bytes and decode to bit-identical values. +- Add a scalar fallback used when SIMD is disabled. - The 64-bit unpack kernel reconstructs one value from a single 64-bit window, so it only handles fields up to 57 bits wide; wider fields are decoded by the scalar tail, which reads two overlapping words. Keep that limit in sync @@ -179,11 +184,14 @@ SIMDe is already a TsFile C++ dependency. The ALP kernels will use the existing - Constant blocks, single-value blocks, partial final blocks, and multi-block pages. - Scalar and SIMD must encode to identical bytes and decode to identical - values. + values. On AVX2-capable x86-64 machines, differential tests also compare the + dispatched AVX2 kernels against the SIMDe kernels directly. - Corrupt/truncated payload tests must return errors without out-of-bounds access. - Benchmarks compare ALP against Gorilla for decimal, normalized, smooth, - repeated, and high-entropy data. + repeated, and high-entropy data. `cpp/bench_mark/alp_avx2_benchmark.cc` + measures the SIMDe and AVX2 kernels per entry point in a default + (no `-mavx2`) build. ## References and Provenance diff --git a/cpp/src/encoding/alp/alp_avx2.h b/cpp/src/encoding/alp/alp_avx2.h new file mode 100644 index 000000000..01a4b4db8 --- /dev/null +++ b/cpp/src/encoding/alp/alp_avx2.h @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_ALP_AVX2_H +#define ENCODING_ALP_AVX2_H + +// Native AVX2 copies of the ALP SIMD kernels. The library is built for the +// portable x86-64 baseline, so the SIMDe kernels in alp_simde.h compile into +// SIMDe's portable fallback (in particular the element-wise emulated gathers +// in the bit-unpack). These copies are compiled with +// __attribute__((target("avx2"))) and selected at runtime by +// common::cpu_has_avx2(); see common/simd_dispatch.h. The portable SIMDe +// kernels remain the fallback for non-x86-64 targets, MSVC, and CPUs without +// AVX2, and both paths must produce identical bytes and values. + +#include + +#include + +#include "alp_traits.h" +#include "common/simd_dispatch.h" + +#ifdef TSFILE_X86_AVX2_DISPATCH +// These kernels use the 64-bit scalar conversion instruction, which 32-bit x86 +// does not provide without undefined behavior, so ALP dispatch is x86-64 only. +// The RLE packers keep the wider TSFILE_X86_AVX2_DISPATCH definition. +#if defined(__x86_64__) +#define ALP_AVX2_DISPATCH 1 + +#include + +namespace storage { +namespace alp { +namespace avx2_detail { + +// AVX2 has no packed double -> int64 conversion (that arrives with AVX-512DQ), +// so each lane uses the scalar SSE2 conversion instruction. It truncates like +// _mm256_cvttps_epi32 and yields the "integer indefinite" value for NaN and +// out-of-range inputs; the callers range-check the result before using it. +TSFILE_TARGET_AVX2 +static inline __m256i AlpAvx2CvttpdEpi64(__m256d v) { + const __m128d lo = _mm256_castpd256_pd128(v); + const __m128d hi = _mm256_extractf128_pd(v, 1); + const long long l0 = _mm_cvttsd_si64(lo); + const long long l1 = _mm_cvttsd_si64(_mm_unpackhi_pd(lo, lo)); + const long long h0 = _mm_cvttsd_si64(hi); + const long long h1 = _mm_cvttsd_si64(_mm_unpackhi_pd(hi, hi)); + return _mm256_setr_epi64x(l0, l1, h0, h1); +} + +// Same story for int64 -> double: no packed AVX2 conversion. The scalar +// conversion is the one the scalar reference and the SIMDe fallback use, so +// the rounding is identical. +TSFILE_TARGET_AVX2 +static inline __m256d AlpAvx2Cvtepi64Pd(__m256i v) { + alignas(32) int64_t lanes[4]; + std::memcpy(lanes, &v, sizeof(lanes)); + return _mm256_setr_pd( + static_cast(lanes[0]), static_cast(lanes[1]), + static_cast(lanes[2]), static_cast(lanes[3])); +} + +template +struct AlpAvx2Kernel; + +template <> +struct AlpAvx2Kernel { + TSFILE_TARGET_AVX2 + static AlpStatus EncodeValues(const float* values, uint32_t count, + uint8_t factor, uint8_t exponent, + int32_t* encoded, uint8_t* bitmap, + float* exceptions, + uint32_t* exception_count) { + typedef AlpTypeTraits Traits; + const float factor_value = static_cast(ALP_FACT[factor]); + const float frac_value = ALP_FLOAT_FRAC[exponent]; + + // Keep the scalar reference's multiplication order -- (value * + // 10^exponent) * 10^-factor -- so both paths emit identical bytes. + const __m256 vexp = _mm256_set1_ps(ALP_FLOAT_EXP[exponent]); + const __m256 vfrac_factor = _mm256_set1_ps(ALP_FLOAT_FRAC[factor]); + const __m256 vmagic = _mm256_set1_ps(ALP_FLOAT_MAGIC); + const __m256 vfactor = _mm256_set1_ps(factor_value); + const __m256 vfrac = _mm256_set1_ps(frac_value); + const __m256 vlower = _mm256_set1_ps(ALP_FLOAT_ENCODING_LOWER_LIMIT); + const __m256 vupper = _mm256_set1_ps(ALP_FLOAT_ENCODING_UPPER_LIMIT); + const __m256i vnegzero = + _mm256_set1_epi32(static_cast(0x80000000u)); + const __m256i vall = _mm256_set1_epi32(-1); + + uint32_t ex_count = 0; + uint32_t i = 0; + for (; i + 7 < count; i += 8) { + const __m256 v = _mm256_loadu_ps(values + i); + const __m256 scaled = + _mm256_mul_ps(_mm256_mul_ps(v, vexp), vfrac_factor); + const __m256 rounded = + _mm256_sub_ps(_mm256_add_ps(scaled, vmagic), vmagic); + const __m256i vi = _mm256_cvttps_epi32(rounded); + const __m256 back = _mm256_mul_ps( + _mm256_mul_ps(_mm256_cvtepi32_ps(vi), vfactor), vfrac); + + const __m256 in_range = + _mm256_and_ps(_mm256_cmp_ps(scaled, vlower, _CMP_GE_OQ), + _mm256_cmp_ps(scaled, vupper, _CMP_LE_OQ)); + const __m256 rounded_in_range = + _mm256_and_ps(_mm256_cmp_ps(rounded, vlower, _CMP_GE_OQ), + _mm256_cmp_ps(rounded, vupper, _CMP_LE_OQ)); + const __m256 ordered = _mm256_cmp_ps(v, v, _CMP_ORD_Q); + const __m256 equal = _mm256_cmp_ps(back, v, _CMP_EQ_OQ); + const __m256i is_negzero = + _mm256_cmpeq_epi32(_mm256_castps_si256(v), vnegzero); + const __m256 not_negzero = + _mm256_castsi256_ps(_mm256_xor_si256(is_negzero, vall)); + const __m256 ok = _mm256_and_ps( + _mm256_and_ps(in_range, rounded_in_range), + _mm256_and_ps(ordered, _mm256_and_ps(equal, not_negzero))); + const int mask = _mm256_movemask_ps(ok); + + _mm256_storeu_si256(reinterpret_cast<__m256i*>(encoded + i), vi); + for (int lane = 0; lane < 8; ++lane) { + if ((mask & (1 << lane)) == 0) { + const uint32_t index = i + static_cast(lane); + bitmap[index >> 3] |= + static_cast(1u << (index & 7u)); + exceptions[ex_count++] = values[index]; + encoded[index] = 0; + } + } + } + + for (; i < count; ++i) { + int32_t value = 0; + if (!Traits::EncodeValue(values[i], factor, exponent, &value) || + !Traits::BitwiseEqual( + Traits::DecodeValue(value, factor, exponent), values[i])) { + bitmap[i >> 3] |= static_cast(1u << (i & 7u)); + exceptions[ex_count++] = values[i]; + encoded[i] = 0; + } else { + encoded[i] = value; + } + } + *exception_count = ex_count; + return ALP_OK; + } + + TSFILE_TARGET_AVX2 + static AlpStatus DecodeValues(const int32_t* encoded, uint32_t count, + uint8_t factor, uint8_t exponent, + const uint8_t* bitmap, + const float* exceptions, + uint32_t exception_count, float* out) { + typedef AlpTypeTraits Traits; + const __m256 vfactor = + _mm256_set1_ps(static_cast(ALP_FACT[factor])); + const __m256 vfrac = _mm256_set1_ps(ALP_FLOAT_FRAC[exponent]); + + uint32_t i = 0; + for (; i + 7 < count; i += 8) { + const __m256i vi = _mm256_loadu_si256( + reinterpret_cast(encoded + i)); + const __m256 value = _mm256_mul_ps( + _mm256_mul_ps(_mm256_cvtepi32_ps(vi), vfactor), vfrac); + _mm256_storeu_ps(out + i, value); + } + for (; i < count; ++i) { + out[i] = Traits::DecodeValue(encoded[i], factor, exponent); + } + + uint32_t exception_index = 0; + for (uint32_t index = 0; index < count; ++index) { + if (bitmap != nullptr && + (bitmap[index >> 3] & + static_cast(1u << (index & 7u))) != 0) { + if (exception_index >= exception_count) { + return ALP_MALFORMED; + } + out[index] = exceptions[exception_index++]; + } + } + return exception_index == exception_count ? ALP_OK : ALP_MALFORMED; + } +}; + +template <> +struct AlpAvx2Kernel { + TSFILE_TARGET_AVX2 + static AlpStatus EncodeValues(const double* values, uint32_t count, + uint8_t factor, uint8_t exponent, + int64_t* encoded, uint8_t* bitmap, + double* exceptions, + uint32_t* exception_count) { + typedef AlpTypeTraits Traits; + const double factor_value = static_cast(ALP_FACT[factor]); + const double frac_value = ALP_DOUBLE_FRAC[exponent]; + + // Same multiplication order as the scalar reference. + const __m256d vexp = _mm256_set1_pd(ALP_DOUBLE_EXP[exponent]); + const __m256d vfrac_factor = _mm256_set1_pd(ALP_DOUBLE_FRAC[factor]); + const __m256d vmagic = _mm256_set1_pd(ALP_DOUBLE_MAGIC); + const __m256d vfactor = _mm256_set1_pd(factor_value); + const __m256d vfrac = _mm256_set1_pd(frac_value); + const __m256d vlower = _mm256_set1_pd(ALP_DOUBLE_ENCODING_LOWER_LIMIT); + const __m256d vupper = _mm256_set1_pd(ALP_DOUBLE_ENCODING_UPPER_LIMIT); + const __m256i vnegzero = + _mm256_set1_epi64x(static_cast(0x8000000000000000ULL)); + const __m256i vall = _mm256_set1_epi64x(-1); + + uint32_t ex_count = 0; + uint32_t i = 0; + for (; i + 3 < count; i += 4) { + const __m256d v = _mm256_loadu_pd(values + i); + const __m256d scaled = + _mm256_mul_pd(_mm256_mul_pd(v, vexp), vfrac_factor); + const __m256d rounded = + _mm256_sub_pd(_mm256_add_pd(scaled, vmagic), vmagic); + const __m256i vi = AlpAvx2CvttpdEpi64(rounded); + const __m256d back = _mm256_mul_pd( + _mm256_mul_pd(AlpAvx2Cvtepi64Pd(vi), vfactor), vfrac); + + const __m256d in_range = + _mm256_and_pd(_mm256_cmp_pd(scaled, vlower, _CMP_GE_OQ), + _mm256_cmp_pd(scaled, vupper, _CMP_LE_OQ)); + const __m256d rounded_in_range = + _mm256_and_pd(_mm256_cmp_pd(rounded, vlower, _CMP_GE_OQ), + _mm256_cmp_pd(rounded, vupper, _CMP_LE_OQ)); + const __m256d ordered = _mm256_cmp_pd(v, v, _CMP_ORD_Q); + const __m256d equal = _mm256_cmp_pd(back, v, _CMP_EQ_OQ); + const __m256i is_negzero = + _mm256_cmpeq_epi64(_mm256_castpd_si256(v), vnegzero); + const __m256d not_negzero = + _mm256_castsi256_pd(_mm256_xor_si256(is_negzero, vall)); + const __m256d ok = _mm256_and_pd( + _mm256_and_pd(in_range, rounded_in_range), + _mm256_and_pd(ordered, _mm256_and_pd(equal, not_negzero))); + const int mask = _mm256_movemask_pd(ok); + + _mm256_storeu_si256(reinterpret_cast<__m256i*>(encoded + i), vi); + for (int lane = 0; lane < 4; ++lane) { + if ((mask & (1 << lane)) == 0) { + const uint32_t index = i + static_cast(lane); + bitmap[index >> 3] |= + static_cast(1u << (index & 7u)); + exceptions[ex_count++] = values[index]; + encoded[index] = 0; + } + } + } + + for (; i < count; ++i) { + int64_t value = 0; + if (!Traits::EncodeValue(values[i], factor, exponent, &value) || + !Traits::BitwiseEqual( + Traits::DecodeValue(value, factor, exponent), values[i])) { + bitmap[i >> 3] |= static_cast(1u << (i & 7u)); + exceptions[ex_count++] = values[i]; + encoded[i] = 0; + } else { + encoded[i] = value; + } + } + *exception_count = ex_count; + return ALP_OK; + } + + TSFILE_TARGET_AVX2 + static AlpStatus DecodeValues(const int64_t* encoded, uint32_t count, + uint8_t factor, uint8_t exponent, + const uint8_t* bitmap, + const double* exceptions, + uint32_t exception_count, double* out) { + typedef AlpTypeTraits Traits; + const __m256d vfactor = + _mm256_set1_pd(static_cast(ALP_FACT[factor])); + const __m256d vfrac = _mm256_set1_pd(ALP_DOUBLE_FRAC[exponent]); + + uint32_t i = 0; + for (; i + 3 < count; i += 4) { + const __m256i vi = _mm256_loadu_si256( + reinterpret_cast(encoded + i)); + const __m256d value = _mm256_mul_pd( + _mm256_mul_pd(AlpAvx2Cvtepi64Pd(vi), vfactor), vfrac); + _mm256_storeu_pd(out + i, value); + } + for (; i < count; ++i) { + out[i] = Traits::DecodeValue(encoded[i], factor, exponent); + } + + uint32_t exception_index = 0; + for (uint32_t index = 0; index < count; ++index) { + if (bitmap != nullptr && + (bitmap[index >> 3] & + static_cast(1u << (index & 7u))) != 0) { + if (exception_index >= exception_count) { + return ALP_MALFORMED; + } + out[index] = exceptions[exception_index++]; + } + } + return exception_index == exception_count ? ALP_OK : ALP_MALFORMED; + } +}; + +// Both unpack kernels gather one 64-bit window per lane, so a lane can read up +// to 8 bytes starting at that value's byte offset. Callers must pass the padded +// ALP body produced by AlpPackBits, which reserves at least 16 zero guard bytes +// past the logical stream; AlpDecodeBlock validates body_bytes against that +// padded size before unpacking. The kernels process whole groups of four only: +// AlpUnpackBits passes count & ~3 and decodes the remaining values itself. +template +struct AlpAvx2UnpackKernel; + +template <> +struct AlpAvx2UnpackKernel { + TSFILE_TARGET_AVX2 + static bool Unpack(const uint8_t* body, uint32_t count, uint8_t bit_width, + uint32_t* out) { + if (bit_width == 0) { + for (uint32_t i = 0; i < count; ++i) { + out[i] = 0; + } + return true; + } + const uint64_t mask64 = + bit_width >= 32 ? static_cast(0xFFFFFFFFu) + : ((static_cast(1) << bit_width) - 1); + const __m256i mask = _mm256_set1_epi64x(static_cast(mask64)); + const __m256i perm = _mm256_setr_epi32(0, 2, 4, 6, 0, 0, 0, 0); + for (uint32_t i = 0; i + 3 < count; i += 4) { + int32_t byte_offsets[4]; + int64_t bit_offsets[4]; + for (int lane = 0; lane < 4; ++lane) { + const uint64_t bit_pos = + static_cast(i + lane) * bit_width; + byte_offsets[lane] = static_cast(bit_pos >> 3); + bit_offsets[lane] = static_cast(bit_pos & 7u); + } + const __m128i indices = + _mm_loadu_si128(reinterpret_cast(byte_offsets)); + const __m256i windows = _mm256_i32gather_epi64( + reinterpret_cast(body), indices, 1); + const __m256i shifts = _mm256_loadu_si256( + reinterpret_cast(bit_offsets)); + const __m256i shifted = _mm256_srlv_epi64(windows, shifts); + const __m256i masked = _mm256_and_si256(shifted, mask); + const __m256i compact = _mm256_permutevar8x32_epi32(masked, perm); + _mm_storeu_si128(reinterpret_cast<__m128i*>(out + i), + _mm256_castsi256_si128(compact)); + } + return true; + } +}; + +template <> +struct AlpAvx2UnpackKernel { + TSFILE_TARGET_AVX2 + static bool Unpack(const uint8_t* body, uint32_t count, uint8_t bit_width, + uint64_t* out) { + if (bit_width == 0) { + for (uint32_t i = 0; i < count; ++i) { + out[i] = 0; + } + return true; + } + const uint64_t mask64 = + bit_width >= 64 ? ~static_cast(0) + : ((static_cast(1) << bit_width) - 1); + const __m256i mask = _mm256_set1_epi64x(static_cast(mask64)); + for (uint32_t i = 0; i + 3 < count; i += 4) { + int32_t byte_offsets[4]; + int64_t bit_offsets[4]; + for (int lane = 0; lane < 4; ++lane) { + const uint64_t bit_pos = + static_cast(i + lane) * bit_width; + byte_offsets[lane] = static_cast(bit_pos >> 3); + bit_offsets[lane] = static_cast(bit_pos & 7u); + } + const __m128i indices = + _mm_loadu_si128(reinterpret_cast(byte_offsets)); + const __m256i windows = _mm256_i32gather_epi64( + reinterpret_cast(body), indices, 1); + const __m256i shifts = _mm256_loadu_si256( + reinterpret_cast(bit_offsets)); + const __m256i shifted = _mm256_srlv_epi64(windows, shifts); + const __m256i masked = _mm256_and_si256(shifted, mask); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(out + i), masked); + } + return true; + } +}; + +} // namespace avx2_detail +} // namespace alp +} // namespace storage + +#endif // __x86_64__ +#endif // TSFILE_X86_AVX2_DISPATCH + +#endif // ENCODING_ALP_AVX2_H diff --git a/cpp/src/encoding/alp/alp_simde.h b/cpp/src/encoding/alp/alp_simde.h index a00aee6d6..e334bd24b 100644 --- a/cpp/src/encoding/alp/alp_simde.h +++ b/cpp/src/encoding/alp/alp_simde.h @@ -23,19 +23,26 @@ #include #include "alp_traits.h" +#include "common/simd_dispatch.h" #include "simde/x86/avx2.h" #include "simde/x86/avx512/cvt.h" #include "simde/x86/avx512/cvtt.h" #include "simde/x86/sse4.1.h" +#ifdef TSFILE_X86_AVX2_DISPATCH +#include "alp_avx2.h" +#endif namespace storage { namespace alp { +// The SIMDe implementation below is the portable fallback. On x86-64 with +// GCC/Clang the dispatchers below pick the native AVX2 copy from alp_avx2.h at +// runtime when the CPU supports it. template -struct AlpSimdKernel; +struct AlpSimdeKernel; template <> -struct AlpSimdKernel { +struct AlpSimdeKernel { static AlpStatus EncodeValues(const float* values, uint32_t count, uint8_t factor, uint8_t exponent, int32_t* encoded, uint8_t* bitmap, @@ -115,6 +122,7 @@ struct AlpSimdKernel { Traits::DecodeValue(value, factor, exponent), values[i])) { bitmap[i >> 3] |= static_cast(1u << (i & 7u)); exceptions[ex_count++] = values[i]; + encoded[i] = 0; } else { encoded[i] = value; } @@ -162,7 +170,7 @@ struct AlpSimdKernel { }; template <> -struct AlpSimdKernel { +struct AlpSimdeKernel { static AlpStatus EncodeValues(const double* values, uint32_t count, uint8_t factor, uint8_t exponent, int64_t* encoded, uint8_t* bitmap, @@ -238,6 +246,7 @@ struct AlpSimdKernel { Traits::DecodeValue(value, factor, exponent), values[i])) { bitmap[i >> 3] |= static_cast(1u << (i & 7u)); exceptions[ex_count++] = values[i]; + encoded[i] = 0; } else { encoded[i] = value; } @@ -283,6 +292,42 @@ struct AlpSimdKernel { } }; +template +struct AlpSimdKernel { + static AlpStatus EncodeValues(const T* values, uint32_t count, + uint8_t factor, uint8_t exponent, + typename AlpTypeTraits::Encoded* encoded, + uint8_t* bitmap, T* exceptions, + uint32_t* exception_count) { +#ifdef ALP_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + return avx2_detail::AlpAvx2Kernel::EncodeValues( + values, count, factor, exponent, encoded, bitmap, exceptions, + exception_count); + } +#endif + return AlpSimdeKernel::EncodeValues(values, count, factor, exponent, + encoded, bitmap, exceptions, + exception_count); + } + + static AlpStatus DecodeValues( + const typename AlpTypeTraits::Encoded* encoded, uint32_t count, + uint8_t factor, uint8_t exponent, const uint8_t* bitmap, + const T* exceptions, uint32_t exception_count, T* out) { +#ifdef ALP_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + return avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, count, factor, exponent, bitmap, exceptions, + exception_count, out); + } +#endif + return AlpSimdeKernel::DecodeValues(encoded, count, factor, exponent, + bitmap, exceptions, + exception_count, out); + } +}; + template inline AlpStatus AlpSimdEncodeValues( const T* values, uint32_t count, uint8_t factor, uint8_t exponent, @@ -304,10 +349,10 @@ inline AlpStatus AlpSimdDecodeValues( } template -struct AlpSimdUnpackKernel; +struct AlpSimdeUnpackKernel; template <> -struct AlpSimdUnpackKernel { +struct AlpSimdeUnpackKernel { // A 32-bit value plus a 0..7 bit offset still fits in the gathered word. static uint8_t SafeBitWidth() { return 32; } @@ -354,7 +399,7 @@ struct AlpSimdUnpackKernel { }; template <> -struct AlpSimdUnpackKernel { +struct AlpSimdeUnpackKernel { // Each lane gathers a single 64-bit window and shifts it right by the bit // offset, so at most 64 - 7 bits survive. Wider fields are decoded by the // scalar tail, which spans two words. @@ -398,6 +443,25 @@ struct AlpSimdUnpackKernel { } }; +template +struct AlpSimdUnpackKernel { + static uint8_t SafeBitWidth() { + return AlpSimdeUnpackKernel::SafeBitWidth(); + } + + static bool Unpack(const uint8_t* body, uint32_t count, uint8_t bit_width, + Unsigned* out) { +#ifdef ALP_AVX2_DISPATCH + if (common::cpu_has_avx2()) { + return avx2_detail::AlpAvx2UnpackKernel::Unpack( + body, count, bit_width, out); + } +#endif + return AlpSimdeUnpackKernel::Unpack(body, count, bit_width, + out); + } +}; + template inline bool AlpSimdUnpackBits(const uint8_t* body, uint32_t count, uint8_t bit_width, Unsigned* out) { diff --git a/cpp/test/encoding/alp_simde_test.cc b/cpp/test/encoding/alp_simde_test.cc index cd67a4964..05f5b0e3e 100644 --- a/cpp/test/encoding/alp_simde_test.cc +++ b/cpp/test/encoding/alp_simde_test.cc @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include @@ -239,6 +240,338 @@ TEST(AlpSimdeTest, UnpackMatchesReferenceAcrossBitWidths) { } } +#ifdef ALP_AVX2_DISPATCH + +// The AlpSimdeTest cases above run through the dispatched entry points, so on +// an AVX2 CPU they exercise the native kernels. These tests compare the AVX2 +// kernels against the SIMDe fallback directly, so a mismatch cannot be hidden +// by the scalar reference agreeing with both. + +template +static bool EncodeWithSimde( + const std::vector& values, uint8_t factor, uint8_t exponent, + std::vector::Encoded>& encoded, + std::vector& bitmap, std::vector& exceptions, + uint32_t* exception_count) { + typedef typename AlpTypeTraits::Encoded Encoded; + const uint32_t count = static_cast(values.size()); + // A sentinel so a slot the kernel forgets to write is visible below. + encoded.assign(count, static_cast(0x5A5A5A5A)); + bitmap.assign((count + 7) / 8, 0); + exceptions.assign(count, T()); + *exception_count = 0; + return AlpSimdeKernel::EncodeValues( + values.empty() ? nullptr : &values[0], count, factor, exponent, + encoded.empty() ? nullptr : &encoded[0], + bitmap.empty() ? nullptr : &bitmap[0], + exceptions.empty() ? nullptr : &exceptions[0], + exception_count) == ALP_OK; +} + +template +static bool EncodeWithAvx2( + const std::vector& values, uint8_t factor, uint8_t exponent, + std::vector::Encoded>& encoded, + std::vector& bitmap, std::vector& exceptions, + uint32_t* exception_count) { + typedef typename AlpTypeTraits::Encoded Encoded; + const uint32_t count = static_cast(values.size()); + encoded.assign(count, static_cast(0x5A5A5A5A)); + bitmap.assign((count + 7) / 8, 0); + exceptions.assign(count, T()); + *exception_count = 0; + return avx2_detail::AlpAvx2Kernel::EncodeValues( + values.empty() ? nullptr : &values[0], count, factor, exponent, + encoded.empty() ? nullptr : &encoded[0], + bitmap.empty() ? nullptr : &bitmap[0], + exceptions.empty() ? nullptr : &exceptions[0], + exception_count) == ALP_OK; +} + +template +static void ExpectAvx2MatchesSimdeForPair(const std::vector& values, + uint8_t factor, uint8_t exponent) { + typedef AlpTypeTraits Traits; + typedef typename Traits::Encoded Encoded; + + std::vector simde_encoded; + std::vector simde_bitmap; + std::vector simde_exceptions; + std::vector avx2_encoded; + std::vector avx2_bitmap; + std::vector avx2_exceptions; + + uint32_t simde_count = 0; + uint32_t avx2_count = 0; + ASSERT_TRUE(EncodeWithSimde(values, factor, exponent, simde_encoded, + simde_bitmap, simde_exceptions, &simde_count)); + ASSERT_TRUE(EncodeWithAvx2(values, factor, exponent, avx2_encoded, + avx2_bitmap, avx2_exceptions, &avx2_count)); + ASSERT_EQ(simde_count, avx2_count); + EXPECT_EQ(simde_bitmap, avx2_bitmap); + EXPECT_EQ(simde_encoded, avx2_encoded); + // Exception slots must be written deterministically even when the caller + // does not pre-zero the output (the buffers start at a sentinel). + for (uint32_t i = 0; i < static_cast(values.size()); ++i) { + if ((simde_bitmap[i >> 3] & static_cast(1u << (i & 7u))) != + 0) { + EXPECT_EQ(0, simde_encoded[i]) << "SIMDe exception slot " << i; + EXPECT_EQ(0, avx2_encoded[i]) << "AVX2 exception slot " << i; + } + } + for (uint32_t i = 0; i < simde_count; ++i) { + EXPECT_TRUE( + Traits::BitwiseEqual(simde_exceptions[i], avx2_exceptions[i])) + << "exception index " << i; + } + + const uint32_t count = static_cast(values.size()); + std::vector simde_decoded(count); + std::vector avx2_decoded(count); + const uint8_t* bitmap_ptr = + simde_bitmap.empty() ? nullptr : &simde_bitmap[0]; + const T* exception_ptr = simde_count == 0 ? nullptr : &simde_exceptions[0]; + ASSERT_EQ(ALP_OK, + AlpSimdeKernel::DecodeValues( + simde_encoded.empty() ? nullptr : &simde_encoded[0], count, + factor, exponent, bitmap_ptr, exception_ptr, simde_count, + simde_decoded.empty() ? nullptr : &simde_decoded[0])); + const uint32_t avx2_status = avx2_detail::AlpAvx2Kernel::DecodeValues( + simde_encoded.empty() ? nullptr : &simde_encoded[0], count, factor, + exponent, bitmap_ptr, exception_ptr, simde_count, + avx2_decoded.empty() ? nullptr : &avx2_decoded[0]); + ASSERT_EQ(ALP_OK, avx2_status); + for (uint32_t i = 0; i < count; ++i) { + EXPECT_TRUE(Traits::BitwiseEqual(simde_decoded[i], avx2_decoded[i])) + << "decoded index " << i; + } +} + +template +static void ExpectAvx2MatchesSimde(const std::vector& values) { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + typedef AlpTypeTraits Traits; + for (uint8_t exponent = 0; exponent <= Traits::MaxExponent(); ++exponent) { + for (uint8_t factor = 0; factor <= exponent; ++factor) { + ExpectAvx2MatchesSimdeForPair(values, factor, exponent); + } + } +} + +TEST(AlpAvx2Test, FloatValuesMatchSimde) { + std::vector values; + for (int i = 0; i < 1024; ++i) { + values.push_back(static_cast(i % 251) * 0.01f - 1.25f); + } + ExpectAvx2MatchesSimde(values); +} + +TEST(AlpAvx2Test, DoubleValuesMatchSimde) { + std::vector values; + for (int i = 0; i < 1024; ++i) { + values.push_back(static_cast(i % 509) * 0.001 - 0.25); + } + ExpectAvx2MatchesSimde(values); +} + +TEST(AlpAvx2Test, SpecialValuesMatchSimde) { + const float nan_f = std::numeric_limits::quiet_NaN(); + const double nan_d = std::numeric_limits::quiet_NaN(); + std::vector floats = {0.0f, + -0.0f, + 1.0f, + -1.0f, + nan_f, + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::denorm_min(), + std::numeric_limits::max()}; + ExpectAvx2MatchesSimde(floats); + + std::vector doubles = {0.0, + -0.0, + 1.0, + -1.0, + nan_d, + std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), + std::numeric_limits::denorm_min(), + std::numeric_limits::max()}; + ExpectAvx2MatchesSimde(doubles); +} + +template +static void ExpectAvx2MatchesSimdeAroundVectorBoundaries() { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + const uint32_t counts[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, 15, 16, 17, 23, 24, 25, 31, 32, + 33, 63, 64, 65, 127, 128, 129, 1024}; + uint64_t state = 0x9e3779b97f4a7c15ULL; + std::vector maximum(counts[sizeof(counts) / sizeof(counts[0]) - 1]); + for (size_t i = 0; i < maximum.size(); ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + const double fraction = static_cast(state % 100000) / 1000.0; + maximum[i] = static_cast(fraction - 50.0); + } + for (size_t c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) { + const std::vector values(maximum.begin(), + maximum.begin() + counts[c]); + // A pair that encodes the decimal data plus one that rejects most of + // it, so both the vector body and the exception path are covered. + ExpectAvx2MatchesSimdeForPair(values, 0, 0); + ExpectAvx2MatchesSimdeForPair(values, 3, 3); + ExpectAvx2MatchesSimdeForPair(values, 2, 2); + } +} + +TEST(AlpAvx2Test, FloatAroundVectorBoundaries) { + ExpectAvx2MatchesSimdeAroundVectorBoundaries(); +} + +TEST(AlpAvx2Test, DoubleAroundVectorBoundaries) { + ExpectAvx2MatchesSimdeAroundVectorBoundaries(); +} + +template +static void ExpectAvx2UnpackMatchesSimde(uint8_t bit_width, uint32_t count) { + const Unsigned mask = + bit_width >= sizeof(Unsigned) * 8 + ? static_cast(~static_cast(0)) + : static_cast((static_cast(1) << bit_width) - + 1); + std::vector values(count); + uint64_t state = 0x243f6a8885a308d3ULL; + for (size_t i = 0; i < values.size(); ++i) { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + values[i] = static_cast(state) & mask; + } + std::vector body; + AlpPackBits(values.empty() ? nullptr : &values[0], count, bit_width, body); + const uint8_t dummy = 0; + const uint8_t* body_ptr = body.empty() ? &dummy : &body[0]; + + std::vector simde_out(count, 0); + std::vector avx2_out(count, 0); + ASSERT_TRUE(AlpSimdeUnpackKernel::Unpack( + body_ptr, count, bit_width, simde_out.data())); + ASSERT_TRUE(avx2_detail::AlpAvx2UnpackKernel::Unpack( + body_ptr, count, bit_width, avx2_out.data())); + EXPECT_EQ(simde_out, avx2_out) + << "bit_width=" << static_cast(bit_width) << " count=" << count; + // The kernels only cover whole groups of four; compare every value they + // did process against the input that was packed, so a mistake shared by + // both implementations cannot hide behind their agreement. + const uint32_t processed = count & ~static_cast(3); + for (uint32_t i = 0; i < processed; ++i) { + EXPECT_EQ(values[i], avx2_out[i]) + << "bit_width=" << static_cast(bit_width) << " count=" << count + << " index=" << i; + } +} + +template +static void ExpectAvx2UnpackMatchesSimdeAcrossBitWidths() { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + const uint32_t counts[] = {0, 1, 2, 3, 4, 5, 7, 8, 9, 31, 64, 65, 1024}; + const uint8_t max_width = AlpSimdeUnpackKernel::SafeBitWidth(); + for (uint8_t bit_width = 0; bit_width <= max_width; ++bit_width) { + for (size_t c = 0; c < sizeof(counts) / sizeof(counts[0]); ++c) { + ExpectAvx2UnpackMatchesSimde(bit_width, counts[c]); + } + } +} + +TEST(AlpAvx2Test, UnpackUint32MatchesSimde) { + ExpectAvx2UnpackMatchesSimdeAcrossBitWidths(); +} + +TEST(AlpAvx2Test, UnpackUint64MatchesSimde) { + ExpectAvx2UnpackMatchesSimdeAcrossBitWidths(); +} + +TEST(AlpAvx2Test, DecodeMalformedStatusMatchesSimde) { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + // Bitmap position 1 is marked as an exception but no exception value is + // supplied: both paths must reject the block rather than read past the + // exception array. + const int32_t encoded[4] = {1, 2, 3, 4}; + const uint8_t bitmap[1] = {0x02}; + float out[4] = {0, 0, 0, 0}; + EXPECT_EQ(ALP_MALFORMED, AlpSimdeKernel::DecodeValues( + encoded, 4, 0, 0, bitmap, nullptr, 0, out)); + EXPECT_EQ(ALP_MALFORMED, avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, 4, 0, 0, bitmap, nullptr, 0, out)); + + // One exception value but no bitmap position to consume it. + const float exceptions[1] = {5.0f}; + EXPECT_EQ(ALP_MALFORMED, AlpSimdeKernel::DecodeValues( + encoded, 4, 0, 0, bitmap, exceptions, 2, out)); + EXPECT_EQ(ALP_MALFORMED, avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, 4, 0, 0, bitmap, exceptions, 2, out)); +} + +// AVX2 has no packed int64 -> double conversion, so the AVX2 kernel uses a +// scalar conversion per lane while SIMDe uses a split-arithmetic trick. Check +// the values where that conversion is most delicate, against SIMDe and against +// the scalar reference (factor 0, exponent 0 decodes to (double)encoded). +TEST(AlpAvx2Test, DoubleDecodeAtInt64ExtremesMatchesReference) { + if (!common::cpu_has_avx2()) { + GTEST_SKIP() << "CPU does not support AVX2"; + return; + } + typedef AlpTypeTraits Traits; + const int64_t kMin = std::numeric_limits::min(); + const int64_t kMax = std::numeric_limits::max(); + const int64_t encoded[16] = { + kMin, + kMin + 1, + -((int64_t)1 << 62), + -((int64_t)1 << 53), + -((int64_t)1 << 52) - 1, + -1, + 0, + 1, + ((int64_t)1 << 52) - 1, + (int64_t)1 << 52, + ((int64_t)1 << 52) + 1, + (int64_t)1 << 53, + ((int64_t)1 << 62) - 1, + (int64_t)1 << 62, + kMax - 1, + kMax, + }; + const uint32_t count = sizeof(encoded) / sizeof(encoded[0]); + std::vector simde_out(count); + std::vector avx2_out(count); + ASSERT_EQ(ALP_OK, + AlpSimdeKernel::DecodeValues( + encoded, count, 0, 0, nullptr, nullptr, 0, &simde_out[0])); + ASSERT_EQ(ALP_OK, + avx2_detail::AlpAvx2Kernel::DecodeValues( + encoded, count, 0, 0, nullptr, nullptr, 0, &avx2_out[0])); + for (uint32_t i = 0; i < count; ++i) { + EXPECT_TRUE(Traits::BitwiseEqual(simde_out[i], avx2_out[i])) + << "SIMDe mismatch at index " << i; + EXPECT_TRUE(Traits::BitwiseEqual(Traits::DecodeValue(encoded[i], 0, 0), + avx2_out[i])) + << "reference mismatch at index " << i; + } +} + +#endif // ALP_AVX2_DISPATCH + #endif // ENABLE_SIMD } // namespace alp